Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # Copyright (C) 2023 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | # |
| 16 | """Call cargo -v, parse its output, and generate a Trusty build system module. |
| 17 | |
| 18 | Usage: Run this script in a crate workspace root directory. The Cargo.toml file |
| 19 | should work at least for the host platform. |
| 20 | |
| 21 | Without other flags, "cargo2rulesmk.py --run" calls cargo clean, calls cargo |
| 22 | build -v, and generates makefile rules. The cargo build only generates crates |
| 23 | for the host without test crates. |
| 24 | |
| 25 | If there are rustc warning messages, this script will add a warning comment to |
| 26 | the owner crate module in rules.mk. |
| 27 | """ |
| 28 | |
| 29 | import argparse |
| 30 | import glob |
| 31 | import json |
| 32 | import os |
| 33 | import os.path |
| 34 | import platform |
| 35 | import re |
| 36 | import shutil |
| 37 | import subprocess |
| 38 | import sys |
| 39 | |
| 40 | from typing import List |
| 41 | |
| 42 | |
| 43 | assert "/development/scripts" in os.path.dirname(__file__) |
| 44 | TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) |
| 45 | |
| 46 | # Some Rust packages include extra unwanted crates. |
| 47 | # This set contains all such excluded crate names. |
| 48 | EXCLUDED_CRATES = {"protobuf_bin_gen_rust_do_not_use"} |
| 49 | |
| 50 | |
| 51 | CUSTOM_MODULE_CRATES = { |
| 52 | # This map tracks Rust crates that have special modules that |
| 53 | # were not generated automatically by this script. Examples |
| 54 | # include compiler builtins and other foundational libraries. |
| 55 | # It also tracks crates tht are not under external/rust/crates. |
| 56 | "compiler_builtins": "trusty/user/base/lib/libcompiler_builtins-rust", |
| 57 | "core": "trusty/user/base/lib/libcore-rust", |
| 58 | } |
| 59 | |
| 60 | RENAME_STEM_MAP = { |
| 61 | # This map includes all changes to the default rust module stem names, |
| 62 | # which is used for output files when different from the module name. |
| 63 | "protoc_gen_rust": "protoc-gen-rust", |
| 64 | } |
| 65 | |
| 66 | # Header added to all generated rules.mk files. |
| 67 | RULES_MK_HEADER = ( |
| 68 | "# This file is generated by cargo2rulesmk.py {args}.\n" |
| 69 | + "# Do not modify this file as changes will be overridden on upgrade.\n\n" |
| 70 | ) |
| 71 | |
| 72 | CARGO_OUT = "cargo.out" # Name of file to keep cargo build -v output. |
| 73 | |
| 74 | # This should be kept in sync with tools/external_updater/crates_updater.py. |
| 75 | ERRORS_LINE = "Errors in " + CARGO_OUT + ":" |
| 76 | |
| 77 | TARGET_TMP = "target.tmp" # Name of temporary output directory. |
| 78 | |
| 79 | # Message to be displayed when this script is called without the --run flag. |
| 80 | DRY_RUN_NOTE = ( |
| 81 | "Dry-run: This script uses ./" |
| 82 | + TARGET_TMP |
| 83 | + " for output directory,\n" |
| 84 | + "runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n" |
| 85 | + "and writes to rules.mk in the current and subdirectories.\n\n" |
| 86 | + "To do do all of the above, use the --run flag.\n" |
| 87 | + "See --help for other flags, and more usage notes in this script.\n" |
| 88 | ) |
| 89 | |
| 90 | # Cargo -v output of a call to rustc. |
Andrei Homescu | e88e4d6 | 2023-08-24 22:11:50 +0000 | [diff] [blame] | 91 | RUSTC_PAT = re.compile("^ +Running `(.*\/)?rustc (.*)`$") |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 92 | |
| 93 | # Cargo -vv output of a call to rustc could be split into multiple lines. |
| 94 | # Assume that the first line will contain some CARGO_* env definition. |
| 95 | RUSTC_VV_PAT = re.compile("^ +Running `.*CARGO_.*=.*$") |
| 96 | # The combined -vv output rustc command line pattern. |
Andrei Homescu | e88e4d6 | 2023-08-24 22:11:50 +0000 | [diff] [blame] | 97 | RUSTC_VV_CMD_ARGS = re.compile("^ *Running `.*CARGO_.*=.* (.*\/)?rustc (.*)`$") |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 98 | |
| 99 | # Cargo -vv output of a "cc" or "ar" command; all in one line. |
| 100 | CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$') |
| 101 | # Some package, such as ring-0.13.5, has pattern '... running "cc"'. |
| 102 | |
| 103 | # Rustc output of file location path pattern for a warning message. |
| 104 | WARNING_FILE_PAT = re.compile("^ *--> ([^:]*):[0-9]+") |
| 105 | |
| 106 | # cargo test --list output of the start of running a binary. |
| 107 | CARGO_TEST_LIST_START_PAT = re.compile(r"^\s*Running (.*) \(.*\)$") |
| 108 | |
| 109 | # cargo test --list output of the end of running a binary. |
| 110 | CARGO_TEST_LIST_END_PAT = re.compile(r"^(\d+) tests?, (\d+) benchmarks$") |
| 111 | |
| 112 | CARGO2ANDROID_RUNNING_PAT = re.compile("^### Running: .*$") |
| 113 | |
| 114 | # Rust package name with suffix -d1.d2.d3(+.*)?. |
| 115 | VERSION_SUFFIX_PAT = re.compile( |
| 116 | r"^(.*)-[0-9]+\.[0-9]+\.[0-9]+(?:-(alpha|beta)\.[0-9]+)?(?:\+.*)?$" |
| 117 | ) |
| 118 | |
| 119 | # Crate types corresponding to a C ABI library |
| 120 | C_LIBRARY_CRATE_TYPES = ["staticlib", "cdylib"] |
| 121 | # Crate types corresponding to a Rust ABI library |
| 122 | RUST_LIBRARY_CRATE_TYPES = ["lib", "rlib", "dylib", "proc-macro"] |
| 123 | # Crate types corresponding to a library |
| 124 | LIBRARY_CRATE_TYPES = C_LIBRARY_CRATE_TYPES + RUST_LIBRARY_CRATE_TYPES |
| 125 | |
| 126 | |
| 127 | def altered_stem(name): |
| 128 | return RENAME_STEM_MAP[name] if (name in RENAME_STEM_MAP) else name |
| 129 | |
| 130 | |
| 131 | def is_build_crate_name(name): |
| 132 | # We added special prefix to build script crate names. |
| 133 | return name.startswith("build_script_") |
| 134 | |
| 135 | |
| 136 | def is_dependent_file_path(path): |
| 137 | # Absolute or dependent '.../' paths are not main files of this crate. |
| 138 | return path.startswith("/") or path.startswith(".../") |
| 139 | |
| 140 | |
| 141 | def get_module_name(crate): # to sort crates in a list |
| 142 | return crate.module_name |
| 143 | |
| 144 | |
| 145 | def pkg2crate_name(s): |
| 146 | return s.replace("-", "_").replace(".", "_") |
| 147 | |
| 148 | |
| 149 | def file_base_name(path): |
| 150 | return os.path.splitext(os.path.basename(path))[0] |
| 151 | |
| 152 | |
| 153 | def test_base_name(path): |
| 154 | return pkg2crate_name(file_base_name(path)) |
| 155 | |
| 156 | |
| 157 | def unquote(s): # remove quotes around str |
| 158 | if s and len(s) > 1 and s[0] == s[-1] and s[0] in ('"', "'"): |
| 159 | return s[1:-1] |
| 160 | return s |
| 161 | |
| 162 | |
| 163 | def remove_version_suffix(s): # remove -d1.d2.d3 suffix |
| 164 | if match := VERSION_SUFFIX_PAT.match(s): |
| 165 | return match.group(1) |
| 166 | return s |
| 167 | |
| 168 | |
| 169 | def short_out_name(pkg, s): # replace /.../pkg-*/out/* with .../out/* |
| 170 | return re.sub("^/.*/" + pkg + "-[0-9a-f]*/out/", ".../out/", s) |
| 171 | |
| 172 | |
| 173 | class Crate(object): |
| 174 | """Information of a Rust crate to collect/emit for a rules.mk module.""" |
| 175 | |
| 176 | def __init__(self, runner, outf_name): |
| 177 | # Remembered global runner and its members. |
| 178 | self.runner = runner |
| 179 | self.debug = runner.args.debug |
| 180 | self.cargo_dir = "" # directory of my Cargo.toml |
| 181 | self.outf_name = outf_name # path to rules.mk |
| 182 | self.outf = None # open file handle of outf_name during dump* |
| 183 | self.has_warning = False |
| 184 | # Trusty module properties derived from rustc parameters. |
| 185 | self.module_name = "" |
| 186 | self.defaults = "" # rust_defaults used by rust_test* modules |
| 187 | self.default_srcs = False # use 'srcs' defined in self.defaults |
| 188 | self.root_pkg = "" # parent package name of a sub/test packge, from -L |
| 189 | self.srcs = [] # main_src or merged multiple source files |
| 190 | self.stem = "" # real base name of output file |
| 191 | # Kept parsed status |
| 192 | self.errors = "" # all errors found during parsing |
| 193 | self.line_num = 1 # runner told input source line number |
| 194 | self.line = "" # original rustc command line parameters |
| 195 | # Parameters collected from rustc command line. |
| 196 | self.crate_name = "" # follows --crate-name |
| 197 | self.main_src = "" # follows crate_name parameter, shortened |
| 198 | self.crate_types = [] # follows --crate-type |
| 199 | self.cfgs = [] # follows --cfg, without feature= prefix |
| 200 | self.features = [] # follows --cfg, name in 'feature="..."' |
| 201 | self.codegens = [] # follows -C, some ignored |
| 202 | self.static_libs = [] # e.g. -l static=host_cpuid |
| 203 | self.shared_libs = [] # e.g. -l dylib=wayland-client, -l z |
| 204 | self.cap_lints = "" # follows --cap-lints |
| 205 | self.emit_list = "" # e.g., --emit=dep-info,metadata,link |
| 206 | self.edition = "2015" # rustc default, e.g., --edition=2018 |
| 207 | self.target = "" # follows --target |
| 208 | self.cargo_env_compat = True |
| 209 | # Parameters collected from cargo metadata output |
| 210 | self.dependencies = [] # crate dependencies output by `cargo metadata` |
| 211 | self.feature_dependencies: dict[str, List[str]] = {} # maps features to |
| 212 | # optional dependencies |
| 213 | |
| 214 | def write(self, s): |
| 215 | """convenient way to output one line at a time with EOL.""" |
| 216 | assert self.outf |
| 217 | self.outf.write(s + "\n") |
| 218 | |
| 219 | def find_cargo_dir(self): |
| 220 | """Deepest directory with Cargo.toml and contains the main_src.""" |
| 221 | if not is_dependent_file_path(self.main_src): |
| 222 | dir_name = os.path.dirname(self.main_src) |
| 223 | while dir_name: |
| 224 | if os.path.exists(dir_name + "/Cargo.toml"): |
| 225 | self.cargo_dir = dir_name |
| 226 | return |
| 227 | dir_name = os.path.dirname(dir_name) |
| 228 | |
| 229 | def add_codegens_flag(self, flag): |
| 230 | """Ignore options not used by Trusty build system""" |
| 231 | # 'prefer-dynamic' may be set by library.mk |
| 232 | # 'embed-bitcode' is ignored; we might control LTO with other flags |
| 233 | # 'codegen-units' is set globally in engine.mk |
| 234 | # 'relocation-model' and 'target-feature=+reserve-x18' may be set by |
| 235 | # common_flags.mk |
| 236 | if not ( |
| 237 | flag.startswith("codegen-units=") |
| 238 | or flag.startswith("debuginfo=") |
| 239 | or flag.startswith("embed-bitcode=") |
| 240 | or flag.startswith("extra-filename=") |
| 241 | or flag.startswith("incremental=") |
| 242 | or flag.startswith("metadata=") |
| 243 | or flag.startswith("relocation-model=") |
| 244 | or flag == "prefer-dynamic" |
| 245 | or flag == "target-feature=+reserve-x18" |
| 246 | ): |
| 247 | self.codegens.append(flag) |
| 248 | |
| 249 | def get_dependencies(self): |
| 250 | """Use output from cargo metadata to determine crate dependencies""" |
| 251 | cargo_metadata = subprocess.run( |
| 252 | [ |
| 253 | self.runner.cargo_path, |
| 254 | "metadata", |
| 255 | "--no-deps", |
| 256 | "--format-version", |
| 257 | "1", |
| 258 | ], |
| 259 | cwd=os.path.abspath(self.cargo_dir), |
| 260 | stdout=subprocess.PIPE, |
| 261 | check=False, |
| 262 | ) |
| 263 | if cargo_metadata.returncode: |
| 264 | self.errors += ( |
| 265 | "ERROR: unable to get cargo metadata to determine " |
| 266 | f"dependencies; return code {cargo_metadata.returncode}\n" |
| 267 | ) |
| 268 | else: |
| 269 | metadata_json = json.loads(cargo_metadata.stdout) |
| 270 | |
| 271 | for package in metadata_json["packages"]: |
| 272 | # package names containing '-' are changed to '_' in crate_name |
| 273 | if package["name"].replace("-", "_") == self.crate_name: |
| 274 | self.dependencies = package["dependencies"] |
| 275 | for feat, props in package["features"].items(): |
| 276 | feat_deps = [ |
| 277 | d[4:] for d in props if d.startswith("dep:") |
| 278 | ] |
| 279 | if feat_deps and feat in self.feature_dependencies: |
| 280 | self.feature_dependencies[feat].extend(feat_deps) |
| 281 | else: |
| 282 | self.feature_dependencies[feat] = feat_deps |
| 283 | break |
| 284 | else: # package name not found in metadata |
| 285 | if is_build_crate_name(self.crate_name): |
| 286 | print( |
| 287 | "### WARNING: unable to determine dependencies for " |
| 288 | + f"{self.crate_name} from cargo metadata" |
| 289 | ) |
| 290 | |
| 291 | def parse(self, line_num, line): |
| 292 | """Find important rustc arguments to convert to makefile rules.""" |
| 293 | self.line_num = line_num |
| 294 | self.line = line |
| 295 | args = [unquote(l) for l in line.split()] |
| 296 | i = 0 |
| 297 | # Loop through every argument of rustc. |
| 298 | while i < len(args): |
| 299 | arg = args[i] |
| 300 | if arg == "--crate-name": |
| 301 | i += 1 |
| 302 | self.crate_name = args[i] |
| 303 | elif arg == "--crate-type": |
| 304 | i += 1 |
| 305 | # cargo calls rustc with multiple --crate-type flags. |
| 306 | # rustc can accept: |
| 307 | # --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro] |
| 308 | self.crate_types.append(args[i]) |
| 309 | elif arg == "--test": |
| 310 | self.crate_types.append("test") |
| 311 | elif arg == "--target": |
| 312 | i += 1 |
| 313 | self.target = args[i] |
| 314 | elif arg == "--cfg": |
| 315 | i += 1 |
| 316 | if args[i].startswith("feature="): |
| 317 | self.features.append( |
| 318 | unquote(args[i].replace("feature=", "")) |
| 319 | ) |
| 320 | else: |
| 321 | self.cfgs.append(args[i]) |
| 322 | elif arg == "--extern": |
| 323 | i += 1 |
| 324 | pass # ignored; get all dependencies from cargo metadata |
| 325 | elif arg == "-C": # codegen options |
| 326 | i += 1 |
| 327 | self.add_codegens_flag(args[i]) |
| 328 | elif arg.startswith("-C"): |
| 329 | # cargo has been passing "-C <xyz>" flag to rustc, |
| 330 | # but newer cargo could pass '-Cembed-bitcode=no' to rustc. |
| 331 | self.add_codegens_flag(arg[2:]) |
| 332 | elif arg == "--cap-lints": |
| 333 | i += 1 |
| 334 | self.cap_lints = args[i] |
| 335 | elif arg == "-L": |
| 336 | i += 1 |
| 337 | if args[i].startswith("dependency=") and args[i].endswith( |
| 338 | "/deps" |
| 339 | ): |
| 340 | if "/" + TARGET_TMP + "/" in args[i]: |
| 341 | self.root_pkg = re.sub( |
| 342 | "^.*/", |
| 343 | "", |
| 344 | re.sub("/" + TARGET_TMP + "/.*/deps$", "", args[i]), |
| 345 | ) |
| 346 | else: |
| 347 | self.root_pkg = re.sub( |
| 348 | "^.*/", |
| 349 | "", |
| 350 | re.sub("/[^/]+/[^/]+/deps$", "", args[i]), |
| 351 | ) |
| 352 | self.root_pkg = remove_version_suffix(self.root_pkg) |
| 353 | elif arg == "-l": |
| 354 | i += 1 |
| 355 | if args[i].startswith("static="): |
| 356 | self.static_libs.append(re.sub("static=", "", args[i])) |
| 357 | elif args[i].startswith("dylib="): |
| 358 | self.shared_libs.append(re.sub("dylib=", "", args[i])) |
| 359 | else: |
| 360 | self.shared_libs.append(args[i]) |
| 361 | elif arg in ("--out-dir", "--color"): # ignored |
| 362 | i += 1 |
| 363 | elif arg.startswith("--error-format=") or arg.startswith("--json="): |
| 364 | pass # ignored |
| 365 | elif arg.startswith("--emit="): |
| 366 | self.emit_list = arg.replace("--emit=", "") |
| 367 | elif arg.startswith("--edition="): |
| 368 | self.edition = arg.replace("--edition=", "") |
| 369 | elif arg.startswith("-Aclippy") or arg.startswith("-Wclippy"): |
| 370 | pass # TODO: emit these flags in rules.mk |
| 371 | elif arg.startswith("-W"): |
| 372 | pass # ignored |
| 373 | elif arg.startswith("-Z"): |
| 374 | pass # ignore unstable flags |
| 375 | elif arg.startswith("-D"): |
| 376 | pass # TODO: emit these flags in rules.mk |
| 377 | elif not arg.startswith("-"): |
| 378 | # shorten imported crate main source paths like $HOME/.cargo/ |
| 379 | # registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/ |
| 380 | # lib.rs |
| 381 | self.main_src = re.sub( |
| 382 | r"^/[^ ]*/registry/src/", ".../", args[i] |
| 383 | ) |
| 384 | self.main_src = re.sub( |
| 385 | r"^\.\.\./github.com-[0-9a-f]*/", ".../", self.main_src |
| 386 | ) |
| 387 | self.find_cargo_dir() |
| 388 | if self.cargo_dir: # for a subdirectory |
| 389 | if ( |
| 390 | self.runner.args.no_subdir |
| 391 | ): # all .mk content to /dev/null |
| 392 | self.outf_name = "/dev/null" |
| 393 | elif not self.runner.args.onefile: |
| 394 | # Write to rules.mk in the subdirectory with Cargo.toml. |
| 395 | self.outf_name = self.cargo_dir + "/rules.mk" |
| 396 | self.main_src = self.main_src[len(self.cargo_dir) + 1 :] |
| 397 | |
| 398 | else: |
| 399 | self.errors += "ERROR: unknown " + arg + "\n" |
| 400 | i += 1 |
| 401 | if not self.crate_name: |
| 402 | self.errors += "ERROR: missing --crate-name\n" |
| 403 | if not self.main_src: |
| 404 | self.errors += "ERROR: missing main source file\n" |
| 405 | else: |
| 406 | self.srcs.append(self.main_src) |
| 407 | if not self.crate_types: |
| 408 | # Treat "--cfg test" as "--test" |
| 409 | if "test" in self.cfgs: |
| 410 | self.crate_types.append("test") |
| 411 | else: |
| 412 | self.errors += "ERROR: missing --crate-type or --test\n" |
| 413 | elif len(self.crate_types) > 1: |
| 414 | if "test" in self.crate_types: |
| 415 | self.errors += ( |
| 416 | "ERROR: cannot handle both --crate-type and --test\n" |
| 417 | ) |
| 418 | if "lib" in self.crate_types and "rlib" in self.crate_types: |
| 419 | self.errors += ( |
| 420 | "ERROR: cannot generate both lib and rlib crate types\n" |
| 421 | ) |
| 422 | if not self.root_pkg: |
| 423 | self.root_pkg = self.crate_name |
| 424 | |
| 425 | # get the package dependencies by running cargo metadata |
| 426 | if not self.skip_crate(): |
| 427 | self.get_dependencies() |
| 428 | self.cfgs = sorted(set(self.cfgs)) |
| 429 | self.features = sorted(set(self.features)) |
| 430 | self.codegens = sorted(set(self.codegens)) |
| 431 | self.static_libs = sorted(set(self.static_libs)) |
| 432 | self.shared_libs = sorted(set(self.shared_libs)) |
| 433 | self.crate_types = sorted(set(self.crate_types)) |
| 434 | self.module_name = self.stem |
| 435 | return self |
| 436 | |
| 437 | def dump_line(self): |
| 438 | self.write("\n// Line " + str(self.line_num) + " " + self.line) |
| 439 | |
| 440 | def feature_list(self): |
| 441 | """Return a string of main_src + "feature_list".""" |
| 442 | pkg = self.main_src |
| 443 | if pkg.startswith(".../"): # keep only the main package name |
| 444 | pkg = re.sub("/.*", "", pkg[4:]) |
| 445 | elif pkg.startswith("/"): # use relative path for a local package |
| 446 | pkg = os.path.relpath(pkg) |
| 447 | if not self.features: |
| 448 | return pkg |
| 449 | return pkg + ' "' + ",".join(self.features) + '"' |
| 450 | |
| 451 | def dump_skip_crate(self, kind): |
| 452 | if self.debug: |
| 453 | self.write("\n// IGNORED: " + kind + " " + self.main_src) |
| 454 | return self |
| 455 | |
| 456 | def skip_crate(self): |
| 457 | """Return crate_name or a message if this crate should be skipped.""" |
| 458 | if ( |
| 459 | is_build_crate_name(self.crate_name) |
| 460 | or self.crate_name in EXCLUDED_CRATES |
| 461 | ): |
| 462 | return self.crate_name |
| 463 | if is_dependent_file_path(self.main_src): |
| 464 | return "dependent crate" |
| 465 | return "" |
| 466 | |
| 467 | def dump(self): |
| 468 | """Dump all error/debug/module code to the output rules.mk file.""" |
| 469 | self.runner.init_rules_file(self.outf_name) |
| 470 | with open(self.outf_name, "a", encoding="utf-8") as outf: |
| 471 | self.outf = outf |
| 472 | if self.errors: |
| 473 | self.dump_line() |
| 474 | self.write(self.errors) |
| 475 | elif self.skip_crate(): |
| 476 | self.dump_skip_crate(self.skip_crate()) |
| 477 | else: |
| 478 | if self.debug: |
| 479 | self.dump_debug_info() |
| 480 | self.dump_trusty_module() |
| 481 | self.outf = None |
| 482 | |
| 483 | def dump_debug_info(self): |
| 484 | """Dump parsed data, when cargo2rulesmk is called with --debug.""" |
| 485 | |
| 486 | def dump(name, value): |
| 487 | self.write(f"//{name:>12} = {value}") |
| 488 | |
| 489 | def opt_dump(name, value): |
| 490 | if value: |
| 491 | dump(name, value) |
| 492 | |
| 493 | def dump_list(fmt, values): |
| 494 | for v in values: |
| 495 | self.write(fmt % v) |
| 496 | |
| 497 | self.dump_line() |
| 498 | dump("module_name", self.module_name) |
| 499 | dump("crate_name", self.crate_name) |
| 500 | dump("crate_types", self.crate_types) |
| 501 | dump("main_src", self.main_src) |
| 502 | dump("has_warning", self.has_warning) |
| 503 | opt_dump("target", self.target) |
| 504 | opt_dump("edition", self.edition) |
| 505 | opt_dump("emit_list", self.emit_list) |
| 506 | opt_dump("cap_lints", self.cap_lints) |
| 507 | dump_list("// cfg = %s", self.cfgs) |
| 508 | dump_list("// cfg = 'feature \"%s\"'", self.features) |
| 509 | # TODO(chh): escape quotes in self.features, but not in other dump_list |
| 510 | dump_list("// codegen = %s", self.codegens) |
| 511 | dump_list("// -l static = %s", self.static_libs) |
| 512 | dump_list("// -l (dylib) = %s", self.shared_libs) |
| 513 | |
| 514 | def dump_trusty_module(self): |
| 515 | """Dump one or more module definitions, depending on crate_types.""" |
Andrei Homescu | fb0b60f | 2023-08-24 21:23:36 +0000 | [diff] [blame] | 516 | if len(self.crate_types) > 1: |
| 517 | if "test" in self.crate_types: |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 518 | self.write("\nERROR: multiple crate types cannot include test type") |
| 519 | return |
Andrei Homescu | fb0b60f | 2023-08-24 21:23:36 +0000 | [diff] [blame] | 520 | |
| 521 | if "lib" in self.crate_types: |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 522 | print(f"### WARNING: crate {self.crate_name} has multiple " |
Andrei Homescu | fb0b60f | 2023-08-24 21:23:36 +0000 | [diff] [blame] | 523 | f"crate types ({str(self.crate_types)}). Treating as 'lib'") |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 524 | self.crate_types = ["lib"] |
Andrei Homescu | fb0b60f | 2023-08-24 21:23:36 +0000 | [diff] [blame] | 525 | else: |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 526 | self.write("\nERROR: don't know how to handle crate types of " |
Andrei Homescu | fb0b60f | 2023-08-24 21:23:36 +0000 | [diff] [blame] | 527 | f"crate {self.crate_name}: {str(self.crate_types)}") |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 528 | return |
| 529 | |
| 530 | self.dump_single_type_trusty_module() |
| 531 | |
| 532 | def dump_srcs_list(self): |
| 533 | """Dump the srcs list, for defaults or regular modules.""" |
| 534 | if len(self.srcs) > 1: |
| 535 | srcs = sorted(set(self.srcs)) # make a copy and dedup |
| 536 | else: |
| 537 | srcs = [self.main_src] |
| 538 | self.write("MODULE_SRCS := \\") |
| 539 | for src in srcs: |
| 540 | self.write(f"\t$(LOCAL_DIR)/{src} \\") |
| 541 | self.write("") |
| 542 | |
| 543 | # add rust file generated by build.rs to MODULE_SRCDEPS, if any |
| 544 | # TODO(perlarsen): is there a need to support more than one output file? |
| 545 | if srcdeps := [ |
| 546 | f for f in self.runner.build_out_files if f.endswith(".rs") |
| 547 | ]: |
| 548 | assert len(srcdeps) == 1 |
| 549 | outfile = srcdeps.pop() |
| 550 | lines = [ |
| 551 | f"OUT_FILE := $(call TOBUILDDIR,$(MODULE))/{outfile}", |
| 552 | f"$(OUT_FILE): $(MODULE)/out/{outfile}", |
| 553 | "\t@echo copying $< to $@", |
| 554 | "\t@$(MKDIR)", |
| 555 | "\tcp $< $@", |
| 556 | "", |
| 557 | "MODULE_RUST_ENV += OUT_DIR=$(dir $(OUT_FILE))", |
| 558 | "", |
| 559 | "MODULE_SRCDEPS := $(OUT_FILE)", |
| 560 | ] |
| 561 | self.write("\n".join(lines)) |
| 562 | |
| 563 | def dump_single_type_trusty_module(self): |
| 564 | """Dump one simple Trusty module, which has only one crate_type.""" |
| 565 | crate_type = self.crate_types[0] |
| 566 | assert crate_type != "test" |
| 567 | self.dump_one_trusty_module(crate_type) |
| 568 | |
| 569 | def dump_one_trusty_module(self, crate_type): |
| 570 | """Dump one Trusty module definition.""" |
| 571 | if crate_type in ["test", "bin"]: # TODO: support test crates |
| 572 | print( |
| 573 | f"### WARNING: ignoring {crate_type} crate: {self.crate_name}") |
| 574 | return |
| 575 | if self.codegens: # TODO: support crates that require codegen flags |
| 576 | print( |
| 577 | f"ERROR: {self.crate_name} uses unexpected codegen flags: " + |
| 578 | str(self.codegens) |
| 579 | ) |
| 580 | return |
| 581 | |
| 582 | self.dump_core_properties() |
| 583 | if not self.defaults: |
| 584 | self.dump_edition_flags_libs() |
| 585 | |
Per Larsen | ab3432c | 2024-01-05 03:11:35 +0000 | [diff] [blame] | 586 | # NOTE: a crate may list the same dependency as required and optional |
| 587 | library_deps = set() |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 588 | for dependency in self.dependencies: |
| 589 | if dependency["kind"] in ["dev", "build"]: |
| 590 | continue |
| 591 | name = ( |
| 592 | rename |
| 593 | if (rename := dependency["rename"]) |
| 594 | else dependency["name"] |
| 595 | ) |
Per Larsen | 70d3787 | 2024-01-12 03:34:29 +0000 | [diff] [blame] | 596 | if dependency["target"]: |
| 597 | print( |
| 598 | f"### WARNING: ignoring target-specific dependency: {name}") |
| 599 | continue |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 600 | path = CUSTOM_MODULE_CRATES.get( |
| 601 | name, f"external/rust/crates/{name}" |
| 602 | ) |
Andrei Homescu | 7afe14e | 2023-10-19 06:54:33 +0000 | [diff] [blame] | 603 | if dependency["optional"]: |
Stephen Crane | f84f5b1 | 2023-12-20 00:10:04 +0000 | [diff] [blame] | 604 | if not any( |
| 605 | name in self.feature_dependencies.get(f, []) |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 606 | for f in self.features |
Stephen Crane | f84f5b1 | 2023-12-20 00:10:04 +0000 | [diff] [blame] | 607 | ): |
Andrei Homescu | 7afe14e | 2023-10-19 06:54:33 +0000 | [diff] [blame] | 608 | continue |
Per Larsen | ab3432c | 2024-01-05 03:11:35 +0000 | [diff] [blame] | 609 | library_deps.add(path) |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 610 | if library_deps: |
| 611 | self.write("MODULE_LIBRARY_DEPS := \\") |
Per Larsen | ab3432c | 2024-01-05 03:11:35 +0000 | [diff] [blame] | 612 | for path in sorted(library_deps): |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 613 | self.write(f"\t{path} \\") |
| 614 | self.write("") |
| 615 | if crate_type == "test" and not self.default_srcs: |
| 616 | raise NotImplementedError("Crates with test data are not supported") |
| 617 | |
| 618 | assert crate_type in LIBRARY_CRATE_TYPES |
| 619 | self.write("include make/library.mk") |
| 620 | |
| 621 | def dump_edition_flags_libs(self): |
| 622 | if self.edition: |
| 623 | self.write(f"MODULE_RUST_EDITION := {self.edition}") |
| 624 | if self.features or self.cfgs: |
| 625 | self.write("MODULE_RUSTFLAGS += \\") |
| 626 | for feature in self.features: |
| 627 | self.write(f"\t--cfg 'feature=\"{feature}\"' \\") |
| 628 | for cfg in self.cfgs: |
| 629 | self.write(f"\t--cfg '{cfg}' \\") |
| 630 | self.write("") |
| 631 | |
| 632 | if self.static_libs or self.shared_libs: |
| 633 | print("### WARNING: Crates with depend on static or shared " |
| 634 | "libraries are not supported") |
| 635 | |
| 636 | def main_src_basename_path(self): |
| 637 | return re.sub("/", "_", re.sub(".rs$", "", self.main_src)) |
| 638 | |
| 639 | def test_module_name(self): |
| 640 | """Return a unique name for a test module.""" |
| 641 | # root_pkg+(_host|_device) + '_test_'+source_file_name |
| 642 | suffix = self.main_src_basename_path() |
| 643 | return self.root_pkg + "_test_" + suffix |
| 644 | |
| 645 | def dump_core_properties(self): |
| 646 | """Dump the module header, name, stem, etc.""" |
| 647 | self.write("LOCAL_DIR := $(GET_LOCAL_DIR)") |
| 648 | self.write("MODULE := $(LOCAL_DIR)") |
| 649 | self.write(f"MODULE_CRATE_NAME := {self.crate_name}") |
| 650 | |
| 651 | # Trusty's module system only supports bin, rlib, and proc-macro so map |
| 652 | # lib->rlib |
| 653 | if self.crate_types != ["lib"]: |
| 654 | crate_types = set( |
| 655 | "rlib" if ct == "lib" else ct for ct in self.crate_types |
| 656 | ) |
| 657 | self.write(f'MODULE_RUST_CRATE_TYPES := {" ".join(crate_types)}') |
| 658 | |
| 659 | if not self.default_srcs: |
| 660 | self.dump_srcs_list() |
| 661 | |
| 662 | if hasattr(self.runner.args, "module_add_implicit_deps"): |
| 663 | if hasattr(self.runner.args, "module_add_implicit_deps_reason"): |
| 664 | self.write(self.runner.args.module_add_implicit_deps_reason) |
| 665 | |
Andrei Homescu | fb0b60f | 2023-08-24 21:23:36 +0000 | [diff] [blame] | 666 | if self.runner.args.module_add_implicit_deps in [True, "yes"]: |
| 667 | self.write("MODULE_ADD_IMPLICIT_DEPS := true") |
| 668 | elif self.runner.args.module_add_implicit_deps in [False, "no"]: |
| 669 | self.write("MODULE_ADD_IMPLICIT_DEPS := false") |
| 670 | else: |
| 671 | sys.exit( |
| 672 | "ERROR: invalid value for module_add_implicit_deps: " + |
| 673 | str(self.runner.args.module_add_implicit_deps) |
| 674 | ) |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 675 | |
| 676 | |
| 677 | class Runner(object): |
| 678 | """Main class to parse cargo -v output and print Trusty makefile modules.""" |
| 679 | |
| 680 | def __init__(self, args): |
| 681 | self.mk_files = set() # Remember all Trusty module files. |
| 682 | self.root_pkg = "" # name of package in ./Cargo.toml |
| 683 | # Saved flags, modes, and data. |
| 684 | self.args = args |
| 685 | self.dry_run = not args.run |
| 686 | self.skip_cargo = args.skipcargo |
| 687 | self.cargo_path = "./cargo" # path to cargo, will be set later |
| 688 | self.checked_out_files = False # to check only once |
| 689 | self.build_out_files = [] # output files generated by build.rs |
| 690 | self.crates: List[Crate] = [] |
| 691 | self.warning_files = set() |
| 692 | # Keep a unique mapping from (module name) to crate |
| 693 | self.name_owners = {} |
| 694 | # Save and dump all errors from cargo to rules.mk. |
| 695 | self.errors = "" |
| 696 | self.test_errors = "" |
| 697 | self.setup_cargo_path() |
| 698 | # Default action is cargo clean, followed by build or user given actions |
| 699 | if args.cargo: |
| 700 | self.cargo = ["clean"] + args.cargo |
| 701 | else: |
| 702 | default_target = "--target x86_64-unknown-linux-gnu" |
| 703 | # Use the same target for both host and default device builds. |
| 704 | # Same target is used as default in host x86_64 Android compilation. |
| 705 | # Note: b/169872957, prebuilt cargo failed to build vsock |
| 706 | # on x86_64-unknown-linux-musl systems. |
| 707 | self.cargo = ["clean", "build " + default_target] |
| 708 | if args.tests: |
| 709 | self.cargo.append("build --tests " + default_target) |
| 710 | self.empty_tests = set() |
| 711 | self.empty_unittests = False |
| 712 | |
| 713 | def setup_cargo_path(self): |
| 714 | """Find cargo in the --cargo_bin or prebuilt rust bin directory.""" |
| 715 | if self.args.cargo_bin: |
| 716 | self.cargo_path = os.path.join(self.args.cargo_bin, "cargo") |
| 717 | if not os.path.isfile(self.cargo_path): |
| 718 | sys.exit("ERROR: cannot find cargo in " + self.args.cargo_bin) |
| 719 | print("INFO: using cargo in " + self.args.cargo_bin) |
| 720 | return |
| 721 | # TODO(perlarsen): try getting cargo from $RUST_BINDIR set in envsetup.sh |
| 722 | # We have only tested this on Linux. |
| 723 | if platform.system() != "Linux": |
| 724 | sys.exit( |
| 725 | "ERROR: this script has only been tested on Linux with cargo." |
| 726 | ) |
| 727 | # Assuming that this script is in development/scripts |
| 728 | linux_dir = os.path.join( |
| 729 | TOP_DIR, "prebuilts", "rust", "linux-x86" |
| 730 | ) |
| 731 | if not os.path.isdir(linux_dir): |
| 732 | sys.exit("ERROR: cannot find directory " + linux_dir) |
| 733 | rust_version = self.find_rust_version(linux_dir) |
| 734 | if self.args.verbose: |
| 735 | print(f"### INFO: using prebuilt rust version {rust_version}") |
| 736 | cargo_bin = os.path.join(linux_dir, rust_version, "bin") |
| 737 | self.cargo_path = os.path.join(cargo_bin, "cargo") |
| 738 | if not os.path.isfile(self.cargo_path): |
| 739 | sys.exit( |
| 740 | "ERROR: cannot find cargo in " |
| 741 | + cargo_bin |
| 742 | + "; please try --cargo_bin= flag." |
| 743 | ) |
| 744 | return |
| 745 | |
| 746 | def find_rust_version(self, linux_dir): |
| 747 | """find newest prebuilt rust version.""" |
| 748 | # find the newest (largest) version number in linux_dir. |
| 749 | rust_version = (0, 0, 0) # the prebuilt version to use |
| 750 | version_pat = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)$") |
| 751 | for dir_name in os.listdir(linux_dir): |
| 752 | result = version_pat.match(dir_name) |
| 753 | if not result: |
| 754 | continue |
| 755 | version = ( |
| 756 | int(result.group(1)), |
| 757 | int(result.group(2)), |
| 758 | int(result.group(3)), |
| 759 | ) |
| 760 | if version > rust_version: |
| 761 | rust_version = version |
| 762 | return ".".join(str(ver) for ver in rust_version) |
| 763 | |
| 764 | def find_out_files(self): |
| 765 | # list1 has build.rs output for normal crates |
| 766 | list1 = glob.glob( |
| 767 | TARGET_TMP + "/*/*/build/" + self.root_pkg + "-*/out/*" |
| 768 | ) |
| 769 | # list2 has build.rs output for proc-macro crates |
| 770 | list2 = glob.glob(TARGET_TMP + "/*/build/" + self.root_pkg + "-*/out/*") |
| 771 | return list1 + list2 |
| 772 | |
| 773 | def copy_out_files(self): |
| 774 | """Copy build.rs output files to ./out and set up build_out_files.""" |
| 775 | if self.checked_out_files: |
| 776 | return |
| 777 | self.checked_out_files = True |
| 778 | cargo_out_files = self.find_out_files() |
| 779 | out_files = set() |
| 780 | if cargo_out_files: |
| 781 | os.makedirs("out", exist_ok=True) |
| 782 | for path in cargo_out_files: |
| 783 | file_name = path.split("/")[-1] |
| 784 | out_files.add(file_name) |
| 785 | shutil.copy(path, "out/" + file_name) |
| 786 | self.build_out_files = sorted(out_files) |
| 787 | |
| 788 | def has_used_out_dir(self): |
| 789 | """Returns true if env!("OUT_DIR") is found.""" |
| 790 | return 0 == os.system( |
| 791 | "grep -rl --exclude build.rs --include \\*.rs" |
| 792 | + " 'env!(\"OUT_DIR\")' * > /dev/null" |
| 793 | ) |
| 794 | |
| 795 | def init_rules_file(self, name): |
| 796 | # name could be rules.mk or sub_dir_path/rules.mk |
| 797 | if name not in self.mk_files: |
| 798 | self.mk_files.add(name) |
| 799 | with open(name, "w", encoding="utf-8") as outf: |
| 800 | print_args = sys.argv[1:].copy() |
| 801 | if "--cargo_bin" in print_args: |
| 802 | index = print_args.index("--cargo_bin") |
| 803 | del print_args[index : index + 2] |
| 804 | outf.write(RULES_MK_HEADER.format(args=" ".join(print_args))) |
| 805 | |
| 806 | def find_root_pkg(self): |
| 807 | """Read name of [package] in ./Cargo.toml.""" |
| 808 | if not os.path.exists("./Cargo.toml"): |
| 809 | return |
| 810 | with open("./Cargo.toml", "r", encoding="utf-8") as inf: |
| 811 | pkg_section = re.compile(r"^ *\[package\]") |
| 812 | name = re.compile('^ *name *= * "([^"]*)"') |
| 813 | in_pkg = False |
| 814 | for line in inf: |
| 815 | if in_pkg: |
| 816 | if match := name.match(line): |
| 817 | self.root_pkg = match.group(1) |
| 818 | break |
| 819 | else: |
| 820 | in_pkg = pkg_section.match(line) is not None |
| 821 | |
| 822 | def run_cargo(self): |
| 823 | """Calls cargo -v and save its output to ./cargo.out.""" |
| 824 | if self.skip_cargo: |
| 825 | return self |
| 826 | cargo_toml = "./Cargo.toml" |
| 827 | cargo_out = "./cargo.out" |
| 828 | |
| 829 | # Do not use Cargo.lock, because Trusty makefile rules are designed |
| 830 | # to run with the latest available vendored crates in Trusty. |
| 831 | cargo_lock = "./Cargo.lock" |
| 832 | cargo_lock_saved = "./cargo.lock.saved" |
| 833 | had_cargo_lock = os.path.exists(cargo_lock) |
| 834 | if not os.access(cargo_toml, os.R_OK): |
| 835 | print("ERROR: Cannot find or read", cargo_toml) |
| 836 | return self |
| 837 | if not self.dry_run: |
| 838 | if os.path.exists(cargo_out): |
| 839 | os.remove(cargo_out) |
| 840 | if not self.args.use_cargo_lock and had_cargo_lock: # save it |
| 841 | os.rename(cargo_lock, cargo_lock_saved) |
| 842 | cmd_tail_target = " --target-dir " + TARGET_TMP |
| 843 | cmd_tail_redir = " >> " + cargo_out + " 2>&1" |
| 844 | # set up search PATH for cargo to find the correct rustc |
| 845 | saved_path = os.environ["PATH"] |
| 846 | os.environ["PATH"] = os.path.dirname(self.cargo_path) + ":" + saved_path |
| 847 | # Add [workspace] to Cargo.toml if it is not there. |
| 848 | added_workspace = False |
| 849 | cargo_toml_lines = None |
| 850 | if self.args.add_workspace: |
| 851 | with open(cargo_toml, "r", encoding="utf-8") as in_file: |
| 852 | cargo_toml_lines = in_file.readlines() |
| 853 | found_workspace = "[workspace]\n" in cargo_toml_lines |
| 854 | if found_workspace: |
| 855 | print("### WARNING: found [workspace] in Cargo.toml") |
| 856 | else: |
| 857 | with open(cargo_toml, "a", encoding="utf-8") as out_file: |
| 858 | out_file.write("\n\n[workspace]\n") |
| 859 | added_workspace = True |
| 860 | if self.args.verbose: |
| 861 | print("### INFO: added [workspace] to Cargo.toml") |
| 862 | features = "" |
| 863 | for c in self.cargo: |
| 864 | features = "" |
| 865 | if c != "clean": |
| 866 | if self.args.features is not None: |
| 867 | features = " --no-default-features" |
| 868 | if self.args.features: |
| 869 | features += " --features " + self.args.features |
| 870 | cmd_v_flag = " -vv " if self.args.vv else " -v " |
| 871 | cmd = self.cargo_path + cmd_v_flag |
| 872 | cmd += c + features + cmd_tail_target + cmd_tail_redir |
| 873 | if self.args.rustflags and c != "clean": |
| 874 | cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cmd |
| 875 | self.run_cmd(cmd, cargo_out) |
| 876 | if self.args.tests: |
| 877 | cmd = ( |
| 878 | self.cargo_path |
| 879 | + " test" |
| 880 | + features |
| 881 | + cmd_tail_target |
| 882 | + " -- --list" |
| 883 | + cmd_tail_redir |
| 884 | ) |
| 885 | self.run_cmd(cmd, cargo_out) |
| 886 | if added_workspace: # restore original Cargo.toml |
| 887 | with open(cargo_toml, "w", encoding="utf-8") as out_file: |
| 888 | assert cargo_toml_lines |
| 889 | out_file.writelines(cargo_toml_lines) |
| 890 | if self.args.verbose: |
| 891 | print("### INFO: restored original Cargo.toml") |
| 892 | os.environ["PATH"] = saved_path |
| 893 | if not self.dry_run: |
| 894 | if not had_cargo_lock: # restore to no Cargo.lock state |
| 895 | if os.path.exists(cargo_lock): |
| 896 | os.remove(cargo_lock) |
| 897 | elif not self.args.use_cargo_lock: # restore saved Cargo.lock |
| 898 | os.rename(cargo_lock_saved, cargo_lock) |
| 899 | return self |
| 900 | |
| 901 | def run_cmd(self, cmd, cargo_out): |
| 902 | if self.dry_run: |
| 903 | print("Dry-run skip:", cmd) |
| 904 | else: |
| 905 | if self.args.verbose: |
| 906 | print("Running:", cmd) |
| 907 | with open(cargo_out, "a+", encoding="utf-8") as out_file: |
| 908 | out_file.write("### Running: " + cmd + "\n") |
| 909 | ret = os.system(cmd) |
| 910 | if ret != 0: |
| 911 | print( |
| 912 | "*** There was an error while running cargo. " |
| 913 | + f"See the {cargo_out} file for details." |
| 914 | ) |
| 915 | |
| 916 | def apply_patch(self): |
| 917 | """Apply local patch file if it is given.""" |
| 918 | if self.args.patch: |
| 919 | if self.dry_run: |
| 920 | print("Dry-run skip patch file:", self.args.patch) |
| 921 | else: |
| 922 | if not os.path.exists(self.args.patch): |
| 923 | self.append_to_rules( |
| 924 | "ERROR cannot find patch file: " + self.args.patch |
| 925 | ) |
| 926 | return self |
| 927 | if self.args.verbose: |
| 928 | print( |
| 929 | "### INFO: applying local patch file:", self.args.patch |
| 930 | ) |
| 931 | subprocess.run( |
| 932 | [ |
| 933 | "patch", |
| 934 | "-s", |
| 935 | "--no-backup-if-mismatch", |
| 936 | "./rules.mk", |
| 937 | self.args.patch, |
| 938 | ], |
| 939 | check=True, |
| 940 | ) |
| 941 | return self |
| 942 | |
| 943 | def gen_rules(self): |
| 944 | """Parse cargo.out and generate Trusty makefile rules""" |
| 945 | if self.dry_run: |
| 946 | print("Dry-run skip: read", CARGO_OUT, "write rules.mk") |
| 947 | elif os.path.exists(CARGO_OUT): |
| 948 | self.find_root_pkg() |
| 949 | if self.args.copy_out: |
| 950 | self.copy_out_files() |
| 951 | elif self.find_out_files() and self.has_used_out_dir(): |
| 952 | print( |
| 953 | "WARNING: " |
| 954 | + self.root_pkg |
| 955 | + " has cargo output files; " |
| 956 | + "please rerun with the --copy-out flag." |
| 957 | ) |
| 958 | with open(CARGO_OUT, "r", encoding="utf-8") as cargo_out: |
| 959 | self.parse(cargo_out, "rules.mk") |
| 960 | self.crates.sort(key=get_module_name) |
| 961 | for crate in self.crates: |
| 962 | crate.dump() |
| 963 | if self.errors: |
| 964 | self.append_to_rules("\n" + ERRORS_LINE + "\n" + self.errors) |
| 965 | if self.test_errors: |
| 966 | self.append_to_rules( |
| 967 | "\n// Errors when listing tests:\n" + self.test_errors |
| 968 | ) |
| 969 | return self |
| 970 | |
| 971 | def add_crate(self, crate: Crate): |
| 972 | """Append crate to list unless it meets criteria for being skipped.""" |
| 973 | if crate.skip_crate(): |
| 974 | if self.args.debug: # include debug info of all crates |
| 975 | self.crates.append(crate) |
| 976 | elif crate.crate_types == set(["bin"]): |
| 977 | print("WARNING: skipping binary crate: " + crate.crate_name) |
| 978 | else: |
| 979 | self.crates.append(crate) |
| 980 | |
| 981 | def find_warning_owners(self): |
| 982 | """For each warning file, find its owner crate.""" |
| 983 | missing_owner = False |
| 984 | for f in self.warning_files: |
| 985 | cargo_dir = "" # find lowest crate, with longest path |
| 986 | owner = None # owner crate of this warning |
| 987 | for c in self.crates: |
| 988 | if f.startswith(c.cargo_dir + "/") and len(cargo_dir) < len( |
| 989 | c.cargo_dir |
| 990 | ): |
| 991 | cargo_dir = c.cargo_dir |
| 992 | owner = c |
| 993 | if owner: |
| 994 | owner.has_warning = True |
| 995 | else: |
| 996 | missing_owner = True |
| 997 | if missing_owner and os.path.exists("Cargo.toml"): |
| 998 | # owner is the root cargo, with empty cargo_dir |
| 999 | for c in self.crates: |
| 1000 | if not c.cargo_dir: |
| 1001 | c.has_warning = True |
| 1002 | |
| 1003 | def rustc_command(self, n, rustc_line, line, outf_name): |
| 1004 | """Process a rustc command line from cargo -vv output.""" |
| 1005 | # cargo build -vv output can have multiple lines for a rustc command |
| 1006 | # due to '\n' in strings for environment variables. |
| 1007 | # strip removes leading spaces and '\n' at the end |
| 1008 | new_rustc = (rustc_line.strip() + line) if rustc_line else line |
| 1009 | # Use an heuristic to detect the completions of a multi-line command. |
| 1010 | # This might fail for some very rare case, but easy to fix manually. |
| 1011 | if not line.endswith("`\n") or (new_rustc.count("`") % 2) != 0: |
| 1012 | return new_rustc |
| 1013 | if match := RUSTC_VV_CMD_ARGS.match(new_rustc): |
Andrei Homescu | e88e4d6 | 2023-08-24 22:11:50 +0000 | [diff] [blame] | 1014 | args = match.group(2) |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 1015 | self.add_crate(Crate(self, outf_name).parse(n, args)) |
| 1016 | else: |
| 1017 | self.assert_empty_vv_line(new_rustc) |
| 1018 | return "" |
| 1019 | |
| 1020 | def append_to_rules(self, line): |
| 1021 | self.init_rules_file("rules.mk") |
| 1022 | with open("rules.mk", "a", encoding="utf-8") as outf: |
| 1023 | outf.write(line) |
| 1024 | |
| 1025 | def assert_empty_vv_line(self, line): |
| 1026 | if line: # report error if line is not empty |
| 1027 | self.append_to_rules("ERROR -vv line: " + line) |
| 1028 | return "" |
| 1029 | |
| 1030 | def add_empty_test(self, name): |
| 1031 | if name.startswith("unittests"): |
| 1032 | self.empty_unittests = True |
| 1033 | else: |
| 1034 | self.empty_tests.add(name) |
| 1035 | |
| 1036 | def should_ignore_test(self, src): |
| 1037 | # cargo test outputs the source file for integration tests but |
| 1038 | # "unittests" for unit tests. To figure out to which crate this |
| 1039 | # corresponds, we check if the current source file is the main source of |
| 1040 | # a non-test crate, e.g., a library or a binary. |
| 1041 | return ( |
| 1042 | src in self.args.test_blocklist |
| 1043 | or src in self.empty_tests |
| 1044 | or ( |
| 1045 | self.empty_unittests |
| 1046 | and src |
| 1047 | in [ |
| 1048 | c.main_src for c in self.crates if c.crate_types != ["test"] |
| 1049 | ] |
| 1050 | ) |
| 1051 | ) |
| 1052 | |
| 1053 | def parse(self, inf, outf_name): |
| 1054 | """Parse rustc, test, and warning messages in input file.""" |
| 1055 | n = 0 # line number |
| 1056 | # We read the file in two passes, where the first simply checks for |
| 1057 | # empty tests. Otherwise we would add and merge tests before seeing |
| 1058 | # they're empty. |
| 1059 | cur_test_name = None |
| 1060 | for line in inf: |
| 1061 | if match := CARGO_TEST_LIST_START_PAT.match(line): |
| 1062 | cur_test_name = match.group(1) |
| 1063 | elif cur_test_name and ( |
| 1064 | match := CARGO_TEST_LIST_END_PAT.match(line) |
| 1065 | ): |
| 1066 | if int(match.group(1)) + int(match.group(2)) == 0: |
| 1067 | self.add_empty_test(cur_test_name) |
| 1068 | cur_test_name = None |
| 1069 | inf.seek(0) |
| 1070 | prev_warning = False # true if the previous line was warning: ... |
| 1071 | rustc_line = "" # previous line(s) matching RUSTC_VV_PAT |
| 1072 | in_tests = False |
| 1073 | for line in inf: |
| 1074 | n += 1 |
| 1075 | if line.startswith("warning: "): |
| 1076 | prev_warning = True |
| 1077 | rustc_line = self.assert_empty_vv_line(rustc_line) |
| 1078 | continue |
| 1079 | new_rustc = "" |
| 1080 | if match := RUSTC_PAT.match(line): |
Andrei Homescu | e88e4d6 | 2023-08-24 22:11:50 +0000 | [diff] [blame] | 1081 | args_line = match.group(2) |
Per Larsen | 7b03617 | 2023-07-06 22:59:34 +0000 | [diff] [blame] | 1082 | self.add_crate(Crate(self, outf_name).parse(n, args_line)) |
| 1083 | self.assert_empty_vv_line(rustc_line) |
| 1084 | elif rustc_line or RUSTC_VV_PAT.match(line): |
| 1085 | new_rustc = self.rustc_command(n, rustc_line, line, outf_name) |
| 1086 | elif CC_AR_VV_PAT.match(line): |
| 1087 | raise NotImplementedError("$CC or $AR commands not supported") |
| 1088 | elif prev_warning and (match := WARNING_FILE_PAT.match(line)): |
| 1089 | self.assert_empty_vv_line(rustc_line) |
| 1090 | fpath = match.group(1) |
| 1091 | if fpath[0] != "/": # ignore absolute path |
| 1092 | self.warning_files.add(fpath) |
| 1093 | elif line.startswith("error: ") or line.startswith("error[E"): |
| 1094 | if not self.args.ignore_cargo_errors: |
| 1095 | if in_tests: |
| 1096 | self.test_errors += "// " + line |
| 1097 | else: |
| 1098 | self.errors += line |
| 1099 | elif CARGO2ANDROID_RUNNING_PAT.match(line): |
| 1100 | in_tests = "cargo test" in line and "--list" in line |
| 1101 | prev_warning = False |
| 1102 | rustc_line = new_rustc |
| 1103 | self.find_warning_owners() |
| 1104 | |
| 1105 | |
| 1106 | def get_parser(): |
| 1107 | """Parse main arguments.""" |
| 1108 | parser = argparse.ArgumentParser("cargo2rulesmk") |
| 1109 | parser.add_argument( |
| 1110 | "--add_workspace", |
| 1111 | action="store_true", |
| 1112 | default=False, |
| 1113 | help=( |
| 1114 | "append [workspace] to Cargo.toml before calling cargo," |
| 1115 | + " to treat current directory as root of package source;" |
| 1116 | + " otherwise the relative source file path in generated" |
| 1117 | + " rules.mk file will be from the parent directory." |
| 1118 | ), |
| 1119 | ) |
| 1120 | parser.add_argument( |
| 1121 | "--cargo", |
| 1122 | action="append", |
| 1123 | metavar="args_string", |
| 1124 | help=( |
| 1125 | "extra cargo build -v args in a string, " |
| 1126 | + "each --cargo flag calls cargo build -v once" |
| 1127 | ), |
| 1128 | ) |
| 1129 | parser.add_argument( |
| 1130 | "--cargo_bin", |
| 1131 | type=str, |
| 1132 | help="use cargo in the cargo_bin directory instead of the prebuilt one", |
| 1133 | ) |
| 1134 | parser.add_argument( |
| 1135 | "--copy-out", |
| 1136 | action="store_true", |
| 1137 | default=False, |
| 1138 | help=( |
| 1139 | "only for root directory, " |
| 1140 | + "copy build.rs output to ./out/* and declare source deps " |
| 1141 | + "for ./out/*.rs; for crates with code pattern: " |
| 1142 | + 'include!(concat!(env!("OUT_DIR"), "/<some_file>.rs"))' |
| 1143 | ), |
| 1144 | ) |
| 1145 | parser.add_argument( |
| 1146 | "--debug", |
| 1147 | action="store_true", |
| 1148 | default=False, |
| 1149 | help="dump debug info into rules.mk", |
| 1150 | ) |
| 1151 | parser.add_argument( |
| 1152 | "--features", |
| 1153 | type=str, |
| 1154 | help=( |
| 1155 | "pass features to cargo build, " |
| 1156 | + "empty string means no default features" |
| 1157 | ), |
| 1158 | ) |
| 1159 | parser.add_argument( |
| 1160 | "--ignore-cargo-errors", |
| 1161 | action="store_true", |
| 1162 | default=False, |
| 1163 | help="do not append cargo/rustc error messages to rules.mk", |
| 1164 | ) |
| 1165 | parser.add_argument( |
| 1166 | "--no-subdir", |
| 1167 | action="store_true", |
| 1168 | default=False, |
| 1169 | help="do not output anything for sub-directories", |
| 1170 | ) |
| 1171 | parser.add_argument( |
| 1172 | "--onefile", |
| 1173 | action="store_true", |
| 1174 | default=False, |
| 1175 | help=( |
| 1176 | "output all into one ./rules.mk, default will generate " |
| 1177 | + "one rules.mk per Cargo.toml in subdirectories" |
| 1178 | ), |
| 1179 | ) |
| 1180 | parser.add_argument( |
| 1181 | "--patch", |
| 1182 | type=str, |
| 1183 | help="apply the given patch file to generated ./rules.mk", |
| 1184 | ) |
| 1185 | parser.add_argument( |
| 1186 | "--run", |
| 1187 | action="store_true", |
| 1188 | default=False, |
| 1189 | help="run it, default is dry-run", |
| 1190 | ) |
| 1191 | parser.add_argument("--rustflags", type=str, help="passing flags to rustc") |
| 1192 | parser.add_argument( |
| 1193 | "--skipcargo", |
| 1194 | action="store_true", |
| 1195 | default=False, |
| 1196 | help="skip cargo command, parse cargo.out, and generate ./rules.mk", |
| 1197 | ) |
| 1198 | parser.add_argument( |
| 1199 | "--tests", |
| 1200 | action="store_true", |
| 1201 | default=False, |
| 1202 | help="run cargo build --tests after normal build", |
| 1203 | ) |
| 1204 | parser.add_argument( |
| 1205 | "--use-cargo-lock", |
| 1206 | action="store_true", |
| 1207 | default=False, |
| 1208 | help=( |
| 1209 | "run cargo build with existing Cargo.lock " |
| 1210 | + "(used when some latest dependent crates failed)" |
| 1211 | ), |
| 1212 | ) |
| 1213 | parser.add_argument( |
| 1214 | "--test-data", |
| 1215 | nargs="*", |
| 1216 | default=[], |
| 1217 | help=( |
| 1218 | "Add the given file to the given test's data property. " |
| 1219 | + "Usage: test-path=data-path" |
| 1220 | ), |
| 1221 | ) |
| 1222 | parser.add_argument( |
| 1223 | "--dependency-blocklist", |
| 1224 | nargs="*", |
| 1225 | default=[], |
| 1226 | help="Do not emit the given dependencies (without lib prefixes).", |
| 1227 | ) |
| 1228 | parser.add_argument( |
| 1229 | "--test-blocklist", |
| 1230 | nargs="*", |
| 1231 | default=[], |
| 1232 | help=( |
| 1233 | "Do not emit the given tests. " |
| 1234 | + "Pass the path to the test file to exclude." |
| 1235 | ), |
| 1236 | ) |
| 1237 | parser.add_argument( |
| 1238 | "--cfg-blocklist", |
| 1239 | nargs="*", |
| 1240 | default=[], |
| 1241 | help="Do not emit the given cfg.", |
| 1242 | ) |
| 1243 | parser.add_argument( |
| 1244 | "--verbose", |
| 1245 | action="store_true", |
| 1246 | default=False, |
| 1247 | help="echo executed commands", |
| 1248 | ) |
| 1249 | parser.add_argument( |
| 1250 | "--vv", |
| 1251 | action="store_true", |
| 1252 | default=False, |
| 1253 | help="run cargo with -vv instead of default -v", |
| 1254 | ) |
| 1255 | parser.add_argument( |
| 1256 | "--dump-config-and-exit", |
| 1257 | type=str, |
| 1258 | help=( |
| 1259 | "Dump command-line arguments (minus this flag) to a config file and" |
| 1260 | " exit. This is intended to help migrate from command line options " |
| 1261 | "to config files." |
| 1262 | ), |
| 1263 | ) |
| 1264 | parser.add_argument( |
| 1265 | "--config", |
| 1266 | type=str, |
| 1267 | help=( |
| 1268 | "Load command-line options from the given config file. Options in " |
| 1269 | "this file will override those passed on the command line." |
| 1270 | ), |
| 1271 | ) |
| 1272 | return parser |
| 1273 | |
| 1274 | |
| 1275 | def parse_args(parser): |
| 1276 | """Parses command-line options.""" |
| 1277 | args = parser.parse_args() |
| 1278 | # Use the values specified in a config file if one was found. |
| 1279 | if args.config: |
| 1280 | with open(args.config, "r", encoding="utf-8") as f: |
| 1281 | config = json.load(f) |
| 1282 | args_dict = vars(args) |
| 1283 | for arg in config: |
| 1284 | args_dict[arg.replace("-", "_")] = config[arg] |
| 1285 | return args |
| 1286 | |
| 1287 | |
| 1288 | def dump_config(parser, args): |
| 1289 | """Writes the non-default command-line options to the specified file.""" |
| 1290 | args_dict = vars(args) |
| 1291 | # Filter out the arguments that have their default value. |
| 1292 | # Also filter certain "temporary" arguments. |
| 1293 | non_default_args = {} |
| 1294 | for arg in args_dict: |
| 1295 | if ( |
| 1296 | args_dict[arg] != parser.get_default(arg) |
| 1297 | and arg != "dump_config_and_exit" |
| 1298 | and arg != "config" |
| 1299 | and arg != "cargo_bin" |
| 1300 | ): |
| 1301 | non_default_args[arg.replace("_", "-")] = args_dict[arg] |
| 1302 | # Write to the specified file. |
| 1303 | with open(args.dump_config_and_exit, "w", encoding="utf-8") as f: |
| 1304 | json.dump(non_default_args, f, indent=2, sort_keys=True) |
| 1305 | |
| 1306 | |
| 1307 | def main(): |
| 1308 | parser = get_parser() |
| 1309 | args = parse_args(parser) |
| 1310 | if not args.run: # default is dry-run |
| 1311 | print(DRY_RUN_NOTE) |
| 1312 | if args.dump_config_and_exit: |
| 1313 | dump_config(parser, args) |
| 1314 | else: |
| 1315 | Runner(args).run_cargo().gen_rules().apply_patch() |
| 1316 | |
| 1317 | |
| 1318 | if __name__ == "__main__": |
| 1319 | main() |