From b1d30d63c5d1b818ea38e77cd155da2016fe8b6c Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Wed, 18 Nov 2020 02:43:19 -0500 Subject: [PATCH] soong_ui kati.go: write more comments. This CL also removes --warn_real_to_phony and --warn_phony_looks_real, since --werror_real_to_phony and --wrror_phony_looks_real are already specified. Test: treehugger Bug: 173474588 Change-Id: Icbb0f53182d1b912c62f8044f52bb79d985bf868 --- ui/build/kati.go | 114 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 8 deletions(-) diff --git a/ui/build/kati.go b/ui/build/kati.go index f6c0f52a3..06ec64607 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -33,12 +33,18 @@ const katiBuildSuffix = "" const katiCleanspecSuffix = "-cleanspec" const katiPackageSuffix = "-package" -// genKatiSuffix creates a suffix for kati-generated files so that we can cache -// them based on their inputs. So this should encode all common changes to Kati -// inputs. Currently that includes the TARGET_PRODUCT, kati-processed command -// line arguments, and the directories specified by mm/mmm. +// genKatiSuffix creates a filename suffix for kati-generated files so that we +// can cache them based on their inputs. Such files include the generated Ninja +// files and env.sh environment variable setup files. +// +// The filename suffix should encode all common changes to Kati inputs. +// Currently that includes the TARGET_PRODUCT and kati-processed command line +// arguments. func genKatiSuffix(ctx Context, config Config) { + // Construct the base suffix. katiSuffix := "-" + config.TargetProduct() + + // Append kati arguments to the suffix. if args := config.KatiArgs(); len(args) > 0 { katiSuffix += "-" + spaceSlashReplacer.Replace(strings.Join(args, "_")) } @@ -60,51 +66,101 @@ func genKatiSuffix(ctx Context, config Config) { } } +// Base function to construct and run the Kati command line with additional +// arguments, and a custom function closure to mutate the environment Kati runs +// in. func runKati(ctx Context, config Config, extraSuffix string, args []string, envFunc func(*Environment)) { executable := config.PrebuiltBuildTool("ckati") + // cKati arguments. args = append([]string{ + // Instead of executing commands directly, generate a Ninja file. "--ninja", + // Generate Ninja files in the output directory. "--ninja_dir=" + config.OutDir(), + // Filename suffix of the generated Ninja file. "--ninja_suffix=" + config.KatiSuffix() + extraSuffix, + // Remove common parts at the beginning of a Ninja file, like build_dir, + // local_pool and _kati_always_build_. Allows Kati to be run multiple + // times, with generated Ninja files combined in a single invocation + // using 'include'. "--no_ninja_prelude", + // Support declaring phony outputs in AOSP Ninja. "--use_ninja_phony_output", + // Support declaring symlink outputs in AOSP Ninja. "--use_ninja_symlink_outputs", + // Regenerate the Ninja file if environment inputs have changed. e.g. + // CLI flags, .mk file timestamps, env vars, $(wildcard ..) and some + // $(shell ..) results. "--regen", + // Skip '-include' directives starting with the specified path. Used to + // ignore generated .mk files. "--ignore_optional_include=" + filepath.Join(config.OutDir(), "%.P"), + // Detect the use of $(shell echo ...). "--detect_android_echo", + // Colorful ANSI-based warning and error messages. "--color_warnings", + // Generate all targets, not just the top level requested ones. "--gen_all_targets", + // Use the built-in emulator of GNU find for better file finding + // performance. Used with $(shell find ...). "--use_find_emulator", + // Fail when the find emulator encounters problems. "--werror_find_emulator", + // Do not provide any built-in rules. "--no_builtin_rules", + // Fail when suffix rules are used. "--werror_suffix_rules", - "--warn_real_to_phony", - "--warn_phony_looks_real", + // Fail when a real target depends on a phony target. "--werror_real_to_phony", - "--werror_phony_looks_real", - "--werror_writable", + // Makes real_to_phony checks assume that any top-level or leaf + // dependencies that does *not* have a '/' in it is a phony target. "--top_level_phony", + // Fail when a phony target contains slashes. + "--werror_phony_looks_real", + // Fail when writing to a read-only directory. + "--werror_writable", + // Print Kati's internal statistics, such as the number of variables, + // implicit/explicit/suffix rules, and so on. "--kati_stats", }, args...) + // Generate a minimal Ninja file. + // + // Used for build_test and multiproduct_kati, which runs Kati several + // hundred times for different configurations to test file generation logic. + // These can result in generating Ninja files reaching ~1GB or more, + // resulting in ~hundreds of GBs of writes. + // + // Since we don't care about executing the Ninja files in these test cases, + // generating the Ninja file content wastes time, so skip writing any + // information out with --empty_ninja_file. + // + // From https://github.com/google/kati/commit/87b8da7af2c8bea28b1d8ab17679453d859f96e5 if config.Environment().IsEnvTrue("EMPTY_NINJA_FILE") { args = append(args, "--empty_ninja_file") } + // Apply 'local_pool' to to all rules that don't specify a pool. if config.UseRemoteBuild() { args = append(args, "--default_pool=local_pool") } cmd := Command(ctx, config, "ckati", executable, args...) + + // Set up the nsjail sandbox. cmd.Sandbox = katiSandbox + + // Set up stdout and stderr. pipe, err := cmd.StdoutPipe() if err != nil { ctx.Fatalln("Error getting output pipe for ckati:", err) } cmd.Stderr = cmd.Stdout + // Apply the caller's function closure to mutate the environment variables. envFunc(cmd.Environment) + // Pass on various build environment metadata to Kati. if _, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { username := "unknown" if u, err := user.Current(); err == nil { @@ -125,6 +181,8 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF } cmd.StartOrFatal() + // Set up the ToolStatus command line reader for Kati for a consistent UI + // for the user. status.KatiReader(ctx.Status.StartTool(), pipe) cmd.WaitOrFatal() } @@ -134,35 +192,57 @@ func runKatiBuild(ctx Context, config Config) { defer ctx.EndTrace() args := []string{ + // Mark the output directory as writable. "--writable", config.OutDir() + "/", + // Fail when encountering implicit rules. e.g. + // %.foo: %.bar + // cp $< $@ "--werror_implicit_rules", + // Entry point for the Kati Ninja file generation. "-f", "build/make/core/main.mk", } if !config.BuildBrokenDupRules() { + // Fail when redefining / duplicating a target. args = append(args, "--werror_overriding_commands") } args = append(args, config.KatiArgs()...) args = append(args, + // Location of the Make vars .mk file generated by Soong. "SOONG_MAKEVARS_MK="+config.SoongMakeVarsMk(), + // Location of the Android.mk file generated by Soong. This + // file contains Soong modules represented as Kati modules, + // allowing Kati modules to depend on Soong modules. "SOONG_ANDROID_MK="+config.SoongAndroidMk(), + // Directory containing outputs for the target device. "TARGET_DEVICE_DIR="+config.TargetDeviceDir(), + // Directory containing .mk files for packaging purposes, such as + // the dist.mk file, containing dist-for-goals data. "KATI_PACKAGE_MK_DIR="+config.KatiPackageMkDir()) runKati(ctx, config, katiBuildSuffix, args, func(env *Environment) {}) + // compress and dist the main build ninja file. distGzipFile(ctx, config, config.KatiBuildNinjaFile()) + // Cleanup steps. cleanCopyHeaders(ctx, config) cleanOldInstalledFiles(ctx, config) } +// Clean out obsolete header files on the disk that were *not copied* during the +// build with BUILD_COPY_HEADERS and LOCAL_COPY_HEADERS. +// +// These should be increasingly uncommon, as it's a deprecated feature and there +// isn't an equivalent feature in Soong. func cleanCopyHeaders(ctx Context, config Config) { ctx.BeginTrace("clean", "clean copy headers") defer ctx.EndTrace() + // Read and parse the list of copied headers from a file in the product + // output directory. data, err := ioutil.ReadFile(filepath.Join(config.ProductOut(), ".copied_headers_list")) if err != nil { if os.IsNotExist(err) { @@ -178,6 +258,8 @@ func cleanCopyHeaders(ctx Context, config Config) { headerDir := headers[0] headers = headers[1:] + // Walk the tree and remove any headers that are not in the list of copied + // headers in the current build. filepath.Walk(headerDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -196,6 +278,8 @@ func cleanCopyHeaders(ctx Context, config Config) { }) } +// Clean out any previously installed files from the disk that are not installed +// in the current build. func cleanOldInstalledFiles(ctx Context, config Config) { ctx.BeginTrace("clean", "clean old installed files") defer ctx.EndTrace() @@ -213,18 +297,27 @@ func cleanOldInstalledFiles(ctx Context, config Config) { cleanOldFiles(ctx, config.HostOut(), ".installable_test_files") } +// Generate the Ninja file containing the packaging command lines for the dist +// dir. func runKatiPackage(ctx Context, config Config) { ctx.BeginTrace(metrics.RunKati, "kati package") defer ctx.EndTrace() args := []string{ + // Mark the dist dir as writable. "--writable", config.DistDir() + "/", + // Fail when encountering implicit rules. e.g. "--werror_implicit_rules", + // Fail when redefining / duplicating a target. "--werror_overriding_commands", + // Entry point. "-f", "build/make/packaging/main.mk", + // Directory containing .mk files for packaging purposes, such as + // the dist.mk file, containing dist-for-goals data. "KATI_PACKAGE_MK_DIR=" + config.KatiPackageMkDir(), } + // Run Kati against a restricted set of environment variables. runKati(ctx, config, katiPackageSuffix, args, func(env *Environment) { env.Allow([]string{ // Some generic basics @@ -251,16 +344,21 @@ func runKatiPackage(ctx Context, config Config) { } }) + // Compress and dist the packaging Ninja file. distGzipFile(ctx, config, config.KatiPackageNinjaFile()) } +// Run Kati on the cleanspec files to clean the build. func runKatiCleanSpec(ctx Context, config Config) { ctx.BeginTrace(metrics.RunKati, "kati cleanspec") defer ctx.EndTrace() runKati(ctx, config, katiCleanspecSuffix, []string{ + // Fail when encountering implicit rules. e.g. "--werror_implicit_rules", + // Fail when redefining / duplicating a target. "--werror_overriding_commands", + // Entry point. "-f", "build/make/core/cleanbuild.mk", "SOONG_MAKEVARS_MK=" + config.SoongMakeVarsMk(), "TARGET_DEVICE_DIR=" + config.TargetDeviceDir(),