From 008b6ea661f39c917a932268100ff422b79a0a96 Mon Sep 17 00:00:00 2001 From: "Lukacs T. Berki" Date: Tue, 17 Aug 2021 17:52:15 +0200 Subject: [PATCH] Do not do glob-related things in RunBlueprint(). From now on, the responsibility of writing build-globs.ninja and adding the glob list files to build.ninja.d will be the responsibility of whoever calls Blueprint. It's nicer this way because the bootstrap Blueprint invocation does not need globs and Blueprint can be invoked multiple times from the soong_build binary depending on what it does. Regrettably, the little Blueprint invocation that generates build-globs.ninja still requires a reference to a BootstrapConfig because GlobRule references ${buildDir}. Side cleanup: remove Args.DocFile, which was unused. Test: Presubmits. Change-Id: I37586966e3d6b7557a490ed1ae81ec596ec66ee5 --- bootstrap/command.go | 10 ---- bootstrap/config.go | 1 - bootstrap/glob.go | 127 ++++++++++++++++++++----------------------- 3 files changed, 58 insertions(+), 80 deletions(-) diff --git a/bootstrap/command.go b/bootstrap/command.go index 507c4c0..1d1b6b7 100644 --- a/bootstrap/command.go +++ b/bootstrap/command.go @@ -32,10 +32,7 @@ import ( type Args struct { OutFile string Subninjas []string - GlobFile string - GlobListDir string DepFile string - DocFile string Cpuprofile string Memprofile string DelveListen string @@ -153,7 +150,6 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []strin topLevelBlueprintsFile: args.TopFile, subninjas: args.Subninjas, - globListDir: args.GlobListDir, runGoTests: args.RunGoTests, useValidations: args.UseValidations, primaryBuilderInvocations: invocations, @@ -165,8 +161,6 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []strin ctx.RegisterModuleType("blueprint_go_binary", newGoBinaryModuleFactory(bootstrapConfig, true)) ctx.RegisterSingletonType("bootstrap", newSingletonFactory(bootstrapConfig)) - ctx.RegisterSingletonType("glob", globSingletonFactory(bootstrapConfig.globListDir, ctx)) - blueprintFiles, errs := ctx.ParseFileList(filepath.Dir(args.TopFile), filesToParse, config) if len(errs) > 0 { fatalErrors(errs) @@ -221,10 +215,6 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []strin out = ioutil.Discard.(io.StringWriter) } - if args.GlobFile != "" { - WriteBuildGlobsNinjaFile(args.GlobListDir, ctx, args, config) - } - err = ctx.WriteBuildFile(out) if err != nil { fatalf("error writing Ninja file contents: %s", err) diff --git a/bootstrap/config.go b/bootstrap/config.go index 01961dd..b6e37e4 100644 --- a/bootstrap/config.go +++ b/bootstrap/config.go @@ -131,7 +131,6 @@ type Config struct { topLevelBlueprintsFile string subninjas []string - globListDir string runGoTests bool useValidations bool diff --git a/bootstrap/glob.go b/bootstrap/glob.go index 617a1d2..0e8caca 100644 --- a/bootstrap/glob.go +++ b/bootstrap/glob.go @@ -145,94 +145,87 @@ func joinWithPrefixAndQuote(strs []string, prefix string) string { return string(ret) } -// globSingleton collects any glob patterns that were seen by Context and writes out rules to +// GlobSingleton collects any glob patterns that were seen by Context and writes out rules to // re-evaluate them whenever the contents of the searched directories change, and retrigger the // primary builder if the results change. -type globSingleton struct { - globListDir string - globLister func() pathtools.MultipleGlobResults - srcDir string - writeRule bool +type GlobSingleton struct { + // A function that returns the glob results of individual glob buckets + GlobLister func() pathtools.MultipleGlobResults + + // Ninja file that contains instructions for validating the glob list files + GlobFile string + + // Directory containing the glob list files + GlobDir string + + // The source directory + SrcDir string } -func globSingletonFactory(globListDir string, ctx *blueprint.Context) func() blueprint.Singleton { - return func() blueprint.Singleton { - return &globSingleton{ - globListDir: globListDir, - globLister: ctx.Globs, - srcDir: ctx.SrcDir(), - } - } +func globBucketName(globDir string, globBucket int) string { + return filepath.Join(globDir, strconv.Itoa(globBucket)) } -func (s *globSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) { +// Returns the directory where glob list files live +func GlobDirectory(buildDir, globListDir string) string { + return filepath.Join(buildDir, bootstrapSubDir, globListDir) +} + +func (s *GlobSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) { // Sort the list of globs into buckets. A hash function is used instead of sharding so that // adding a new glob doesn't force rerunning all the buckets by shifting them all by 1. globBuckets := make([]pathtools.MultipleGlobResults, numGlobBuckets) - for _, g := range s.globLister() { + for _, g := range s.GlobLister() { bucket := globToBucket(g) globBuckets[bucket] = append(globBuckets[bucket], g) } - // The directory for the intermediates needs to be different for bootstrap and the primary - // builder. - bootstrapConfig := ctx.Config().(BootstrapConfig) - globsDir := globsDir(bootstrapConfig, s.globListDir) - for i, globs := range globBuckets { - fileListFile := filepath.Join(globsDir, strconv.Itoa(i)) + fileListFile := globBucketName(s.GlobDir, i) - if s.writeRule { - // Called from generateGlobNinjaFile. Write out the file list to disk, and add a ninja - // rule to run bpglob if any of the dependencies (usually directories that contain - // globbed files) have changed. The file list produced by bpglob should match exactly - // with the file written here so that restat can prevent rerunning the primary builder. - // - // We need to write the file list here so that it has an older modified date - // than the build.ninja (otherwise we'd run the primary builder twice on - // every new glob) - // - // We don't need to write the depfile because we're guaranteed that ninja - // will run the command at least once (to record it into the ninja_log), so - // the depfile will be loaded from that execution. - absoluteFileListFile := joinPath(s.srcDir, fileListFile) - err := pathtools.WriteFileIfChanged(absoluteFileListFile, globs.FileList(), 0666) - if err != nil { - panic(fmt.Errorf("error writing %s: %s", fileListFile, err)) - } - - // Write out the ninja rule to run bpglob. - multipleGlobFilesRule(ctx, fileListFile, i, globs) - } else { - // Called from the main Context, make build.ninja depend on the fileListFile. - ctx.AddNinjaFileDeps(fileListFile) + // Called from generateGlobNinjaFile. Write out the file list to disk, and add a ninja + // rule to run bpglob if any of the dependencies (usually directories that contain + // globbed files) have changed. The file list produced by bpglob should match exactly + // with the file written here so that restat can prevent rerunning the primary builder. + // + // We need to write the file list here so that it has an older modified date + // than the build.ninja (otherwise we'd run the primary builder twice on + // every new glob) + // + // We don't need to write the depfile because we're guaranteed that ninja + // will run the command at least once (to record it into the ninja_log), so + // the depfile will be loaded from that execution. + absoluteFileListFile := joinPath(s.SrcDir, fileListFile) + err := pathtools.WriteFileIfChanged(absoluteFileListFile, globs.FileList(), 0666) + if err != nil { + panic(fmt.Errorf("error writing %s: %s", fileListFile, err)) } + + // Write out the ninja rule to run bpglob. + multipleGlobFilesRule(ctx, fileListFile, i, globs) } } -func WriteBuildGlobsNinjaFile(globListDir string, ctx *blueprint.Context, args Args, config interface{}) { - buffer, errs := generateGlobNinjaFile(ctx.SrcDir(), globListDir, config, ctx.Globs) +// Writes a .ninja file that contains instructions for regenerating the glob +// files that contain the results of every glob that was run. The list of files +// is available as the result of GlobFileListFiles(). +func WriteBuildGlobsNinjaFile(glob *GlobSingleton, config interface{}) { + buffer, errs := generateGlobNinjaFile(glob, config) if len(errs) > 0 { fatalErrors(errs) } const outFilePermissions = 0666 - err := ioutil.WriteFile(joinPath(ctx.SrcDir(), args.GlobFile), buffer, outFilePermissions) + err := ioutil.WriteFile(joinPath(glob.SrcDir, glob.GlobFile), buffer, outFilePermissions) if err != nil { - fatalf("error writing %s: %s", args.GlobFile, err) + fatalf("error writing %s: %s", glob.GlobFile, err) } } -func generateGlobNinjaFile(srcDir, globListDir string, config interface{}, - globLister func() pathtools.MultipleGlobResults) ([]byte, []error) { +func generateGlobNinjaFile(glob *GlobSingleton, config interface{}) ([]byte, []error) { ctx := blueprint.NewContext() ctx.RegisterSingletonType("glob", func() blueprint.Singleton { - return &globSingleton{ - globListDir: globListDir, - globLister: globLister, - srcDir: srcDir, - writeRule: true, - } + return glob }) extraDeps, errs := ctx.ResolveDependencies(config) @@ -260,19 +253,15 @@ func generateGlobNinjaFile(srcDir, globListDir string, config interface{}, return buf.Bytes(), nil } -// globsDir returns a different directory to store glob intermediates for the bootstrap and -// primary builder executions. -func globsDir(config BootstrapConfig, globsDir string) string { - buildDir := config.BuildDir() - return filepath.Join(buildDir, bootstrapSubDir, globsDir) -} - -// GlobFileListFiles returns the list of sharded glob file list files for the main stage. -func GlobFileListFiles(config BootstrapConfig, globListDir string) []string { - globsDir := globsDir(config, globListDir) +// GlobFileListFiles returns the list of files that contain the result of globs +// in the build. It is suitable for inclusion in build.ninja.d (so that +// build.ninja is regenerated if the globs change). The instructions to +// regenerate these files are written by WriteBuildGlobsNinjaFile(). +func GlobFileListFiles(globDir string) []string { var fileListFiles []string for i := 0; i < numGlobBuckets; i++ { - fileListFiles = append(fileListFiles, filepath.Join(globsDir, strconv.Itoa(i))) + fileListFile := globBucketName(globDir, i) + fileListFiles = append(fileListFiles, fileListFile) } return fileListFiles }