Merge "Do not do glob-related things in RunBlueprint()." am: 3bec89465e am: 58c28580fd

Original change: https://android-review.googlesource.com/c/platform/build/blueprint/+/1799767

Change-Id: I6cee2482f510ecfce8e9aa4c21e00e06269d883d

bootstrap/command.go

@@ -32,10 +32,7 @@ import (
 type Args struct {
 	OutFile                  string
 	Subninjas                []string
-	GlobFile                 string
-	GlobListDir              string
 	DepFile                  string
-	DocFile                  string
 	Cpuprofile               string
 	Memprofile               string
 	DelveListen              string
@@ -153,7 +150,6 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []strin
 
 		topLevelBlueprintsFile:    args.TopFile,
 		subninjas:                 args.Subninjas,
-		globListDir:               args.GlobListDir,
 		runGoTests:                args.RunGoTests,
 		useValidations:            args.UseValidations,
 		primaryBuilderInvocations: invocations,
@@ -165,8 +161,6 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []strin
 	ctx.RegisterModuleType("blueprint_go_binary", newGoBinaryModuleFactory(bootstrapConfig, true))
 	ctx.RegisterSingletonType("bootstrap", newSingletonFactory(bootstrapConfig))
 
-	ctx.RegisterSingletonType("glob", globSingletonFactory(bootstrapConfig.globListDir, ctx))
-
 	blueprintFiles, errs := ctx.ParseFileList(filepath.Dir(args.TopFile), filesToParse, config)
 	if len(errs) > 0 {
 		fatalErrors(errs)
@@ -221,10 +215,6 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []strin
 		out = ioutil.Discard.(io.StringWriter)
 	}
 
-	if args.GlobFile != "" {
-		WriteBuildGlobsNinjaFile(args.GlobListDir, ctx, args, config)
-	}
-
 	err = ctx.WriteBuildFile(out)
 	if err != nil {
 		fatalf("error writing Ninja file contents: %s", err)

bootstrap/config.go

@@ -131,7 +131,6 @@ type Config struct {
 
 	topLevelBlueprintsFile string
 	subninjas              []string
-	globListDir            string
 
 	runGoTests     bool
 	useValidations bool

bootstrap/glob.go

@@ -145,94 +145,87 @@ func joinWithPrefixAndQuote(strs []string, prefix string) string {
 	return string(ret)
 }
 
-// globSingleton collects any glob patterns that were seen by Context and writes out rules to
+// GlobSingleton collects any glob patterns that were seen by Context and writes out rules to
 // re-evaluate them whenever the contents of the searched directories change, and retrigger the
 // primary builder if the results change.
-type globSingleton struct {
+type GlobSingleton struct {
-	globListDir string
+	// A function that returns the glob results of individual glob buckets
-	globLister  func() pathtools.MultipleGlobResults
+	GlobLister func() pathtools.MultipleGlobResults
-	srcDir      string
+
-	writeRule   bool
+	// Ninja file that contains instructions for validating the glob list files
+	GlobFile string
+
+	// Directory containing the glob list files
+	GlobDir string
+
+	// The source directory
+	SrcDir string
 }
 
-func globSingletonFactory(globListDir string, ctx *blueprint.Context) func() blueprint.Singleton {
+func globBucketName(globDir string, globBucket int) string {
-	return func() blueprint.Singleton {
+	return filepath.Join(globDir, strconv.Itoa(globBucket))
-		return &globSingleton{
-			globListDir: globListDir,
-			globLister:  ctx.Globs,
-			srcDir:      ctx.SrcDir(),
-		}
-	}
 }
 
-func (s *globSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
+// Returns the directory where glob list files live
+func GlobDirectory(buildDir, globListDir string) string {
+	return filepath.Join(buildDir, bootstrapSubDir, globListDir)
+}
+
+func (s *GlobSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 	// Sort the list of globs into buckets.  A hash function is used instead of sharding so that
 	// adding a new glob doesn't force rerunning all the buckets by shifting them all by 1.
 	globBuckets := make([]pathtools.MultipleGlobResults, numGlobBuckets)
-	for _, g := range s.globLister() {
+	for _, g := range s.GlobLister() {
 		bucket := globToBucket(g)
 		globBuckets[bucket] = append(globBuckets[bucket], g)
 	}
 
-	// The directory for the intermediates needs to be different for bootstrap and the primary
-	// builder.
-	bootstrapConfig := ctx.Config().(BootstrapConfig)
-	globsDir := globsDir(bootstrapConfig, s.globListDir)
-
 	for i, globs := range globBuckets {
-		fileListFile := filepath.Join(globsDir, strconv.Itoa(i))
+		fileListFile := globBucketName(s.GlobDir, i)
 
-		if s.writeRule {
+		// Called from generateGlobNinjaFile.  Write out the file list to disk, and add a ninja
-			// Called from generateGlobNinjaFile.  Write out the file list to disk, and add a ninja
+		// rule to run bpglob if any of the dependencies (usually directories that contain
-			// rule to run bpglob if any of the dependencies (usually directories that contain
+		// globbed files) have changed.  The file list produced by bpglob should match exactly
-			// globbed files) have changed.  The file list produced by bpglob should match exactly
+		// with the file written here so that restat can prevent rerunning the primary builder.
-			// with the file written here so that restat can prevent rerunning the primary builder.
+		//
-			//
+		// We need to write the file list here so that it has an older modified date
-			// We need to write the file list here so that it has an older modified date
+		// than the build.ninja (otherwise we'd run the primary builder twice on
-			// than the build.ninja (otherwise we'd run the primary builder twice on
+		// every new glob)
-			// every new glob)
+		//
-			//
+		// We don't need to write the depfile because we're guaranteed that ninja
-			// We don't need to write the depfile because we're guaranteed that ninja
+		// will run the command at least once (to record it into the ninja_log), so
-			// will run the command at least once (to record it into the ninja_log), so
+		// the depfile will be loaded from that execution.
-			// the depfile will be loaded from that execution.
+		absoluteFileListFile := joinPath(s.SrcDir, fileListFile)
-			absoluteFileListFile := joinPath(s.srcDir, fileListFile)
+		err := pathtools.WriteFileIfChanged(absoluteFileListFile, globs.FileList(), 0666)
-			err := pathtools.WriteFileIfChanged(absoluteFileListFile, globs.FileList(), 0666)
+		if err != nil {
-			if err != nil {
+			panic(fmt.Errorf("error writing %s: %s", fileListFile, err))
-				panic(fmt.Errorf("error writing %s: %s", fileListFile, err))
-			}
-
-			// Write out the ninja rule to run bpglob.
-			multipleGlobFilesRule(ctx, fileListFile, i, globs)
-		} else {
-			// Called from the main Context, make build.ninja depend on the fileListFile.
-			ctx.AddNinjaFileDeps(fileListFile)
 		}
+
+		// Write out the ninja rule to run bpglob.
+		multipleGlobFilesRule(ctx, fileListFile, i, globs)
 	}
 }
 
-func WriteBuildGlobsNinjaFile(globListDir string, ctx *blueprint.Context, args Args, config interface{}) {
+// Writes a .ninja file that contains instructions for regenerating the glob
-	buffer, errs := generateGlobNinjaFile(ctx.SrcDir(), globListDir, config, ctx.Globs)
+// files that contain the results of every glob that was run. The list of files
+// is available as the result of GlobFileListFiles().
+func WriteBuildGlobsNinjaFile(glob *GlobSingleton, config interface{}) {
+	buffer, errs := generateGlobNinjaFile(glob, config)
 	if len(errs) > 0 {
 		fatalErrors(errs)
 	}
 
 	const outFilePermissions = 0666
-	err := ioutil.WriteFile(joinPath(ctx.SrcDir(), args.GlobFile), buffer, outFilePermissions)
+	err := ioutil.WriteFile(joinPath(glob.SrcDir, glob.GlobFile), buffer, outFilePermissions)
 	if err != nil {
-		fatalf("error writing %s: %s", args.GlobFile, err)
+		fatalf("error writing %s: %s", glob.GlobFile, err)
 	}
 }
-func generateGlobNinjaFile(srcDir, globListDir string, config interface{},
+func generateGlobNinjaFile(glob *GlobSingleton, config interface{}) ([]byte, []error) {
-	globLister func() pathtools.MultipleGlobResults) ([]byte, []error) {
 
 	ctx := blueprint.NewContext()
 	ctx.RegisterSingletonType("glob", func() blueprint.Singleton {
-		return &globSingleton{
+		return glob
-			globListDir: globListDir,
-			globLister:  globLister,
-			srcDir:      srcDir,
-			writeRule:   true,
-		}
 	})
 
 	extraDeps, errs := ctx.ResolveDependencies(config)
@@ -260,19 +253,15 @@ func generateGlobNinjaFile(srcDir, globListDir string, config interface{},
 	return buf.Bytes(), nil
 }
 
-// globsDir returns a different directory to store glob intermediates for the bootstrap and
+// GlobFileListFiles returns the list of files that contain the result of globs
-// primary builder executions.
+// in the build. It is suitable for inclusion in build.ninja.d (so that
-func globsDir(config BootstrapConfig, globsDir string) string {
+// build.ninja is regenerated if the globs change). The instructions to
-	buildDir := config.BuildDir()
+// regenerate these files are written by WriteBuildGlobsNinjaFile().
-	return filepath.Join(buildDir, bootstrapSubDir, globsDir)
+func GlobFileListFiles(globDir string) []string {
-}
-
-// GlobFileListFiles returns the list of sharded glob file list files for the main stage.
-func GlobFileListFiles(config BootstrapConfig, globListDir string) []string {
-	globsDir := globsDir(config, globListDir)
 	var fileListFiles []string
 	for i := 0; i < numGlobBuckets; i++ {
-		fileListFiles = append(fileListFiles, filepath.Join(globsDir, strconv.Itoa(i)))
+		fileListFile := globBucketName(globDir, i)
+		fileListFiles = append(fileListFiles, fileListFile)
 	}
 	return fileListFiles
 }