tequilaOS/platform_build_blueprint
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Do not do glob-related things in RunBlueprint()."

Browse source
This commit is contained in:
Lukács T. Berki 2021-08-19 06:17:32 +00:00 committed by Gerrit Code Review
commit 3bec89465e
3 changed files with 58 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
bootstrap/command.go
View file

@ -32,10 +32,7 @@ import (
type Args struct {
OutFile string
Subninjas []string
GlobFile string
GlobListDir string
DepFile string
DocFile string
Cpuprofile string
Memprofile string
DelveListen string
@ -153,7 +150,6 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []strin
topLevelBlueprintsFile: args.TopFile,
subninjas: args.Subninjas,
globListDir: args.GlobListDir,
runGoTests: args.RunGoTests,
useValidations: args.UseValidations,
primaryBuilderInvocations: invocations,
@ -165,8 +161,6 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []strin
ctx.RegisterModuleType("blueprint_go_binary", newGoBinaryModuleFactory(bootstrapConfig, true))
ctx.RegisterSingletonType("bootstrap", newSingletonFactory(bootstrapConfig))
ctx.RegisterSingletonType("glob", globSingletonFactory(bootstrapConfig.globListDir, ctx))
blueprintFiles, errs := ctx.ParseFileList(filepath.Dir(args.TopFile), filesToParse, config)
if len(errs) > 0 {
fatalErrors(errs)
@ -221,10 +215,6 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []strin
out = ioutil.Discard.(io.StringWriter)
}
if args.GlobFile != "" {
WriteBuildGlobsNinjaFile(args.GlobListDir, ctx, args, config)
}
err = ctx.WriteBuildFile(out)
if err != nil {
fatalf("error writing Ninja file contents: %s", err)

1
bootstrap/config.go
View file

@ -131,7 +131,6 @@ type Config struct {
topLevelBlueprintsFile string
subninjas []string
globListDir string
runGoTests bool
useValidations bool

127
bootstrap/glob.go
View file

@ -145,94 +145,87 @@ func joinWithPrefixAndQuote(strs []string, prefix string) string {
return string(ret)
}
// globSingleton collects any glob patterns that were seen by Context and writes out rules to
// GlobSingleton collects any glob patterns that were seen by Context and writes out rules to
// re-evaluate them whenever the contents of the searched directories change, and retrigger the
// primary builder if the results change.
type globSingleton struct {
globListDir string
globLister func() pathtools.MultipleGlobResults
srcDir string
writeRule bool
type GlobSingleton struct {
// A function that returns the glob results of individual glob buckets
GlobLister func() pathtools.MultipleGlobResults
// Ninja file that contains instructions for validating the glob list files
GlobFile string
// Directory containing the glob list files
GlobDir string
// The source directory
SrcDir string
}
func globSingletonFactory(globListDir string, ctx *blueprint.Context) func() blueprint.Singleton {
return func() blueprint.Singleton {
return &globSingleton{
globListDir: globListDir,
globLister: ctx.Globs,
srcDir: ctx.SrcDir(),
}
}
func globBucketName(globDir string, globBucket int) string {
return filepath.Join(globDir, strconv.Itoa(globBucket))
}
func (s *globSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
// Returns the directory where glob list files live
func GlobDirectory(buildDir, globListDir string) string {
return filepath.Join(buildDir, bootstrapSubDir, globListDir)
}
func (s *GlobSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
// Sort the list of globs into buckets. A hash function is used instead of sharding so that
// adding a new glob doesn't force rerunning all the buckets by shifting them all by 1.
globBuckets := make([]pathtools.MultipleGlobResults, numGlobBuckets)
for _, g := range s.globLister() {
for _, g := range s.GlobLister() {
bucket := globToBucket(g)
globBuckets[bucket] = append(globBuckets[bucket], g)
}
// The directory for the intermediates needs to be different for bootstrap and the primary
// builder.
bootstrapConfig := ctx.Config().(BootstrapConfig)
globsDir := globsDir(bootstrapConfig, s.globListDir)
for i, globs := range globBuckets {
fileListFile := filepath.Join(globsDir, strconv.Itoa(i))
fileListFile := globBucketName(s.GlobDir, i)
if s.writeRule {
// Called from generateGlobNinjaFile. Write out the file list to disk, and add a ninja
// rule to run bpglob if any of the dependencies (usually directories that contain
// globbed files) have changed. The file list produced by bpglob should match exactly
// with the file written here so that restat can prevent rerunning the primary builder.
//
// We need to write the file list here so that it has an older modified date
// than the build.ninja (otherwise we'd run the primary builder twice on
// every new glob)
//
// We don't need to write the depfile because we're guaranteed that ninja
// will run the command at least once (to record it into the ninja_log), so
// the depfile will be loaded from that execution.
absoluteFileListFile := joinPath(s.srcDir, fileListFile)
err := pathtools.WriteFileIfChanged(absoluteFileListFile, globs.FileList(), 0666)
if err != nil {
panic(fmt.Errorf("error writing %s: %s", fileListFile, err))
}
// Write out the ninja rule to run bpglob.
multipleGlobFilesRule(ctx, fileListFile, i, globs)
} else {
// Called from the main Context, make build.ninja depend on the fileListFile.
ctx.AddNinjaFileDeps(fileListFile)
// Called from generateGlobNinjaFile. Write out the file list to disk, and add a ninja
// rule to run bpglob if any of the dependencies (usually directories that contain
// globbed files) have changed. The file list produced by bpglob should match exactly
// with the file written here so that restat can prevent rerunning the primary builder.
//
// We need to write the file list here so that it has an older modified date
// than the build.ninja (otherwise we'd run the primary builder twice on
// every new glob)
//
// We don't need to write the depfile because we're guaranteed that ninja
// will run the command at least once (to record it into the ninja_log), so
// the depfile will be loaded from that execution.
absoluteFileListFile := joinPath(s.SrcDir, fileListFile)
err := pathtools.WriteFileIfChanged(absoluteFileListFile, globs.FileList(), 0666)
if err != nil {
panic(fmt.Errorf("error writing %s: %s", fileListFile, err))
}
// Write out the ninja rule to run bpglob.
multipleGlobFilesRule(ctx, fileListFile, i, globs)
}
}
func WriteBuildGlobsNinjaFile(globListDir string, ctx *blueprint.Context, args Args, config interface{}) {
buffer, errs := generateGlobNinjaFile(ctx.SrcDir(), globListDir, config, ctx.Globs)
// Writes a .ninja file that contains instructions for regenerating the glob
// files that contain the results of every glob that was run. The list of files
// is available as the result of GlobFileListFiles().
func WriteBuildGlobsNinjaFile(glob *GlobSingleton, config interface{}) {
buffer, errs := generateGlobNinjaFile(glob, config)
if len(errs) > 0 {
fatalErrors(errs)
}
const outFilePermissions = 0666
err := ioutil.WriteFile(joinPath(ctx.SrcDir(), args.GlobFile), buffer, outFilePermissions)
err := ioutil.WriteFile(joinPath(glob.SrcDir, glob.GlobFile), buffer, outFilePermissions)
if err != nil {
fatalf("error writing %s: %s", args.GlobFile, err)
fatalf("error writing %s: %s", glob.GlobFile, err)
}
}
func generateGlobNinjaFile(srcDir, globListDir string, config interface{},
globLister func() pathtools.MultipleGlobResults) ([]byte, []error) {
func generateGlobNinjaFile(glob *GlobSingleton, config interface{}) ([]byte, []error) {
ctx := blueprint.NewContext()
ctx.RegisterSingletonType("glob", func() blueprint.Singleton {
return &globSingleton{
globListDir: globListDir,
globLister: globLister,
srcDir: srcDir,
writeRule: true,
}
return glob
})
extraDeps, errs := ctx.ResolveDependencies(config)
@ -260,19 +253,15 @@ func generateGlobNinjaFile(srcDir, globListDir string, config interface{},
return buf.Bytes(), nil
}
// globsDir returns a different directory to store glob intermediates for the bootstrap and
// primary builder executions.
func globsDir(config BootstrapConfig, globsDir string) string {
buildDir := config.BuildDir()
return filepath.Join(buildDir, bootstrapSubDir, globsDir)
}
// GlobFileListFiles returns the list of sharded glob file list files for the main stage.
func GlobFileListFiles(config BootstrapConfig, globListDir string) []string {
globsDir := globsDir(config, globListDir)
// GlobFileListFiles returns the list of files that contain the result of globs
// in the build. It is suitable for inclusion in build.ninja.d (so that
// build.ninja is regenerated if the globs change). The instructions to
// regenerate these files are written by WriteBuildGlobsNinjaFile().
func GlobFileListFiles(globDir string) []string {
var fileListFiles []string
for i := 0; i < numGlobBuckets; i++ {
fileListFiles = append(fileListFiles, filepath.Join(globsDir, strconv.Itoa(i)))
fileListFile := globBucketName(globDir, i)
fileListFiles = append(fileListFiles, fileListFile)
}
return fileListFiles
}