Allow build.ninja regeneration (fix Soong globs)

Soong (and soon, Blueprint) embeds the results of filesystem globs into
the build.ninja, and uses a helper tool to detect when the glob changes
and we need to rebuild the build.ninja file. This is more flexible than
listing the affected directories in the depfiles, since it can check to
see if a file we actually cared about was added, instead of re-running
anytime any file was added/removed (which happens on atomic file
modifications as well).

My recent bootstrap simplification broke this, since the helper rules
are in the main build.ninja, but I removed the ability to regenerate
that file from itself. So keep the current model, but add a rule into
the primary and main stages that allow themselves to re-run their
generator and write out a new build.ninja file if necessary. The actual
build rules of the generator aren't necessary, since we already built
them in the previous stage.

Change-Id: Ib51245920b2ec3ee5306c0c269361a5a8733caa8

bootstrap/bootstrap.go

@@ -96,6 +96,16 @@ var (
 		},
 		"depfile", "generator")
 
+	generateBuildNinja = pctx.StaticRule("build.ninja",
+		blueprint.RuleParams{
+			Command:     "$builder $extra -b $buildDir -d $out.d -o $out $in",
+			CommandDeps: []string{"$builder"},
+			Description: "$builder $out",
+			Depfile:     "$out.d",
+			Restat:      true,
+		},
+		"builder", "extra", "generator")
+
 	// Work around a Ninja issue.  See https://github.com/martine/ninja/pull/634
 	phony = pctx.StaticRule("phony",
 		blueprint.RuleParams{
@@ -611,6 +621,11 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 			}
 		})
 
+	var extraTestFlags string
+	if s.config.runGoTests {
+		extraTestFlags = " -t"
+	}
+
 	var primaryBuilderName, primaryBuilderExtraFlags string
 	switch len(primaryBuilders) {
 	case 0:
@@ -618,7 +633,7 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 		// as the primary builder.  We can trigger its primary builder mode with
 		// the -p flag.
 		primaryBuilderName = "minibp"
-		primaryBuilderExtraFlags = "-p"
+		primaryBuilderExtraFlags = "-p" + extraTestFlags
 
 	case 1:
 		primaryBuilderName = ctx.ModuleName(primaryBuilders[0])
@@ -634,10 +649,6 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 
 	primaryBuilderFile := filepath.Join("$BinDir", primaryBuilderName)
 
-	if s.config.runGoTests {
-		primaryBuilderExtraFlags += " -t"
-	}
-
 	// Get the filename of the top-level Blueprints file to pass to minibp.
 	topLevelBlueprints := filepath.Join("$srcDir",
 		filepath.Base(s.config.topLevelBlueprintsFile))
@@ -680,56 +691,27 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 		// cleanup process will remove files from the other builds.
 		ctx.SetNinjaBuildDir(pctx, miniBootstrapDir)
 
-		// Generate the Ninja file to build the primary builder. Save the
-		// timestamps and deps, so that we can come back to this stage if
-		// it needs to be regenerated.
-		primarybp := ctx.Rule(pctx, "primarybp",
-			blueprint.RuleParams{
-				Command: fmt.Sprintf("%s --build-primary $runTests "+
-					"-b $buildDir -d $out.d -o $out $in", minibpFile),
-				Description: "minibp $out",
-				Depfile:     "$out.d",
-			},
-			"runTests")
-
-		args := make(map[string]string)
-
-		if s.config.runGoTests {
-			args["runTests"] = "-t"
-		}
-
+		// Generate the Ninja file to build the primary builder.
 		ctx.Build(pctx, blueprint.BuildParams{
-			Rule:      primarybp,
+			Rule:      generateBuildNinja,
 			Outputs:   []string{primaryBuilderNinjaFile},
 			Inputs:    []string{topLevelBlueprints},
 			Implicits: bootstrapDeps,
-			Args:      args,
+			Args: map[string]string{
+				"builder": minibpFile,
+				"extra":   "--build-primary" + extraTestFlags,
+			},
 		})
 
 		// Rebuild the bootstrap Ninja file using the minibp that we just built.
-		minibp := ctx.Rule(pctx, "minibp",
-			blueprint.RuleParams{
-				Command: fmt.Sprintf("%s $runTests "+
-					"-b $buildDir -d $out.d -o $out $in", minibpFile),
-				CommandDeps: []string{minibpFile},
-				Description: "minibp $out",
-				Depfile:     "$out.d",
-				// So that we don't trigger a restart if this hasn't changed
-				Restat: true,
-			},
-			"runTests")
-
-		args = map[string]string{}
-
-		if s.config.runGoTests {
-			args["runTests"] = "-t"
-		}
-
 		ctx.Build(pctx, blueprint.BuildParams{
-			Rule:    minibp,
+			Rule:    generateBuildNinja,
 			Outputs: []string{bootstrapNinjaFileTemplate},
 			Inputs:  []string{topLevelBlueprints},
-			Args:    args,
+			Args: map[string]string{
+				"builder": minibpFile,
+				"extra":   extraTestFlags,
+			},
 		})
 
 		ctx.Build(pctx, blueprint.BuildParams{
@@ -746,23 +728,29 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 		// cleanup process will remove files from the other builds.
 		ctx.SetNinjaBuildDir(pctx, bootstrapDir)
 
-		// We generate the depfile here that includes the dependencies for all
-		// the Blueprints files that contribute to generating the big build
-		// manifest (build.ninja file).
-		bigbp := ctx.Rule(pctx, "bigbp",
-			blueprint.RuleParams{
-				Command: fmt.Sprintf("%s %s "+
-					"-b $buildDir -d $out.d -o $out $in", primaryBuilderFile,
-					primaryBuilderExtraFlags),
-				Description: fmt.Sprintf("%s $out", primaryBuilderName),
-				Depfile:     "$out.d",
-			})
-
+		// Add a way to rebuild the primary build.ninja so that globs works
 		ctx.Build(pctx, blueprint.BuildParams{
-			Rule:      bigbp,
+			Rule:      generateBuildNinja,
+			Outputs:   []string{primaryBuilderNinjaFile},
+			Inputs:    []string{topLevelBlueprints},
+			Implicits: bootstrapDeps,
+			Args: map[string]string{
+				"builder":   minibpFile,
+				"extra":     "--build-primary" + extraTestFlags,
+				"generator": "true",
+			},
+		})
+
+		// Build the main build.ninja
+		ctx.Build(pctx, blueprint.BuildParams{
+			Rule:      generateBuildNinja,
 			Outputs:   []string{mainNinjaFile},
 			Inputs:    []string{topLevelBlueprints},
 			Implicits: primaryBootstrapDeps,
+			Args: map[string]string{
+				"builder": primaryBuilderFile,
+				"extra":   primaryBuilderExtraFlags,
+			},
 		})
 
 		// Generate build system docs for the primary builder.  Generating docs reads the source
@@ -786,6 +774,20 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 	case StageMain:
 		ctx.SetNinjaBuildDir(pctx, "${buildDir}")
 
+		// Add a way to rebuild the main build.ninja in case it creates rules that
+		// it will depend on itself. (In Android, globs with soong_glob)
+		ctx.Build(pctx, blueprint.BuildParams{
+			Rule:      generateBuildNinja,
+			Outputs:   []string{mainNinjaFile},
+			Inputs:    []string{topLevelBlueprints},
+			Implicits: primaryBootstrapDeps,
+			Args: map[string]string{
+				"builder":   primaryBuilderFile,
+				"extra":     primaryBuilderExtraFlags,
+				"generator": "true",
+			},
+		})
+
 		if primaryBuilderName == "minibp" {
 			// This is a standalone Blueprint build, so we copy the minibp
 			// binary to the "bin" directory to make it easier to find.

build.ninja.in

@@ -30,6 +30,12 @@ rule g.bootstrap.bootstrap
     description = bootstrap ${in}
     generator = true
 
+rule g.bootstrap.build.ninja
+    command = ${builder} ${extra} -b ${g.bootstrap.buildDir} -d ${out}.d -o ${out} ${in}
+    depfile = ${out}.d
+    description = ${builder} ${out}
+    restat = true
+
 rule g.bootstrap.compile
     command = GOROOT='${g.bootstrap.goRoot}' ${g.bootstrap.compileCmd} -o ${out} -p ${pkgPath} -complete ${incFlags} -pack ${in}
     description = compile ${out}
@@ -263,26 +269,18 @@ default ${g.bootstrap.BinDir}/minibp
 # Singleton: bootstrap
 # Factory:   github.com/google/blueprint/bootstrap.newSingletonFactory.func1
 
-rule s.bootstrap.primarybp
-    command = ${g.bootstrap.BinDir}/minibp --build-primary ${runTests} -b ${g.bootstrap.buildDir} -d ${out}.d -o ${out} ${in}
-    depfile = ${out}.d
-    description = minibp ${out}
-
-rule s.bootstrap.minibp
-    command = ${g.bootstrap.BinDir}/minibp ${runTests} -b ${g.bootstrap.buildDir} -d ${out}.d -o ${out} ${in}
-    depfile = ${out}.d
-    description = minibp ${out}
-    restat = true
-
-build ${g.bootstrap.buildDir}/.bootstrap/build.ninja: s.bootstrap.primarybp $
-        ${g.bootstrap.srcDir}/Blueprints | ${g.bootstrap.BinDir}/gotestmain $
-        ${g.bootstrap.BinDir}/gotestrunner ${g.bootstrap.BinDir}/minibp $
-        ${g.bootstrap.srcDir}/Blueprints
+build ${g.bootstrap.buildDir}/.bootstrap/build.ninja: g.bootstrap.build.ninja $
+        ${g.bootstrap.srcDir}/Blueprints | ${builder} $
+        ${g.bootstrap.BinDir}/gotestmain ${g.bootstrap.BinDir}/gotestrunner $
+        ${g.bootstrap.BinDir}/minibp ${g.bootstrap.srcDir}/Blueprints
+    builder = ${g.bootstrap.BinDir}/minibp
+    extra = --build-primary
 default ${g.bootstrap.buildDir}/.bootstrap/build.ninja
 
 build ${g.bootstrap.buildDir}/.minibootstrap/build.ninja.in: $
-        s.bootstrap.minibp ${g.bootstrap.srcDir}/Blueprints | $
-        ${g.bootstrap.BinDir}/minibp
+        g.bootstrap.build.ninja ${g.bootstrap.srcDir}/Blueprints | ${builder}
+    builder = ${g.bootstrap.BinDir}/minibp
+    extra = 
 default ${g.bootstrap.buildDir}/.minibootstrap/build.ninja.in
 
 build ${g.bootstrap.buildDir}/.minibootstrap/build.ninja: $

tests/test.sh

@@ -17,7 +17,7 @@ if [[ -d .bootstrap/blueprint/test ]]; then
 fi
 
 sleep 2
-sed -i 's/${runTests}/-t/' src.build.ninja.in
+sed -i 's/extra =/extra = -t/' src.build.ninja.in
 ./blueprint.bash
 
 if [[ ! -d .bootstrap/blueprint/test ]]; then