Move primary builder command line computation.

This makes it possible to invoke the primary builder more than once in a
single Ninja file.

This required adding arguments that were hard-wired in the build.ninja
rule to extraArgs and adding a "set of primary builder invocations"
member to bootstrap.Config . The only command line arguments that remain
hard-wired are those which are expected to be relevant to every
invocation of the primary builder.

As a welcome side effect, I was able to remove a number of random
environment variables that were so far special-cased ($SOONG_DELVE,
$SOONG_DELVE_PATH and $SOONG_OUTDIR). I was also able to move writing
the empty stub ninja glob file to command.go from the bowels of the
Blueprint machinery.

In theory, $TOP and $BUILDER could be removed, too, but this would
require hard-coding the value of $TOP into build.ninja and I don't know
what would break if I did that (it's okay to hard-wired $SOONG_DELVE and
$SOONG_DELVE_PATH because those are only used for debugging and
$SOONG_OUTDIR turned out to be superfluous)

Test: Presubmits.
Change-Id: Idbfd9976c4b270bc3e5a8926c8c760a8534596cf

bootstrap/bootstrap.go

@@ -17,8 +17,6 @@ package bootstrap
 import (
 	"fmt"
 	"go/build"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -134,24 +132,18 @@ var (
 				`BUILDER="$$PWD/$$(basename "$builder")" && ` +
 				`cd / && ` +
 				`env -i "$$BUILDER" ` +
-				`    $extra ` +
 				`    --top "$$TOP" ` +
-				`    --out "$$SOONG_OUTDIR" ` +
-				`    --delve_listen "$$SOONG_DELVE" ` +
-				`    --delve_path "$$SOONG_DELVE_PATH" ` +
-				`    -b "$buildDir" ` +
+				`    --out "$buildDir" ` +
 				`    -n "$ninjaBuildDir" ` +
 				`    -d "$out.d" ` +
-				`    -globFile "$globFile" ` +
-				`    -o "$out" ` +
-				`    "$in" `,
+				`    $extra`,
 			CommandDeps: []string{"$builder"},
 			Description: "$builder $out",
 			Deps:        blueprint.DepsGCC,
 			Depfile:     "$out.d",
 			Restat:      true,
 		},
-		"builder", "extra", "generator", "globFile")
+		"builder", "extra")
 
 	// Work around a Ninja issue.  See https://github.com/martine/ninja/pull/634
 	phony = pctx.StaticRule("phony",
@@ -708,73 +700,48 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 			}
 		})
 
-	var extraSharedFlagArray []string
-	if s.config.runGoTests {
-		extraSharedFlagArray = append(extraSharedFlagArray, "-t")
-	}
-	if s.config.moduleListFile != "" {
-		extraSharedFlagArray = append(extraSharedFlagArray, "-l", s.config.moduleListFile)
-	}
-	if s.config.emptyNinjaFile {
-		extraSharedFlagArray = append(extraSharedFlagArray, "--empty-ninja-file")
-	}
-	extraSharedFlagString := strings.Join(extraSharedFlagArray, " ")
+	var primaryBuilderCmdlinePrefix []string
+	var primaryBuilderName string
 
-	var primaryBuilderName, primaryBuilderExtraFlags string
-	switch len(primaryBuilders) {
-	case 0:
+	if len(primaryBuilders) == 0 {
 		// If there's no primary builder module then that means we'll use minibp
 		// as the primary builder.  We can trigger its primary builder mode with
 		// the -p flag.
 		primaryBuilderName = "minibp"
-		primaryBuilderExtraFlags = "-p " + extraSharedFlagString
-
-	case 1:
-		primaryBuilderName = ctx.ModuleName(primaryBuilders[0])
-		primaryBuilderExtraFlags = extraSharedFlagString
-
-	default:
+		primaryBuilderCmdlinePrefix = append(primaryBuilderCmdlinePrefix, "-p")
+	} else if len(primaryBuilders) > 1 {
 		ctx.Errorf("multiple primary builder modules present:")
 		for _, primaryBuilder := range primaryBuilders {
 			ctx.ModuleErrorf(primaryBuilder, "<-- module %s",
 				ctx.ModuleName(primaryBuilder))
 		}
 		return
+	} else {
+		primaryBuilderName = ctx.ModuleName(primaryBuilders[0])
 	}
 
 	primaryBuilderFile := filepath.Join("$BinDir", primaryBuilderName)
-
-	// Get the filename of the top-level Blueprints file to pass to minibp.
-	topLevelBlueprints := filepath.Join("$srcDir",
-		filepath.Base(s.config.topLevelBlueprintsFile))
 	ctx.SetNinjaBuildDir(pctx, "${ninjaBuildDir}")
 
-	buildDir := ctx.Config().(BootstrapConfig).BuildDir()
-
 	if s.config.stage == StagePrimary {
-		mainNinjaFile := filepath.Join("$buildDir", "build.ninja")
-		primaryBuilderNinjaGlobFile := absolutePath(filepath.Join(buildDir, bootstrapSubDir, "build-globs.ninja"))
+		ctx.AddSubninja(s.config.globFile)
 
-		if _, err := os.Stat(primaryBuilderNinjaGlobFile); os.IsNotExist(err) {
-			err = ioutil.WriteFile(primaryBuilderNinjaGlobFile, nil, 0666)
-			if err != nil {
-				ctx.Errorf("Failed to create empty ninja file: %s", err)
-			}
+		for _, i := range s.config.primaryBuilderInvocations {
+			flags := make([]string, 0)
+			flags = append(flags, primaryBuilderCmdlinePrefix...)
+			flags = append(flags, i.Args...)
+
+			// Build the main build.ninja
+			ctx.Build(pctx, blueprint.BuildParams{
+				Rule:    generateBuildNinja,
+				Outputs: i.Outputs,
+				Inputs:  i.Inputs,
+				Args: map[string]string{
+					"builder": primaryBuilderFile,
+					"extra":   strings.Join(flags, " "),
+				},
+			})
 		}
-
-		ctx.AddSubninja(primaryBuilderNinjaGlobFile)
-
-		// Build the main build.ninja
-		ctx.Build(pctx, blueprint.BuildParams{
-			Rule:    generateBuildNinja,
-			Outputs: []string{mainNinjaFile},
-			Inputs:  []string{topLevelBlueprints},
-			Args: map[string]string{
-				"builder":  primaryBuilderFile,
-				"extra":    primaryBuilderExtraFlags,
-				"globFile": primaryBuilderNinjaGlobFile,
-			},
-		})
 	}
 
 	if s.config.stage == StageMain {
@@ -798,7 +765,7 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
 		bigbpDocs := ctx.Rule(pctx, "bigbpDocs",
 			blueprint.RuleParams{
 				Command: fmt.Sprintf("%s %s -b $buildDir --docs $out %s", primaryBuilderFile,
-					primaryBuilderExtraFlags, topLevelBlueprints),
+					primaryBuilderExtraFlags, s.config.topLevelBlueprintsFile),
 				CommandDeps: []string{primaryBuilderFile},
 				Description: fmt.Sprintf("%s docs $out", primaryBuilderName),
 			})

bootstrap/command.go

@@ -38,6 +38,8 @@ type Args struct {
 	DocFile                  string
 	Cpuprofile               string
 	Memprofile               string
+	DelveListen              string
+	DelvePath                string
 	TraceFile                string
 	RunGoTests               bool
 	UseValidations           bool
@@ -104,6 +106,46 @@ func Main(ctx *blueprint.Context, config interface{}, generatingPrimaryBuilder b
 	RunBlueprint(cmdline, ctx, config, extraNinjaFileDeps...)
 }
 
+func primaryBuilderExtraFlags(args Args, globFile, mainNinjaFile string) []string {
+	result := make([]string, 0)
+
+	if args.RunGoTests {
+		result = append(result, "-t")
+	}
+
+	result = append(result, "-l", args.ModuleListFile)
+	result = append(result, "-globFile", globFile)
+	result = append(result, "-o", mainNinjaFile)
+
+	if args.EmptyNinjaFile {
+		result = append(result, "--empty-ninja-file")
+	}
+
+	if args.DelveListen != "" {
+		result = append(result, "--delve_listen", args.DelveListen)
+	}
+
+	if args.DelvePath != "" {
+		result = append(result, "--delve_path", args.DelvePath)
+	}
+
+	result = append(result, args.TopFile)
+	return result
+}
+
+func writeEmptyGlobFile(path string) {
+	err := os.MkdirAll(filepath.Dir(path), 0777)
+	if err != nil {
+		fatalf("Failed to create parent directories of empty ninja glob file '%s': %s", path, err)
+	}
+
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		err = ioutil.WriteFile(path, nil, 0666)
+		if err != nil {
+			fatalf("Failed to create empty ninja glob file '%s': %s", path, err)
+		}
+	}
+}
 func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}, extraNinjaFileDeps ...string) {
 	runtime.GOMAXPROCS(runtime.NumCPU())
 
@@ -153,14 +195,25 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}, extraNi
 		stage = StagePrimary
 	}
 
+	primaryBuilderNinjaGlobFile := absolutePath(filepath.Join(args.BuildDir, bootstrapSubDir, "build-globs.ninja"))
+	mainNinjaFile := filepath.Join("$buildDir", "build.ninja")
+
+	writeEmptyGlobFile(primaryBuilderNinjaGlobFile)
+
 	bootstrapConfig := &Config{
 		stage: stage,
 
 		topLevelBlueprintsFile: args.TopFile,
-		emptyNinjaFile:         args.EmptyNinjaFile,
+		globFile:               primaryBuilderNinjaGlobFile,
 		runGoTests:             args.RunGoTests,
 		useValidations:         args.UseValidations,
-		moduleListFile:         args.ModuleListFile,
+		primaryBuilderInvocations: []PrimaryBuilderInvocation{
+			{
+				Inputs:  []string{args.TopFile},
+				Outputs: []string{mainNinjaFile},
+				Args:    primaryBuilderExtraFlags(args, primaryBuilderNinjaGlobFile, mainNinjaFile),
+			},
+		},
 	}
 
 	ctx.RegisterBottomUpMutator("bootstrap_plugin_deps", pluginDeps)
@@ -171,7 +224,7 @@ func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}, extraNi
 
 	ctx.RegisterSingletonType("glob", globSingletonFactory(ctx))
 
-	deps, errs := ctx.ParseFileList(filepath.Dir(bootstrapConfig.topLevelBlueprintsFile), filesToParse, config)
+	deps, errs := ctx.ParseFileList(filepath.Dir(args.TopFile), filesToParse, config)
 	if len(errs) > 0 {
 		fatalErrors(errs)
 	}

bootstrap/config.go

@@ -123,13 +123,20 @@ const (
 	StageMain
 )
 
+type PrimaryBuilderInvocation struct {
+	Inputs  []string
+	Outputs []string
+	Args    []string
+}
+
 type Config struct {
 	stage Stage
 
 	topLevelBlueprintsFile string
+	globFile               string
 
-	emptyNinjaFile bool
 	runGoTests     bool
 	useValidations bool
-	moduleListFile string
+
+	primaryBuilderInvocations []PrimaryBuilderInvocation
 }