platform_build_blueprint/bootstrap/bootstrap.go

788 lines
22 KiB
Go
Raw Normal View History

// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bootstrap
import (
"fmt"
"path/filepath"
"strings"
"github.com/google/blueprint"
"github.com/google/blueprint/pathtools"
)
const bootstrapSubDir = ".bootstrap"
const miniBootstrapSubDir = ".minibootstrap"
var (
2015-03-21 03:39:29 +01:00
pctx = blueprint.NewPackageContext("github.com/google/blueprint/bootstrap")
goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain"))
goTestRunnerCmd = pctx.StaticVariable("goTestRunnerCmd", filepath.Join(bootstrapDir, "bin", "gotestrunner"))
pluginGenSrcCmd = pctx.StaticVariable("pluginGenSrcCmd", filepath.Join(bootstrapDir, "bin", "loadplugins"))
compile = pctx.StaticRule("compile",
blueprint.RuleParams{
Command: "GOROOT='$goRoot' $compileCmd -o $out -p $pkgPath -complete " +
"$incFlags -pack $in",
CommandDeps: []string{"$compileCmd"},
Description: "compile $out",
},
"pkgPath", "incFlags")
link = pctx.StaticRule("link",
blueprint.RuleParams{
Command: "GOROOT='$goRoot' $linkCmd -o $out $libDirFlags $in",
CommandDeps: []string{"$linkCmd"},
Description: "link $out",
},
"libDirFlags")
goTestMain = pctx.StaticRule("gotestmain",
blueprint.RuleParams{
Command: "$goTestMainCmd -o $out -pkg $pkg $in",
CommandDeps: []string{"$goTestMainCmd"},
Description: "gotestmain $out",
},
"pkg")
pluginGenSrc = pctx.StaticRule("pluginGenSrc",
blueprint.RuleParams{
Command: "$pluginGenSrcCmd -o $out -p $pkg $plugins",
CommandDeps: []string{"$pluginGenSrcCmd"},
Description: "create $out",
},
"pkg", "plugins")
test = pctx.StaticRule("test",
blueprint.RuleParams{
Command: "$goTestRunnerCmd -p $pkgSrcDir -f $out -- $in -test.short",
CommandDeps: []string{"$goTestRunnerCmd"},
Description: "test $pkg",
},
"pkg", "pkgSrcDir")
cp = pctx.StaticRule("cp",
blueprint.RuleParams{
Command: "cp $in $out",
Description: "cp $out",
},
"generator")
bootstrap = pctx.StaticRule("bootstrap",
blueprint.RuleParams{
Command: "BUILDDIR=$buildDir $bootstrapCmd -i $in",
CommandDeps: []string{"$bootstrapCmd"},
Description: "bootstrap $in",
Generator: true,
})
Enhance bootstrap stage selection This simplifies the bootstrap process while making it more flexible by moving the stage selection into a go binary(choosestage). It will now be possible to have more than two build stages. Now each stage has a ninja template(main.ninja.in) and a timestamp file(main.ninja.in.timestamp). The timestamp file may be updated by any build stage that wishes to regenerate the ninja template. If the choosestage binaries sees that the timestamp is newer than the template, it will choose the prior stage. The main stage no longer writes to the source tree to update the build.ninja.in file. This was a problem for read-only source trees. Instead, the choosestage binary first checks to see if that file is newer than the last bootstrap.ninja.in, copies it in place, and starts the boostrap stage. The bootstrap stage regenerates it's own ninja template, but that required a loop through the main stage to actually run it. The choosestage binary now detects if the template has changed for the current stage, and will restart the stage. One change is that if dependencies do get messed up, instead of silently failing, there's a higher chance that the bootstrap step will just continue looping, doing nothing. This can happen if the main stage has a dependency that triggers the bootstrap stage, but the bootstrap stage doesn't see anything required to rebuild the main ninja file. A side effect of this requirement is that changes to test code will now rebuild the main ninja file. Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
touch = pctx.StaticRule("touch",
blueprint.RuleParams{
Command: "touch $out",
Description: "touch $out",
},
"depfile", "generator")
generateBuildNinja = pctx.StaticRule("build.ninja",
blueprint.RuleParams{
Command: "$builder $extra -b $buildDir -d $out.d -o $out $in",
CommandDeps: []string{"$builder"},
Description: "$builder $out",
Depfile: "$out.d",
Restat: true,
},
"builder", "extra", "generator")
// Work around a Ninja issue. See https://github.com/martine/ninja/pull/634
phony = pctx.StaticRule("phony",
blueprint.RuleParams{
Command: "# phony $out",
Description: "phony $out",
Generator: true,
},
"depfile")
binDir = pctx.StaticVariable("BinDir", filepath.Join(bootstrapDir, "bin"))
minibpFile = filepath.Join("$BinDir", "minibp")
docsDir = filepath.Join(bootstrapDir, "docs")
toolDir = pctx.VariableFunc("ToolDir", func(config interface{}) (string, error) {
if c, ok := config.(ConfigBlueprintToolLocation); ok {
return c.BlueprintToolLocation(), nil
}
return filepath.Join("$buildDir", "bin"), nil
})
bootstrapDir = filepath.Join("$buildDir", bootstrapSubDir)
miniBootstrapDir = filepath.Join("$buildDir", miniBootstrapSubDir)
)
type bootstrapGoCore interface {
BuildStage() Stage
SetBuildStage(Stage)
}
func propagateStageBootstrap(mctx blueprint.TopDownMutatorContext) {
if mod, ok := mctx.Module().(bootstrapGoCore); ok {
stage := mod.BuildStage()
mctx.VisitDirectDeps(func(mod blueprint.Module) {
if m, ok := mod.(bootstrapGoCore); ok && m.BuildStage() > stage {
m.SetBuildStage(stage)
}
})
}
}
func pluginDeps(ctx blueprint.BottomUpMutatorContext) {
if pkg, ok := ctx.Module().(*goPackage); ok {
for _, plugin := range pkg.properties.PluginFor {
ctx.AddReverseDependency(ctx.Module(), nil, plugin)
}
}
}
type goPackageProducer interface {
GoPkgRoot() string
GoPackageTarget() string
GoTestTargets() []string
}
func isGoPackageProducer(module blueprint.Module) bool {
_, ok := module.(goPackageProducer)
return ok
}
type goPluginProvider interface {
GoPkgPath() string
IsPluginFor(string) bool
}
func isGoPluginFor(name string) func(blueprint.Module) bool {
return func(module blueprint.Module) bool {
if plugin, ok := module.(goPluginProvider); ok {
return plugin.IsPluginFor(name)
}
return false
}
}
func isBootstrapModule(module blueprint.Module) bool {
_, isPackage := module.(*goPackage)
_, isBinary := module.(*goBinary)
return isPackage || isBinary
}
func isBootstrapBinaryModule(module blueprint.Module) bool {
_, isBinary := module.(*goBinary)
return isBinary
}
// A goPackage is a module for building Go packages.
type goPackage struct {
blueprint.SimpleName
properties struct {
Deps []string
PkgPath string
Srcs []string
TestSrcs []string
PluginFor []string
// The stage in which this module should be built
BuildStage Stage `blueprint:"mutated"`
}
// The root dir in which the package .a file is located. The full .a file
// path will be "packageRoot/PkgPath.a"
pkgRoot string
// The path of the .a file that is to be built.
archiveFile string
// The path of the test result file.
testResultFile []string
// The bootstrap Config
config *Config
}
var _ goPackageProducer = (*goPackage)(nil)
func newGoPackageModuleFactory(config *Config) func() (blueprint.Module, []interface{}) {
return func() (blueprint.Module, []interface{}) {
module := &goPackage{
config: config,
}
module.properties.BuildStage = StageMain
return module, []interface{}{&module.properties, &module.SimpleName.Properties}
}
}
func (g *goPackage) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string {
return g.properties.Deps
}
func (g *goPackage) GoPkgPath() string {
return g.properties.PkgPath
}
func (g *goPackage) GoPkgRoot() string {
return g.pkgRoot
}
func (g *goPackage) GoPackageTarget() string {
return g.archiveFile
}
func (g *goPackage) GoTestTargets() []string {
return g.testResultFile
}
func (g *goPackage) BuildStage() Stage {
return g.properties.BuildStage
}
func (g *goPackage) SetBuildStage(buildStage Stage) {
g.properties.BuildStage = buildStage
}
func (g *goPackage) IsPluginFor(name string) bool {
for _, plugin := range g.properties.PluginFor {
if plugin == name {
return true
}
}
return false
}
func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
var (
name = ctx.ModuleName()
hasPlugins = false
pluginSrc = ""
genSrcs = []string{}
)
if g.properties.PkgPath == "" {
ctx.ModuleErrorf("module %s did not specify a valid pkgPath", name)
return
}
g.pkgRoot = packageRoot(ctx)
g.archiveFile = filepath.Join(g.pkgRoot,
filepath.FromSlash(g.properties.PkgPath)+".a")
var testArchiveFile string
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
testArchiveFile = filepath.Join(testRoot(ctx),
filepath.FromSlash(g.properties.PkgPath)+".a")
}
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) { hasPlugins = true })
if hasPlugins {
pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
genSrcs = append(genSrcs, pluginSrc)
}
// We only actually want to build the builder modules if we're running as
// minibp (i.e. we're generating a bootstrap Ninja file). This is to break
// the circular dependence that occurs when the builder requires a new Ninja
// file to be built, but building a new ninja file requires the builder to
// be built.
if g.config.stage == g.BuildStage() {
if hasPlugins && !buildGoPluginLoader(ctx, g.properties.PkgPath, pluginSrc, g.config.stage) {
return
}
if g.config.runGoTests {
g.testResultFile = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
g.properties.PkgPath, g.properties.Srcs, genSrcs,
g.properties.TestSrcs)
}
buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile,
g.properties.Srcs, genSrcs)
}
}
// A goBinary is a module for building executable binaries from Go sources.
type goBinary struct {
blueprint.SimpleName
properties struct {
Deps []string
Srcs []string
TestSrcs []string
PrimaryBuilder bool
// The stage in which this module should be built
BuildStage Stage `blueprint:"mutated"`
}
// The bootstrap Config
config *Config
}
func newGoBinaryModuleFactory(config *Config, buildStage Stage) func() (blueprint.Module, []interface{}) {
return func() (blueprint.Module, []interface{}) {
module := &goBinary{
config: config,
}
module.properties.BuildStage = buildStage
return module, []interface{}{&module.properties, &module.SimpleName.Properties}
}
}
func (g *goBinary) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string {
return g.properties.Deps
}
func (g *goBinary) BuildStage() Stage {
return g.properties.BuildStage
}
func (g *goBinary) SetBuildStage(buildStage Stage) {
g.properties.BuildStage = buildStage
}
func (g *goBinary) InstallPath() string {
if g.BuildStage() == StageMain {
return "$ToolDir"
}
return "$BinDir"
}
func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
var (
name = ctx.ModuleName()
objDir = moduleObjDir(ctx)
archiveFile = filepath.Join(objDir, name+".a")
testArchiveFile = filepath.Join(testRoot(ctx), name+".a")
aoutFile = filepath.Join(objDir, "a.out")
binaryFile = filepath.Join(g.InstallPath(), name)
hasPlugins = false
pluginSrc = ""
genSrcs = []string{}
)
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) { hasPlugins = true })
if hasPlugins {
pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
genSrcs = append(genSrcs, pluginSrc)
}
if g.config.stage == g.BuildStage() {
var deps []string
if hasPlugins && !buildGoPluginLoader(ctx, "main", pluginSrc, g.config.stage) {
return
}
if g.config.runGoTests {
deps = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
name, g.properties.Srcs, genSrcs, g.properties.TestSrcs)
}
buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, genSrcs)
var libDirFlags []string
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
func(module blueprint.Module) {
dep := module.(goPackageProducer)
libDir := dep.GoPkgRoot()
libDirFlags = append(libDirFlags, "-L "+libDir)
deps = append(deps, dep.GoTestTargets()...)
})
linkArgs := map[string]string{}
if len(libDirFlags) > 0 {
linkArgs["libDirFlags"] = strings.Join(libDirFlags, " ")
}
ctx.Build(pctx, blueprint.BuildParams{
Rule: link,
Outputs: []string{aoutFile},
Inputs: []string{archiveFile},
Args: linkArgs,
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: cp,
Outputs: []string{binaryFile},
Inputs: []string{aoutFile},
OrderOnly: deps,
})
}
}
func buildGoPluginLoader(ctx blueprint.ModuleContext, pkgPath, pluginSrc string, stage Stage) bool {
ret := true
name := ctx.ModuleName()
var pluginPaths []string
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) {
plugin := module.(goPluginProvider)
pluginPaths = append(pluginPaths, plugin.GoPkgPath())
if stage == StageBootstrap {
ctx.OtherModuleErrorf(module, "plugin %q may not be included in core module %q",
ctx.OtherModuleName(module), name)
ret = false
}
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: pluginGenSrc,
Outputs: []string{pluginSrc},
Args: map[string]string{
"pkg": pkgPath,
"plugins": strings.Join(pluginPaths, " "),
},
})
return ret
}
func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
pkgPath string, archiveFile string, srcs []string, genSrcs []string) {
srcDir := moduleSrcDir(ctx)
srcFiles := pathtools.PrefixPaths(srcs, srcDir)
srcFiles = append(srcFiles, genSrcs...)
var incFlags []string
var deps []string
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
func(module blueprint.Module) {
dep := module.(goPackageProducer)
incDir := dep.GoPkgRoot()
target := dep.GoPackageTarget()
incFlags = append(incFlags, "-I "+incDir)
deps = append(deps, target)
})
compileArgs := map[string]string{
"pkgPath": pkgPath,
}
if len(incFlags) > 0 {
compileArgs["incFlags"] = strings.Join(incFlags, " ")
}
ctx.Build(pctx, blueprint.BuildParams{
Rule: compile,
Outputs: []string{archiveFile},
Inputs: srcFiles,
Implicits: deps,
Args: compileArgs,
})
}
func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
pkgPath string, srcs, genSrcs, testSrcs []string) []string {
if len(testSrcs) == 0 {
return nil
}
srcDir := moduleSrcDir(ctx)
testFiles := pathtools.PrefixPaths(testSrcs, srcDir)
mainFile := filepath.Join(testRoot, "test.go")
testArchive := filepath.Join(testRoot, "test.a")
testFile := filepath.Join(testRoot, "test")
testPassed := filepath.Join(testRoot, "test.passed")
buildGoPackage(ctx, testRoot, pkgPath, testPkgArchive,
append(srcs, testSrcs...), genSrcs)
ctx.Build(pctx, blueprint.BuildParams{
Rule: goTestMain,
Outputs: []string{mainFile},
Inputs: testFiles,
Args: map[string]string{
"pkg": pkgPath,
},
})
libDirFlags := []string{"-L " + testRoot}
testDeps := []string{}
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
func(module blueprint.Module) {
dep := module.(goPackageProducer)
libDir := dep.GoPkgRoot()
libDirFlags = append(libDirFlags, "-L "+libDir)
testDeps = append(testDeps, dep.GoTestTargets()...)
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: compile,
Outputs: []string{testArchive},
Inputs: []string{mainFile},
Implicits: []string{testPkgArchive},
Args: map[string]string{
"pkgPath": "main",
"incFlags": "-I " + testRoot,
},
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: link,
Outputs: []string{testFile},
Inputs: []string{testArchive},
Args: map[string]string{
"libDirFlags": strings.Join(libDirFlags, " "),
},
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: test,
Outputs: []string{testPassed},
Inputs: []string{testFile},
OrderOnly: testDeps,
Args: map[string]string{
"pkg": pkgPath,
"pkgSrcDir": filepath.Dir(testFiles[0]),
},
})
return []string{testPassed}
}
type singleton struct {
// The bootstrap Config
config *Config
}
func newSingletonFactory(config *Config) func() blueprint.Singleton {
return func() blueprint.Singleton {
return &singleton{
config: config,
}
}
}
func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
// Find the module that's marked as the "primary builder", which means it's
// creating the binary that we'll use to generate the non-bootstrap
// build.ninja file.
var primaryBuilders []*goBinary
// blueprintTools contains blueprint go binaries that will be built in StageMain
var blueprintTools []string
ctx.VisitAllModulesIf(isBootstrapBinaryModule,
func(module blueprint.Module) {
binaryModule := module.(*goBinary)
binaryModuleName := ctx.ModuleName(binaryModule)
installPath := filepath.Join(binaryModule.InstallPath(), binaryModuleName)
if binaryModule.BuildStage() == StageMain {
blueprintTools = append(blueprintTools, installPath)
}
if binaryModule.properties.PrimaryBuilder {
primaryBuilders = append(primaryBuilders, binaryModule)
}
})
var extraTestFlags string
if s.config.runGoTests {
extraTestFlags = " -t"
}
var primaryBuilderName, primaryBuilderExtraFlags string
switch len(primaryBuilders) {
case 0:
// If there's no primary builder module then that means we'll use minibp
// as the primary builder. We can trigger its primary builder mode with
// the -p flag.
primaryBuilderName = "minibp"
primaryBuilderExtraFlags = "-p" + extraTestFlags
case 1:
primaryBuilderName = ctx.ModuleName(primaryBuilders[0])
primaryBuilderExtraFlags = extraTestFlags
default:
ctx.Errorf("multiple primary builder modules present:")
for _, primaryBuilder := range primaryBuilders {
ctx.ModuleErrorf(primaryBuilder, "<-- module %s",
ctx.ModuleName(primaryBuilder))
}
return
}
primaryBuilderFile := filepath.Join("$BinDir", primaryBuilderName)
// Get the filename of the top-level Blueprints file to pass to minibp.
topLevelBlueprints := filepath.Join("$srcDir",
filepath.Base(s.config.topLevelBlueprintsFile))
Simplify bootstrap tl;dr: Read if you don't use the wrapper or use SKIP_NINJA Previously, we were relying on the ninja behavior of restarting the build when the build.ninja file was updated to switch between different bootstrap stages. But that means that every step that could produce a build.ninja must pass in order to switch to a different stage. That wasn't a big problem when we had a two stage build -- there was very little that could fail in the second stage before we chose to go back to the first stage. But when we had a three stage build, it was possible to get into a state (usually during development) where you were in the second stage, but the build was failing because the first stage needed to be run. This was fixed in d79f1af7423e0ef7a13573efdae5100a57fabc82 by adding a wrapper that always started building at the first stage. But this kept all of the complexity of using ninja restarts without any of the benefits, so this change removes that complexity and just runs each stage sequentially in the wrapper. So the wrapper is now required. Since we're no longer going through choosestage, we can also skip the template parsing for the later stages that don't need to be templated -- this can save a couple of seconds for large files. In addition to all of the above, this also lets Soong reduce the number of times the main ninja file is loaded. We had been running the wrapper once (3 stages), then running ninja again after combining the Soong-generated build.ninja with the Kati-generated build.ninja. This change lets us removing the intermediate parsing of Soong's build.ninja, so that we only execute ninja 3 times per build. It also lets us have dependencies on pools or rules from Kati in the primary builder, since we're never executing the main build.ninja without the Kati build.ninja. The wrapper has a new option, NINJA to provide the path to ninja. This used to be hardcoded to `ninja`, and will still default to that. But we'll be running the first two bootstrap stages with $NINJA even if SKIP_NINJA is set. The wrapper passes "-w dupbuild=err" to ninja now -- this really should always be turned on if you care about reliable builds. Change-Id: I6f656b74eb3d064b8b9e69d1d6dac1129d72b747
2016-08-13 21:42:11 +02:00
mainNinjaFile := filepath.Join("$buildDir", "build.ninja")
primaryBuilderNinjaFile := filepath.Join(bootstrapDir, "build.ninja")
bootstrapNinjaFileTemplate := filepath.Join(miniBootstrapDir, "build.ninja.in")
bootstrapNinjaFile := filepath.Join(miniBootstrapDir, "build.ninja")
docsFile := filepath.Join(docsDir, primaryBuilderName+".html")
Enhance bootstrap stage selection This simplifies the bootstrap process while making it more flexible by moving the stage selection into a go binary(choosestage). It will now be possible to have more than two build stages. Now each stage has a ninja template(main.ninja.in) and a timestamp file(main.ninja.in.timestamp). The timestamp file may be updated by any build stage that wishes to regenerate the ninja template. If the choosestage binaries sees that the timestamp is newer than the template, it will choose the prior stage. The main stage no longer writes to the source tree to update the build.ninja.in file. This was a problem for read-only source trees. Instead, the choosestage binary first checks to see if that file is newer than the last bootstrap.ninja.in, copies it in place, and starts the boostrap stage. The bootstrap stage regenerates it's own ninja template, but that required a loop through the main stage to actually run it. The choosestage binary now detects if the template has changed for the current stage, and will restart the stage. One change is that if dependencies do get messed up, instead of silently failing, there's a higher chance that the bootstrap step will just continue looping, doing nothing. This can happen if the main stage has a dependency that triggers the bootstrap stage, but the bootstrap stage doesn't see anything required to rebuild the main ninja file. A side effect of this requirement is that changes to test code will now rebuild the main ninja file. Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
switch s.config.stage {
case StageBootstrap:
// We're generating a bootstrapper Ninja file, so we need to set things
// up to rebuild the build.ninja file using the primary builder.
// BuildDir must be different between the three stages, otherwise the
// cleanup process will remove files from the other builds.
ctx.SetNinjaBuildDir(pctx, miniBootstrapDir)
// Generate the Ninja file to build the primary builder.
ctx.Build(pctx, blueprint.BuildParams{
Rule: generateBuildNinja,
Outputs: []string{primaryBuilderNinjaFile},
Inputs: []string{topLevelBlueprints},
Args: map[string]string{
"builder": minibpFile,
"extra": "--build-primary" + extraTestFlags,
},
})
Enhance bootstrap stage selection This simplifies the bootstrap process while making it more flexible by moving the stage selection into a go binary(choosestage). It will now be possible to have more than two build stages. Now each stage has a ninja template(main.ninja.in) and a timestamp file(main.ninja.in.timestamp). The timestamp file may be updated by any build stage that wishes to regenerate the ninja template. If the choosestage binaries sees that the timestamp is newer than the template, it will choose the prior stage. The main stage no longer writes to the source tree to update the build.ninja.in file. This was a problem for read-only source trees. Instead, the choosestage binary first checks to see if that file is newer than the last bootstrap.ninja.in, copies it in place, and starts the boostrap stage. The bootstrap stage regenerates it's own ninja template, but that required a loop through the main stage to actually run it. The choosestage binary now detects if the template has changed for the current stage, and will restart the stage. One change is that if dependencies do get messed up, instead of silently failing, there's a higher chance that the bootstrap step will just continue looping, doing nothing. This can happen if the main stage has a dependency that triggers the bootstrap stage, but the bootstrap stage doesn't see anything required to rebuild the main ninja file. A side effect of this requirement is that changes to test code will now rebuild the main ninja file. Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
// Rebuild the bootstrap Ninja file using the minibp that we just built.
ctx.Build(pctx, blueprint.BuildParams{
Rule: generateBuildNinja,
Simplify bootstrap tl;dr: Read if you don't use the wrapper or use SKIP_NINJA Previously, we were relying on the ninja behavior of restarting the build when the build.ninja file was updated to switch between different bootstrap stages. But that means that every step that could produce a build.ninja must pass in order to switch to a different stage. That wasn't a big problem when we had a two stage build -- there was very little that could fail in the second stage before we chose to go back to the first stage. But when we had a three stage build, it was possible to get into a state (usually during development) where you were in the second stage, but the build was failing because the first stage needed to be run. This was fixed in d79f1af7423e0ef7a13573efdae5100a57fabc82 by adding a wrapper that always started building at the first stage. But this kept all of the complexity of using ninja restarts without any of the benefits, so this change removes that complexity and just runs each stage sequentially in the wrapper. So the wrapper is now required. Since we're no longer going through choosestage, we can also skip the template parsing for the later stages that don't need to be templated -- this can save a couple of seconds for large files. In addition to all of the above, this also lets Soong reduce the number of times the main ninja file is loaded. We had been running the wrapper once (3 stages), then running ninja again after combining the Soong-generated build.ninja with the Kati-generated build.ninja. This change lets us removing the intermediate parsing of Soong's build.ninja, so that we only execute ninja 3 times per build. It also lets us have dependencies on pools or rules from Kati in the primary builder, since we're never executing the main build.ninja without the Kati build.ninja. The wrapper has a new option, NINJA to provide the path to ninja. This used to be hardcoded to `ninja`, and will still default to that. But we'll be running the first two bootstrap stages with $NINJA even if SKIP_NINJA is set. The wrapper passes "-w dupbuild=err" to ninja now -- this really should always be turned on if you care about reliable builds. Change-Id: I6f656b74eb3d064b8b9e69d1d6dac1129d72b747
2016-08-13 21:42:11 +02:00
Outputs: []string{bootstrapNinjaFileTemplate},
Enhance bootstrap stage selection This simplifies the bootstrap process while making it more flexible by moving the stage selection into a go binary(choosestage). It will now be possible to have more than two build stages. Now each stage has a ninja template(main.ninja.in) and a timestamp file(main.ninja.in.timestamp). The timestamp file may be updated by any build stage that wishes to regenerate the ninja template. If the choosestage binaries sees that the timestamp is newer than the template, it will choose the prior stage. The main stage no longer writes to the source tree to update the build.ninja.in file. This was a problem for read-only source trees. Instead, the choosestage binary first checks to see if that file is newer than the last bootstrap.ninja.in, copies it in place, and starts the boostrap stage. The bootstrap stage regenerates it's own ninja template, but that required a loop through the main stage to actually run it. The choosestage binary now detects if the template has changed for the current stage, and will restart the stage. One change is that if dependencies do get messed up, instead of silently failing, there's a higher chance that the bootstrap step will just continue looping, doing nothing. This can happen if the main stage has a dependency that triggers the bootstrap stage, but the bootstrap stage doesn't see anything required to rebuild the main ninja file. A side effect of this requirement is that changes to test code will now rebuild the main ninja file. Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
Inputs: []string{topLevelBlueprints},
Args: map[string]string{
"builder": minibpFile,
"extra": extraTestFlags,
},
Enhance bootstrap stage selection This simplifies the bootstrap process while making it more flexible by moving the stage selection into a go binary(choosestage). It will now be possible to have more than two build stages. Now each stage has a ninja template(main.ninja.in) and a timestamp file(main.ninja.in.timestamp). The timestamp file may be updated by any build stage that wishes to regenerate the ninja template. If the choosestage binaries sees that the timestamp is newer than the template, it will choose the prior stage. The main stage no longer writes to the source tree to update the build.ninja.in file. This was a problem for read-only source trees. Instead, the choosestage binary first checks to see if that file is newer than the last bootstrap.ninja.in, copies it in place, and starts the boostrap stage. The bootstrap stage regenerates it's own ninja template, but that required a loop through the main stage to actually run it. The choosestage binary now detects if the template has changed for the current stage, and will restart the stage. One change is that if dependencies do get messed up, instead of silently failing, there's a higher chance that the bootstrap step will just continue looping, doing nothing. This can happen if the main stage has a dependency that triggers the bootstrap stage, but the bootstrap stage doesn't see anything required to rebuild the main ninja file. A side effect of this requirement is that changes to test code will now rebuild the main ninja file. Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
})
ctx.Build(pctx, blueprint.BuildParams{
Simplify bootstrap tl;dr: Read if you don't use the wrapper or use SKIP_NINJA Previously, we were relying on the ninja behavior of restarting the build when the build.ninja file was updated to switch between different bootstrap stages. But that means that every step that could produce a build.ninja must pass in order to switch to a different stage. That wasn't a big problem when we had a two stage build -- there was very little that could fail in the second stage before we chose to go back to the first stage. But when we had a three stage build, it was possible to get into a state (usually during development) where you were in the second stage, but the build was failing because the first stage needed to be run. This was fixed in d79f1af7423e0ef7a13573efdae5100a57fabc82 by adding a wrapper that always started building at the first stage. But this kept all of the complexity of using ninja restarts without any of the benefits, so this change removes that complexity and just runs each stage sequentially in the wrapper. So the wrapper is now required. Since we're no longer going through choosestage, we can also skip the template parsing for the later stages that don't need to be templated -- this can save a couple of seconds for large files. In addition to all of the above, this also lets Soong reduce the number of times the main ninja file is loaded. We had been running the wrapper once (3 stages), then running ninja again after combining the Soong-generated build.ninja with the Kati-generated build.ninja. This change lets us removing the intermediate parsing of Soong's build.ninja, so that we only execute ninja 3 times per build. It also lets us have dependencies on pools or rules from Kati in the primary builder, since we're never executing the main build.ninja without the Kati build.ninja. The wrapper has a new option, NINJA to provide the path to ninja. This used to be hardcoded to `ninja`, and will still default to that. But we'll be running the first two bootstrap stages with $NINJA even if SKIP_NINJA is set. The wrapper passes "-w dupbuild=err" to ninja now -- this really should always be turned on if you care about reliable builds. Change-Id: I6f656b74eb3d064b8b9e69d1d6dac1129d72b747
2016-08-13 21:42:11 +02:00
Rule: bootstrap,
Outputs: []string{bootstrapNinjaFile},
Inputs: []string{bootstrapNinjaFileTemplate},
})
case StagePrimary:
// We're generating a bootstrapper Ninja file, so we need to set things
// up to rebuild the build.ninja file using the primary builder.
// BuildDir must be different between the three stages, otherwise the
// cleanup process will remove files from the other builds.
ctx.SetNinjaBuildDir(pctx, bootstrapDir)
// Add a way to rebuild the primary build.ninja so that globs works
ctx.Build(pctx, blueprint.BuildParams{
Rule: generateBuildNinja,
Outputs: []string{primaryBuilderNinjaFile},
Inputs: []string{topLevelBlueprints},
Args: map[string]string{
"builder": minibpFile,
"extra": "--build-primary" + extraTestFlags,
"generator": "true",
},
})
Enhance bootstrap stage selection This simplifies the bootstrap process while making it more flexible by moving the stage selection into a go binary(choosestage). It will now be possible to have more than two build stages. Now each stage has a ninja template(main.ninja.in) and a timestamp file(main.ninja.in.timestamp). The timestamp file may be updated by any build stage that wishes to regenerate the ninja template. If the choosestage binaries sees that the timestamp is newer than the template, it will choose the prior stage. The main stage no longer writes to the source tree to update the build.ninja.in file. This was a problem for read-only source trees. Instead, the choosestage binary first checks to see if that file is newer than the last bootstrap.ninja.in, copies it in place, and starts the boostrap stage. The bootstrap stage regenerates it's own ninja template, but that required a loop through the main stage to actually run it. The choosestage binary now detects if the template has changed for the current stage, and will restart the stage. One change is that if dependencies do get messed up, instead of silently failing, there's a higher chance that the bootstrap step will just continue looping, doing nothing. This can happen if the main stage has a dependency that triggers the bootstrap stage, but the bootstrap stage doesn't see anything required to rebuild the main ninja file. A side effect of this requirement is that changes to test code will now rebuild the main ninja file. Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
// Build the main build.ninja
Enhance bootstrap stage selection This simplifies the bootstrap process while making it more flexible by moving the stage selection into a go binary(choosestage). It will now be possible to have more than two build stages. Now each stage has a ninja template(main.ninja.in) and a timestamp file(main.ninja.in.timestamp). The timestamp file may be updated by any build stage that wishes to regenerate the ninja template. If the choosestage binaries sees that the timestamp is newer than the template, it will choose the prior stage. The main stage no longer writes to the source tree to update the build.ninja.in file. This was a problem for read-only source trees. Instead, the choosestage binary first checks to see if that file is newer than the last bootstrap.ninja.in, copies it in place, and starts the boostrap stage. The bootstrap stage regenerates it's own ninja template, but that required a loop through the main stage to actually run it. The choosestage binary now detects if the template has changed for the current stage, and will restart the stage. One change is that if dependencies do get messed up, instead of silently failing, there's a higher chance that the bootstrap step will just continue looping, doing nothing. This can happen if the main stage has a dependency that triggers the bootstrap stage, but the bootstrap stage doesn't see anything required to rebuild the main ninja file. A side effect of this requirement is that changes to test code will now rebuild the main ninja file. Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
ctx.Build(pctx, blueprint.BuildParams{
Rule: generateBuildNinja,
Outputs: []string{mainNinjaFile},
Inputs: []string{topLevelBlueprints},
Args: map[string]string{
"builder": primaryBuilderFile,
"extra": primaryBuilderExtraFlags,
},
Enhance bootstrap stage selection This simplifies the bootstrap process while making it more flexible by moving the stage selection into a go binary(choosestage). It will now be possible to have more than two build stages. Now each stage has a ninja template(main.ninja.in) and a timestamp file(main.ninja.in.timestamp). The timestamp file may be updated by any build stage that wishes to regenerate the ninja template. If the choosestage binaries sees that the timestamp is newer than the template, it will choose the prior stage. The main stage no longer writes to the source tree to update the build.ninja.in file. This was a problem for read-only source trees. Instead, the choosestage binary first checks to see if that file is newer than the last bootstrap.ninja.in, copies it in place, and starts the boostrap stage. The bootstrap stage regenerates it's own ninja template, but that required a loop through the main stage to actually run it. The choosestage binary now detects if the template has changed for the current stage, and will restart the stage. One change is that if dependencies do get messed up, instead of silently failing, there's a higher chance that the bootstrap step will just continue looping, doing nothing. This can happen if the main stage has a dependency that triggers the bootstrap stage, but the bootstrap stage doesn't see anything required to rebuild the main ninja file. A side effect of this requirement is that changes to test code will now rebuild the main ninja file. Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
})
// Generate build system docs for the primary builder. Generating docs reads the source
// files used to build the primary builder, but that dependency will be picked up through
// the dependency on the primary builder itself. There are no dependencies on the
// Blueprints files, as any relevant changes to the Blueprints files would have caused
// a rebuild of the primary builder.
bigbpDocs := ctx.Rule(pctx, "bigbpDocs",
blueprint.RuleParams{
Command: fmt.Sprintf("%s %s -b $buildDir --docs $out %s", primaryBuilderFile,
primaryBuilderExtraFlags, topLevelBlueprints),
CommandDeps: []string{primaryBuilderFile},
Description: fmt.Sprintf("%s docs $out", primaryBuilderName),
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: bigbpDocs,
Outputs: []string{docsFile},
})
Enhance bootstrap stage selection This simplifies the bootstrap process while making it more flexible by moving the stage selection into a go binary(choosestage). It will now be possible to have more than two build stages. Now each stage has a ninja template(main.ninja.in) and a timestamp file(main.ninja.in.timestamp). The timestamp file may be updated by any build stage that wishes to regenerate the ninja template. If the choosestage binaries sees that the timestamp is newer than the template, it will choose the prior stage. The main stage no longer writes to the source tree to update the build.ninja.in file. This was a problem for read-only source trees. Instead, the choosestage binary first checks to see if that file is newer than the last bootstrap.ninja.in, copies it in place, and starts the boostrap stage. The bootstrap stage regenerates it's own ninja template, but that required a loop through the main stage to actually run it. The choosestage binary now detects if the template has changed for the current stage, and will restart the stage. One change is that if dependencies do get messed up, instead of silently failing, there's a higher chance that the bootstrap step will just continue looping, doing nothing. This can happen if the main stage has a dependency that triggers the bootstrap stage, but the bootstrap stage doesn't see anything required to rebuild the main ninja file. A side effect of this requirement is that changes to test code will now rebuild the main ninja file. Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
case StageMain:
ctx.SetNinjaBuildDir(pctx, "${buildDir}")
// Add a way to rebuild the main build.ninja in case it creates rules that
// it will depend on itself. (In Android, globs with soong_glob)
ctx.Build(pctx, blueprint.BuildParams{
Rule: generateBuildNinja,
Outputs: []string{mainNinjaFile},
Inputs: []string{topLevelBlueprints},
Args: map[string]string{
"builder": primaryBuilderFile,
"extra": primaryBuilderExtraFlags,
"generator": "true",
},
})
if primaryBuilderName == "minibp" {
// This is a standalone Blueprint build, so we copy the minibp
// binary to the "bin" directory to make it easier to find.
finalMinibp := filepath.Join("$buildDir", "bin", primaryBuilderName)
ctx.Build(pctx, blueprint.BuildParams{
Rule: cp,
Inputs: []string{primaryBuilderFile},
Outputs: []string{finalMinibp},
})
}
ctx.Build(pctx, blueprint.BuildParams{
Rule: blueprint.Phony,
Outputs: []string{"blueprint_tools"},
Inputs: blueprintTools,
})
}
}
// packageRoot returns the module-specific package root directory path. This
// directory is where the final package .a files are output and where dependant
// modules search for this package via -I arguments.
func packageRoot(ctx blueprint.ModuleContext) string {
return filepath.Join(bootstrapDir, ctx.ModuleName(), "pkg")
}
// testRoot returns the module-specific package root directory path used for
// building tests. The .a files generated here will include everything from
// packageRoot, plus the test-only code.
func testRoot(ctx blueprint.ModuleContext) string {
return filepath.Join(bootstrapDir, ctx.ModuleName(), "test")
}
// moduleSrcDir returns the path of the directory that all source file paths are
// specified relative to.
func moduleSrcDir(ctx blueprint.ModuleContext) string {
return filepath.Join("$srcDir", ctx.ModuleDir())
}
// moduleObjDir returns the module-specific object directory path.
func moduleObjDir(ctx blueprint.ModuleContext) string {
return filepath.Join(bootstrapDir, ctx.ModuleName(), "obj")
}
// moduleGenSrcDir returns the module-specific generated sources path.
func moduleGenSrcDir(ctx blueprint.ModuleContext) string {
return filepath.Join(bootstrapDir, ctx.ModuleName(), "gen")
}