2015-01-23 23:15:10 +01:00
|
|
|
// Copyright 2014 Google Inc. All rights reserved.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
package bootstrap
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"path/filepath"
|
|
|
|
"strings"
|
2015-05-13 23:36:24 +02:00
|
|
|
|
|
|
|
"github.com/google/blueprint"
|
|
|
|
"github.com/google/blueprint/pathtools"
|
2014-05-28 01:34:41 +02:00
|
|
|
)
|
|
|
|
|
2015-09-12 01:55:53 +02:00
|
|
|
const bootstrapSubDir = ".bootstrap"
|
|
|
|
const miniBootstrapSubDir = ".minibootstrap"
|
2014-06-06 23:21:57 +02:00
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
var (
|
2015-03-21 03:39:29 +01:00
|
|
|
pctx = blueprint.NewPackageContext("github.com/google/blueprint/bootstrap")
|
2014-05-28 01:34:41 +02:00
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain"))
|
2015-09-18 05:59:51 +02:00
|
|
|
goTestRunnerCmd = pctx.StaticVariable("goTestRunnerCmd", filepath.Join(bootstrapDir, "bin", "gotestrunner"))
|
2015-07-25 01:53:27 +02:00
|
|
|
pluginGenSrcCmd = pctx.StaticVariable("pluginGenSrcCmd", filepath.Join(bootstrapDir, "bin", "loadplugins"))
|
2014-10-03 11:49:58 +02:00
|
|
|
|
2015-08-02 00:07:27 +02:00
|
|
|
compile = pctx.StaticRule("compile",
|
2014-05-28 01:34:41 +02:00
|
|
|
blueprint.RuleParams{
|
2015-08-02 00:07:27 +02:00
|
|
|
Command: "GOROOT='$goRoot' $compileCmd -o $out -p $pkgPath -complete " +
|
2014-10-22 19:55:28 +02:00
|
|
|
"$incFlags -pack $in",
|
2015-11-17 23:21:45 +01:00
|
|
|
CommandDeps: []string{"$compileCmd"},
|
2015-08-02 00:07:27 +02:00
|
|
|
Description: "compile $out",
|
2014-05-28 01:34:41 +02:00
|
|
|
},
|
|
|
|
"pkgPath", "incFlags")
|
|
|
|
|
2014-10-03 11:49:58 +02:00
|
|
|
link = pctx.StaticRule("link",
|
2014-05-28 01:34:41 +02:00
|
|
|
blueprint.RuleParams{
|
2014-11-09 20:52:56 +01:00
|
|
|
Command: "GOROOT='$goRoot' $linkCmd -o $out $libDirFlags $in",
|
2015-11-17 23:21:45 +01:00
|
|
|
CommandDeps: []string{"$linkCmd"},
|
2015-08-02 00:07:27 +02:00
|
|
|
Description: "link $out",
|
2014-05-28 01:34:41 +02:00
|
|
|
},
|
|
|
|
"libDirFlags")
|
|
|
|
|
2015-06-24 02:21:00 +02:00
|
|
|
goTestMain = pctx.StaticRule("gotestmain",
|
|
|
|
blueprint.RuleParams{
|
|
|
|
Command: "$goTestMainCmd -o $out -pkg $pkg $in",
|
2015-11-17 23:21:45 +01:00
|
|
|
CommandDeps: []string{"$goTestMainCmd"},
|
2015-06-24 02:21:00 +02:00
|
|
|
Description: "gotestmain $out",
|
|
|
|
},
|
|
|
|
"pkg")
|
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
pluginGenSrc = pctx.StaticRule("pluginGenSrc",
|
|
|
|
blueprint.RuleParams{
|
|
|
|
Command: "$pluginGenSrcCmd -o $out -p $pkg $plugins",
|
2015-11-17 23:21:45 +01:00
|
|
|
CommandDeps: []string{"$pluginGenSrcCmd"},
|
2015-07-25 01:53:27 +02:00
|
|
|
Description: "create $out",
|
|
|
|
},
|
|
|
|
"pkg", "plugins")
|
|
|
|
|
2015-06-24 02:21:00 +02:00
|
|
|
test = pctx.StaticRule("test",
|
|
|
|
blueprint.RuleParams{
|
2015-09-18 05:59:51 +02:00
|
|
|
Command: "$goTestRunnerCmd -p $pkgSrcDir -f $out -- $in -test.short",
|
2015-11-17 23:21:45 +01:00
|
|
|
CommandDeps: []string{"$goTestRunnerCmd"},
|
2015-06-24 02:21:00 +02:00
|
|
|
Description: "test $pkg",
|
|
|
|
},
|
|
|
|
"pkg", "pkgSrcDir")
|
|
|
|
|
2014-10-03 11:49:58 +02:00
|
|
|
cp = pctx.StaticRule("cp",
|
2014-05-28 01:34:41 +02:00
|
|
|
blueprint.RuleParams{
|
|
|
|
Command: "cp $in $out",
|
|
|
|
Description: "cp $out",
|
2014-06-06 05:00:22 +02:00
|
|
|
},
|
|
|
|
"generator")
|
2014-05-28 01:34:41 +02:00
|
|
|
|
2014-10-03 11:49:58 +02:00
|
|
|
bootstrap = pctx.StaticRule("bootstrap",
|
2014-05-28 01:34:41 +02:00
|
|
|
blueprint.RuleParams{
|
2015-09-18 07:48:04 +02:00
|
|
|
Command: "BUILDDIR=$buildDir $bootstrapCmd -i $in",
|
2015-11-17 23:21:45 +01:00
|
|
|
CommandDeps: []string{"$bootstrapCmd"},
|
2014-05-28 01:34:41 +02:00
|
|
|
Description: "bootstrap $in",
|
|
|
|
Generator: true,
|
|
|
|
})
|
|
|
|
|
Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
|
|
|
touch = pctx.StaticRule("touch",
|
|
|
|
blueprint.RuleParams{
|
|
|
|
Command: "touch $out",
|
|
|
|
Description: "touch $out",
|
|
|
|
},
|
|
|
|
"depfile", "generator")
|
|
|
|
|
2016-11-02 01:12:28 +01:00
|
|
|
generateBuildNinja = pctx.StaticRule("build.ninja",
|
|
|
|
blueprint.RuleParams{
|
|
|
|
Command: "$builder $extra -b $buildDir -d $out.d -o $out $in",
|
|
|
|
CommandDeps: []string{"$builder"},
|
|
|
|
Description: "$builder $out",
|
|
|
|
Depfile: "$out.d",
|
|
|
|
Restat: true,
|
|
|
|
},
|
|
|
|
"builder", "extra", "generator")
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
// Work around a Ninja issue. See https://github.com/martine/ninja/pull/634
|
2014-10-03 11:49:58 +02:00
|
|
|
phony = pctx.StaticRule("phony",
|
2014-05-28 01:34:41 +02:00
|
|
|
blueprint.RuleParams{
|
|
|
|
Command: "# phony $out",
|
|
|
|
Description: "phony $out",
|
|
|
|
Generator: true,
|
|
|
|
},
|
|
|
|
"depfile")
|
|
|
|
|
2015-07-14 08:28:37 +02:00
|
|
|
binDir = pctx.StaticVariable("BinDir", filepath.Join(bootstrapDir, "bin"))
|
|
|
|
minibpFile = filepath.Join("$BinDir", "minibp")
|
2015-05-13 23:36:24 +02:00
|
|
|
|
|
|
|
docsDir = filepath.Join(bootstrapDir, "docs")
|
2016-05-26 19:07:59 +02:00
|
|
|
toolDir = pctx.VariableFunc("ToolDir", func(config interface{}) (string, error) {
|
|
|
|
if c, ok := config.(ConfigBlueprintToolLocation); ok {
|
|
|
|
return c.BlueprintToolLocation(), nil
|
|
|
|
}
|
|
|
|
return filepath.Join("$buildDir", "bin"), nil
|
|
|
|
})
|
2015-09-12 01:55:53 +02:00
|
|
|
|
2015-09-18 05:59:51 +02:00
|
|
|
bootstrapDir = filepath.Join("$buildDir", bootstrapSubDir)
|
2015-09-12 01:55:53 +02:00
|
|
|
miniBootstrapDir = filepath.Join("$buildDir", miniBootstrapSubDir)
|
2014-05-28 01:34:41 +02:00
|
|
|
)
|
|
|
|
|
2015-07-23 02:06:06 +02:00
|
|
|
type bootstrapGoCore interface {
|
|
|
|
BuildStage() Stage
|
|
|
|
SetBuildStage(Stage)
|
|
|
|
}
|
|
|
|
|
|
|
|
func propagateStageBootstrap(mctx blueprint.TopDownMutatorContext) {
|
2016-05-26 19:07:59 +02:00
|
|
|
if mod, ok := mctx.Module().(bootstrapGoCore); ok {
|
|
|
|
stage := mod.BuildStage()
|
2015-07-23 02:06:06 +02:00
|
|
|
|
2016-05-26 19:07:59 +02:00
|
|
|
mctx.VisitDirectDeps(func(mod blueprint.Module) {
|
|
|
|
if m, ok := mod.(bootstrapGoCore); ok && m.BuildStage() > stage {
|
|
|
|
m.SetBuildStage(stage)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
2015-07-23 02:06:06 +02:00
|
|
|
}
|
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
func pluginDeps(ctx blueprint.BottomUpMutatorContext) {
|
|
|
|
if pkg, ok := ctx.Module().(*goPackage); ok {
|
|
|
|
for _, plugin := range pkg.properties.PluginFor {
|
2016-04-12 00:47:28 +02:00
|
|
|
ctx.AddReverseDependency(ctx.Module(), nil, plugin)
|
2015-07-25 01:53:27 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
type goPackageProducer interface {
|
|
|
|
GoPkgRoot() string
|
|
|
|
GoPackageTarget() string
|
2016-11-02 08:43:00 +01:00
|
|
|
GoTestTargets() []string
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func isGoPackageProducer(module blueprint.Module) bool {
|
|
|
|
_, ok := module.(goPackageProducer)
|
|
|
|
return ok
|
|
|
|
}
|
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
type goPluginProvider interface {
|
|
|
|
GoPkgPath() string
|
|
|
|
IsPluginFor(string) bool
|
|
|
|
}
|
|
|
|
|
|
|
|
func isGoPluginFor(name string) func(blueprint.Module) bool {
|
|
|
|
return func(module blueprint.Module) bool {
|
|
|
|
if plugin, ok := module.(goPluginProvider); ok {
|
|
|
|
return plugin.IsPluginFor(name)
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
func isBootstrapModule(module blueprint.Module) bool {
|
|
|
|
_, isPackage := module.(*goPackage)
|
|
|
|
_, isBinary := module.(*goBinary)
|
|
|
|
return isPackage || isBinary
|
|
|
|
}
|
|
|
|
|
|
|
|
func isBootstrapBinaryModule(module blueprint.Module) bool {
|
|
|
|
_, isBinary := module.(*goBinary)
|
|
|
|
return isBinary
|
|
|
|
}
|
|
|
|
|
|
|
|
// A goPackage is a module for building Go packages.
|
|
|
|
type goPackage struct {
|
2016-05-17 23:58:05 +02:00
|
|
|
blueprint.SimpleName
|
2014-05-28 01:34:41 +02:00
|
|
|
properties struct {
|
2016-05-17 23:58:05 +02:00
|
|
|
Deps []string
|
2015-07-25 01:53:27 +02:00
|
|
|
PkgPath string
|
|
|
|
Srcs []string
|
|
|
|
TestSrcs []string
|
|
|
|
PluginFor []string
|
2016-04-12 00:41:52 +02:00
|
|
|
|
|
|
|
// The stage in which this module should be built
|
|
|
|
BuildStage Stage `blueprint:"mutated"`
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// The root dir in which the package .a file is located. The full .a file
|
|
|
|
// path will be "packageRoot/PkgPath.a"
|
|
|
|
pkgRoot string
|
|
|
|
|
|
|
|
// The path of the .a file that is to be built.
|
|
|
|
archiveFile string
|
2015-06-25 04:21:21 +02:00
|
|
|
|
2016-11-02 08:43:00 +01:00
|
|
|
// The path of the test result file.
|
|
|
|
testResultFile []string
|
2015-06-24 02:21:00 +02:00
|
|
|
|
2015-06-25 04:21:21 +02:00
|
|
|
// The bootstrap Config
|
|
|
|
config *Config
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
var _ goPackageProducer = (*goPackage)(nil)
|
|
|
|
|
2015-06-25 04:21:21 +02:00
|
|
|
func newGoPackageModuleFactory(config *Config) func() (blueprint.Module, []interface{}) {
|
|
|
|
return func() (blueprint.Module, []interface{}) {
|
|
|
|
module := &goPackage{
|
2016-04-12 00:41:52 +02:00
|
|
|
config: config,
|
2015-06-25 04:21:21 +02:00
|
|
|
}
|
2016-05-26 19:07:59 +02:00
|
|
|
module.properties.BuildStage = StageMain
|
2016-05-17 23:58:05 +02:00
|
|
|
return module, []interface{}{&module.properties, &module.SimpleName.Properties}
|
2015-06-25 04:21:21 +02:00
|
|
|
}
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
2016-05-17 23:58:05 +02:00
|
|
|
func (g *goPackage) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string {
|
|
|
|
return g.properties.Deps
|
|
|
|
}
|
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
func (g *goPackage) GoPkgPath() string {
|
|
|
|
return g.properties.PkgPath
|
|
|
|
}
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
func (g *goPackage) GoPkgRoot() string {
|
|
|
|
return g.pkgRoot
|
|
|
|
}
|
|
|
|
|
|
|
|
func (g *goPackage) GoPackageTarget() string {
|
|
|
|
return g.archiveFile
|
|
|
|
}
|
|
|
|
|
2016-11-02 08:43:00 +01:00
|
|
|
func (g *goPackage) GoTestTargets() []string {
|
|
|
|
return g.testResultFile
|
2015-06-24 02:21:00 +02:00
|
|
|
}
|
|
|
|
|
2015-07-23 02:06:06 +02:00
|
|
|
func (g *goPackage) BuildStage() Stage {
|
2016-04-12 00:41:52 +02:00
|
|
|
return g.properties.BuildStage
|
2015-07-23 02:06:06 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func (g *goPackage) SetBuildStage(buildStage Stage) {
|
2016-04-12 00:41:52 +02:00
|
|
|
g.properties.BuildStage = buildStage
|
2015-07-23 02:06:06 +02:00
|
|
|
}
|
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
func (g *goPackage) IsPluginFor(name string) bool {
|
|
|
|
for _, plugin := range g.properties.PluginFor {
|
|
|
|
if plugin == name {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
|
2015-07-25 01:53:27 +02:00
|
|
|
var (
|
|
|
|
name = ctx.ModuleName()
|
|
|
|
hasPlugins = false
|
|
|
|
pluginSrc = ""
|
|
|
|
genSrcs = []string{}
|
|
|
|
)
|
2014-05-28 01:34:41 +02:00
|
|
|
|
|
|
|
if g.properties.PkgPath == "" {
|
|
|
|
ctx.ModuleErrorf("module %s did not specify a valid pkgPath", name)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
g.pkgRoot = packageRoot(ctx)
|
2015-06-24 02:21:00 +02:00
|
|
|
g.archiveFile = filepath.Join(g.pkgRoot,
|
|
|
|
filepath.FromSlash(g.properties.PkgPath)+".a")
|
2016-11-02 08:43:00 +01:00
|
|
|
var testArchiveFile string
|
2015-06-24 02:21:00 +02:00
|
|
|
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
|
2016-11-02 08:43:00 +01:00
|
|
|
testArchiveFile = filepath.Join(testRoot(ctx),
|
2015-06-24 02:21:00 +02:00
|
|
|
filepath.FromSlash(g.properties.PkgPath)+".a")
|
|
|
|
}
|
2014-05-28 01:34:41 +02:00
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
|
|
|
|
func(module blueprint.Module) { hasPlugins = true })
|
|
|
|
if hasPlugins {
|
|
|
|
pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
|
|
|
|
genSrcs = append(genSrcs, pluginSrc)
|
|
|
|
}
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
// We only actually want to build the builder modules if we're running as
|
|
|
|
// minibp (i.e. we're generating a bootstrap Ninja file). This is to break
|
|
|
|
// the circular dependence that occurs when the builder requires a new Ninja
|
|
|
|
// file to be built, but building a new ninja file requires the builder to
|
|
|
|
// be built.
|
2015-07-23 02:06:06 +02:00
|
|
|
if g.config.stage == g.BuildStage() {
|
2015-07-25 01:53:27 +02:00
|
|
|
if hasPlugins && !buildGoPluginLoader(ctx, g.properties.PkgPath, pluginSrc, g.config.stage) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-06-24 02:21:00 +02:00
|
|
|
if g.config.runGoTests {
|
2016-11-02 08:43:00 +01:00
|
|
|
g.testResultFile = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
|
2015-07-25 01:53:27 +02:00
|
|
|
g.properties.PkgPath, g.properties.Srcs, genSrcs,
|
2015-06-24 02:21:00 +02:00
|
|
|
g.properties.TestSrcs)
|
|
|
|
}
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile,
|
2016-11-02 08:43:00 +01:00
|
|
|
g.properties.Srcs, genSrcs)
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// A goBinary is a module for building executable binaries from Go sources.
|
|
|
|
type goBinary struct {
|
2016-05-17 23:58:05 +02:00
|
|
|
blueprint.SimpleName
|
2014-05-28 01:34:41 +02:00
|
|
|
properties struct {
|
2016-05-17 23:58:05 +02:00
|
|
|
Deps []string
|
2014-05-28 01:34:41 +02:00
|
|
|
Srcs []string
|
2015-06-24 02:21:00 +02:00
|
|
|
TestSrcs []string
|
2014-05-28 01:34:41 +02:00
|
|
|
PrimaryBuilder bool
|
2016-04-12 00:41:52 +02:00
|
|
|
|
|
|
|
// The stage in which this module should be built
|
|
|
|
BuildStage Stage `blueprint:"mutated"`
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
2015-06-25 04:21:21 +02:00
|
|
|
|
|
|
|
// The bootstrap Config
|
|
|
|
config *Config
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
2015-07-23 02:06:06 +02:00
|
|
|
func newGoBinaryModuleFactory(config *Config, buildStage Stage) func() (blueprint.Module, []interface{}) {
|
2015-06-25 04:21:21 +02:00
|
|
|
return func() (blueprint.Module, []interface{}) {
|
|
|
|
module := &goBinary{
|
2016-04-12 00:41:52 +02:00
|
|
|
config: config,
|
2015-06-25 04:21:21 +02:00
|
|
|
}
|
2016-04-12 00:41:52 +02:00
|
|
|
module.properties.BuildStage = buildStage
|
2016-05-17 23:58:05 +02:00
|
|
|
return module, []interface{}{&module.properties, &module.SimpleName.Properties}
|
2015-06-25 04:21:21 +02:00
|
|
|
}
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
2016-05-17 23:58:05 +02:00
|
|
|
func (g *goBinary) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string {
|
|
|
|
return g.properties.Deps
|
|
|
|
}
|
|
|
|
|
2015-07-23 02:06:06 +02:00
|
|
|
func (g *goBinary) BuildStage() Stage {
|
2016-04-12 00:41:52 +02:00
|
|
|
return g.properties.BuildStage
|
2015-07-23 02:06:06 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func (g *goBinary) SetBuildStage(buildStage Stage) {
|
2016-04-12 00:41:52 +02:00
|
|
|
g.properties.BuildStage = buildStage
|
2015-07-23 02:06:06 +02:00
|
|
|
}
|
|
|
|
|
2016-05-26 19:07:59 +02:00
|
|
|
func (g *goBinary) InstallPath() string {
|
|
|
|
if g.BuildStage() == StageMain {
|
|
|
|
return "$ToolDir"
|
|
|
|
}
|
|
|
|
return "$BinDir"
|
|
|
|
}
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
|
|
|
|
var (
|
2016-11-02 08:43:00 +01:00
|
|
|
name = ctx.ModuleName()
|
|
|
|
objDir = moduleObjDir(ctx)
|
|
|
|
archiveFile = filepath.Join(objDir, name+".a")
|
|
|
|
testArchiveFile = filepath.Join(testRoot(ctx), name+".a")
|
|
|
|
aoutFile = filepath.Join(objDir, "a.out")
|
|
|
|
binaryFile = filepath.Join(g.InstallPath(), name)
|
|
|
|
hasPlugins = false
|
|
|
|
pluginSrc = ""
|
|
|
|
genSrcs = []string{}
|
2014-05-28 01:34:41 +02:00
|
|
|
)
|
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
|
|
|
|
func(module blueprint.Module) { hasPlugins = true })
|
|
|
|
if hasPlugins {
|
|
|
|
pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
|
|
|
|
genSrcs = append(genSrcs, pluginSrc)
|
|
|
|
}
|
|
|
|
|
2015-07-23 02:06:06 +02:00
|
|
|
if g.config.stage == g.BuildStage() {
|
2015-06-24 02:21:00 +02:00
|
|
|
var deps []string
|
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
if hasPlugins && !buildGoPluginLoader(ctx, "main", pluginSrc, g.config.stage) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-06-24 02:21:00 +02:00
|
|
|
if g.config.runGoTests {
|
2016-11-02 08:43:00 +01:00
|
|
|
deps = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
|
2015-07-25 01:53:27 +02:00
|
|
|
name, g.properties.Srcs, genSrcs, g.properties.TestSrcs)
|
2015-06-24 02:21:00 +02:00
|
|
|
}
|
|
|
|
|
2016-11-02 08:43:00 +01:00
|
|
|
buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, genSrcs)
|
2014-05-28 01:34:41 +02:00
|
|
|
|
|
|
|
var libDirFlags []string
|
|
|
|
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
|
|
|
|
func(module blueprint.Module) {
|
|
|
|
dep := module.(goPackageProducer)
|
|
|
|
libDir := dep.GoPkgRoot()
|
|
|
|
libDirFlags = append(libDirFlags, "-L "+libDir)
|
2016-11-02 08:43:00 +01:00
|
|
|
deps = append(deps, dep.GoTestTargets()...)
|
2014-05-28 01:34:41 +02:00
|
|
|
})
|
|
|
|
|
|
|
|
linkArgs := map[string]string{}
|
|
|
|
if len(libDirFlags) > 0 {
|
|
|
|
linkArgs["libDirFlags"] = strings.Join(libDirFlags, " ")
|
|
|
|
}
|
|
|
|
|
2014-10-03 11:49:58 +02:00
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2015-11-17 23:21:45 +01:00
|
|
|
Rule: link,
|
|
|
|
Outputs: []string{aoutFile},
|
|
|
|
Inputs: []string{archiveFile},
|
|
|
|
Args: linkArgs,
|
2014-05-28 01:34:41 +02:00
|
|
|
})
|
|
|
|
|
2014-10-03 11:49:58 +02:00
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2016-11-02 08:43:00 +01:00
|
|
|
Rule: cp,
|
|
|
|
Outputs: []string{binaryFile},
|
|
|
|
Inputs: []string{aoutFile},
|
|
|
|
OrderOnly: deps,
|
2014-05-28 01:34:41 +02:00
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
func buildGoPluginLoader(ctx blueprint.ModuleContext, pkgPath, pluginSrc string, stage Stage) bool {
|
|
|
|
ret := true
|
|
|
|
name := ctx.ModuleName()
|
|
|
|
|
|
|
|
var pluginPaths []string
|
|
|
|
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
|
|
|
|
func(module blueprint.Module) {
|
|
|
|
plugin := module.(goPluginProvider)
|
|
|
|
pluginPaths = append(pluginPaths, plugin.GoPkgPath())
|
|
|
|
if stage == StageBootstrap {
|
|
|
|
ctx.OtherModuleErrorf(module, "plugin %q may not be included in core module %q",
|
|
|
|
ctx.OtherModuleName(module), name)
|
|
|
|
ret = false
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2015-11-17 23:21:45 +01:00
|
|
|
Rule: pluginGenSrc,
|
|
|
|
Outputs: []string{pluginSrc},
|
2015-07-25 01:53:27 +02:00
|
|
|
Args: map[string]string{
|
|
|
|
"pkg": pkgPath,
|
|
|
|
"plugins": strings.Join(pluginPaths, " "),
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
return ret
|
|
|
|
}
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
|
2016-11-02 08:43:00 +01:00
|
|
|
pkgPath string, archiveFile string, srcs []string, genSrcs []string) {
|
2014-05-28 01:34:41 +02:00
|
|
|
|
2014-11-09 20:52:56 +01:00
|
|
|
srcDir := moduleSrcDir(ctx)
|
2014-06-06 23:37:07 +02:00
|
|
|
srcFiles := pathtools.PrefixPaths(srcs, srcDir)
|
2015-07-25 01:53:27 +02:00
|
|
|
srcFiles = append(srcFiles, genSrcs...)
|
2014-05-28 01:34:41 +02:00
|
|
|
|
|
|
|
var incFlags []string
|
2015-11-17 23:21:45 +01:00
|
|
|
var deps []string
|
2014-05-28 01:34:41 +02:00
|
|
|
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
|
|
|
|
func(module blueprint.Module) {
|
|
|
|
dep := module.(goPackageProducer)
|
|
|
|
incDir := dep.GoPkgRoot()
|
|
|
|
target := dep.GoPackageTarget()
|
|
|
|
incFlags = append(incFlags, "-I "+incDir)
|
2014-10-22 22:22:22 +02:00
|
|
|
deps = append(deps, target)
|
2014-05-28 01:34:41 +02:00
|
|
|
})
|
|
|
|
|
2015-08-02 00:07:27 +02:00
|
|
|
compileArgs := map[string]string{
|
2014-05-28 01:34:41 +02:00
|
|
|
"pkgPath": pkgPath,
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(incFlags) > 0 {
|
2015-08-02 00:07:27 +02:00
|
|
|
compileArgs["incFlags"] = strings.Join(incFlags, " ")
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
2014-10-03 11:49:58 +02:00
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2015-08-02 00:07:27 +02:00
|
|
|
Rule: compile,
|
2014-10-22 19:55:28 +02:00
|
|
|
Outputs: []string{archiveFile},
|
2014-05-28 01:34:41 +02:00
|
|
|
Inputs: srcFiles,
|
2014-10-22 22:22:22 +02:00
|
|
|
Implicits: deps,
|
2015-08-02 00:07:27 +02:00
|
|
|
Args: compileArgs,
|
2014-05-28 01:34:41 +02:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2015-07-25 01:53:27 +02:00
|
|
|
func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
|
|
|
|
pkgPath string, srcs, genSrcs, testSrcs []string) []string {
|
2015-06-24 02:21:00 +02:00
|
|
|
|
|
|
|
if len(testSrcs) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
srcDir := moduleSrcDir(ctx)
|
|
|
|
testFiles := pathtools.PrefixPaths(testSrcs, srcDir)
|
|
|
|
|
|
|
|
mainFile := filepath.Join(testRoot, "test.go")
|
|
|
|
testArchive := filepath.Join(testRoot, "test.a")
|
|
|
|
testFile := filepath.Join(testRoot, "test")
|
|
|
|
testPassed := filepath.Join(testRoot, "test.passed")
|
|
|
|
|
|
|
|
buildGoPackage(ctx, testRoot, pkgPath, testPkgArchive,
|
2016-11-02 08:43:00 +01:00
|
|
|
append(srcs, testSrcs...), genSrcs)
|
2015-06-24 02:21:00 +02:00
|
|
|
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2015-11-17 23:21:45 +01:00
|
|
|
Rule: goTestMain,
|
|
|
|
Outputs: []string{mainFile},
|
|
|
|
Inputs: testFiles,
|
2015-06-24 02:21:00 +02:00
|
|
|
Args: map[string]string{
|
|
|
|
"pkg": pkgPath,
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
libDirFlags := []string{"-L " + testRoot}
|
2016-11-02 08:43:00 +01:00
|
|
|
testDeps := []string{}
|
2015-06-24 02:21:00 +02:00
|
|
|
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
|
|
|
|
func(module blueprint.Module) {
|
|
|
|
dep := module.(goPackageProducer)
|
|
|
|
libDir := dep.GoPkgRoot()
|
|
|
|
libDirFlags = append(libDirFlags, "-L "+libDir)
|
2016-11-02 08:43:00 +01:00
|
|
|
testDeps = append(testDeps, dep.GoTestTargets()...)
|
2015-06-24 02:21:00 +02:00
|
|
|
})
|
|
|
|
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2015-08-02 00:07:27 +02:00
|
|
|
Rule: compile,
|
2015-06-24 02:21:00 +02:00
|
|
|
Outputs: []string{testArchive},
|
|
|
|
Inputs: []string{mainFile},
|
2015-11-17 23:21:45 +01:00
|
|
|
Implicits: []string{testPkgArchive},
|
2015-06-24 02:21:00 +02:00
|
|
|
Args: map[string]string{
|
|
|
|
"pkgPath": "main",
|
|
|
|
"incFlags": "-I " + testRoot,
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2015-11-17 23:21:45 +01:00
|
|
|
Rule: link,
|
|
|
|
Outputs: []string{testFile},
|
|
|
|
Inputs: []string{testArchive},
|
2015-06-24 02:21:00 +02:00
|
|
|
Args: map[string]string{
|
|
|
|
"libDirFlags": strings.Join(libDirFlags, " "),
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2016-11-02 08:43:00 +01:00
|
|
|
Rule: test,
|
|
|
|
Outputs: []string{testPassed},
|
|
|
|
Inputs: []string{testFile},
|
|
|
|
OrderOnly: testDeps,
|
2015-06-24 02:21:00 +02:00
|
|
|
Args: map[string]string{
|
|
|
|
"pkg": pkgPath,
|
|
|
|
"pkgSrcDir": filepath.Dir(testFiles[0]),
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
return []string{testPassed}
|
|
|
|
}
|
|
|
|
|
2015-06-25 04:21:21 +02:00
|
|
|
type singleton struct {
|
|
|
|
// The bootstrap Config
|
|
|
|
config *Config
|
|
|
|
}
|
2014-05-28 01:34:41 +02:00
|
|
|
|
2015-06-25 04:21:21 +02:00
|
|
|
func newSingletonFactory(config *Config) func() blueprint.Singleton {
|
|
|
|
return func() blueprint.Singleton {
|
|
|
|
return &singleton{
|
|
|
|
config: config,
|
|
|
|
}
|
|
|
|
}
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
|
|
|
|
// Find the module that's marked as the "primary builder", which means it's
|
|
|
|
// creating the binary that we'll use to generate the non-bootstrap
|
|
|
|
// build.ninja file.
|
|
|
|
var primaryBuilders []*goBinary
|
2016-05-26 19:07:59 +02:00
|
|
|
// blueprintTools contains blueprint go binaries that will be built in StageMain
|
|
|
|
var blueprintTools []string
|
2014-05-28 01:34:41 +02:00
|
|
|
ctx.VisitAllModulesIf(isBootstrapBinaryModule,
|
|
|
|
func(module blueprint.Module) {
|
|
|
|
binaryModule := module.(*goBinary)
|
2014-10-23 00:54:20 +02:00
|
|
|
binaryModuleName := ctx.ModuleName(binaryModule)
|
2016-05-26 19:07:59 +02:00
|
|
|
installPath := filepath.Join(binaryModule.InstallPath(), binaryModuleName)
|
|
|
|
|
2016-11-02 06:44:25 +01:00
|
|
|
if binaryModule.BuildStage() == StageMain {
|
2016-05-26 19:07:59 +02:00
|
|
|
blueprintTools = append(blueprintTools, installPath)
|
2015-07-23 02:06:06 +02:00
|
|
|
}
|
2014-05-28 01:34:41 +02:00
|
|
|
if binaryModule.properties.PrimaryBuilder {
|
|
|
|
primaryBuilders = append(primaryBuilders, binaryModule)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
2016-11-02 01:12:28 +01:00
|
|
|
var extraTestFlags string
|
|
|
|
if s.config.runGoTests {
|
|
|
|
extraTestFlags = " -t"
|
|
|
|
}
|
|
|
|
|
2014-05-28 01:34:41 +02:00
|
|
|
var primaryBuilderName, primaryBuilderExtraFlags string
|
|
|
|
switch len(primaryBuilders) {
|
|
|
|
case 0:
|
|
|
|
// If there's no primary builder module then that means we'll use minibp
|
|
|
|
// as the primary builder. We can trigger its primary builder mode with
|
|
|
|
// the -p flag.
|
|
|
|
primaryBuilderName = "minibp"
|
2016-11-02 01:12:28 +01:00
|
|
|
primaryBuilderExtraFlags = "-p" + extraTestFlags
|
2014-05-28 01:34:41 +02:00
|
|
|
|
|
|
|
case 1:
|
|
|
|
primaryBuilderName = ctx.ModuleName(primaryBuilders[0])
|
2017-01-18 23:34:46 +01:00
|
|
|
primaryBuilderExtraFlags = extraTestFlags
|
2014-05-28 01:34:41 +02:00
|
|
|
|
|
|
|
default:
|
|
|
|
ctx.Errorf("multiple primary builder modules present:")
|
|
|
|
for _, primaryBuilder := range primaryBuilders {
|
|
|
|
ctx.ModuleErrorf(primaryBuilder, "<-- module %s",
|
|
|
|
ctx.ModuleName(primaryBuilder))
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-07-14 08:28:37 +02:00
|
|
|
primaryBuilderFile := filepath.Join("$BinDir", primaryBuilderName)
|
2014-05-28 01:34:41 +02:00
|
|
|
|
|
|
|
// Get the filename of the top-level Blueprints file to pass to minibp.
|
2014-11-09 20:52:56 +01:00
|
|
|
topLevelBlueprints := filepath.Join("$srcDir",
|
2015-06-25 04:21:21 +02:00
|
|
|
filepath.Base(s.config.topLevelBlueprintsFile))
|
2014-05-28 01:34:41 +02:00
|
|
|
|
Simplify bootstrap
tl;dr: Read if you don't use the wrapper or use SKIP_NINJA
Previously, we were relying on the ninja behavior of restarting the
build when the build.ninja file was updated to switch between different
bootstrap stages. But that means that every step that could produce a
build.ninja must pass in order to switch to a different stage. That
wasn't a big problem when we had a two stage build -- there was very
little that could fail in the second stage before we chose to go back to
the first stage. But when we had a three stage build, it was possible to
get into a state (usually during development) where you were in the
second stage, but the build was failing because the first stage needed
to be run. This was fixed in d79f1af7423e0ef7a13573efdae5100a57fabc82
by adding a wrapper that always started building at the first stage.
But this kept all of the complexity of using ninja restarts without any
of the benefits, so this change removes that complexity and just runs
each stage sequentially in the wrapper. So the wrapper is now required.
Since we're no longer going through choosestage, we can also skip the
template parsing for the later stages that don't need to be templated --
this can save a couple of seconds for large files.
In addition to all of the above, this also lets Soong reduce the number
of times the main ninja file is loaded. We had been running the wrapper
once (3 stages), then running ninja again after combining the
Soong-generated build.ninja with the Kati-generated build.ninja. This
change lets us removing the intermediate parsing of Soong's build.ninja,
so that we only execute ninja 3 times per build. It also lets us have
dependencies on pools or rules from Kati in the primary builder, since
we're never executing the main build.ninja without the Kati build.ninja.
The wrapper has a new option, NINJA to provide the path to ninja. This
used to be hardcoded to `ninja`, and will still default to that. But
we'll be running the first two bootstrap stages with $NINJA even if
SKIP_NINJA is set.
The wrapper passes "-w dupbuild=err" to ninja now -- this really should
always be turned on if you care about reliable builds.
Change-Id: I6f656b74eb3d064b8b9e69d1d6dac1129d72b747
2016-08-13 21:42:11 +02:00
|
|
|
mainNinjaFile := filepath.Join("$buildDir", "build.ninja")
|
|
|
|
primaryBuilderNinjaFile := filepath.Join(bootstrapDir, "build.ninja")
|
|
|
|
bootstrapNinjaFileTemplate := filepath.Join(miniBootstrapDir, "build.ninja.in")
|
|
|
|
bootstrapNinjaFile := filepath.Join(miniBootstrapDir, "build.ninja")
|
2015-05-13 23:36:24 +02:00
|
|
|
docsFile := filepath.Join(docsDir, primaryBuilderName+".html")
|
|
|
|
|
Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
|
|
|
switch s.config.stage {
|
|
|
|
case StageBootstrap:
|
2014-05-28 01:34:41 +02:00
|
|
|
// We're generating a bootstrapper Ninja file, so we need to set things
|
|
|
|
// up to rebuild the build.ninja file using the primary builder.
|
|
|
|
|
2015-07-23 02:06:06 +02:00
|
|
|
// BuildDir must be different between the three stages, otherwise the
|
|
|
|
// cleanup process will remove files from the other builds.
|
2015-11-19 01:01:01 +01:00
|
|
|
ctx.SetNinjaBuildDir(pctx, miniBootstrapDir)
|
2015-07-23 02:06:06 +02:00
|
|
|
|
2016-11-02 01:12:28 +01:00
|
|
|
// Generate the Ninja file to build the primary builder.
|
2015-07-23 02:06:06 +02:00
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2016-11-02 06:44:25 +01:00
|
|
|
Rule: generateBuildNinja,
|
|
|
|
Outputs: []string{primaryBuilderNinjaFile},
|
|
|
|
Inputs: []string{topLevelBlueprints},
|
2016-11-02 01:12:28 +01:00
|
|
|
Args: map[string]string{
|
|
|
|
"builder": minibpFile,
|
|
|
|
"extra": "--build-primary" + extraTestFlags,
|
|
|
|
},
|
2015-07-23 02:06:06 +02:00
|
|
|
})
|
2014-06-06 05:00:22 +02:00
|
|
|
|
Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
|
|
|
// Rebuild the bootstrap Ninja file using the minibp that we just built.
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2016-11-02 01:12:28 +01:00
|
|
|
Rule: generateBuildNinja,
|
Simplify bootstrap
tl;dr: Read if you don't use the wrapper or use SKIP_NINJA
Previously, we were relying on the ninja behavior of restarting the
build when the build.ninja file was updated to switch between different
bootstrap stages. But that means that every step that could produce a
build.ninja must pass in order to switch to a different stage. That
wasn't a big problem when we had a two stage build -- there was very
little that could fail in the second stage before we chose to go back to
the first stage. But when we had a three stage build, it was possible to
get into a state (usually during development) where you were in the
second stage, but the build was failing because the first stage needed
to be run. This was fixed in d79f1af7423e0ef7a13573efdae5100a57fabc82
by adding a wrapper that always started building at the first stage.
But this kept all of the complexity of using ninja restarts without any
of the benefits, so this change removes that complexity and just runs
each stage sequentially in the wrapper. So the wrapper is now required.
Since we're no longer going through choosestage, we can also skip the
template parsing for the later stages that don't need to be templated --
this can save a couple of seconds for large files.
In addition to all of the above, this also lets Soong reduce the number
of times the main ninja file is loaded. We had been running the wrapper
once (3 stages), then running ninja again after combining the
Soong-generated build.ninja with the Kati-generated build.ninja. This
change lets us removing the intermediate parsing of Soong's build.ninja,
so that we only execute ninja 3 times per build. It also lets us have
dependencies on pools or rules from Kati in the primary builder, since
we're never executing the main build.ninja without the Kati build.ninja.
The wrapper has a new option, NINJA to provide the path to ninja. This
used to be hardcoded to `ninja`, and will still default to that. But
we'll be running the first two bootstrap stages with $NINJA even if
SKIP_NINJA is set.
The wrapper passes "-w dupbuild=err" to ninja now -- this really should
always be turned on if you care about reliable builds.
Change-Id: I6f656b74eb3d064b8b9e69d1d6dac1129d72b747
2016-08-13 21:42:11 +02:00
|
|
|
Outputs: []string{bootstrapNinjaFileTemplate},
|
Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
|
|
|
Inputs: []string{topLevelBlueprints},
|
2016-11-02 01:12:28 +01:00
|
|
|
Args: map[string]string{
|
|
|
|
"builder": minibpFile,
|
|
|
|
"extra": extraTestFlags,
|
|
|
|
},
|
Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
|
|
|
})
|
|
|
|
|
2015-07-23 02:06:06 +02:00
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
Simplify bootstrap
tl;dr: Read if you don't use the wrapper or use SKIP_NINJA
Previously, we were relying on the ninja behavior of restarting the
build when the build.ninja file was updated to switch between different
bootstrap stages. But that means that every step that could produce a
build.ninja must pass in order to switch to a different stage. That
wasn't a big problem when we had a two stage build -- there was very
little that could fail in the second stage before we chose to go back to
the first stage. But when we had a three stage build, it was possible to
get into a state (usually during development) where you were in the
second stage, but the build was failing because the first stage needed
to be run. This was fixed in d79f1af7423e0ef7a13573efdae5100a57fabc82
by adding a wrapper that always started building at the first stage.
But this kept all of the complexity of using ninja restarts without any
of the benefits, so this change removes that complexity and just runs
each stage sequentially in the wrapper. So the wrapper is now required.
Since we're no longer going through choosestage, we can also skip the
template parsing for the later stages that don't need to be templated --
this can save a couple of seconds for large files.
In addition to all of the above, this also lets Soong reduce the number
of times the main ninja file is loaded. We had been running the wrapper
once (3 stages), then running ninja again after combining the
Soong-generated build.ninja with the Kati-generated build.ninja. This
change lets us removing the intermediate parsing of Soong's build.ninja,
so that we only execute ninja 3 times per build. It also lets us have
dependencies on pools or rules from Kati in the primary builder, since
we're never executing the main build.ninja without the Kati build.ninja.
The wrapper has a new option, NINJA to provide the path to ninja. This
used to be hardcoded to `ninja`, and will still default to that. But
we'll be running the first two bootstrap stages with $NINJA even if
SKIP_NINJA is set.
The wrapper passes "-w dupbuild=err" to ninja now -- this really should
always be turned on if you care about reliable builds.
Change-Id: I6f656b74eb3d064b8b9e69d1d6dac1129d72b747
2016-08-13 21:42:11 +02:00
|
|
|
Rule: bootstrap,
|
|
|
|
Outputs: []string{bootstrapNinjaFile},
|
|
|
|
Inputs: []string{bootstrapNinjaFileTemplate},
|
2015-07-23 02:06:06 +02:00
|
|
|
})
|
|
|
|
|
|
|
|
case StagePrimary:
|
|
|
|
// We're generating a bootstrapper Ninja file, so we need to set things
|
|
|
|
// up to rebuild the build.ninja file using the primary builder.
|
|
|
|
|
|
|
|
// BuildDir must be different between the three stages, otherwise the
|
|
|
|
// cleanup process will remove files from the other builds.
|
2015-11-19 01:01:01 +01:00
|
|
|
ctx.SetNinjaBuildDir(pctx, bootstrapDir)
|
2015-07-23 02:06:06 +02:00
|
|
|
|
2016-11-02 01:12:28 +01:00
|
|
|
// Add a way to rebuild the primary build.ninja so that globs works
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2016-11-02 06:44:25 +01:00
|
|
|
Rule: generateBuildNinja,
|
|
|
|
Outputs: []string{primaryBuilderNinjaFile},
|
|
|
|
Inputs: []string{topLevelBlueprints},
|
2016-11-02 01:12:28 +01:00
|
|
|
Args: map[string]string{
|
|
|
|
"builder": minibpFile,
|
|
|
|
"extra": "--build-primary" + extraTestFlags,
|
|
|
|
"generator": "true",
|
|
|
|
},
|
|
|
|
})
|
Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
|
|
|
|
2016-11-02 01:12:28 +01:00
|
|
|
// Build the main build.ninja
|
Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2016-11-02 06:44:25 +01:00
|
|
|
Rule: generateBuildNinja,
|
|
|
|
Outputs: []string{mainNinjaFile},
|
|
|
|
Inputs: []string{topLevelBlueprints},
|
2016-11-02 01:12:28 +01:00
|
|
|
Args: map[string]string{
|
|
|
|
"builder": primaryBuilderFile,
|
|
|
|
"extra": primaryBuilderExtraFlags,
|
|
|
|
},
|
Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
|
|
|
})
|
|
|
|
|
2015-05-13 23:36:24 +02:00
|
|
|
// Generate build system docs for the primary builder. Generating docs reads the source
|
|
|
|
// files used to build the primary builder, but that dependency will be picked up through
|
|
|
|
// the dependency on the primary builder itself. There are no dependencies on the
|
|
|
|
// Blueprints files, as any relevant changes to the Blueprints files would have caused
|
|
|
|
// a rebuild of the primary builder.
|
|
|
|
bigbpDocs := ctx.Rule(pctx, "bigbpDocs",
|
|
|
|
blueprint.RuleParams{
|
2015-07-14 03:11:49 +02:00
|
|
|
Command: fmt.Sprintf("%s %s -b $buildDir --docs $out %s", primaryBuilderFile,
|
2015-05-13 23:36:24 +02:00
|
|
|
primaryBuilderExtraFlags, topLevelBlueprints),
|
2015-11-17 23:21:45 +01:00
|
|
|
CommandDeps: []string{primaryBuilderFile},
|
2015-05-13 23:36:24 +02:00
|
|
|
Description: fmt.Sprintf("%s docs $out", primaryBuilderName),
|
|
|
|
})
|
|
|
|
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2015-11-17 23:21:45 +01:00
|
|
|
Rule: bigbpDocs,
|
|
|
|
Outputs: []string{docsFile},
|
2015-05-13 23:36:24 +02:00
|
|
|
})
|
|
|
|
|
Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
2015-07-23 02:05:59 +02:00
|
|
|
case StageMain:
|
2015-11-19 01:01:01 +01:00
|
|
|
ctx.SetNinjaBuildDir(pctx, "${buildDir}")
|
2015-07-14 03:11:49 +02:00
|
|
|
|
2016-11-02 01:12:28 +01:00
|
|
|
// Add a way to rebuild the main build.ninja in case it creates rules that
|
|
|
|
// it will depend on itself. (In Android, globs with soong_glob)
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2016-11-02 06:44:25 +01:00
|
|
|
Rule: generateBuildNinja,
|
|
|
|
Outputs: []string{mainNinjaFile},
|
|
|
|
Inputs: []string{topLevelBlueprints},
|
2016-11-02 01:12:28 +01:00
|
|
|
Args: map[string]string{
|
|
|
|
"builder": primaryBuilderFile,
|
|
|
|
"extra": primaryBuilderExtraFlags,
|
|
|
|
"generator": "true",
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
2014-06-12 00:51:08 +02:00
|
|
|
if primaryBuilderName == "minibp" {
|
|
|
|
// This is a standalone Blueprint build, so we copy the minibp
|
|
|
|
// binary to the "bin" directory to make it easier to find.
|
2015-07-14 03:11:49 +02:00
|
|
|
finalMinibp := filepath.Join("$buildDir", "bin", primaryBuilderName)
|
2014-10-03 11:49:58 +02:00
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
2014-06-12 00:51:08 +02:00
|
|
|
Rule: cp,
|
|
|
|
Inputs: []string{primaryBuilderFile},
|
|
|
|
Outputs: []string{finalMinibp},
|
|
|
|
})
|
|
|
|
}
|
2016-05-26 19:07:59 +02:00
|
|
|
|
|
|
|
ctx.Build(pctx, blueprint.BuildParams{
|
|
|
|
Rule: blueprint.Phony,
|
|
|
|
Outputs: []string{"blueprint_tools"},
|
|
|
|
Inputs: blueprintTools,
|
|
|
|
})
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// packageRoot returns the module-specific package root directory path. This
|
|
|
|
// directory is where the final package .a files are output and where dependant
|
|
|
|
// modules search for this package via -I arguments.
|
|
|
|
func packageRoot(ctx blueprint.ModuleContext) string {
|
2014-06-06 23:21:57 +02:00
|
|
|
return filepath.Join(bootstrapDir, ctx.ModuleName(), "pkg")
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
2015-06-24 02:21:00 +02:00
|
|
|
// testRoot returns the module-specific package root directory path used for
|
|
|
|
// building tests. The .a files generated here will include everything from
|
|
|
|
// packageRoot, plus the test-only code.
|
|
|
|
func testRoot(ctx blueprint.ModuleContext) string {
|
|
|
|
return filepath.Join(bootstrapDir, ctx.ModuleName(), "test")
|
|
|
|
}
|
|
|
|
|
2014-11-09 20:52:56 +01:00
|
|
|
// moduleSrcDir returns the path of the directory that all source file paths are
|
2014-05-28 01:34:41 +02:00
|
|
|
// specified relative to.
|
2014-11-09 20:52:56 +01:00
|
|
|
func moduleSrcDir(ctx blueprint.ModuleContext) string {
|
|
|
|
return filepath.Join("$srcDir", ctx.ModuleDir())
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
|
|
|
|
2014-11-09 20:52:56 +01:00
|
|
|
// moduleObjDir returns the module-specific object directory path.
|
|
|
|
func moduleObjDir(ctx blueprint.ModuleContext) string {
|
2014-06-06 23:21:57 +02:00
|
|
|
return filepath.Join(bootstrapDir, ctx.ModuleName(), "obj")
|
2014-05-28 01:34:41 +02:00
|
|
|
}
|
2015-07-25 01:53:27 +02:00
|
|
|
|
|
|
|
// moduleGenSrcDir returns the module-specific generated sources path.
|
|
|
|
func moduleGenSrcDir(ctx blueprint.ModuleContext) string {
|
|
|
|
return filepath.Join(bootstrapDir, ctx.ModuleName(), "gen")
|
|
|
|
}
|