tequilaOS/platform_build_blueprint
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #132 from danw/bootstrap

Browse source 
Fix Soong glob regeneration and other bootstrap cleanups
This commit is contained in:
Dan Willemsen 2016-11-02 16:12:36 -07:00 committed by GitHub
commit b589835c0d
4 changed files with 112 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
blueprint.bash
View file

@ -53,10 +53,10 @@ else
fi
# Build minibp and the primary build.ninja
"${NINJA}" -w dupbuild=err -f "${BUILDDIR}/.minibootstrap/build.ninja" "${BUILDDIR}/.bootstrap/build.ninja"
"${NINJA}" -w dupbuild=err -f "${BUILDDIR}/.minibootstrap/build.ninja"
# Build the primary builder and the main build.ninja
"${NINJA}" -w dupbuild=err -f "${BUILDDIR}/.bootstrap/build.ninja" "${BUILDDIR}/build.ninja"
"${NINJA}" -w dupbuild=err -f "${BUILDDIR}/.bootstrap/build.ninja"
# SKIP_NINJA can be used by wrappers that wish to run ninja themselves.
if [ -z "$SKIP_NINJA" ]; then

242
bootstrap/bootstrap.go
View file

@ -96,6 +96,16 @@ var (
},
"depfile", "generator")
generateBuildNinja = pctx.StaticRule("build.ninja",
blueprint.RuleParams{
Command: "$builder $extra -b $buildDir -d $out.d -o $out $in",
CommandDeps: []string{"$builder"},
Description: "$builder $out",
Depfile: "$out.d",
Restat: true,
},
"builder", "extra", "generator")
// Work around a Ninja issue. See https://github.com/martine/ninja/pull/634
phony = pctx.StaticRule("phony",
blueprint.RuleParams{
@ -148,6 +158,7 @@ func pluginDeps(ctx blueprint.BottomUpMutatorContext) {
type goPackageProducer interface {
GoPkgRoot() string
GoPackageTarget() string
GoTestTargets() []string
}
func isGoPackageProducer(module blueprint.Module) bool {
@ -155,16 +166,6 @@ func isGoPackageProducer(module blueprint.Module) bool {
return ok
}
type goTestProducer interface {
GoTestTarget() string
BuildStage() Stage
}
func isGoTestProducer(module blueprint.Module) bool {
_, ok := module.(goTestProducer)
return ok
}
type goPluginProvider interface {
GoPkgPath() string
IsPluginFor(string) bool
@ -211,8 +212,8 @@ type goPackage struct {
// The path of the .a file that is to be built.
archiveFile string
// The path of the test .a file that is to be built.
testArchiveFile string
// The path of the test result file.
testResultFile []string
// The bootstrap Config
config *Config
@ -246,8 +247,8 @@ func (g *goPackage) GoPackageTarget() string {
return g.archiveFile
}
func (g *goPackage) GoTestTarget() string {
return g.testArchiveFile
func (g *goPackage) GoTestTargets() []string {
return g.testResultFile
}
func (g *goPackage) BuildStage() Stage {
@ -283,8 +284,9 @@ func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
g.pkgRoot = packageRoot(ctx)
g.archiveFile = filepath.Join(g.pkgRoot,
filepath.FromSlash(g.properties.PkgPath)+".a")
var testArchiveFile string
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
g.testArchiveFile = filepath.Join(testRoot(ctx),
testArchiveFile = filepath.Join(testRoot(ctx),
filepath.FromSlash(g.properties.PkgPath)+".a")
}
@ -301,20 +303,18 @@ func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
// file to be built, but building a new ninja file requires the builder to
// be built.
if g.config.stage == g.BuildStage() {
var deps []string
if hasPlugins && !buildGoPluginLoader(ctx, g.properties.PkgPath, pluginSrc, g.config.stage) {
return
}
if g.config.runGoTests {
deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
g.testResultFile = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
g.properties.PkgPath, g.properties.Srcs, genSrcs,
g.properties.TestSrcs)
}
buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile,
g.properties.Srcs, genSrcs, deps)
g.properties.Srcs, genSrcs)
}
}
@ -331,9 +331,6 @@ type goBinary struct {
BuildStage Stage `blueprint:"mutated"`
}
// The path of the test .a file that is to be built.
testArchiveFile string
// The bootstrap Config
config *Config
}
@ -352,10 +349,6 @@ func (g *goBinary) DynamicDependencies(ctx blueprint.DynamicDependerModuleContex
return g.properties.Deps
}
func (g *goBinary) GoTestTarget() string {
return g.testArchiveFile
}
func (g *goBinary) BuildStage() Stage {
return g.properties.BuildStage
}
@ -373,20 +366,17 @@ func (g *goBinary) InstallPath() string {
func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
var (
name = ctx.ModuleName()
objDir = moduleObjDir(ctx)
archiveFile = filepath.Join(objDir, name+".a")
aoutFile = filepath.Join(objDir, "a.out")
binaryFile = filepath.Join(g.InstallPath(), name)
hasPlugins = false
pluginSrc = ""
genSrcs = []string{}
name = ctx.ModuleName()
objDir = moduleObjDir(ctx)
archiveFile = filepath.Join(objDir, name+".a")
testArchiveFile = filepath.Join(testRoot(ctx), name+".a")
aoutFile = filepath.Join(objDir, "a.out")
binaryFile = filepath.Join(g.InstallPath(), name)
hasPlugins = false
pluginSrc = ""
genSrcs = []string{}
)
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
g.testArchiveFile = filepath.Join(testRoot(ctx), name+".a")
}
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) { hasPlugins = true })
if hasPlugins {
@ -402,11 +392,11 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
}
if g.config.runGoTests {
deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
deps = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
name, g.properties.Srcs, genSrcs, g.properties.TestSrcs)
}
buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, genSrcs, deps)
buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, genSrcs)
var libDirFlags []string
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
@ -414,6 +404,7 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
dep := module.(goPackageProducer)
libDir := dep.GoPkgRoot()
libDirFlags = append(libDirFlags, "-L "+libDir)
deps = append(deps, dep.GoTestTargets()...)
})
linkArgs := map[string]string{}
@ -429,9 +420,10 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: cp,
Outputs: []string{binaryFile},
Inputs: []string{aoutFile},
Rule: cp,
Outputs: []string{binaryFile},
Inputs: []string{aoutFile},
OrderOnly: deps,
})
}
}
@ -465,7 +457,7 @@ func buildGoPluginLoader(ctx blueprint.ModuleContext, pkgPath, pluginSrc string,
}
func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
pkgPath string, archiveFile string, srcs []string, genSrcs []string, orderDeps []string) {
pkgPath string, archiveFile string, srcs []string, genSrcs []string) {
srcDir := moduleSrcDir(ctx)
srcFiles := pathtools.PrefixPaths(srcs, srcDir)
@ -494,7 +486,6 @@ func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
Rule: compile,
Outputs: []string{archiveFile},
Inputs: srcFiles,
OrderOnly: orderDeps,
Implicits: deps,
Args: compileArgs,
})
@ -516,7 +507,7 @@ func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
testPassed := filepath.Join(testRoot, "test.passed")
buildGoPackage(ctx, testRoot, pkgPath, testPkgArchive,
append(srcs, testSrcs...), genSrcs, nil)
append(srcs, testSrcs...), genSrcs)
ctx.Build(pctx, blueprint.BuildParams{
Rule: goTestMain,
@ -528,11 +519,13 @@ func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
})
libDirFlags := []string{"-L " + testRoot}
testDeps := []string{}
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
func(module blueprint.Module) {
dep := module.(goPackageProducer)
libDir := dep.GoPkgRoot()
libDirFlags = append(libDirFlags, "-L "+libDir)
testDeps = append(testDeps, dep.GoTestTargets()...)
})
ctx.Build(pctx, blueprint.BuildParams{
@ -556,9 +549,10 @@ func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: test,
Outputs: []string{testPassed},
Inputs: []string{testFile},
Rule: test,
Outputs: []string{testPassed},
Inputs: []string{testFile},
OrderOnly: testDeps,
Args: map[string]string{
"pkg": pkgPath,
"pkgSrcDir": filepath.Dir(testFiles[0]),
@ -586,10 +580,6 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
// creating the binary that we'll use to generate the non-bootstrap
// build.ninja file.
var primaryBuilders []*goBinary
// bootstrapDeps contains modules that will be built in StageBootstrap
var bootstrapDeps []string
// primaryBootstrapDeps contains modules that will be built in StagePrimary
var primaryBootstrapDeps []string
// blueprintTools contains blueprint go binaries that will be built in StageMain
var blueprintTools []string
ctx.VisitAllModulesIf(isBootstrapBinaryModule,
@ -598,12 +588,7 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
binaryModuleName := ctx.ModuleName(binaryModule)
installPath := filepath.Join(binaryModule.InstallPath(), binaryModuleName)
switch binaryModule.BuildStage() {
case StageBootstrap:
bootstrapDeps = append(bootstrapDeps, installPath)
case StagePrimary:
primaryBootstrapDeps = append(primaryBootstrapDeps, installPath)
case StageMain:
if binaryModule.BuildStage() == StageMain {
blueprintTools = append(blueprintTools, installPath)
}
if binaryModule.properties.PrimaryBuilder {
@ -611,6 +596,11 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
}
})
var extraTestFlags string
if s.config.runGoTests {
extraTestFlags = " -t"
}
var primaryBuilderName, primaryBuilderExtraFlags string
switch len(primaryBuilders) {
case 0:
@ -618,7 +608,7 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
// as the primary builder. We can trigger its primary builder mode with
// the -p flag.
primaryBuilderName = "minibp"
primaryBuilderExtraFlags = "-p"
primaryBuilderExtraFlags = "-p" + extraTestFlags
case 1:
primaryBuilderName = ctx.ModuleName(primaryBuilders[0])
@ -634,43 +624,16 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
primaryBuilderFile := filepath.Join("$BinDir", primaryBuilderName)
if s.config.runGoTests {
primaryBuilderExtraFlags += " -t"
}
// Get the filename of the top-level Blueprints file to pass to minibp.
topLevelBlueprints := filepath.Join("$srcDir",
filepath.Base(s.config.topLevelBlueprintsFile))
bootstrapDeps = append(bootstrapDeps, topLevelBlueprints)
mainNinjaFile := filepath.Join("$buildDir", "build.ninja")
primaryBuilderNinjaFile := filepath.Join(bootstrapDir, "build.ninja")
bootstrapNinjaFileTemplate := filepath.Join(miniBootstrapDir, "build.ninja.in")
bootstrapNinjaFile := filepath.Join(miniBootstrapDir, "build.ninja")
docsFile := filepath.Join(docsDir, primaryBuilderName+".html")
primaryBootstrapDeps = append(primaryBootstrapDeps, docsFile)
// If the tests change, be sure to re-run them. These need to be
// dependencies for the ninja file so that it's updated after these
// run.
ctx.VisitAllModulesIf(isGoTestProducer,
func(module blueprint.Module) {
testModule := module.(goTestProducer)
target := testModule.GoTestTarget()
if target != "" {
switch testModule.BuildStage() {
case StageBootstrap:
bootstrapDeps = append(bootstrapDeps, target)
case StagePrimary:
primaryBootstrapDeps = append(primaryBootstrapDeps, target)
case StageMain:
blueprintTools = append(blueprintTools, target)
}
}
})
switch s.config.stage {
case StageBootstrap:
// We're generating a bootstrapper Ninja file, so we need to set things
@ -680,56 +643,26 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
// cleanup process will remove files from the other builds.
ctx.SetNinjaBuildDir(pctx, miniBootstrapDir)
// Generate the Ninja file to build the primary builder. Save the
// timestamps and deps, so that we can come back to this stage if
// it needs to be regenerated.
primarybp := ctx.Rule(pctx, "primarybp",
blueprint.RuleParams{
Command: fmt.Sprintf("%s --build-primary $runTests "+
"-b $buildDir -d $out.d -o $out $in", minibpFile),
Description: "minibp $out",
Depfile: "$out.d",
},
"runTests")
args := make(map[string]string)
if s.config.runGoTests {
args["runTests"] = "-t"
}
// Generate the Ninja file to build the primary builder.
ctx.Build(pctx, blueprint.BuildParams{
Rule: primarybp,
Outputs: []string{primaryBuilderNinjaFile},
Inputs: []string{topLevelBlueprints},
Implicits: bootstrapDeps,
Args: args,
Rule: generateBuildNinja,
Outputs: []string{primaryBuilderNinjaFile},
Inputs: []string{topLevelBlueprints},
Args: map[string]string{
"builder": minibpFile,
"extra": "--build-primary" + extraTestFlags,
},
})
// Rebuild the bootstrap Ninja file using the minibp that we just built.
minibp := ctx.Rule(pctx, "minibp",
blueprint.RuleParams{
Command: fmt.Sprintf("%s $runTests "+
"-b $buildDir -d $out.d -o $out $in", minibpFile),
CommandDeps: []string{minibpFile},
Description: "minibp $out",
Depfile: "$out.d",
// So that we don't trigger a restart if this hasn't changed
Restat: true,
},
"runTests")
args = map[string]string{}
if s.config.runGoTests {
args["runTests"] = "-t"
}
ctx.Build(pctx, blueprint.BuildParams{
Rule: minibp,
Rule: generateBuildNinja,
Outputs: []string{bootstrapNinjaFileTemplate},
Inputs: []string{topLevelBlueprints},
Args: args,
Args: map[string]string{
"builder": minibpFile,
"extra": extraTestFlags,
},
})
ctx.Build(pctx, blueprint.BuildParams{
@ -746,23 +679,27 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
// cleanup process will remove files from the other builds.
ctx.SetNinjaBuildDir(pctx, bootstrapDir)
// We generate the depfile here that includes the dependencies for all
// the Blueprints files that contribute to generating the big build
// manifest (build.ninja file).
bigbp := ctx.Rule(pctx, "bigbp",
blueprint.RuleParams{
Command: fmt.Sprintf("%s %s "+
"-b $buildDir -d $out.d -o $out $in", primaryBuilderFile,
primaryBuilderExtraFlags),
Description: fmt.Sprintf("%s $out", primaryBuilderName),
Depfile: "$out.d",
})
// Add a way to rebuild the primary build.ninja so that globs works
ctx.Build(pctx, blueprint.BuildParams{
Rule: bigbp,
Outputs: []string{mainNinjaFile},
Inputs: []string{topLevelBlueprints},
Implicits: primaryBootstrapDeps,
Rule: generateBuildNinja,
Outputs: []string{primaryBuilderNinjaFile},
Inputs: []string{topLevelBlueprints},
Args: map[string]string{
"builder": minibpFile,
"extra": "--build-primary" + extraTestFlags,
"generator": "true",
},
})
// Build the main build.ninja
ctx.Build(pctx, blueprint.BuildParams{
Rule: generateBuildNinja,
Outputs: []string{mainNinjaFile},
Inputs: []string{topLevelBlueprints},
Args: map[string]string{
"builder": primaryBuilderFile,
"extra": primaryBuilderExtraFlags,
},
})
// Generate build system docs for the primary builder. Generating docs reads the source
@ -786,6 +723,19 @@ func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
case StageMain:
ctx.SetNinjaBuildDir(pctx, "${buildDir}")
// Add a way to rebuild the main build.ninja in case it creates rules that
// it will depend on itself. (In Android, globs with soong_glob)
ctx.Build(pctx, blueprint.BuildParams{
Rule: generateBuildNinja,
Outputs: []string{mainNinjaFile},
Inputs: []string{topLevelBlueprints},
Args: map[string]string{
"builder": primaryBuilderFile,
"extra": primaryBuilderExtraFlags,
"generator": "true",
},
})
if primaryBuilderName == "minibp" {
// This is a standalone Blueprint build, so we copy the minibp
// binary to the "bin" directory to make it easier to find.

30
build.ninja.in
View file

@ -30,6 +30,12 @@ rule g.bootstrap.bootstrap
description = bootstrap ${in}
generator = true
rule g.bootstrap.build.ninja
command = ${builder} ${extra} -b ${g.bootstrap.buildDir} -d ${out}.d -o ${out} ${in}
depfile = ${out}.d
description = ${builder} ${out}
restat = true
rule g.bootstrap.compile
command = GOROOT='${g.bootstrap.goRoot}' ${g.bootstrap.compileCmd} -o ${out} -p ${pkgPath} -complete ${incFlags} -pack ${in}
description = compile ${out}
@ -263,26 +269,16 @@ default ${g.bootstrap.BinDir}/minibp
# Singleton: bootstrap
# Factory: github.com/google/blueprint/bootstrap.newSingletonFactory.func1
rule s.bootstrap.primarybp
command = ${g.bootstrap.BinDir}/minibp --build-primary ${runTests} -b ${g.bootstrap.buildDir} -d ${out}.d -o ${out} ${in}
depfile = ${out}.d
description = minibp ${out}
rule s.bootstrap.minibp
command = ${g.bootstrap.BinDir}/minibp ${runTests} -b ${g.bootstrap.buildDir} -d ${out}.d -o ${out} ${in}
depfile = ${out}.d
description = minibp ${out}
restat = true
build ${g.bootstrap.buildDir}/.bootstrap/build.ninja: s.bootstrap.primarybp $
${g.bootstrap.srcDir}/Blueprints | ${g.bootstrap.BinDir}/gotestmain $
${g.bootstrap.BinDir}/gotestrunner ${g.bootstrap.BinDir}/minibp $
${g.bootstrap.srcDir}/Blueprints
build ${g.bootstrap.buildDir}/.bootstrap/build.ninja: g.bootstrap.build.ninja $
${g.bootstrap.srcDir}/Blueprints | ${builder}
builder = ${g.bootstrap.BinDir}/minibp
extra = --build-primary
default ${g.bootstrap.buildDir}/.bootstrap/build.ninja
build ${g.bootstrap.buildDir}/.minibootstrap/build.ninja.in: $
s.bootstrap.minibp ${g.bootstrap.srcDir}/Blueprints | $
${g.bootstrap.BinDir}/minibp
g.bootstrap.build.ninja ${g.bootstrap.srcDir}/Blueprints | ${builder}
builder = ${g.bootstrap.BinDir}/minibp
extra =
default ${g.bootstrap.buildDir}/.minibootstrap/build.ninja.in
build ${g.bootstrap.buildDir}/.minibootstrap/build.ninja: $

2
tests/test.sh
View file

@ -17,7 +17,7 @@ if [[ -d .bootstrap/blueprint/test ]]; then
fi
sleep 2
sed -i 's/${runTests}/-t/' src.build.ninja.in
sed -i 's/extra =/extra = -t/' src.build.ninja.in
./blueprint.bash
if [[ ! -d .bootstrap/blueprint/test ]]; then