// Copyright 2014 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bootstrap import ( "fmt" "go/build" "path/filepath" "runtime" "strings" "github.com/google/blueprint" "github.com/google/blueprint/pathtools" ) var ( pctx = blueprint.NewPackageContext("github.com/google/blueprint/bootstrap") goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join("$ToolDir", "gotestmain")) goTestRunnerCmd = pctx.StaticVariable("goTestRunnerCmd", filepath.Join("$ToolDir", "gotestrunner")) pluginGenSrcCmd = pctx.StaticVariable("pluginGenSrcCmd", filepath.Join("$ToolDir", "loadplugins")) parallelCompile = pctx.StaticVariable("parallelCompile", func() string { // Parallel compilation is only supported on >= go1.9 for _, r := range build.Default.ReleaseTags { if r == "go1.9" { numCpu := runtime.NumCPU() // This will cause us to recompile all go programs if the // number of cpus changes. We don't get a lot of benefit from // higher values, so cap this to make it cheaper to move trees // between machines. if numCpu > 8 { numCpu = 8 } return fmt.Sprintf("-c %d", numCpu) } } return "" }()) compile = pctx.StaticRule("compile", blueprint.RuleParams{ Command: "GOROOT='$goRoot' $compileCmd $parallelCompile -o $out.tmp " + "$debugFlags -p $pkgPath -complete $incFlags -pack $in && " + "if cmp --quiet $out.tmp $out; then rm $out.tmp; else mv -f $out.tmp $out; fi", CommandDeps: []string{"$compileCmd"}, Description: "compile $out", Restat: true, }, "pkgPath", "incFlags") link = pctx.StaticRule("link", blueprint.RuleParams{ Command: "GOROOT='$goRoot' $linkCmd -o $out.tmp $libDirFlags $in && " + "if cmp --quiet $out.tmp $out; then rm $out.tmp; else mv -f $out.tmp $out; fi", CommandDeps: []string{"$linkCmd"}, Description: "link $out", Restat: true, }, "libDirFlags") goTestMain = pctx.StaticRule("gotestmain", blueprint.RuleParams{ Command: "$goTestMainCmd -o $out -pkg $pkg $in", CommandDeps: []string{"$goTestMainCmd"}, Description: "gotestmain $out", }, "pkg") pluginGenSrc = pctx.StaticRule("pluginGenSrc", blueprint.RuleParams{ Command: "$pluginGenSrcCmd -o $out -p $pkg $plugins", CommandDeps: []string{"$pluginGenSrcCmd"}, Description: "create $out", }, "pkg", "plugins") test = pctx.StaticRule("test", blueprint.RuleParams{ Command: "$goTestRunnerCmd -p $pkgSrcDir -f $out -- $in -test.short", CommandDeps: []string{"$goTestRunnerCmd"}, Description: "test $pkg", }, "pkg", "pkgSrcDir") cp = pctx.StaticRule("cp", blueprint.RuleParams{ Command: "cp $in $out", Description: "cp $out", }, "generator") bootstrap = pctx.StaticRule("bootstrap", blueprint.RuleParams{ Command: "BUILDDIR=$soongOutDir $bootstrapCmd -i $in", CommandDeps: []string{"$bootstrapCmd"}, Description: "bootstrap $in", Generator: true, }) touch = pctx.StaticRule("touch", blueprint.RuleParams{ Command: "touch $out", Description: "touch $out", }, "depfile", "generator") generateBuildNinja = pctx.StaticRule("build.ninja", blueprint.RuleParams{ // TODO: it's kinda ugly that some parameters are computed from // environment variables and some from Ninja parameters, but it's probably // better to not to touch that while Blueprint and Soong are separate // NOTE: The spaces at EOL are important because otherwise Ninja would // omit all spaces between the different options. Command: `cd "$$(dirname "$builder")" && ` + `BUILDER="$$PWD/$$(basename "$builder")" && ` + `cd / && ` + `env -i "$$BUILDER" ` + ` --top "$$TOP" ` + ` --soong_out "$soongOutDir" ` + ` --out "$outDir" ` + ` $extra`, CommandDeps: []string{"$builder"}, Description: "$builder $out", Deps: blueprint.DepsGCC, Depfile: "$out.d", Restat: true, }, "builder", "extra") // Work around a Ninja issue. See https://github.com/martine/ninja/pull/634 phony = pctx.StaticRule("phony", blueprint.RuleParams{ Command: "# phony $out", Description: "phony $out", Generator: true, }, "depfile") _ = pctx.VariableFunc("ToolDir", func(config interface{}) (string, error) { return config.(BootstrapConfig).HostToolDir(), nil }) ) type GoBinaryTool interface { InstallPath() string // So that other packages can't implement this interface isGoBinary() } func pluginDeps(ctx blueprint.BottomUpMutatorContext) { if pkg, ok := ctx.Module().(*goPackage); ok { if ctx.PrimaryModule() == ctx.Module() { for _, plugin := range pkg.properties.PluginFor { ctx.AddReverseDependency(ctx.Module(), nil, plugin) } } } } type goPackageProducer interface { GoPkgRoot() string GoPackageTarget() string GoTestTargets() []string } func isGoPackageProducer(module blueprint.Module) bool { _, ok := module.(goPackageProducer) return ok } type goPluginProvider interface { GoPkgPath() string IsPluginFor(string) bool } func isGoPluginFor(name string) func(blueprint.Module) bool { return func(module blueprint.Module) bool { if plugin, ok := module.(goPluginProvider); ok { return plugin.IsPluginFor(name) } return false } } func IsBootstrapModule(module blueprint.Module) bool { _, isPackage := module.(*goPackage) _, isBinary := module.(*goBinary) return isPackage || isBinary } func isBootstrapBinaryModule(module blueprint.Module) bool { _, isBinary := module.(*goBinary) return isBinary } // A goPackage is a module for building Go packages. type goPackage struct { blueprint.SimpleName properties struct { Deps []string PkgPath string Srcs []string TestSrcs []string PluginFor []string Darwin struct { Srcs []string TestSrcs []string } Linux struct { Srcs []string TestSrcs []string } } // The root dir in which the package .a file is located. The full .a file // path will be "packageRoot/PkgPath.a" pkgRoot string // The path of the .a file that is to be built. archiveFile string // The path of the test result file. testResultFile []string } var _ goPackageProducer = (*goPackage)(nil) func newGoPackageModuleFactory() func() (blueprint.Module, []interface{}) { return func() (blueprint.Module, []interface{}) { module := &goPackage{} return module, []interface{}{&module.properties, &module.SimpleName.Properties} } } func (g *goPackage) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string { if ctx.Module() != ctx.PrimaryModule() { return nil } return g.properties.Deps } func (g *goPackage) GoPkgPath() string { return g.properties.PkgPath } func (g *goPackage) GoPkgRoot() string { return g.pkgRoot } func (g *goPackage) GoPackageTarget() string { return g.archiveFile } func (g *goPackage) GoTestTargets() []string { return g.testResultFile } func (g *goPackage) IsPluginFor(name string) bool { for _, plugin := range g.properties.PluginFor { if plugin == name { return true } } return false } func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) { // Allow the primary builder to create multiple variants. Any variants after the first // will copy outputs from the first. if ctx.Module() != ctx.PrimaryModule() { primary := ctx.PrimaryModule().(*goPackage) g.pkgRoot = primary.pkgRoot g.archiveFile = primary.archiveFile g.testResultFile = primary.testResultFile return } var ( name = ctx.ModuleName() hasPlugins = false pluginSrc = "" genSrcs = []string{} ) if g.properties.PkgPath == "" { ctx.ModuleErrorf("module %s did not specify a valid pkgPath", name) return } g.pkgRoot = packageRoot(ctx) g.archiveFile = filepath.Join(g.pkgRoot, filepath.FromSlash(g.properties.PkgPath)+".a") ctx.VisitDepsDepthFirstIf(isGoPluginFor(name), func(module blueprint.Module) { hasPlugins = true }) if hasPlugins { pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go") genSrcs = append(genSrcs, pluginSrc) } if hasPlugins && !buildGoPluginLoader(ctx, g.properties.PkgPath, pluginSrc) { return } var srcs, testSrcs []string if runtime.GOOS == "darwin" { srcs = append(g.properties.Srcs, g.properties.Darwin.Srcs...) testSrcs = append(g.properties.TestSrcs, g.properties.Darwin.TestSrcs...) } else if runtime.GOOS == "linux" { srcs = append(g.properties.Srcs, g.properties.Linux.Srcs...) testSrcs = append(g.properties.TestSrcs, g.properties.Linux.TestSrcs...) } if ctx.Config().(BootstrapConfig).RunGoTests() { testArchiveFile := filepath.Join(testRoot(ctx), filepath.FromSlash(g.properties.PkgPath)+".a") g.testResultFile = buildGoTest(ctx, testRoot(ctx), testArchiveFile, g.properties.PkgPath, srcs, genSrcs, testSrcs) } // Don't build for test-only packages if len(srcs) == 0 && len(genSrcs) == 0 { ctx.Build(pctx, blueprint.BuildParams{ Rule: touch, Outputs: []string{g.archiveFile}, Optional: true, }) return } buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile, srcs, genSrcs) } // A goBinary is a module for building executable binaries from Go sources. type goBinary struct { blueprint.SimpleName properties struct { Deps []string Srcs []string TestSrcs []string PrimaryBuilder bool Default bool Darwin struct { Srcs []string TestSrcs []string } Linux struct { Srcs []string TestSrcs []string } } installPath string } var _ GoBinaryTool = (*goBinary)(nil) func newGoBinaryModuleFactory() func() (blueprint.Module, []interface{}) { return func() (blueprint.Module, []interface{}) { module := &goBinary{} return module, []interface{}{&module.properties, &module.SimpleName.Properties} } } func (g *goBinary) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string { if ctx.Module() != ctx.PrimaryModule() { return nil } return g.properties.Deps } func (g *goBinary) isGoBinary() {} func (g *goBinary) InstallPath() string { return g.installPath } func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) { // Allow the primary builder to create multiple variants. Any variants after the first // will copy outputs from the first. if ctx.Module() != ctx.PrimaryModule() { primary := ctx.PrimaryModule().(*goBinary) g.installPath = primary.installPath return } var ( name = ctx.ModuleName() objDir = moduleObjDir(ctx) archiveFile = filepath.Join(objDir, name+".a") testArchiveFile = filepath.Join(testRoot(ctx), name+".a") aoutFile = filepath.Join(objDir, "a.out") hasPlugins = false pluginSrc = "" genSrcs = []string{} ) g.installPath = filepath.Join(ctx.Config().(BootstrapConfig).HostToolDir(), name) ctx.VisitDepsDepthFirstIf(isGoPluginFor(name), func(module blueprint.Module) { hasPlugins = true }) if hasPlugins { pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go") genSrcs = append(genSrcs, pluginSrc) } var testDeps []string if hasPlugins && !buildGoPluginLoader(ctx, "main", pluginSrc) { return } var srcs, testSrcs []string if runtime.GOOS == "darwin" { srcs = append(g.properties.Srcs, g.properties.Darwin.Srcs...) testSrcs = append(g.properties.TestSrcs, g.properties.Darwin.TestSrcs...) } else if runtime.GOOS == "linux" { srcs = append(g.properties.Srcs, g.properties.Linux.Srcs...) testSrcs = append(g.properties.TestSrcs, g.properties.Linux.TestSrcs...) } if ctx.Config().(BootstrapConfig).RunGoTests() { testDeps = buildGoTest(ctx, testRoot(ctx), testArchiveFile, name, srcs, genSrcs, testSrcs) } buildGoPackage(ctx, objDir, "main", archiveFile, srcs, genSrcs) var linkDeps []string var libDirFlags []string ctx.VisitDepsDepthFirstIf(isGoPackageProducer, func(module blueprint.Module) { dep := module.(goPackageProducer) linkDeps = append(linkDeps, dep.GoPackageTarget()) libDir := dep.GoPkgRoot() libDirFlags = append(libDirFlags, "-L "+libDir) testDeps = append(testDeps, dep.GoTestTargets()...) }) linkArgs := map[string]string{} if len(libDirFlags) > 0 { linkArgs["libDirFlags"] = strings.Join(libDirFlags, " ") } ctx.Build(pctx, blueprint.BuildParams{ Rule: link, Outputs: []string{aoutFile}, Inputs: []string{archiveFile}, Implicits: linkDeps, Args: linkArgs, Optional: true, }) var orderOnlyDeps, validationDeps []string if ctx.Config().(BootstrapConfig).UseValidationsForGoTests() { validationDeps = testDeps } else { orderOnlyDeps = testDeps } ctx.Build(pctx, blueprint.BuildParams{ Rule: cp, Outputs: []string{g.installPath}, Inputs: []string{aoutFile}, OrderOnly: orderOnlyDeps, Validations: validationDeps, Optional: !g.properties.Default, }) } func buildGoPluginLoader(ctx blueprint.ModuleContext, pkgPath, pluginSrc string) bool { ret := true name := ctx.ModuleName() var pluginPaths []string ctx.VisitDepsDepthFirstIf(isGoPluginFor(name), func(module blueprint.Module) { plugin := module.(goPluginProvider) pluginPaths = append(pluginPaths, plugin.GoPkgPath()) }) ctx.Build(pctx, blueprint.BuildParams{ Rule: pluginGenSrc, Outputs: []string{pluginSrc}, Args: map[string]string{ "pkg": pkgPath, "plugins": strings.Join(pluginPaths, " "), }, Optional: true, }) return ret } func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string, pkgPath string, archiveFile string, srcs []string, genSrcs []string) { srcDir := moduleSrcDir(ctx) srcFiles := pathtools.PrefixPaths(srcs, srcDir) srcFiles = append(srcFiles, genSrcs...) var incFlags []string var deps []string ctx.VisitDepsDepthFirstIf(isGoPackageProducer, func(module blueprint.Module) { dep := module.(goPackageProducer) incDir := dep.GoPkgRoot() target := dep.GoPackageTarget() incFlags = append(incFlags, "-I "+incDir) deps = append(deps, target) }) compileArgs := map[string]string{ "pkgPath": pkgPath, } if len(incFlags) > 0 { compileArgs["incFlags"] = strings.Join(incFlags, " ") } ctx.Build(pctx, blueprint.BuildParams{ Rule: compile, Outputs: []string{archiveFile}, Inputs: srcFiles, Implicits: deps, Args: compileArgs, Optional: true, }) } func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive, pkgPath string, srcs, genSrcs, testSrcs []string) []string { if len(testSrcs) == 0 { return nil } srcDir := moduleSrcDir(ctx) testFiles := pathtools.PrefixPaths(testSrcs, srcDir) mainFile := filepath.Join(testRoot, "test.go") testArchive := filepath.Join(testRoot, "test.a") testFile := filepath.Join(testRoot, "test") testPassed := filepath.Join(testRoot, "test.passed") buildGoPackage(ctx, testRoot, pkgPath, testPkgArchive, append(srcs, testSrcs...), genSrcs) ctx.Build(pctx, blueprint.BuildParams{ Rule: goTestMain, Outputs: []string{mainFile}, Inputs: testFiles, Args: map[string]string{ "pkg": pkgPath, }, Optional: true, }) linkDeps := []string{testPkgArchive} libDirFlags := []string{"-L " + testRoot} testDeps := []string{} ctx.VisitDepsDepthFirstIf(isGoPackageProducer, func(module blueprint.Module) { dep := module.(goPackageProducer) linkDeps = append(linkDeps, dep.GoPackageTarget()) libDir := dep.GoPkgRoot() libDirFlags = append(libDirFlags, "-L "+libDir) testDeps = append(testDeps, dep.GoTestTargets()...) }) ctx.Build(pctx, blueprint.BuildParams{ Rule: compile, Outputs: []string{testArchive}, Inputs: []string{mainFile}, Implicits: []string{testPkgArchive}, Args: map[string]string{ "pkgPath": "main", "incFlags": "-I " + testRoot, }, Optional: true, }) ctx.Build(pctx, blueprint.BuildParams{ Rule: link, Outputs: []string{testFile}, Inputs: []string{testArchive}, Implicits: linkDeps, Args: map[string]string{ "libDirFlags": strings.Join(libDirFlags, " "), }, Optional: true, }) var orderOnlyDeps, validationDeps []string if ctx.Config().(BootstrapConfig).UseValidationsForGoTests() { validationDeps = testDeps } else { orderOnlyDeps = testDeps } ctx.Build(pctx, blueprint.BuildParams{ Rule: test, Outputs: []string{testPassed}, Inputs: []string{testFile}, OrderOnly: orderOnlyDeps, Validations: validationDeps, Args: map[string]string{ "pkg": pkgPath, "pkgSrcDir": filepath.Dir(testFiles[0]), }, Optional: true, }) return []string{testPassed} } type singleton struct { } func newSingletonFactory() func() blueprint.Singleton { return func() blueprint.Singleton { return &singleton{} } } func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) { // Find the module that's marked as the "primary builder", which means it's // creating the binary that we'll use to generate the non-bootstrap // build.ninja file. var primaryBuilders []*goBinary // blueprintTools contains blueprint go binaries that will be built in StageMain var blueprintTools []string // blueprintGoPackages contains all blueprint go packages that can be built in StageMain var blueprintGoPackages []string ctx.VisitAllModulesIf(IsBootstrapModule, func(module blueprint.Module) { if ctx.PrimaryModule(module) == module { if binaryModule, ok := module.(*goBinary); ok { blueprintTools = append(blueprintTools, binaryModule.InstallPath()) if binaryModule.properties.PrimaryBuilder { primaryBuilders = append(primaryBuilders, binaryModule) } } if packageModule, ok := module.(*goPackage); ok { blueprintGoPackages = append(blueprintGoPackages, packageModule.GoPackageTarget()) blueprintGoPackages = append(blueprintGoPackages, packageModule.GoTestTargets()...) } } }) var primaryBuilderCmdlinePrefix []string var primaryBuilderName string if len(primaryBuilders) == 0 { ctx.Errorf("no primary builder module present") return } else if len(primaryBuilders) > 1 { ctx.Errorf("multiple primary builder modules present:") for _, primaryBuilder := range primaryBuilders { ctx.ModuleErrorf(primaryBuilder, "<-- module %s", ctx.ModuleName(primaryBuilder)) } return } else { primaryBuilderName = ctx.ModuleName(primaryBuilders[0]) } primaryBuilderFile := filepath.Join("$ToolDir", primaryBuilderName) ctx.SetOutDir(pctx, "${outDir}") for _, subninja := range ctx.Config().(BootstrapConfig).Subninjas() { ctx.AddSubninja(subninja) } for _, i := range ctx.Config().(BootstrapConfig).PrimaryBuilderInvocations() { flags := make([]string, 0) flags = append(flags, primaryBuilderCmdlinePrefix...) flags = append(flags, i.Args...) // Build the main build.ninja ctx.Build(pctx, blueprint.BuildParams{ Rule: generateBuildNinja, Outputs: i.Outputs, Inputs: i.Inputs, Args: map[string]string{ "builder": primaryBuilderFile, "extra": strings.Join(flags, " "), }, // soong_ui explicitly requests what it wants to be build. This is // because the same Ninja file contains instructions to run // soong_build, run bp2build and to generate the JSON module graph. Optional: true, }) } // Add a phony target for building various tools that are part of blueprint ctx.Build(pctx, blueprint.BuildParams{ Rule: blueprint.Phony, Outputs: []string{"blueprint_tools"}, Inputs: blueprintTools, }) // Add a phony target for running go tests ctx.Build(pctx, blueprint.BuildParams{ Rule: blueprint.Phony, Outputs: []string{"blueprint_go_packages"}, Inputs: blueprintGoPackages, Optional: true, }) } // packageRoot returns the module-specific package root directory path. This // directory is where the final package .a files are output and where dependant // modules search for this package via -I arguments. func packageRoot(ctx blueprint.ModuleContext) string { toolDir := ctx.Config().(BootstrapConfig).HostToolDir() return filepath.Join(toolDir, "go", ctx.ModuleName(), "pkg") } // testRoot returns the module-specific package root directory path used for // building tests. The .a files generated here will include everything from // packageRoot, plus the test-only code. func testRoot(ctx blueprint.ModuleContext) string { toolDir := ctx.Config().(BootstrapConfig).HostToolDir() return filepath.Join(toolDir, "go", ctx.ModuleName(), "test") } // moduleSrcDir returns the path of the directory that all source file paths are // specified relative to. func moduleSrcDir(ctx blueprint.ModuleContext) string { return ctx.ModuleDir() } // moduleObjDir returns the module-specific object directory path. func moduleObjDir(ctx blueprint.ModuleContext) string { toolDir := ctx.Config().(BootstrapConfig).HostToolDir() return filepath.Join(toolDir, "go", ctx.ModuleName(), "obj") } // moduleGenSrcDir returns the module-specific generated sources path. func moduleGenSrcDir(ctx blueprint.ModuleContext) string { toolDir := ctx.Config().(BootstrapConfig).HostToolDir() return filepath.Join(toolDir, "go", ctx.ModuleName(), "gen") }