Implement plugins for bootstrap go modules

Now that we have multi-stage bootstrapping, we can make the primary
builder build more dynamic. Add the concept of plugins that will be
linked and loaded into bootstrap_go_binary or bootstrap_go_package
modules. It's expected that the plugin's init() functions will do
whatever registration is necessary.

Example Blueprint definition:

    bootstrap_go_binary {
      name: "builder",
      ...
    }

    bootstrap_go_package {
      name: "plugin1",
      pluginFor: ["builder"],
    }

A package may specify more than one plugin if it will be inserted into
more than one go module.

Change-Id: I109835f444196b66fc4018c3fa36ba0875823184
This commit is contained in:
Dan Willemsen 2015-07-24 16:53:27 -07:00
parent 9cdb70b93b
commit fdeb724f74
7 changed files with 281 additions and 52 deletions

View file

@ -128,3 +128,8 @@ bootstrap_core_go_binary(
name = "choosestage",
srcs = ["choosestage/choosestage.go"],
)
bootstrap_go_binary{
name = "loadplugins",
srcs = ["loadplugins/loadplugins.go"],
}

View file

@ -29,8 +29,9 @@ const miniBootstrapDir = "$buildDir/.minibootstrap"
var (
pctx = blueprint.NewPackageContext("github.com/google/blueprint/bootstrap")
goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain"))
chooseStageCmd = pctx.StaticVariable("chooseStageCmd", filepath.Join(bootstrapDir, "bin", "choosestage"))
goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain"))
chooseStageCmd = pctx.StaticVariable("chooseStageCmd", filepath.Join(bootstrapDir, "bin", "choosestage"))
pluginGenSrcCmd = pctx.StaticVariable("pluginGenSrcCmd", filepath.Join(bootstrapDir, "bin", "loadplugins"))
compile = pctx.StaticRule("compile",
blueprint.RuleParams{
@ -54,6 +55,13 @@ var (
},
"pkg")
pluginGenSrc = pctx.StaticRule("pluginGenSrc",
blueprint.RuleParams{
Command: "$pluginGenSrcCmd -o $out -p $pkg $plugins",
Description: "create $out",
},
"pkg", "plugins")
test = pctx.StaticRule("test",
blueprint.RuleParams{
Command: "(cd $pkgSrcDir && $$OLDPWD/$in -test.short) && touch $out",
@ -121,6 +129,14 @@ func propagateStageBootstrap(mctx blueprint.TopDownMutatorContext) {
})
}
func pluginDeps(ctx blueprint.BottomUpMutatorContext) {
if pkg, ok := ctx.Module().(*goPackage); ok {
for _, plugin := range pkg.properties.PluginFor {
ctx.AddReverseDependency(ctx.Module(), plugin)
}
}
}
type goPackageProducer interface {
GoPkgRoot() string
GoPackageTarget() string
@ -141,6 +157,20 @@ func isGoTestProducer(module blueprint.Module) bool {
return ok
}
type goPluginProvider interface {
GoPkgPath() string
IsPluginFor(string) bool
}
func isGoPluginFor(name string) func(blueprint.Module) bool {
return func(module blueprint.Module) bool {
if plugin, ok := module.(goPluginProvider); ok {
return plugin.IsPluginFor(name)
}
return false
}
}
func isBootstrapModule(module blueprint.Module) bool {
_, isPackage := module.(*goPackage)
_, isBinary := module.(*goBinary)
@ -155,9 +185,10 @@ func isBootstrapBinaryModule(module blueprint.Module) bool {
// A goPackage is a module for building Go packages.
type goPackage struct {
properties struct {
PkgPath string
Srcs []string
TestSrcs []string
PkgPath string
Srcs []string
TestSrcs []string
PluginFor []string
}
// The root dir in which the package .a file is located. The full .a file
@ -189,6 +220,10 @@ func newGoPackageModuleFactory(config *Config) func() (blueprint.Module, []inter
}
}
func (g *goPackage) GoPkgPath() string {
return g.properties.PkgPath
}
func (g *goPackage) GoPkgRoot() string {
return g.pkgRoot
}
@ -209,8 +244,22 @@ func (g *goPackage) SetBuildStage(buildStage Stage) {
g.buildStage = buildStage
}
func (g *goPackage) IsPluginFor(name string) bool {
for _, plugin := range g.properties.PluginFor {
if plugin == name {
return true
}
}
return false
}
func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
name := ctx.ModuleName()
var (
name = ctx.ModuleName()
hasPlugins = false
pluginSrc = ""
genSrcs = []string{}
)
if g.properties.PkgPath == "" {
ctx.ModuleErrorf("module %s did not specify a valid pkgPath", name)
@ -225,6 +274,13 @@ func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
filepath.FromSlash(g.properties.PkgPath)+".a")
}
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) { hasPlugins = true })
if hasPlugins {
pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
genSrcs = append(genSrcs, pluginSrc)
}
// We only actually want to build the builder modules if we're running as
// minibp (i.e. we're generating a bootstrap Ninja file). This is to break
// the circular dependence that occurs when the builder requires a new Ninja
@ -233,19 +289,23 @@ func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
if g.config.stage == g.BuildStage() {
var deps []string
if hasPlugins && !buildGoPluginLoader(ctx, g.properties.PkgPath, pluginSrc, g.config.stage) {
return
}
if g.config.runGoTests {
deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
g.properties.PkgPath, g.properties.Srcs,
g.properties.PkgPath, g.properties.Srcs, genSrcs,
g.properties.TestSrcs)
}
buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile,
g.properties.Srcs, deps)
g.properties.Srcs, genSrcs, deps)
} else if g.config.stage != StageBootstrap {
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil)
phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil, nil)
}
phonyGoTarget(ctx, g.archiveFile, g.properties.Srcs, nil)
phonyGoTarget(ctx, g.archiveFile, g.properties.Srcs, genSrcs, nil)
}
}
@ -296,12 +356,22 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
archiveFile = filepath.Join(objDir, name+".a")
aoutFile = filepath.Join(objDir, "a.out")
binaryFile = filepath.Join("$BinDir", name)
hasPlugins = false
pluginSrc = ""
genSrcs = []string{}
)
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
g.testArchiveFile = filepath.Join(testRoot(ctx), name+".a")
}
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) { hasPlugins = true })
if hasPlugins {
pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
genSrcs = append(genSrcs, pluginSrc)
}
// We only actually want to build the builder modules if we're running as
// minibp (i.e. we're generating a bootstrap Ninja file). This is to break
// the circular dependence that occurs when the builder requires a new Ninja
@ -310,12 +380,16 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
if g.config.stage == g.BuildStage() {
var deps []string
if g.config.runGoTests {
deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
name, g.properties.Srcs, g.properties.TestSrcs)
if hasPlugins && !buildGoPluginLoader(ctx, "main", pluginSrc, g.config.stage) {
return
}
buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, deps)
if g.config.runGoTests {
deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
name, g.properties.Srcs, genSrcs, g.properties.TestSrcs)
}
buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, genSrcs, deps)
var libDirFlags []string
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
@ -345,19 +419,49 @@ func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
})
} else if g.config.stage != StageBootstrap {
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil)
phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil, nil)
}
intermediates := []string{aoutFile, archiveFile}
phonyGoTarget(ctx, binaryFile, g.properties.Srcs, intermediates)
phonyGoTarget(ctx, binaryFile, g.properties.Srcs, genSrcs, intermediates)
}
}
func buildGoPluginLoader(ctx blueprint.ModuleContext, pkgPath, pluginSrc string, stage Stage) bool {
ret := true
name := ctx.ModuleName()
var pluginPaths []string
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) {
plugin := module.(goPluginProvider)
pluginPaths = append(pluginPaths, plugin.GoPkgPath())
if stage == StageBootstrap {
ctx.OtherModuleErrorf(module, "plugin %q may not be included in core module %q",
ctx.OtherModuleName(module), name)
ret = false
}
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: pluginGenSrc,
Outputs: []string{pluginSrc},
Implicits: []string{"$pluginGenSrcCmd"},
Args: map[string]string{
"pkg": pkgPath,
"plugins": strings.Join(pluginPaths, " "),
},
})
return ret
}
func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
pkgPath string, archiveFile string, srcs []string, orderDeps []string) {
pkgPath string, archiveFile string, srcs []string, genSrcs []string, orderDeps []string) {
srcDir := moduleSrcDir(ctx)
srcFiles := pathtools.PrefixPaths(srcs, srcDir)
srcFiles = append(srcFiles, genSrcs...)
var incFlags []string
deps := []string{"$compileCmd"}
@ -388,9 +492,8 @@ func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
})
}
func buildGoTest(ctx blueprint.ModuleContext, testRoot string,
testPkgArchive string, pkgPath string, srcs []string,
testSrcs []string) []string {
func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
pkgPath string, srcs, genSrcs, testSrcs []string) []string {
if len(testSrcs) == 0 {
return nil
@ -405,7 +508,7 @@ func buildGoTest(ctx blueprint.ModuleContext, testRoot string,
testPassed := filepath.Join(testRoot, "test.passed")
buildGoPackage(ctx, testRoot, pkgPath, testPkgArchive,
append(srcs, testSrcs...), nil)
append(srcs, testSrcs...), genSrcs, nil)
ctx.Build(pctx, blueprint.BuildParams{
Rule: goTestMain,
@ -460,7 +563,7 @@ func buildGoTest(ctx blueprint.ModuleContext, testRoot string,
}
func phonyGoTarget(ctx blueprint.ModuleContext, target string, srcs []string,
intermediates []string) {
gensrcs []string, intermediates []string) {
var depTargets []string
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
@ -472,6 +575,7 @@ func phonyGoTarget(ctx blueprint.ModuleContext, target string, srcs []string,
moduleDir := ctx.ModuleDir()
srcs = pathtools.PrefixPaths(srcs, filepath.Join("$srcDir", moduleDir))
srcs = append(srcs, gensrcs...)
ctx.Build(pctx, blueprint.BuildParams{
Rule: phony,
@ -884,3 +988,8 @@ func moduleSrcDir(ctx blueprint.ModuleContext) string {
func moduleObjDir(ctx blueprint.ModuleContext) string {
return filepath.Join(bootstrapDir, ctx.ModuleName(), "obj")
}
// moduleGenSrcDir returns the module-specific generated sources path.
func moduleGenSrcDir(ctx blueprint.ModuleContext) string {
return filepath.Join(bootstrapDir, ctx.ModuleName(), "gen")
}

View file

@ -90,6 +90,7 @@ func Main(ctx *blueprint.Context, config interface{}, extraNinjaFileDeps ...stri
runGoTests: runGoTests,
}
ctx.RegisterBottomUpMutator("bootstrap_plugin_deps", pluginDeps)
ctx.RegisterModuleType("bootstrap_go_package", newGoPackageModuleFactory(bootstrapConfig))
ctx.RegisterModuleType("bootstrap_core_go_binary", newGoBinaryModuleFactory(bootstrapConfig, StageBootstrap))
ctx.RegisterModuleType("bootstrap_go_binary", newGoBinaryModuleFactory(bootstrapConfig, StagePrimary))

View file

@ -54,7 +54,7 @@ rule g.bootstrap.link
# Module: blueprint
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.func·002
# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:1:1
build $
@ -80,7 +80,7 @@ default $
# Module: blueprint-bootstrap
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.func·002
# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:70:1
build $
@ -107,7 +107,7 @@ default $
# Module: blueprint-bootstrap-bpdoc
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.func·002
# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:89:1
build $
@ -127,7 +127,7 @@ default $
# Module: blueprint-deptools
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.func·002
# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:46:1
build $
@ -142,7 +142,7 @@ default $
# Module: blueprint-parser
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.func·002
# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:31:1
build $
@ -159,7 +159,7 @@ default $
# Module: blueprint-pathtools
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.func·002
# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:52:1
build $
@ -174,7 +174,7 @@ default $
# Module: blueprint-proptools
# Variant:
# Type: bootstrap_go_package
# Factory: github.com/google/blueprint/bootstrap.func·002
# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:64:1
build $
@ -189,7 +189,7 @@ default $
# Module: choosestage
# Variant:
# Type: bootstrap_core_go_binary
# Factory: github.com/google/blueprint/bootstrap.func·003
# Factory: github.com/google/blueprint/bootstrap.func·005
# Defined: Blueprints:127:1
build ${g.bootstrap.buildDir}/.bootstrap/choosestage/obj/choosestage.a: $
@ -211,7 +211,7 @@ default ${g.bootstrap.BinDir}/choosestage
# Module: gotestmain
# Variant:
# Type: bootstrap_core_go_binary
# Factory: github.com/google/blueprint/bootstrap.func·003
# Factory: github.com/google/blueprint/bootstrap.func·005
# Defined: Blueprints:122:1
build ${g.bootstrap.buildDir}/.bootstrap/gotestmain/obj/gotestmain.a: $
@ -233,7 +233,7 @@ default ${g.bootstrap.BinDir}/gotestmain
# Module: minibp
# Variant:
# Type: bootstrap_core_go_binary
# Factory: github.com/google/blueprint/bootstrap.func·003
# Factory: github.com/google/blueprint/bootstrap.func·005
# Defined: Blueprints:101:1
build ${g.bootstrap.buildDir}/.bootstrap/minibp/obj/minibp.a: $
@ -262,7 +262,7 @@ default ${g.bootstrap.BinDir}/minibp
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Singleton: bootstrap
# Factory: github.com/google/blueprint/bootstrap.func·008
# Factory: github.com/google/blueprint/bootstrap.func·012
rule s.bootstrap.primarybp
command = ${g.bootstrap.BinDir}/minibp --build-primary ${runTests} -m ${g.bootstrap.bootstrapManifest} --timestamp ${timestamp} --timestampdep ${timestampdep} -b ${g.bootstrap.buildDir} -d ${outfile}.d -o ${outfile} ${in}

View file

@ -1077,6 +1077,11 @@ func (c *Context) ResolveDependencies(config interface{}) []error {
return errs
}
errs = c.runMutators(config)
if len(errs) > 0 {
return errs
}
c.dependenciesReady = true
return nil
}
@ -1151,6 +1156,22 @@ func (c *Context) resolveDependencies(config interface{}) (errs []error) {
return
}
// findMatchingVariant searches the moduleGroup for a module with the same variant as module,
// and returns the matching module, or nil if one is not found.
func (c *Context) findMatchingVariant(module *moduleInfo, group *moduleGroup) *moduleInfo {
if len(group.modules) == 1 {
return group.modules[0]
} else {
for _, m := range group.modules {
if m.variant.equal(module.dependencyVariant) {
return m
}
}
}
return nil
}
func (c *Context) addDependency(module *moduleInfo, depName string) []error {
depsPos := module.propertyPos["deps"]
@ -1176,16 +1197,9 @@ func (c *Context) addDependency(module *moduleInfo, depName string) []error {
}
}
if len(depInfo.modules) == 1 {
module.directDeps = append(module.directDeps, depInfo.modules[0])
if m := c.findMatchingVariant(module, depInfo); m != nil {
module.directDeps = append(module.directDeps, m)
return nil
} else {
for _, m := range depInfo.modules {
if m.variant.equal(module.dependencyVariant) {
module.directDeps = append(module.directDeps, m)
return nil
}
}
}
return []error{&Error{
@ -1196,6 +1210,36 @@ func (c *Context) addDependency(module *moduleInfo, depName string) []error {
}}
}
func (c *Context) addReverseDependency(module *moduleInfo, destName string) []error {
if destName == module.properties.Name {
return []error{&Error{
Err: fmt.Errorf("%q depends on itself", destName),
Pos: module.pos,
}}
}
destInfo, ok := c.moduleGroups[destName]
if !ok {
return []error{&Error{
Err: fmt.Errorf("%q has a reverse dependency on undefined module %q",
module.properties.Name, destName),
Pos: module.pos,
}}
}
if m := c.findMatchingVariant(module, destInfo); m != nil {
m.directDeps = append(m.directDeps, module)
return nil
}
return []error{&Error{
Err: fmt.Errorf("reverse dependency %q of %q missing variant %q",
destName, module.properties.Name,
c.prettyPrintVariant(module.dependencyVariant)),
Pos: module.pos,
}}
}
func (c *Context) addVariationDependency(module *moduleInfo, variations []Variation,
depName string, far bool) []error {
@ -1435,11 +1479,6 @@ func (c *Context) PrepareBuildActions(config interface{}) (deps []string, errs [
}
}
errs = c.runMutators(config)
if len(errs) > 0 {
return nil, errs
}
liveGlobals := newLiveTracker(config)
c.initSpecialVariables()

View file

@ -0,0 +1,67 @@
// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"text/template"
)
var (
output = flag.String("o", "", "output filename")
pkg = flag.String("p", "main", "package name")
)
func main() {
flag.Parse()
if flag.NArg() == 0 {
fmt.Fprintln(os.Stderr, "error: must pass at least one input")
os.Exit(1)
}
buf := &bytes.Buffer{}
err := pluginTmpl.Execute(buf, struct {
Package string
Plugins []string
}{
filepath.Base(*pkg),
flag.Args(),
})
if err != nil {
panic(err)
}
err = ioutil.WriteFile(*output, buf.Bytes(), 0666)
if err != nil {
panic(err)
}
}
var pluginTmpl = template.Must(template.New("pluginloader").Parse(`
package {{.Package}}
import (
{{range .Plugins}}
_ "{{.}}"
{{end}}
)
`))

View file

@ -359,8 +359,7 @@ func (mctx *dynamicDependerModuleContext) AddFarVariationDependencies(variations
type mutatorContext struct {
baseModuleContext
name string
dependenciesModified bool
name string
}
type baseMutatorContext interface {
@ -389,6 +388,7 @@ type BottomUpMutatorContext interface {
baseMutatorContext
AddDependency(module Module, name string)
AddReverseDependency(module Module, name string)
CreateVariations(...string) []Module
CreateLocalVariations(...string) []Module
SetDependencyVariation(string)
@ -464,8 +464,7 @@ func (mctx *mutatorContext) Module() Module {
return mctx.module.logicModule
}
// Add a dependency to the given module. The depender can be a specific variant
// of a module, but the dependee must be a module that has no variations.
// Add a dependency to the given module.
// Does not affect the ordering of the current mutator pass, but will be ordered
// correctly for all future mutator passes.
func (mctx *mutatorContext) AddDependency(module Module, depName string) {
@ -473,7 +472,16 @@ func (mctx *mutatorContext) AddDependency(module Module, depName string) {
if len(errs) > 0 {
mctx.errs = append(mctx.errs, errs...)
}
mctx.dependenciesModified = true
}
// Add a dependency from the destination to the given module.
// Does not affect the ordering of the current mutator pass, but will be ordered
// correctly for all future mutator passes.
func (mctx *mutatorContext) AddReverseDependency(module Module, destName string) {
errs := mctx.context.addReverseDependency(mctx.context.moduleInfo[module], destName)
if len(errs) > 0 {
mctx.errs = append(mctx.errs, errs...)
}
}
func (mctx *mutatorContext) VisitDirectDeps(visit func(Module)) {