diff --git a/bootstrap/command.go b/bootstrap/command.go index 3c41cce..08a0e43 100644 --- a/bootstrap/command.go +++ b/bootstrap/command.go @@ -91,7 +91,7 @@ func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, conf ctx.RegisterBottomUpMutator("bootstrap_plugin_deps", pluginDeps) ctx.RegisterModuleType("bootstrap_go_package", newGoPackageModuleFactory()) ctx.RegisterModuleType("blueprint_go_binary", newGoBinaryModuleFactory()) - ctx.RegisterSingletonType("bootstrap", newSingletonFactory()) + ctx.RegisterSingletonType("bootstrap", newSingletonFactory(), false) blueprint.RegisterPackageIncludesModuleType(ctx) ctx.BeginEvent("parse_bp") diff --git a/bootstrap/glob.go b/bootstrap/glob.go index a766676..ef68090 100644 --- a/bootstrap/glob.go +++ b/bootstrap/glob.go @@ -227,7 +227,7 @@ func generateGlobNinjaFile(glob *GlobSingleton, config interface{}) ([]byte, []e ctx := blueprint.NewContext() ctx.RegisterSingletonType("glob", func() blueprint.Singleton { return glob - }) + }, false) extraDeps, errs := ctx.ResolveDependencies(config) if len(extraDeps) > 0 { diff --git a/context.go b/context.go index 3997633..ae35dac 100644 --- a/context.go +++ b/context.go @@ -441,6 +441,7 @@ type singletonInfo struct { factory SingletonFactory singleton Singleton name string + parallel bool // set during PrepareBuildActions actionDefs localBuildActions @@ -566,13 +567,15 @@ type SingletonFactory func() Singleton // RegisterSingletonType registers a singleton type that will be invoked to // generate build actions. Each registered singleton type is instantiated -// and invoked exactly once as part of the generate phase. Each registered -// singleton is invoked in registration order. +// and invoked exactly once as part of the generate phase. +// +// Those singletons registered with parallel=true are run in parallel, after +// which the other registered singletons are run in registration order. // // The singleton type names given here must be unique for the context. The // factory function should be a named function so that its package and name can // be included in the generated Ninja file for debugging purposes. -func (c *Context) RegisterSingletonType(name string, factory SingletonFactory) { +func (c *Context) RegisterSingletonType(name string, factory SingletonFactory, parallel bool) { for _, s := range c.singletonInfo { if s.name == name { panic(fmt.Errorf("singleton %q is already registered", name)) @@ -583,6 +586,7 @@ func (c *Context) RegisterSingletonType(name string, factory SingletonFactory) { factory: factory, singleton: factory(), name: name, + parallel: parallel, }) } @@ -605,6 +609,7 @@ func (c *Context) RegisterPreSingletonType(name string, factory SingletonFactory factory: factory, singleton: factory(), name: name, + parallel: false, }) } @@ -3314,6 +3319,8 @@ func spliceModules(modules modulesOrAliases, i int, newModules modulesOrAliases) func (c *Context) generateModuleBuildActions(config interface{}, liveGlobals *liveTracker) ([]string, []error) { + c.BeginEvent("generateModuleBuildActions") + defer c.EndEvent("generateModuleBuildActions") var deps []string var errs []error @@ -3411,56 +3418,131 @@ func (c *Context) generateModuleBuildActions(config interface{}, return deps, errs } -func (c *Context) generateSingletonBuildActions(config interface{}, - singletons []*singletonInfo, liveGlobals *liveTracker) ([]string, []error) { +func (c *Context) generateOneSingletonBuildActions(config interface{}, + info *singletonInfo, liveGlobals *liveTracker) ([]string, []error) { var deps []string var errs []error - for _, info := range singletons { - // The parent scope of the singletonContext's local scope gets overridden to be that of the - // calling Go package on a per-call basis. Since the initial parent scope doesn't matter we - // just set it to nil. - scope := newLocalScope(nil, singletonNamespacePrefix(info.name)) + // The parent scope of the singletonContext's local scope gets overridden to be that of the + // calling Go package on a per-call basis. Since the initial parent scope doesn't matter we + // just set it to nil. + scope := newLocalScope(nil, singletonNamespacePrefix(info.name)) - sctx := &singletonContext{ - name: info.name, - context: c, - config: config, - scope: scope, - globals: liveGlobals, - } + sctx := &singletonContext{ + name: info.name, + context: c, + config: config, + scope: scope, + globals: liveGlobals, + } - func() { - defer func() { - if r := recover(); r != nil { - in := fmt.Sprintf("GenerateBuildActions for singleton %s", info.name) - if err, ok := r.(panicError); ok { - err.addIn(in) - sctx.error(err) - } else { - sctx.error(newPanicErrorf(r, in)) - } + func() { + defer func() { + if r := recover(); r != nil { + in := fmt.Sprintf("GenerateBuildActions for singleton %s", info.name) + if err, ok := r.(panicError); ok { + err.addIn(in) + sctx.error(err) + } else { + sctx.error(newPanicErrorf(r, in)) } - }() - info.singleton.GenerateBuildActions(sctx) + } }() + info.singleton.GenerateBuildActions(sctx) + }() - if len(sctx.errs) > 0 { - errs = append(errs, sctx.errs...) + if len(sctx.errs) > 0 { + errs = append(errs, sctx.errs...) + return deps, errs + } + + deps = append(deps, sctx.ninjaFileDeps...) + + newErrs := c.processLocalBuildActions(&info.actionDefs, + &sctx.actionDefs, liveGlobals) + errs = append(errs, newErrs...) + return deps, errs +} + +func (c *Context) generateParallelSingletonBuildActions(config interface{}, + singletons []*singletonInfo, liveGlobals *liveTracker) ([]string, []error) { + + c.BeginEvent("generateParallelSingletonBuildActions") + defer c.EndEvent("generateParallelSingletonBuildActions") + + var deps []string + var errs []error + + wg := sync.WaitGroup{} + cancelCh := make(chan struct{}) + depsCh := make(chan []string) + errsCh := make(chan []error) + + go func() { + for { + select { + case <-cancelCh: + close(cancelCh) + return + case dep := <-depsCh: + deps = append(deps, dep...) + case newErrs := <-errsCh: + if len(errs) <= maxErrors { + errs = append(errs, newErrs...) + } + } + } + }() + + for _, info := range singletons { + if !info.parallel { + // Skip any singletons registered with parallel=false. + continue + } + wg.Add(1) + go func(inf *singletonInfo) { + defer wg.Done() + newDeps, newErrs := c.generateOneSingletonBuildActions(config, inf, liveGlobals) + depsCh <- newDeps + errsCh <- newErrs + }(info) + } + wg.Wait() + + cancelCh <- struct{}{} + <-cancelCh + + return deps, errs +} + +func (c *Context) generateSingletonBuildActions(config interface{}, + singletons []*singletonInfo, liveGlobals *liveTracker) ([]string, []error) { + + c.BeginEvent("generateSingletonBuildActions") + defer c.EndEvent("generateSingletonBuildActions") + + var deps []string + var errs []error + + // Run one singleton. Use a variable to simplify manual validation testing. + var runSingleton = func(info *singletonInfo) { + c.BeginEvent("singleton:" + info.name) + defer c.EndEvent("singleton:" + info.name) + newDeps, newErrs := c.generateOneSingletonBuildActions(config, info, liveGlobals) + deps = append(deps, newDeps...) + errs = append(errs, newErrs...) + } + + // First, take care of any singletons that want to run in parallel. + deps, errs = c.generateParallelSingletonBuildActions(config, singletons, liveGlobals) + + for _, info := range singletons { + if !info.parallel { + runSingleton(info) if len(errs) > maxErrors { break } - continue - } - - deps = append(deps, sctx.ninjaFileDeps...) - - newErrs := c.processLocalBuildActions(&info.actionDefs, - &sctx.actionDefs, liveGlobals) - errs = append(errs, newErrs...) - if len(errs) > maxErrors { - break } } diff --git a/module_ctx_test.go b/module_ctx_test.go index af23be7..d1477ff 100644 --- a/module_ctx_test.go +++ b/module_ctx_test.go @@ -590,7 +590,7 @@ func TestAddNinjaFileDeps(t *testing.T) { ctx.RegisterBottomUpMutator("testBottomUpMutator", addNinjaDepsTestBottomUpMutator) ctx.RegisterTopDownMutator("testTopDownMutator", addNinjaDepsTestTopDownMutator) ctx.RegisterPreSingletonType("testPreSingleton", addNinjaDepsTestPreSingletonFactory) - ctx.RegisterSingletonType("testSingleton", addNinjaDepsTestSingletonFactory) + ctx.RegisterSingletonType("testSingleton", addNinjaDepsTestSingletonFactory, false) parseDeps, errs := ctx.ParseBlueprintsFiles("Android.bp", nil) if len(errs) > 0 { t.Errorf("unexpected parse errors:")