tequilaOS/platform_build_blueprint
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #113 from colincross/parallel

Browse source 
Add concurrency to BottomUpMutators and cloneModules
This commit is contained in:
colincross 2016-08-11 11:18:15 -07:00 committed by GitHub
commit 056963fe71
3 changed files with 194 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

275
context.go
View file

@ -70,7 +70,7 @@ type Context struct {
modulesSorted []*moduleInfo
singletonInfo []*singletonInfo
mutatorInfo []*mutatorInfo
earlyMutatorInfo []*earlyMutatorInfo
earlyMutatorInfo []*mutatorInfo
variantMutatorNames []string
moduleNinjaNames map[string]*moduleGroup
@ -224,12 +224,7 @@ type mutatorInfo struct {
topDownMutator TopDownMutator
bottomUpMutator BottomUpMutator
name string
}
type earlyMutatorInfo struct {
// set during RegisterEarlyMutator
mutator EarlyMutator
name string
parallel bool
}
func (e *Error) Error() string {
@ -399,19 +394,35 @@ func (c *Context) RegisterTopDownMutator(name string, mutator TopDownMutator) {
//
// The mutator type names given here must be unique to all bottom up or early
// mutators in the Context.
func (c *Context) RegisterBottomUpMutator(name string, mutator BottomUpMutator) {
//
// Returns a BottomUpMutatorHandle, on which Parallel can be called to set
// the mutator to visit modules in parallel while maintaining ordering.
func (c *Context) RegisterBottomUpMutator(name string, mutator BottomUpMutator) BottomUpMutatorHandle {
for _, m := range c.variantMutatorNames {
if m == name {
panic(fmt.Errorf("mutator name %s is already registered", name))
}
}
c.mutatorInfo = append(c.mutatorInfo, &mutatorInfo{
info := &mutatorInfo{
bottomUpMutator: mutator,
name: name,
})
}
c.mutatorInfo = append(c.mutatorInfo, info)
c.variantMutatorNames = append(c.variantMutatorNames, name)
return info
}
type BottomUpMutatorHandle interface {
// Set the mutator to visit modules in parallel while maintaining ordering
Parallel() BottomUpMutatorHandle
}
func (mutator *mutatorInfo) Parallel() BottomUpMutatorHandle {
mutator.parallel = true
return mutator
}
// RegisterEarlyMutator registers a mutator that will be invoked to split
@ -436,9 +447,11 @@ func (c *Context) RegisterEarlyMutator(name string, mutator EarlyMutator) {
}
}
c.earlyMutatorInfo = append(c.earlyMutatorInfo, &earlyMutatorInfo{
mutator: mutator,
name: name,
c.earlyMutatorInfo = append(c.earlyMutatorInfo, &mutatorInfo{
bottomUpMutator: func(mctx BottomUpMutatorContext) {
mutator(mctx)
},
name: name,
})
c.variantMutatorNames = append(c.variantMutatorNames, name)
@ -995,11 +1008,6 @@ func (c *Context) createVariations(origModule *moduleInfo, mutatorName string,
newModules = append(newModules, newModule)
// Insert the new variant into the global module map. If this is the first variant then
// it reuses logicModule from the original module, which causes this to replace the
// original module in the global module map.
c.moduleInfo[newModule.logicModule] = newModule
newErrs := c.convertDepsToVariation(newModule, mutatorName, variationName)
if len(newErrs) > 0 {
errs = append(errs, newErrs...)
@ -1139,7 +1147,12 @@ func (c *Context) addModule(module *moduleInfo) []error {
// the modules depended upon are defined and that no circular dependencies
// exist.
func (c *Context) ResolveDependencies(config interface{}) []error {
errs := c.runMutators(config)
errs := c.updateDependencies()
if len(errs) > 0 {
return errs
}
errs = c.runMutators(config)
if len(errs) > 0 {
return errs
}
@ -1351,6 +1364,16 @@ func (c *Context) addInterVariantDependency(origModule *moduleInfo, tag Dependen
fromInfo.directDeps = append(fromInfo.directDeps, depInfo{toInfo, tag})
}
func (c *Context) visitAllBottomUp(visit func(group *moduleInfo) bool) {
for _, module := range c.modulesSorted {
if visit(module) {
return
}
}
}
// Calls visit on each module, guaranteeing that visit is not called on a module until visit on all
// of its dependencies has finished.
func (c *Context) parallelVisitAllBottomUp(visit func(group *moduleInfo) bool) {
doneCh := make(chan *moduleInfo)
count := 0
@ -1571,69 +1594,17 @@ func (c *Context) PrepareBuildActions(config interface{}) (deps []string, errs [
return deps, nil
}
func (c *Context) runEarlyMutators(config interface{}) (errs []error) {
for _, mutator := range c.earlyMutatorInfo {
for _, group := range c.moduleGroups {
newModules := make([]*moduleInfo, 0, len(group.modules))
for _, module := range group.modules {
mctx := &mutatorContext{
baseModuleContext: baseModuleContext{
context: c,
config: config,
module: module,
},
name: mutator.name,
}
func() {
defer func() {
if r := recover(); r != nil {
in := fmt.Sprintf("early mutator %q for %s", mutator.name, module)
if err, ok := r.(panicError); ok {
err.addIn(in)
mctx.error(err)
} else {
mctx.error(newPanicErrorf(r, in))
}
}
}()
mutator.mutator(mctx)
}()
if len(mctx.errs) > 0 {
errs = append(errs, mctx.errs...)
return errs
}
if module.splitModules != nil {
newModules = append(newModules, module.splitModules...)
} else {
newModules = append(newModules, module)
}
}
group.modules = newModules
}
}
errs = c.updateDependencies()
if len(errs) > 0 {
return errs
}
return nil
}
func (c *Context) runMutators(config interface{}) (errs []error) {
errs = c.runEarlyMutators(config)
if len(errs) > 0 {
return errs
}
var mutators []*mutatorInfo
for _, mutator := range c.mutatorInfo {
mutators = append(mutators, c.earlyMutatorInfo...)
mutators = append(mutators, c.mutatorInfo...)
for _, mutator := range mutators {
if mutator.topDownMutator != nil {
errs = c.runTopDownMutator(config, mutator.name, mutator.topDownMutator)
errs = c.runTopDownMutator(config, mutator)
} else if mutator.bottomUpMutator != nil {
errs = c.runBottomUpMutator(config, mutator.name, mutator.bottomUpMutator)
errs = c.runBottomUpMutator(config, mutator)
} else {
panic("no mutator set on " + mutator.name)
}
@ -1645,8 +1616,7 @@ func (c *Context) runMutators(config interface{}) (errs []error) {
return nil
}
func (c *Context) runTopDownMutator(config interface{},
name string, mutator TopDownMutator) (errs []error) {
func (c *Context) runTopDownMutator(config interface{}, mutator *mutatorInfo) (errs []error) {
for i := 0; i < len(c.modulesSorted); i++ {
module := c.modulesSorted[len(c.modulesSorted)-1-i]
@ -1656,12 +1626,12 @@ func (c *Context) runTopDownMutator(config interface{},
config: config,
module: module,
},
name: name,
name: mutator.name,
}
func() {
defer func() {
if r := recover(); r != nil {
in := fmt.Sprintf("top down mutator %q for %s", name, module)
in := fmt.Sprintf("top down mutator %q for %s", mutator.name, module)
if err, ok := r.(panicError); ok {
err.addIn(in)
mctx.error(err)
@ -1670,7 +1640,7 @@ func (c *Context) runTopDownMutator(config interface{},
}
}
}()
mutator(mctx)
mutator.topDownMutator(mctx)
}()
if len(mctx.errs) > 0 {
@ -1682,14 +1652,27 @@ func (c *Context) runTopDownMutator(config interface{},
return errs
}
type reverseDep struct {
module *moduleInfo
dep depInfo
}
func (c *Context) runBottomUpMutator(config interface{},
name string, mutator BottomUpMutator) (errs []error) {
mutator *mutatorInfo) (errs []error) {
newModuleInfo := make(map[Module]*moduleInfo)
for k, v := range c.moduleInfo {
newModuleInfo[k] = v
}
reverseDeps := make(map[*moduleInfo][]depInfo)
for _, module := range c.modulesSorted {
newModules := make([]*moduleInfo, 0, 1)
errsCh := make(chan []error)
reverseDepsCh := make(chan []reverseDep)
newModulesCh := make(chan []*moduleInfo)
done := make(chan bool)
visit := func(module *moduleInfo) bool {
if module.splitModules != nil {
panic("split module found in sorted module list")
}
@ -1700,14 +1683,13 @@ func (c *Context) runBottomUpMutator(config interface{},
config: config,
module: module,
},
name: name,
reverseDeps: reverseDeps,
name: mutator.name,
}
func() {
defer func() {
if r := recover(); r != nil {
in := fmt.Sprintf("bottom up mutator %q for %s", name, module)
in := fmt.Sprintf("bottom up mutator %q for %s", mutator.name, module)
if err, ok := r.(panicError); ok {
err.addIn(in)
mctx.error(err)
@ -1716,28 +1698,76 @@ func (c *Context) runBottomUpMutator(config interface{},
}
}
}()
mutator(mctx)
mutator.bottomUpMutator(mctx)
}()
if len(mctx.errs) > 0 {
errs = append(errs, mctx.errs...)
return errs
errsCh <- errs
return true
}
// Fix up any remaining dependencies on modules that were split into variants
// by replacing them with the first variant
for i, dep := range module.directDeps {
if dep.module.logicModule == nil {
module.directDeps[i].module = dep.module.splitModules[0]
if len(mctx.newModules) > 0 {
newModulesCh <- mctx.newModules
}
if len(mctx.reverseDeps) > 0 {
reverseDepsCh <- mctx.reverseDeps
}
return false
}
// Process errs and reverseDeps in a single goroutine
go func() {
for {
select {
case newErrs := <-errsCh:
errs = append(errs, newErrs...)
case newReverseDeps := <-reverseDepsCh:
for _, r := range newReverseDeps {
reverseDeps[r.module] = append(reverseDeps[r.module], r.dep)
}
case newModules := <-newModulesCh:
for _, m := range newModules {
newModuleInfo[m.logicModule] = m
}
case <-done:
return
}
}
}()
if module.splitModules != nil {
newModules = append(newModules, module.splitModules...)
} else {
newModules = append(newModules, module)
if mutator.parallel {
c.parallelVisitAllBottomUp(visit)
} else {
c.visitAllBottomUp(visit)
}
done <- true
if len(errs) > 0 {
return errs
}
c.moduleInfo = newModuleInfo
for _, group := range c.moduleGroups {
for i := 0; i < len(group.modules); i++ {
module := group.modules[i]
// Update module group to contain newly split variants
if module.splitModules != nil {
group.modules, i = spliceModules(group.modules, i, module.splitModules)
}
// Fix up any remaining dependencies on modules that were split into variants
// by replacing them with the first variant
for j, dep := range module.directDeps {
if dep.module.logicModule == nil {
module.directDeps[j].module = dep.module.splitModules[0]
}
}
}
module.group.modules = spliceModules(module.group.modules, module, newModules)
}
for module, deps := range reverseDeps {
@ -1745,6 +1775,7 @@ func (c *Context) runBottomUpMutator(config interface{},
module.directDeps = append(module.directDeps, deps...)
}
// TODO(ccross): update can be elided if no dependencies were modified
errs = c.updateDependencies()
if len(errs) > 0 {
return errs
@ -1757,26 +1788,30 @@ func (c *Context) runBottomUpMutator(config interface{},
// a mutator sets a non-property member variable on a module, which works until a later mutator
// creates variants of that module.
func (c *Context) cloneModules() {
type update struct {
orig Module
clone *moduleInfo
}
ch := make(chan update, 100)
for _, m := range c.modulesSorted {
origLogicModule := m.logicModule
m.logicModule, m.moduleProperties = c.cloneLogicModule(m)
delete(c.moduleInfo, origLogicModule)
c.moduleInfo[m.logicModule] = m
go func(m *moduleInfo) {
origLogicModule := m.logicModule
m.logicModule, m.moduleProperties = c.cloneLogicModule(m)
ch <- update{origLogicModule, m}
}(m)
}
for i := 0; i < len(c.modulesSorted); i++ {
update := <-ch
delete(c.moduleInfo, update.orig)
c.moduleInfo[update.clone.logicModule] = update.clone
}
}
func spliceModules(modules []*moduleInfo, origModule *moduleInfo,
newModules []*moduleInfo) []*moduleInfo {
for i, m := range modules {
if m == origModule {
return spliceModulesAtIndex(modules, i, newModules)
}
}
panic("failed to find original module to splice")
}
func spliceModulesAtIndex(modules []*moduleInfo, i int, newModules []*moduleInfo) []*moduleInfo {
// Removes modules[i] from the list and inserts newModules... where it was located, returning
// the new slice and the index of the last inserted element
func spliceModules(modules []*moduleInfo, i int, newModules []*moduleInfo) ([]*moduleInfo, int) {
spliceSize := len(newModules)
newLen := len(modules) + spliceSize - 1
var dest []*moduleInfo
@ -1794,7 +1829,7 @@ func spliceModulesAtIndex(modules []*moduleInfo, i int, newModules []*moduleInfo
// Copy the new modules into the slice
copy(dest[i:], newModules)
return dest
return dest, i + spliceSize - 1
}
func (c *Context) initSpecialVariables() {

16
module_ctx.go
View file

@ -429,7 +429,8 @@ func (m *moduleContext) GetMissingDependencies() []string {
type mutatorContext struct {
baseModuleContext
name string
reverseDeps map[*moduleInfo][]depInfo
reverseDeps []reverseDep
newModules []*moduleInfo
}
type baseMutatorContext interface {
@ -460,7 +461,7 @@ type TopDownMutatorContext interface {
}
type BottomUpMutatorContext interface {
TopDownMutatorContext
baseMutatorContext
AddDependency(module Module, tag DependencyTag, name ...string)
AddReverseDependency(module Module, tag DependencyTag, name string)
@ -541,6 +542,11 @@ func (mctx *mutatorContext) createVariations(variationNames []string, local bool
}
}
if mctx.newModules != nil {
panic("module already has variations from this mutator")
}
mctx.newModules = modules
if len(ret) != len(variationNames) {
panic("oops!")
}
@ -582,8 +588,10 @@ func (mctx *mutatorContext) AddReverseDependency(module Module, tag DependencyTa
return
}
mctx.reverseDeps[destModule] = append(mctx.reverseDeps[destModule],
depInfo{mctx.context.moduleInfo[module], tag})
mctx.reverseDeps = append(mctx.reverseDeps, reverseDep{
destModule,
depInfo{mctx.context.moduleInfo[module], tag},
})
}
// AddVariationDependencies adds deps as dependencies of the current module, but uses the variations

39
splice_modules_test.go
View file

@ -30,73 +30,82 @@ var (
var spliceModulesTestCases = []struct {
in []*moduleInfo
replace *moduleInfo
at int
with []*moduleInfo
out []*moduleInfo
outAt int
reallocate bool
}{
{
// Insert at the beginning
in: []*moduleInfo{testModuleA, testModuleB, testModuleC},
replace: testModuleA,
at: 0,
with: []*moduleInfo{testModuleD, testModuleE},
out: []*moduleInfo{testModuleD, testModuleE, testModuleB, testModuleC},
outAt: 1,
reallocate: true,
},
{
// Insert in the middle
in: []*moduleInfo{testModuleA, testModuleB, testModuleC},
replace: testModuleB,
at: 1,
with: []*moduleInfo{testModuleD, testModuleE},
out: []*moduleInfo{testModuleA, testModuleD, testModuleE, testModuleC},
outAt: 2,
reallocate: true,
},
{
// Insert at the end
in: []*moduleInfo{testModuleA, testModuleB, testModuleC},
replace: testModuleC,
at: 2,
with: []*moduleInfo{testModuleD, testModuleE},
out: []*moduleInfo{testModuleA, testModuleB, testModuleD, testModuleE},
outAt: 3,
reallocate: true,
},
{
// Insert over a single element
in: []*moduleInfo{testModuleA},
replace: testModuleA,
at: 0,
with: []*moduleInfo{testModuleD, testModuleE},
out: []*moduleInfo{testModuleD, testModuleE},
outAt: 1,
reallocate: true,
},
{
// Insert at the beginning without reallocating
in: []*moduleInfo{testModuleA, testModuleB, testModuleC, nil}[0:3],
replace: testModuleA,
at: 0,
with: []*moduleInfo{testModuleD, testModuleE},
out: []*moduleInfo{testModuleD, testModuleE, testModuleB, testModuleC},
outAt: 1,
reallocate: false,
},
{
// Insert in the middle without reallocating
in: []*moduleInfo{testModuleA, testModuleB, testModuleC, nil}[0:3],
replace: testModuleB,
at: 1,
with: []*moduleInfo{testModuleD, testModuleE},
out: []*moduleInfo{testModuleA, testModuleD, testModuleE, testModuleC},
outAt: 2,
reallocate: false,
},
{
// Insert at the end without reallocating
in: []*moduleInfo{testModuleA, testModuleB, testModuleC, nil}[0:3],
replace: testModuleC,
at: 2,
with: []*moduleInfo{testModuleD, testModuleE},
out: []*moduleInfo{testModuleA, testModuleB, testModuleD, testModuleE},
outAt: 3,
reallocate: false,
},
{
// Insert over a single element without reallocating
in: []*moduleInfo{testModuleA, nil}[0:1],
replace: testModuleA,
at: 0,
with: []*moduleInfo{testModuleD, testModuleE},
out: []*moduleInfo{testModuleD, testModuleE},
outAt: 1,
reallocate: false,
},
}
@ -106,15 +115,21 @@ func TestSpliceModules(t *testing.T) {
in := make([]*moduleInfo, len(testCase.in), cap(testCase.in))
copy(in, testCase.in)
origIn := in
got := spliceModules(in, testCase.replace, testCase.with)
got, gotAt := spliceModules(in, testCase.at, testCase.with)
if !reflect.DeepEqual(got, testCase.out) {
t.Errorf("test case: %v, %v -> %v", testCase.in, testCase.replace, testCase.with)
t.Errorf("test case: %v, %v -> %v", testCase.in, testCase.at, testCase.with)
t.Errorf("incorrect output:")
t.Errorf(" expected: %v", testCase.out)
t.Errorf(" got: %v", got)
}
if gotAt != testCase.outAt {
t.Errorf("test case: %v, %v -> %v", testCase.in, testCase.at, testCase.with)
t.Errorf("incorrect index:")
t.Errorf(" expected: %d", testCase.outAt)
t.Errorf(" got: %d", gotAt)
}
if sameArray(origIn, got) != !testCase.reallocate {
t.Errorf("test case: %v, %v -> %v", testCase.in, testCase.replace, testCase.with)
t.Errorf("test case: %v, %v -> %v", testCase.in, testCase.at, testCase.with)
not := ""
if !testCase.reallocate {
not = " not"