tequilaOS/platform_build_blueprint
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #231 from colincross/pprof

Browse source 
Performance improvements
This commit is contained in:
colincross 2019-01-24 10:32:25 -08:00 committed by GitHub
commit 3eeabc7991
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 210 additions and 152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
bootstrap/command.go
View file

@ -15,9 +15,10 @@
package bootstrap
import (
"bytes"
"bufio"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -173,20 +174,39 @@ func Main(ctx *blueprint.Context, config interface{}, extraNinjaFileDeps ...stri
}
deps = append(deps, extraDeps...)
buf := bytes.NewBuffer(nil)
err = ctx.WriteBuildFile(buf)
if err != nil {
fatalf("error generating Ninja file contents: %s", err)
}
if stage == StageMain && emptyNinjaFile {
buf.Reset()
}
const outFilePermissions = 0666
err = ioutil.WriteFile(outFile, buf.Bytes(), outFilePermissions)
var out io.Writer
var f *os.File
var buf *bufio.Writer
if stage != StageMain || !emptyNinjaFile {
f, err = os.OpenFile(outFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, outFilePermissions)
if err != nil {
fatalf("error opening Ninja file: %s", err)
}
buf = bufio.NewWriter(f)
out = buf
} else {
out = ioutil.Discard
}
err = ctx.WriteBuildFile(out)
if err != nil {
fatalf("error writing %s: %s", outFile, err)
fatalf("error writing Ninja file contents: %s", err)
}
if buf != nil {
err = buf.Flush()
if err != nil {
fatalf("error flushing Ninja file contents: %s", err)
}
}
if f != nil {
err = f.Close()
if err != nil {
fatalf("error closing Ninja file: %s", err)
}
}
if globFile != "" {

260
context.go
View file

@ -16,6 +16,7 @@ package blueprint
import (
"bytes"
"context"
"errors"
"fmt"
"io"
@ -24,6 +25,7 @@ import (
"path/filepath"
"reflect"
"runtime"
"runtime/pprof"
"sort"
"strings"
"sync"
@ -66,6 +68,8 @@ const MockModuleListFile = "bplist"
// write phase generates the Ninja manifest text based on the generated build
// actions.
type Context struct {
context.Context
// set at instantiation
moduleFactories map[string]ModuleFactory
nameInterface NameInterface
@ -275,6 +279,7 @@ type mutatorInfo struct {
func newContext() *Context {
return &Context{
Context: context.Background(),
moduleFactories: make(map[string]ModuleFactory),
nameInterface: NewSimpleNameInterface(),
moduleInfo: make(map[Module]*moduleInfo),
@ -1329,27 +1334,39 @@ func (c *Context) addModule(module *moduleInfo) []error {
// the modules depended upon are defined and that no circular dependencies
// exist.
func (c *Context) ResolveDependencies(config interface{}) (deps []string, errs []error) {
c.liveGlobals = newLiveTracker(config)
return c.resolveDependencies(c.Context, config)
}
func (c *Context) resolveDependencies(ctx context.Context, config interface{}) (deps []string, errs []error) {
pprof.Do(ctx, pprof.Labels("blueprint", "ResolveDependencies"), func(ctx context.Context) {
c.liveGlobals = newLiveTracker(config)
deps, errs = c.generateSingletonBuildActions(config, c.preSingletonInfo, c.liveGlobals)
if len(errs) > 0 {
return
}
errs = c.updateDependencies()
if len(errs) > 0 {
return
}
var mutatorDeps []string
mutatorDeps, errs = c.runMutators(ctx, config)
if len(errs) > 0 {
return
}
deps = append(deps, mutatorDeps...)
c.cloneModules()
c.dependenciesReady = true
})
deps, errs = c.generateSingletonBuildActions(config, c.preSingletonInfo, c.liveGlobals)
if len(errs) > 0 {
return nil, errs
}
errs = c.updateDependencies()
if len(errs) > 0 {
return nil, errs
}
mutatorDeps, errs := c.runMutators(config)
if len(errs) > 0 {
return nil, errs
}
deps = append(deps, mutatorDeps...)
c.cloneModules()
c.dependenciesReady = true
return deps, nil
}
@ -1863,69 +1880,93 @@ func (c *Context) updateDependencies() (errs []error) {
// SingletonContext.AddNinjaFileDeps(), and PackageContext.AddNinjaFileDeps()
// methods.
func (c *Context) PrepareBuildActions(config interface{}) (deps []string, errs []error) {
c.buildActionsReady = false
pprof.Do(c.Context, pprof.Labels("blueprint", "PrepareBuildActions"), func(ctx context.Context) {
c.buildActionsReady = false
if !c.dependenciesReady {
extraDeps, errs := c.ResolveDependencies(config)
if len(errs) > 0 {
return nil, errs
if !c.dependenciesReady {
var extraDeps []string
extraDeps, errs = c.resolveDependencies(ctx, config)
if len(errs) > 0 {
return
}
deps = append(deps, extraDeps...)
}
deps = append(deps, extraDeps...)
}
depsModules, errs := c.generateModuleBuildActions(config, c.liveGlobals)
var depsModules []string
depsModules, errs = c.generateModuleBuildActions(config, c.liveGlobals)
if len(errs) > 0 {
return
}
var depsSingletons []string
depsSingletons, errs = c.generateSingletonBuildActions(config, c.singletonInfo, c.liveGlobals)
if len(errs) > 0 {
return
}
deps = append(deps, depsModules...)
deps = append(deps, depsSingletons...)
if c.ninjaBuildDir != nil {
err := c.liveGlobals.addNinjaStringDeps(c.ninjaBuildDir)
if err != nil {
errs = []error{err}
return
}
}
pkgNames, depsPackages := c.makeUniquePackageNames(c.liveGlobals)
deps = append(deps, depsPackages...)
// This will panic if it finds a problem since it's a programming error.
c.checkForVariableReferenceCycles(c.liveGlobals.variables, pkgNames)
c.pkgNames = pkgNames
c.globalVariables = c.liveGlobals.variables
c.globalPools = c.liveGlobals.pools
c.globalRules = c.liveGlobals.rules
c.buildActionsReady = true
})
if len(errs) > 0 {
return nil, errs
}
depsSingletons, errs := c.generateSingletonBuildActions(config, c.singletonInfo, c.liveGlobals)
if len(errs) > 0 {
return nil, errs
}
deps = append(deps, depsModules...)
deps = append(deps, depsSingletons...)
if c.ninjaBuildDir != nil {
c.liveGlobals.addNinjaStringDeps(c.ninjaBuildDir)
}
pkgNames, depsPackages := c.makeUniquePackageNames(c.liveGlobals)
deps = append(deps, depsPackages...)
// This will panic if it finds a problem since it's a programming error.
c.checkForVariableReferenceCycles(c.liveGlobals.variables, pkgNames)
c.pkgNames = pkgNames
c.globalVariables = c.liveGlobals.variables
c.globalPools = c.liveGlobals.pools
c.globalRules = c.liveGlobals.rules
c.buildActionsReady = true
return deps, nil
}
func (c *Context) runMutators(config interface{}) (deps []string, errs []error) {
func (c *Context) runMutators(ctx context.Context, config interface{}) (deps []string, errs []error) {
var mutators []*mutatorInfo
mutators = append(mutators, c.earlyMutatorInfo...)
mutators = append(mutators, c.mutatorInfo...)
pprof.Do(ctx, pprof.Labels("blueprint", "runMutators"), func(ctx context.Context) {
mutators = append(mutators, c.earlyMutatorInfo...)
mutators = append(mutators, c.mutatorInfo...)
for _, mutator := range mutators {
var newDeps []string
if mutator.topDownMutator != nil {
newDeps, errs = c.runMutator(config, mutator, topDownMutator)
} else if mutator.bottomUpMutator != nil {
newDeps, errs = c.runMutator(config, mutator, bottomUpMutator)
} else {
panic("no mutator set on " + mutator.name)
for _, mutator := range mutators {
pprof.Do(ctx, pprof.Labels("mutator", mutator.name), func(context.Context) {
var newDeps []string
if mutator.topDownMutator != nil {
newDeps, errs = c.runMutator(config, mutator, topDownMutator)
} else if mutator.bottomUpMutator != nil {
newDeps, errs = c.runMutator(config, mutator, bottomUpMutator)
} else {
panic("no mutator set on " + mutator.name)
}
if len(errs) > 0 {
return
}
deps = append(deps, newDeps...)
})
if len(errs) > 0 {
return
}
}
if len(errs) > 0 {
return nil, errs
}
deps = append(deps, newDeps...)
})
if len(errs) > 0 {
return nil, errs
}
return deps, nil
@ -2932,55 +2973,63 @@ func (c *Context) VisitAllModuleVariants(module Module,
// actions to w. If this is called before PrepareBuildActions successfully
// completes then ErrBuildActionsNotReady is returned.
func (c *Context) WriteBuildFile(w io.Writer) error {
if !c.buildActionsReady {
return ErrBuildActionsNotReady
}
var err error
pprof.Do(c.Context, pprof.Labels("blueprint", "WriteBuildFile"), func(ctx context.Context) {
if !c.buildActionsReady {
err = ErrBuildActionsNotReady
return
}
nw := newNinjaWriter(w)
nw := newNinjaWriter(w)
err := c.writeBuildFileHeader(nw)
if err != nil {
return err
}
err = c.writeBuildFileHeader(nw)
if err != nil {
return
}
err = c.writeNinjaRequiredVersion(nw)
if err != nil {
return err
}
err = c.writeNinjaRequiredVersion(nw)
if err != nil {
return
}
err = c.writeSubninjas(nw)
if err != nil {
return err
}
err = c.writeSubninjas(nw)
if err != nil {
return
}
// TODO: Group the globals by package.
// TODO: Group the globals by package.
err = c.writeGlobalVariables(nw)
if err != nil {
return err
}
err = c.writeGlobalVariables(nw)
if err != nil {
return
}
err = c.writeGlobalPools(nw)
if err != nil {
return err
}
err = c.writeGlobalPools(nw)
if err != nil {
return
}
err = c.writeBuildDir(nw)
if err != nil {
return err
}
err = c.writeBuildDir(nw)
if err != nil {
return
}
err = c.writeGlobalRules(nw)
if err != nil {
return err
}
err = c.writeGlobalRules(nw)
if err != nil {
return
}
err = c.writeAllModuleActions(nw)
if err != nil {
return err
}
err = c.writeAllModuleActions(nw)
if err != nil {
return
}
err = c.writeAllSingletonActions(nw)
if err != nil {
return
}
})
err = c.writeAllSingletonActions(nw)
if err != nil {
return err
}
@ -3064,7 +3113,10 @@ func (c *Context) writeNinjaRequiredVersion(nw *ninjaWriter) error {
func (c *Context) writeSubninjas(nw *ninjaWriter) error {
for _, subninja := range c.subninjas {
nw.Subninja(subninja)
err := nw.Subninja(subninja)
if err != nil {
return err
}
}
return nw.BlankLine()
}

5
ninja_defs.go
View file

@ -410,7 +410,10 @@ func (b *buildDef) WriteTo(nw *ninjaWriter, pkgNames map[*packageContext]string)
}
if !b.Optional {
nw.Default(outputs...)
err = nw.Default(outputs...)
if err != nil {
return err
}
}
return nw.BlankLine()

10
ninja_writer.go
View file

@ -117,7 +117,10 @@ func (n *ninjaWriter) Build(comment string, rule string, outputs, implicitOuts,
}
if comment != "" {
wrapper.Comment(comment)
err := wrapper.Comment(comment)
if err != nil {
return err
}
}
wrapper.WriteString("build")
@ -237,7 +240,10 @@ func (n *ninjaWriterWithWrap) writeString(s string, space bool) {
n.writtenLen = indentWidth * 2
s = strings.TrimLeftFunc(s, unicode.IsSpace)
} else if space {
io.WriteString(n.writer, " ")
_, n.err = io.WriteString(n.writer, " ")
if n.err != nil {
return
}
n.writtenLen++
}

41
proptools/clone.go
View file

@ -18,7 +18,6 @@ import (
"fmt"
"reflect"
"sync"
"sync/atomic"
)
func CloneProperties(structValue reflect.Value) reflect.Value {
@ -248,28 +247,17 @@ func cloneEmptyProperties(dstValue, srcValue reflect.Value) {
}
}
// reflect.Type.Field allocates a []int{} to hold the index every time it is called, which ends up
// being a significant portion of the GC pressure. It can't reuse the same one in case a caller
// modifies the backing array through the slice. Since we don't modify it, cache the result
// locally to reduce allocations.
type typeFieldMap map[reflect.Type][]reflect.StructField
var (
// Stores an atomic pointer to map caching Type to its StructField
typeFieldCache atomic.Value
// Lock used by slow path updating the cache pointer
typeFieldCacheWriterLock sync.Mutex
)
func init() {
typeFieldCache.Store(make(typeFieldMap))
}
var typeFieldCache sync.Map
func typeFields(typ reflect.Type) []reflect.StructField {
// reflect.Type.Field allocates a []int{} to hold the index every time it is called, which ends up
// being a significant portion of the GC pressure. It can't reuse the same one in case a caller
// modifies the backing array through the slice. Since we don't modify it, cache the result
// locally to reduce allocations.
// Fast path
cache := typeFieldCache.Load().(typeFieldMap)
if typeFields, ok := cache[typ]; ok {
return typeFields
if typeFields, ok := typeFieldCache.Load(typ); ok {
return typeFields.([]reflect.StructField)
}
// Slow path
@ -279,18 +267,7 @@ func typeFields(typ reflect.Type) []reflect.StructField {
typeFields[i] = typ.Field(i)
}
typeFieldCacheWriterLock.Lock()
defer typeFieldCacheWriterLock.Unlock()
old := typeFieldCache.Load().(typeFieldMap)
cache = make(typeFieldMap)
for k, v := range old {
cache[k] = v
}
cache[typ] = typeFields
typeFieldCache.Store(cache)
typeFieldCache.Store(typ, typeFields)
return typeFields
}