95bec3331c
Storing every string without ninja variable references through simpleNinjaString costs 24 bytes and a heap allocation. 16 bytes is used for the ninjaString.str string, 8 bytes for the ninjaString.variables *[]variableReference. An additional 8 bytes is used for the resulting pointer into the heap. The vast majority of calls to simpleNinjaString originate in blueprint.parseBuildParams, which converts all of the parameters passed to ctx.Build into ninjaStrings. All together this was allocating 1.575 GB of *ninjaString objects. Add a parseNinjaOrSimpleStrings function that converts input strings into ninjaStrings if they have ninja variable references, but also returns a slice of plain strings for input strings without any ninja variable references. That still results in 1.39 GB of allocations just for the output string slice, so also add an optimization that reuses the input string slice as the output slice if all of the strings had no variable references. Plumb the resulting strings through everywhere that the []*ninjaStrings were used. This reduces the total memory allocations inside blueprint.parseBuildParams in my AOSP aosp_cf_x86_64_phone-userdebug build from 3.337 GB to 1.786 GB. Test: ninja_strings_test.go Change-Id: I51bc138a2a6b1cc7383c7df0a483ccb067ffa02b
210 lines
5.5 KiB
Go
210 lines
5.5 KiB
Go
// Copyright 2014 Google Inc. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package bootstrap
|
|
|
|
import (
|
|
"bufio"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"runtime/debug"
|
|
"runtime/pprof"
|
|
"runtime/trace"
|
|
|
|
"github.com/google/blueprint"
|
|
)
|
|
|
|
type Args struct {
|
|
ModuleListFile string
|
|
OutFile string
|
|
|
|
EmptyNinjaFile bool
|
|
|
|
NoGC bool
|
|
Cpuprofile string
|
|
Memprofile string
|
|
TraceFile string
|
|
}
|
|
|
|
// RegisterGoModuleTypes adds module types to build tools written in golang
|
|
func RegisterGoModuleTypes(ctx *blueprint.Context) {
|
|
ctx.RegisterModuleType("bootstrap_go_package", newGoPackageModuleFactory())
|
|
ctx.RegisterModuleType("blueprint_go_binary", newGoBinaryModuleFactory())
|
|
}
|
|
|
|
// RunBlueprint emits `args.OutFile` (a Ninja file) and returns the list of
|
|
// its dependencies. These can be written to a `${args.OutFile}.d` file
|
|
// so that it is correctly rebuilt when needed in case Blueprint is itself
|
|
// invoked from Ninja
|
|
func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, config interface{}) ([]string, error) {
|
|
runtime.GOMAXPROCS(runtime.NumCPU())
|
|
|
|
if args.NoGC {
|
|
debug.SetGCPercent(-1)
|
|
}
|
|
|
|
if args.Cpuprofile != "" {
|
|
f, err := os.Create(joinPath(ctx.SrcDir(), args.Cpuprofile))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error opening cpuprofile: %s", err)
|
|
}
|
|
pprof.StartCPUProfile(f)
|
|
defer f.Close()
|
|
defer pprof.StopCPUProfile()
|
|
}
|
|
|
|
if args.TraceFile != "" {
|
|
f, err := os.Create(joinPath(ctx.SrcDir(), args.TraceFile))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error opening trace: %s", err)
|
|
}
|
|
trace.Start(f)
|
|
defer f.Close()
|
|
defer trace.Stop()
|
|
}
|
|
|
|
if args.ModuleListFile == "" {
|
|
return nil, fmt.Errorf("-l <moduleListFile> is required and must be nonempty")
|
|
}
|
|
ctx.SetModuleListFile(args.ModuleListFile)
|
|
|
|
var ninjaDeps []string
|
|
ninjaDeps = append(ninjaDeps, args.ModuleListFile)
|
|
|
|
ctx.BeginEvent("list_modules")
|
|
var filesToParse []string
|
|
if f, err := ctx.ListModulePaths("."); err != nil {
|
|
return nil, fmt.Errorf("could not enumerate files: %v\n", err.Error())
|
|
} else {
|
|
filesToParse = f
|
|
}
|
|
ctx.EndEvent("list_modules")
|
|
|
|
ctx.RegisterBottomUpMutator("bootstrap_plugin_deps", pluginDeps)
|
|
ctx.RegisterSingletonType("bootstrap", newSingletonFactory(), false)
|
|
RegisterGoModuleTypes(ctx)
|
|
blueprint.RegisterPackageIncludesModuleType(ctx)
|
|
|
|
ctx.BeginEvent("parse_bp")
|
|
if blueprintFiles, errs := ctx.ParseFileList(".", filesToParse, config); len(errs) > 0 {
|
|
return nil, fatalErrors(errs)
|
|
} else {
|
|
ctx.EndEvent("parse_bp")
|
|
ninjaDeps = append(ninjaDeps, blueprintFiles...)
|
|
}
|
|
|
|
if resolvedDeps, errs := ctx.ResolveDependencies(config); len(errs) > 0 {
|
|
return nil, fatalErrors(errs)
|
|
} else {
|
|
ninjaDeps = append(ninjaDeps, resolvedDeps...)
|
|
}
|
|
|
|
if stopBefore == StopBeforePrepareBuildActions {
|
|
return ninjaDeps, nil
|
|
}
|
|
|
|
if ctx.BeforePrepareBuildActionsHook != nil {
|
|
if err := ctx.BeforePrepareBuildActionsHook(); err != nil {
|
|
return nil, fatalErrors([]error{err})
|
|
}
|
|
}
|
|
|
|
if buildActionsDeps, errs := ctx.PrepareBuildActions(config); len(errs) > 0 {
|
|
return nil, fatalErrors(errs)
|
|
} else {
|
|
ninjaDeps = append(ninjaDeps, buildActionsDeps...)
|
|
}
|
|
|
|
if stopBefore == StopBeforeWriteNinja {
|
|
return ninjaDeps, nil
|
|
}
|
|
|
|
const outFilePermissions = 0666
|
|
var out blueprint.StringWriterWriter
|
|
var f *os.File
|
|
var buf *bufio.Writer
|
|
|
|
ctx.BeginEvent("write_files")
|
|
defer ctx.EndEvent("write_files")
|
|
if args.EmptyNinjaFile {
|
|
if err := os.WriteFile(joinPath(ctx.SrcDir(), args.OutFile), []byte(nil), outFilePermissions); err != nil {
|
|
return nil, fmt.Errorf("error writing empty Ninja file: %s", err)
|
|
}
|
|
out = io.Discard.(blueprint.StringWriterWriter)
|
|
} else {
|
|
f, err := os.OpenFile(joinPath(ctx.SrcDir(), args.OutFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, outFilePermissions)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error opening Ninja file: %s", err)
|
|
}
|
|
defer f.Close()
|
|
buf = bufio.NewWriterSize(f, 16*1024*1024)
|
|
out = buf
|
|
}
|
|
|
|
if err := ctx.WriteBuildFile(out); err != nil {
|
|
return nil, fmt.Errorf("error writing Ninja file contents: %s", err)
|
|
}
|
|
|
|
if buf != nil {
|
|
if err := buf.Flush(); err != nil {
|
|
return nil, fmt.Errorf("error flushing Ninja file contents: %s", err)
|
|
}
|
|
}
|
|
|
|
if f != nil {
|
|
if err := f.Close(); err != nil {
|
|
return nil, fmt.Errorf("error closing Ninja file: %s", err)
|
|
}
|
|
}
|
|
|
|
if args.Memprofile != "" {
|
|
f, err := os.Create(joinPath(ctx.SrcDir(), args.Memprofile))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error opening memprofile: %s", err)
|
|
}
|
|
defer f.Close()
|
|
pprof.WriteHeapProfile(f)
|
|
}
|
|
|
|
return ninjaDeps, nil
|
|
}
|
|
|
|
func fatalErrors(errs []error) error {
|
|
red := "\x1b[31m"
|
|
unred := "\x1b[0m"
|
|
|
|
for _, err := range errs {
|
|
switch err := err.(type) {
|
|
case *blueprint.BlueprintError,
|
|
*blueprint.ModuleError,
|
|
*blueprint.PropertyError:
|
|
fmt.Printf("%serror:%s %s\n", red, unred, err.Error())
|
|
default:
|
|
fmt.Printf("%sinternal error:%s %s\n", red, unred, err)
|
|
}
|
|
}
|
|
|
|
return errors.New("fatal errors encountered")
|
|
}
|
|
|
|
func joinPath(base, path string) string {
|
|
if filepath.IsAbs(path) {
|
|
return path
|
|
}
|
|
return filepath.Join(base, path)
|
|
}
|