platform_build_blueprint/ninja_writer.go

451 lines
11 KiB
Go
Raw Normal View History

// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blueprint
import (
"io"
"strings"
"unicode"
)
const (
indentWidth = 4
maxIndentDepth = 2
lineWidth = 80
)
var indentString = strings.Repeat(" ", indentWidth*maxIndentDepth)
type StringWriterWriter interface {
io.StringWriter
io.Writer
}
type ninjaWriter struct {
writer StringWriterWriter
justDidBlankLine bool // true if the last operation was a BlankLine
}
func newNinjaWriter(writer StringWriterWriter) *ninjaWriter {
return &ninjaWriter{
writer: writer,
}
}
func (n *ninjaWriter) Comment(comment string) error {
n.justDidBlankLine = false
const lineHeaderLen = len("# ")
const maxLineLen = lineWidth - lineHeaderLen
var lineStart, lastSplitPoint int
for i, r := range comment {
if unicode.IsSpace(r) {
// We know we can safely split the line here.
lastSplitPoint = i + 1
}
var line string
var writeLine bool
switch {
case r == '\n':
// Output the line without trimming the left so as to allow comments
// to contain their own indentation.
line = strings.TrimRightFunc(comment[lineStart:i], unicode.IsSpace)
writeLine = true
case (i-lineStart > maxLineLen) && (lastSplitPoint > lineStart):
// The line has grown too long and is splittable. Split it at the
// last split point.
line = strings.TrimSpace(comment[lineStart:lastSplitPoint])
writeLine = true
}
if writeLine {
line = strings.TrimSpace("# "+line) + "\n"
_, err := n.writer.WriteString(line)
if err != nil {
return err
}
lineStart = lastSplitPoint
}
}
if lineStart != len(comment) {
line := strings.TrimSpace(comment[lineStart:])
_, err := n.writer.WriteString("# ")
if err != nil {
return err
}
_, err = n.writer.WriteString(line)
if err != nil {
return err
}
_, err = n.writer.WriteString("\n")
if err != nil {
return err
}
}
return nil
}
func (n *ninjaWriter) Pool(name string) error {
n.justDidBlankLine = false
return n.writeStatement("pool", name)
}
func (n *ninjaWriter) Rule(name string) error {
n.justDidBlankLine = false
return n.writeStatement("rule", name)
}
func (n *ninjaWriter) Build(comment string, rule string, outputs, implicitOuts,
Optimize memory usage of ninjaString ninjaString is an interface, which uses 16 bytes of memory on top of the size of the concrete type. A literalNinjaString is a string, which is another 16 bytes for the string header for a total of 32 bytes. A varNinjaString is two slices, which are 24 bytes each for the slice headers, for a total of 64 bytes. The slices contain the first constant string, and then altenrating variable and string parts of the ninjaString, resulting in 16 bytes plus 32 bytes per variable. This patch replaces the ninjaString interface with a *ninjaString concrete struct type. The ninjaString struct is a string and a pointer to a slice of variable references, for a total of 24 bytes. ninjaStrings with no variable references (the equivalent of the old literalNinjaString) have a nil slice, and now use 24 bytes instead of 32 bytes. ninjaStrings with variable references allocate a slice of variable references that contain 32-bit start and end offsets and a Variable interface, but reuse the original string and so avoid the extra string headers, resulting in 24 bytes for the slice header, and 24 bytes per variable. These savings reduce the peak memory usage averaged across 10 runs of /bin/time -v build/soong/soong_ui.bash --make-mode nothing on the internal master branch cf_x86_64_phone-userdebug build from 50114842kB to 45577638kB, a savings of 4537204kB or 9%. The new Benchmark_parseNinjaString shows savings in both time and memory. Before: Benchmark_parseNinjaString/constant/1-128 594251787 2.006 ns/op 0 B/op 0 allocs/op Benchmark_parseNinjaString/constant/10-128 21191347 65.57 ns/op 16 B/op 1 allocs/op Benchmark_parseNinjaString/constant/100-128 9983748 130.2 ns/op 112 B/op 1 allocs/op Benchmark_parseNinjaString/constant/1000-128 2632527 445.1 ns/op 1024 B/op 1 allocs/op Benchmark_parseNinjaString/variable/1-128 2964896 419.4 ns/op 176 B/op 4 allocs/op Benchmark_parseNinjaString/variable/10-128 1807341 670.6 ns/op 192 B/op 7 allocs/op Benchmark_parseNinjaString/variable/100-128 1000000 1092 ns/op 352 B/op 7 allocs/op Benchmark_parseNinjaString/variable/1000-128 300649 3773 ns/op 1584 B/op 7 allocs/op Benchmark_parseNinjaString/variables/1-128 2858432 441.6 ns/op 176 B/op 4 allocs/op Benchmark_parseNinjaString/variables/2-128 2360505 513.4 ns/op 208 B/op 4 allocs/op Benchmark_parseNinjaString/variables/3-128 1867136 635.6 ns/op 240 B/op 4 allocs/op Benchmark_parseNinjaString/variables/4-128 1584045 752.1 ns/op 272 B/op 4 allocs/op Benchmark_parseNinjaString/variables/5-128 1338189 885.8 ns/op 304 B/op 4 allocs/op Benchmark_parseNinjaString/variables/10-128 1000000 1468 ns/op 464 B/op 4 allocs/op Benchmark_parseNinjaString/variables/100-128 88768 12895 ns/op 3712 B/op 4 allocs/op Benchmark_parseNinjaString/variables/1000-128 8972 133627 ns/op 32896 B/op 4 allocs/op After: Benchmark_parseNinjaString/constant/1-128 584600864 2.004 ns/op 0 B/op 0 allocs/op Benchmark_parseNinjaString/constant/10-128 19274581 64.84 ns/op 16 B/op 1 allocs/op Benchmark_parseNinjaString/constant/100-128 9017640 127.6 ns/op 112 B/op 1 allocs/op Benchmark_parseNinjaString/constant/1000-128 2630797 453.0 ns/op 1024 B/op 1 allocs/op Benchmark_parseNinjaString/variable/1-128 3460422 347.0 ns/op 136 B/op 4 allocs/op Benchmark_parseNinjaString/variable/10-128 2103404 519.9 ns/op 152 B/op 7 allocs/op Benchmark_parseNinjaString/variable/100-128 1315778 906.5 ns/op 312 B/op 7 allocs/op Benchmark_parseNinjaString/variable/1000-128 354812 3284 ns/op 1544 B/op 7 allocs/op Benchmark_parseNinjaString/variables/1-128 3386868 361.5 ns/op 136 B/op 4 allocs/op Benchmark_parseNinjaString/variables/2-128 2675594 456.9 ns/op 160 B/op 4 allocs/op Benchmark_parseNinjaString/variables/3-128 2344670 520.0 ns/op 192 B/op 4 allocs/op Benchmark_parseNinjaString/variables/4-128 1919482 648.1 ns/op 208 B/op 4 allocs/op Benchmark_parseNinjaString/variables/5-128 1560556 723.9 ns/op 240 B/op 4 allocs/op Benchmark_parseNinjaString/variables/10-128 1000000 1169 ns/op 352 B/op 4 allocs/op Benchmark_parseNinjaString/variables/100-128 116738 10168 ns/op 2800 B/op 4 allocs/op Benchmark_parseNinjaString/variables/1000-128 10000 105646 ns/op 24688 B/op 4 allocs/op Bug: 286423944 Test: ninja_strings_test.go Test: out/soong/build*.ninja is the same before and after this change Change-Id: I1ecffbaccb0d0469a41fa31255c1b17311e01687
2023-06-15 01:14:10 +02:00
explicitDeps, implicitDeps, orderOnlyDeps, validations []*ninjaString,
outputStrings, implicitOutStrings, explicitDepStrings,
implicitDepStrings, orderOnlyDepStrings, validationStrings []string,
nameTracker *nameTracker) error {
n.justDidBlankLine = false
const lineWrapLen = len(" $")
const maxLineLen = lineWidth - lineWrapLen
wrapper := &ninjaWriterWithWrap{
ninjaWriter: n,
maxLineLen: maxLineLen,
}
if comment != "" {
err := wrapper.Comment(comment)
if err != nil {
return err
}
}
wrapper.WriteString("build")
for _, output := range outputStrings {
wrapper.Space()
outputEscaper.WriteString(wrapper, output)
}
for _, output := range outputs {
wrapper.Space()
output.ValueWithEscaper(wrapper, nameTracker, outputEscaper)
}
if len(implicitOuts) > 0 || len(implicitOutStrings) > 0 {
wrapper.WriteStringWithSpace("|")
for _, out := range implicitOutStrings {
wrapper.Space()
outputEscaper.WriteString(wrapper, out)
}
for _, out := range implicitOuts {
wrapper.Space()
out.ValueWithEscaper(wrapper, nameTracker, outputEscaper)
}
}
wrapper.WriteString(":")
wrapper.WriteStringWithSpace(rule)
for _, dep := range explicitDepStrings {
wrapper.Space()
inputEscaper.WriteString(wrapper, dep)
}
for _, dep := range explicitDeps {
wrapper.Space()
dep.ValueWithEscaper(wrapper, nameTracker, inputEscaper)
}
if len(implicitDeps) > 0 || len(implicitDepStrings) > 0 {
wrapper.WriteStringWithSpace("|")
for _, dep := range implicitDepStrings {
wrapper.Space()
inputEscaper.WriteString(wrapper, dep)
}
for _, dep := range implicitDeps {
wrapper.Space()
dep.ValueWithEscaper(wrapper, nameTracker, inputEscaper)
}
}
if len(orderOnlyDeps) > 0 || len(orderOnlyDepStrings) > 0 {
wrapper.WriteStringWithSpace("||")
for _, dep := range orderOnlyDepStrings {
wrapper.Space()
inputEscaper.WriteString(wrapper, dep)
}
for _, dep := range orderOnlyDeps {
wrapper.Space()
dep.ValueWithEscaper(wrapper, nameTracker, inputEscaper)
}
}
if len(validations) > 0 || len(validationStrings) > 0 {
wrapper.WriteStringWithSpace("|@")
for _, dep := range validationStrings {
wrapper.Space()
inputEscaper.WriteString(wrapper, dep)
}
for _, dep := range validations {
wrapper.Space()
dep.ValueWithEscaper(wrapper, nameTracker, inputEscaper)
}
}
return wrapper.Flush()
}
func (n *ninjaWriter) Assign(name, value string) error {
n.justDidBlankLine = false
_, err := n.writer.WriteString(name)
if err != nil {
return err
}
_, err = n.writer.WriteString(" = ")
if err != nil {
return err
}
_, err = n.writer.WriteString(value)
if err != nil {
return err
}
_, err = n.writer.WriteString("\n")
if err != nil {
return err
}
return nil
}
func (n *ninjaWriter) ScopedAssign(name, value string) error {
n.justDidBlankLine = false
_, err := n.writer.WriteString(indentString[:indentWidth])
if err != nil {
return err
}
_, err = n.writer.WriteString(name)
if err != nil {
return err
}
_, err = n.writer.WriteString(" = ")
if err != nil {
return err
}
_, err = n.writer.WriteString(value)
if err != nil {
return err
}
_, err = n.writer.WriteString("\n")
if err != nil {
return err
}
return nil
}
func (n *ninjaWriter) Default(nameTracker *nameTracker, targets []*ninjaString, targetStrings []string) error {
n.justDidBlankLine = false
const lineWrapLen = len(" $")
const maxLineLen = lineWidth - lineWrapLen
wrapper := &ninjaWriterWithWrap{
ninjaWriter: n,
maxLineLen: maxLineLen,
}
wrapper.WriteString("default")
for _, target := range targetStrings {
wrapper.Space()
outputEscaper.WriteString(wrapper, target)
}
for _, target := range targets {
wrapper.Space()
target.ValueWithEscaper(wrapper, nameTracker, outputEscaper)
}
return wrapper.Flush()
}
func (n *ninjaWriter) Subninja(file string) error {
n.justDidBlankLine = false
return n.writeStatement("subninja", file)
}
func (n *ninjaWriter) BlankLine() (err error) {
// We don't output multiple blank lines in a row.
if !n.justDidBlankLine {
n.justDidBlankLine = true
_, err = n.writer.WriteString("\n")
}
return err
}
func (n *ninjaWriter) writeStatement(directive, name string) error {
_, err := n.writer.WriteString(directive + " ")
if err != nil {
return err
}
_, err = n.writer.WriteString(name)
if err != nil {
return err
}
_, err = n.writer.WriteString("\n")
if err != nil {
return err
}
return nil
}
// ninjaWriterWithWrap is an io.StringWriter that writes through to a ninjaWriter, but supports
// user-readable line wrapping on boundaries when ninjaWriterWithWrap.Space is called.
// It collects incoming calls to WriteString until either the line length is exceeded, in which case
// it inserts a wrap before the pending strings and then writes them, or the next call to Space, in
// which case it writes out the pending strings.
//
// WriteString never returns an error, all errors are held until Flush is called. Once an error has
// occurred all writes become noops.
type ninjaWriterWithWrap struct {
*ninjaWriter
// pending lists the strings that have been written since the last call to Space.
pending []string
// pendingLen accumulates the lengths of the strings in pending.
pendingLen int
// lineLen accumulates the number of bytes on the current line.
lineLen int
// maxLineLen is the length of the line before wrapping.
maxLineLen int
// space is true if the strings in pending should be preceded by a space.
space bool
// err holds any error that has occurred to return in Flush.
err error
}
// WriteString writes the string to buffer, wrapping on a previous Space call if necessary.
// It never returns an error, all errors are held until Flush is called.
func (n *ninjaWriterWithWrap) WriteString(s string) (written int, noError error) {
// Always return the full length of the string and a nil error.
// ninjaWriterWithWrap doesn't return errors to the caller, it saves them until Flush()
written = len(s)
if n.err != nil {
return
}
const spaceLen = 1
if !n.space {
// No space is pending, so a line wrap can't be inserted before this, so just write
// the string.
n.lineLen += len(s)
_, n.err = n.writer.WriteString(s)
} else if n.lineLen+len(s)+spaceLen > n.maxLineLen {
// A space is pending, and the pending strings plus the current string would exceed the
// maximum line length. Wrap and indent before the pending space and strings, then write
// the pending and current strings.
_, n.err = n.writer.WriteString(" $\n")
if n.err != nil {
return
}
_, n.err = n.writer.WriteString(indentString[:indentWidth*2])
if n.err != nil {
return
}
n.lineLen = indentWidth*2 + n.pendingLen
s = strings.TrimLeftFunc(s, unicode.IsSpace)
n.pending = append(n.pending, s)
n.lineLen += len(s)
n.writePending()
n.space = false
} else {
// A space is pending but the current string would not reach the maximum line length,
// add it to the pending list.
n.pending = append(n.pending, s)
n.pendingLen += len(s)
n.lineLen += len(s)
}
return
}
func (n *ninjaWriterWithWrap) Write(p []byte) (written int, noError error) {
// Write is rarely used, implement it via WriteString.
return n.WriteString(string(p))
}
// Space inserts a space that is also a possible wrapping point into the string.
func (n *ninjaWriterWithWrap) Space() {
if n.err != nil {
return
}
if n.space {
// A space was already pending, and the space plus any strings written after the space did
// not reach the maxmimum line length, so write out the old space and pending strings.
_, n.err = n.writer.WriteString(" ")
n.lineLen++
n.writePending()
}
n.space = true
}
// writePending writes out all the strings stored in pending and resets it.
func (n *ninjaWriterWithWrap) writePending() {
if n.err != nil {
return
}
for _, pending := range n.pending {
_, n.err = n.writer.WriteString(pending)
if n.err != nil {
return
}
}
// Reset the length of pending back to 0 without reducing its capacity to avoid reallocating
// the backing array.
n.pending = n.pending[:0]
n.pendingLen = 0
}
// WriteStringWithSpace is a helper that calls Space and WriteString.
func (n *ninjaWriterWithWrap) WriteStringWithSpace(s string) {
n.Space()
_, _ = n.WriteString(s)
}
// Flush writes out any pending space or strings and then a newline. It also returns any errors
// that have previously occurred.
func (n *ninjaWriterWithWrap) Flush() error {
if n.space {
_, n.err = n.writer.WriteString(" ")
}
n.writePending()
if n.err != nil {
return n.err
}
_, err := n.writer.WriteString("\n")
return err
}