platform_build_blueprint/ninja_strings_test.go
Colin Cross 6126fe8067 Optimize memory usage of ninjaString
ninjaString is an interface, which uses 16 bytes of memory on top
of the size of the concrete type.  A literalNinjaString is a string,
which is another 16 bytes for the string header for a total of 32
bytes.  A varNinjaString is two slices, which are 24 bytes each
for the slice headers, for a total of 64 bytes.  The slices contain
the first constant string, and then altenrating variable and string
parts of the ninjaString, resulting in 16 bytes plus 32 bytes per
variable.

This patch replaces the ninjaString interface with a *ninjaString
concrete struct type.  The ninjaString struct is a string and a
pointer to a slice of variable references, for a total of 24 bytes.

ninjaStrings with no variable references (the equivalent of the old
literalNinjaString) have a nil slice, and now use 24 bytes instead
of 32 bytes.

ninjaStrings with variable references allocate a slice of variable
references that contain 32-bit start and end offsets and a Variable
interface, but reuse the original string and so avoid the extra
string headers, resulting in 24 bytes for the slice header, and
24 bytes per variable.

These savings reduce the peak memory usage averaged across 10 runs of
/bin/time -v build/soong/soong_ui.bash --make-mode nothing
on the internal master branch cf_x86_64_phone-userdebug build
from 50114842kB to 45577638kB, a savings of 4537204kB or 9%.

The new Benchmark_parseNinjaString shows savings in both time and
memory.  Before:
Benchmark_parseNinjaString/constant/1-128       	594251787	         2.006 ns/op	       0 B/op	       0 allocs/op
Benchmark_parseNinjaString/constant/10-128      	21191347	        65.57 ns/op	      16 B/op	       1 allocs/op
Benchmark_parseNinjaString/constant/100-128     	 9983748	       130.2 ns/op	     112 B/op	       1 allocs/op
Benchmark_parseNinjaString/constant/1000-128    	 2632527	       445.1 ns/op	    1024 B/op	       1 allocs/op
Benchmark_parseNinjaString/variable/1-128       	 2964896	       419.4 ns/op	     176 B/op	       4 allocs/op
Benchmark_parseNinjaString/variable/10-128      	 1807341	       670.6 ns/op	     192 B/op	       7 allocs/op
Benchmark_parseNinjaString/variable/100-128     	 1000000	      1092 ns/op	     352 B/op	       7 allocs/op
Benchmark_parseNinjaString/variable/1000-128    	  300649	      3773 ns/op	    1584 B/op	       7 allocs/op
Benchmark_parseNinjaString/variables/1-128      	 2858432	       441.6 ns/op	     176 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/2-128      	 2360505	       513.4 ns/op	     208 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/3-128      	 1867136	       635.6 ns/op	     240 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/4-128      	 1584045	       752.1 ns/op	     272 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/5-128      	 1338189	       885.8 ns/op	     304 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/10-128     	 1000000	      1468 ns/op	     464 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/100-128    	   88768	     12895 ns/op	    3712 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/1000-128   	    8972	    133627 ns/op	   32896 B/op	       4 allocs/op

After:
Benchmark_parseNinjaString/constant/1-128       	584600864	         2.004 ns/op	       0 B/op	       0 allocs/op
Benchmark_parseNinjaString/constant/10-128      	19274581	        64.84 ns/op	      16 B/op	       1 allocs/op
Benchmark_parseNinjaString/constant/100-128     	 9017640	       127.6 ns/op	     112 B/op	       1 allocs/op
Benchmark_parseNinjaString/constant/1000-128    	 2630797	       453.0 ns/op	    1024 B/op	       1 allocs/op
Benchmark_parseNinjaString/variable/1-128       	 3460422	       347.0 ns/op	     136 B/op	       4 allocs/op
Benchmark_parseNinjaString/variable/10-128      	 2103404	       519.9 ns/op	     152 B/op	       7 allocs/op
Benchmark_parseNinjaString/variable/100-128     	 1315778	       906.5 ns/op	     312 B/op	       7 allocs/op
Benchmark_parseNinjaString/variable/1000-128    	  354812	      3284 ns/op	    1544 B/op	       7 allocs/op
Benchmark_parseNinjaString/variables/1-128      	 3386868	       361.5 ns/op	     136 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/2-128      	 2675594	       456.9 ns/op	     160 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/3-128      	 2344670	       520.0 ns/op	     192 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/4-128      	 1919482	       648.1 ns/op	     208 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/5-128      	 1560556	       723.9 ns/op	     240 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/10-128     	 1000000	      1169 ns/op	     352 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/100-128    	  116738	     10168 ns/op	    2800 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/1000-128   	   10000	    105646 ns/op	   24688 B/op	       4 allocs/op

Bug: 286423944
Test: ninja_strings_test.go
Test: out/soong/build*.ninja is the same before and after this change
Change-Id: I1ecffbaccb0d0469a41fa31255c1b17311e01687
2023-06-15 21:53:56 -07:00

303 lines
7.5 KiB
Go

// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blueprint
import (
"reflect"
"strconv"
"strings"
"testing"
)
type testVariableRef struct {
start, end int
name string
}
func TestParseNinjaString(t *testing.T) {
testCases := []struct {
input string
vars []string
value string
eval string
err string
}{
{
input: "abc def $ghi jkl",
vars: []string{"ghi"},
value: "abc def ${namespace.ghi} jkl",
eval: "abc def GHI jkl",
},
{
input: "abc def $ghi$jkl",
vars: []string{"ghi", "jkl"},
value: "abc def ${namespace.ghi}${namespace.jkl}",
eval: "abc def GHIJKL",
},
{
input: "foo $012_-345xyz_! bar",
vars: []string{"012_-345xyz_"},
value: "foo ${namespace.012_-345xyz_}! bar",
eval: "foo 012_-345XYZ_! bar",
},
{
input: "foo ${012_-345xyz_} bar",
vars: []string{"012_-345xyz_"},
value: "foo ${namespace.012_-345xyz_} bar",
eval: "foo 012_-345XYZ_ bar",
},
{
input: "foo ${012_-345xyz_} bar",
vars: []string{"012_-345xyz_"},
value: "foo ${namespace.012_-345xyz_} bar",
eval: "foo 012_-345XYZ_ bar",
},
{
input: "foo $$ bar",
vars: nil,
value: "foo $$ bar",
eval: "foo $$ bar",
},
{
input: "$foo${bar}",
vars: []string{"foo", "bar"},
value: "${namespace.foo}${namespace.bar}",
eval: "FOOBAR",
},
{
input: "$foo$$",
vars: []string{"foo"},
value: "${namespace.foo}$$",
eval: "FOO$$",
},
{
input: "foo bar",
vars: nil,
value: "foo bar",
eval: "foo bar",
},
{
input: " foo ",
vars: nil,
value: "$ foo ",
eval: "$ foo ",
},
{
input: "\tfoo ",
vars: nil,
value: "\tfoo ",
eval: "\tfoo ",
},
{
input: "\nfoo ",
vars: nil,
value: "$\nfoo ",
eval: "\nfoo ",
},
{
input: " $foo ",
vars: []string{"foo"},
value: "$ ${namespace.foo} ",
eval: " FOO ",
},
{
input: "\t$foo ",
vars: []string{"foo"},
value: "\t${namespace.foo} ",
eval: "\tFOO ",
},
{
input: "\n$foo ",
vars: []string{"foo"},
value: "$\n${namespace.foo} ",
eval: "\nFOO ",
},
{
input: "foo $ bar",
err: `error parsing ninja string "foo $ bar": invalid character after '$' at byte offset 5`,
},
{
input: "foo $",
err: "unexpected end of string after '$'",
},
{
input: "foo ${} bar",
err: `error parsing ninja string "foo ${} bar": empty variable name at byte offset 6`,
},
{
input: "foo ${abc!} bar",
err: `error parsing ninja string "foo ${abc!} bar": invalid character in variable name at byte offset 9`,
},
{
input: "foo ${abc",
err: "unexpected end of string in variable name",
},
}
for _, testCase := range testCases {
t.Run(testCase.input, func(t *testing.T) {
scope := newLocalScope(nil, "namespace.")
variablesMap := map[Variable]*ninjaString{}
for _, varName := range testCase.vars {
_, err := scope.LookupVariable(varName)
if err != nil {
v, err := scope.AddLocalVariable(varName, strings.ToUpper(varName))
if err != nil {
t.Fatalf("error creating scope: %s", err)
}
variablesMap[v] = simpleNinjaString(strings.ToUpper(varName))
}
}
output, err := parseNinjaString(scope, testCase.input)
if err == nil {
if g, w := output.Value(nil), testCase.value; g != w {
t.Errorf("incorrect Value output, want %q, got %q", w, g)
}
eval, err := output.Eval(variablesMap)
if err != nil {
t.Errorf("unexpected error in Eval: %s", err)
}
if g, w := eval, testCase.eval; g != w {
t.Errorf("incorrect Eval output, want %q, got %q", w, g)
}
}
var errStr string
if err != nil {
errStr = err.Error()
}
if err != nil && err.Error() != testCase.err {
t.Errorf("unexpected error:")
t.Errorf(" input: %q", testCase.input)
t.Errorf(" expected: %q", testCase.err)
t.Errorf(" got: %q", errStr)
}
})
}
}
func TestParseNinjaStringWithImportedVar(t *testing.T) {
ImpVar := &staticVariable{name_: "ImpVar", fullName_: "g.impPkg.ImpVar"}
impScope := newScope(nil)
impScope.AddVariable(ImpVar)
scope := newScope(nil)
scope.AddImport("impPkg", impScope)
input := "abc def ${impPkg.ImpVar} ghi"
output, err := parseNinjaString(scope, input)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
expect := []variableReference{{8, 24, ImpVar}}
if !reflect.DeepEqual(*output.variables, expect) {
t.Errorf("incorrect output:")
t.Errorf(" input: %q", input)
t.Errorf(" expected: %#v", expect)
t.Errorf(" got: %#v", *output.variables)
}
if g, w := output.Value(nil), "abc def ${g.impPkg.ImpVar} ghi"; g != w {
t.Errorf("incorrect Value output, want %q got %q", w, g)
}
}
func Benchmark_parseNinjaString(b *testing.B) {
b.Run("constant", func(b *testing.B) {
for _, l := range []int{1, 10, 100, 1000} {
b.Run(strconv.Itoa(l), func(b *testing.B) {
b.ReportAllocs()
for n := 0; n < b.N; n++ {
_ = simpleNinjaString(strings.Repeat("a", l))
}
})
}
})
b.Run("variable", func(b *testing.B) {
for _, l := range []int{1, 10, 100, 1000} {
scope := newLocalScope(nil, "")
scope.AddLocalVariable("a", strings.Repeat("b", l/3))
b.Run(strconv.Itoa(l), func(b *testing.B) {
b.ReportAllocs()
for n := 0; n < b.N; n++ {
_, _ = parseNinjaString(scope, strings.Repeat("a", l/3)+"${a}"+strings.Repeat("a", l/3))
}
})
}
})
b.Run("variables", func(b *testing.B) {
for _, l := range []int{1, 2, 3, 4, 5, 10, 100, 1000} {
scope := newLocalScope(nil, "")
str := strings.Repeat("a", 10)
for i := 0; i < l; i++ {
scope.AddLocalVariable("a"+strconv.Itoa(i), strings.Repeat("b", 10))
str += "${a" + strconv.Itoa(i) + "}"
}
b.Run(strconv.Itoa(l), func(b *testing.B) {
b.ReportAllocs()
for n := 0; n < b.N; n++ {
_, _ = parseNinjaString(scope, str)
}
})
}
})
}
func BenchmarkNinjaString_Value(b *testing.B) {
b.Run("constant", func(b *testing.B) {
for _, l := range []int{1, 10, 100, 1000} {
ns := simpleNinjaString(strings.Repeat("a", l))
b.Run(strconv.Itoa(l), func(b *testing.B) {
b.ReportAllocs()
for n := 0; n < b.N; n++ {
ns.Value(nil)
}
})
}
})
b.Run("variable", func(b *testing.B) {
for _, l := range []int{1, 10, 100, 1000} {
scope := newLocalScope(nil, "")
scope.AddLocalVariable("a", strings.Repeat("b", l/3))
ns, _ := parseNinjaString(scope, strings.Repeat("a", l/3)+"${a}"+strings.Repeat("a", l/3))
b.Run(strconv.Itoa(l), func(b *testing.B) {
b.ReportAllocs()
for n := 0; n < b.N; n++ {
ns.Value(nil)
}
})
}
})
b.Run("variables", func(b *testing.B) {
for _, l := range []int{1, 2, 3, 4, 5, 10, 100, 1000} {
scope := newLocalScope(nil, "")
str := strings.Repeat("a", 10)
for i := 0; i < l; i++ {
scope.AddLocalVariable("a"+strconv.Itoa(i), strings.Repeat("b", 10))
str += "${a" + strconv.Itoa(i) + "}"
}
ns, _ := parseNinjaString(scope, str)
b.Run(strconv.Itoa(l), func(b *testing.B) {
b.ReportAllocs()
for n := 0; n < b.N; n++ {
ns.Value(nil)
}
})
}
})
}