Optimize memory usage of ninjaString

ninjaString is an interface, which uses 16 bytes of memory on top
of the size of the concrete type.  A literalNinjaString is a string,
which is another 16 bytes for the string header for a total of 32
bytes.  A varNinjaString is two slices, which are 24 bytes each
for the slice headers, for a total of 64 bytes.  The slices contain
the first constant string, and then altenrating variable and string
parts of the ninjaString, resulting in 16 bytes plus 32 bytes per
variable.

This patch replaces the ninjaString interface with a *ninjaString
concrete struct type.  The ninjaString struct is a string and a
pointer to a slice of variable references, for a total of 24 bytes.

ninjaStrings with no variable references (the equivalent of the old
literalNinjaString) have a nil slice, and now use 24 bytes instead
of 32 bytes.

ninjaStrings with variable references allocate a slice of variable
references that contain 32-bit start and end offsets and a Variable
interface, but reuse the original string and so avoid the extra
string headers, resulting in 24 bytes for the slice header, and
24 bytes per variable.

These savings reduce the peak memory usage averaged across 10 runs of
/bin/time -v build/soong/soong_ui.bash --make-mode nothing
on the internal master branch cf_x86_64_phone-userdebug build
from 50114842kB to 45577638kB, a savings of 4537204kB or 9%.

The new Benchmark_parseNinjaString shows savings in both time and
memory.  Before:
Benchmark_parseNinjaString/constant/1-128       	594251787	         2.006 ns/op	       0 B/op	       0 allocs/op
Benchmark_parseNinjaString/constant/10-128      	21191347	        65.57 ns/op	      16 B/op	       1 allocs/op
Benchmark_parseNinjaString/constant/100-128     	 9983748	       130.2 ns/op	     112 B/op	       1 allocs/op
Benchmark_parseNinjaString/constant/1000-128    	 2632527	       445.1 ns/op	    1024 B/op	       1 allocs/op
Benchmark_parseNinjaString/variable/1-128       	 2964896	       419.4 ns/op	     176 B/op	       4 allocs/op
Benchmark_parseNinjaString/variable/10-128      	 1807341	       670.6 ns/op	     192 B/op	       7 allocs/op
Benchmark_parseNinjaString/variable/100-128     	 1000000	      1092 ns/op	     352 B/op	       7 allocs/op
Benchmark_parseNinjaString/variable/1000-128    	  300649	      3773 ns/op	    1584 B/op	       7 allocs/op
Benchmark_parseNinjaString/variables/1-128      	 2858432	       441.6 ns/op	     176 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/2-128      	 2360505	       513.4 ns/op	     208 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/3-128      	 1867136	       635.6 ns/op	     240 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/4-128      	 1584045	       752.1 ns/op	     272 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/5-128      	 1338189	       885.8 ns/op	     304 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/10-128     	 1000000	      1468 ns/op	     464 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/100-128    	   88768	     12895 ns/op	    3712 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/1000-128   	    8972	    133627 ns/op	   32896 B/op	       4 allocs/op

After:
Benchmark_parseNinjaString/constant/1-128       	584600864	         2.004 ns/op	       0 B/op	       0 allocs/op
Benchmark_parseNinjaString/constant/10-128      	19274581	        64.84 ns/op	      16 B/op	       1 allocs/op
Benchmark_parseNinjaString/constant/100-128     	 9017640	       127.6 ns/op	     112 B/op	       1 allocs/op
Benchmark_parseNinjaString/constant/1000-128    	 2630797	       453.0 ns/op	    1024 B/op	       1 allocs/op
Benchmark_parseNinjaString/variable/1-128       	 3460422	       347.0 ns/op	     136 B/op	       4 allocs/op
Benchmark_parseNinjaString/variable/10-128      	 2103404	       519.9 ns/op	     152 B/op	       7 allocs/op
Benchmark_parseNinjaString/variable/100-128     	 1315778	       906.5 ns/op	     312 B/op	       7 allocs/op
Benchmark_parseNinjaString/variable/1000-128    	  354812	      3284 ns/op	    1544 B/op	       7 allocs/op
Benchmark_parseNinjaString/variables/1-128      	 3386868	       361.5 ns/op	     136 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/2-128      	 2675594	       456.9 ns/op	     160 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/3-128      	 2344670	       520.0 ns/op	     192 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/4-128      	 1919482	       648.1 ns/op	     208 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/5-128      	 1560556	       723.9 ns/op	     240 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/10-128     	 1000000	      1169 ns/op	     352 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/100-128    	  116738	     10168 ns/op	    2800 B/op	       4 allocs/op
Benchmark_parseNinjaString/variables/1000-128   	   10000	    105646 ns/op	   24688 B/op	       4 allocs/op

Bug: 286423944
Test: ninja_strings_test.go
Test: out/soong/build*.ninja is the same before and after this change
Change-Id: I1ecffbaccb0d0469a41fa31255c1b17311e01687

context.go

@@ -103,12 +103,12 @@ type Context struct {
 	// set during PrepareBuildActions
 	pkgNames        map[*packageContext]string
 	liveGlobals     *liveTracker
-	globalVariables map[Variable]ninjaString
+	globalVariables map[Variable]*ninjaString
 	globalPools     map[Pool]*poolDef
 	globalRules     map[Rule]*ruleDef
 
 	// set during PrepareBuildActions
-	outDir             ninjaString // The builddir special Ninja variable
+	outDir             *ninjaString // The builddir special Ninja variable
 	requiredNinjaMajor int          // For the ninja_required_version variable
 	requiredNinjaMinor int          // For the ninja_required_version variable
 	requiredNinjaMicro int          // For the ninja_required_version variable
@@ -2814,7 +2814,7 @@ func jsonModuleWithActionsFromModuleInfo(m *moduleInfo) *JsonModule {
 
 // Gets a list of strings from the given list of ninjaStrings by invoking ninjaString.Value with
 // nil pkgNames on each of the input ninjaStrings.
-func getNinjaStringsWithNilPkgNames(nStrs []ninjaString) []string {
+func getNinjaStringsWithNilPkgNames(nStrs []*ninjaString) []string {
 	var strs []string
 	for _, nstr := range nStrs {
 		strs = append(strs, nstr.Value(nil))
@@ -3820,7 +3820,7 @@ func (c *Context) requireNinjaVersion(major, minor, micro int) {
 	}
 }
 
-func (c *Context) setOutDir(value ninjaString) {
+func (c *Context) setOutDir(value *ninjaString) {
 	if c.outDir == nil {
 		c.outDir = value
 	}
@@ -3901,7 +3901,7 @@ func (c *Context) memoizeFullNames(liveGlobals *liveTracker, pkgNames map[*packa
 }
 
 func (c *Context) checkForVariableReferenceCycles(
-	variables map[Variable]ninjaString, pkgNames map[*packageContext]string) {
+	variables map[Variable]*ninjaString, pkgNames map[*packageContext]string) {
 
 	visited := make(map[Variable]bool)  // variables that were already checked
 	checking := make(map[Variable]bool) // variables actively being checked
@@ -4696,7 +4696,7 @@ type phonyCandidate struct {
 // keyForPhonyCandidate gives a unique identifier for a set of deps.
 // If any of the deps use a variable, we return an empty string to signal
 // that this set of deps is ineligible for extraction.
-func keyForPhonyCandidate(deps []ninjaString) string {
+func keyForPhonyCandidate(deps []*ninjaString) string {
 	hasher := sha256.New()
 	for _, d := range deps {
 		if len(d.Variables()) != 0 {
@@ -4727,7 +4727,7 @@ func scanBuildDef(wg *sync.WaitGroup, candidates *sync.Map, phonyCount *atomic.U
 			phonyCount.Add(1)
 			m.phony = &buildDef{
 				Rule:     Phony,
-				Outputs:  []ninjaString{simpleNinjaString("dedup-" + key)},
+				Outputs:  []*ninjaString{simpleNinjaString("dedup-" + key)},
 				Inputs:   m.first.OrderOnly, //we could also use b.OrderOnly
 				Optional: true,
 			}

context_test.go

@@ -1159,10 +1159,10 @@ func TestPackageIncludes(t *testing.T) {
 }
 
 func TestDeduplicateOrderOnlyDeps(t *testing.T) {
-	outputs := func(names ...string) []ninjaString {
-		r := make([]ninjaString, len(names))
+	outputs := func(names ...string) []*ninjaString {
+		r := make([]*ninjaString, len(names))
 		for i, name := range names {
-			r[i] = literalNinjaString(name)
+			r[i] = simpleNinjaString(name)
 		}
 		return r
 	}
@@ -1179,7 +1179,7 @@ func TestDeduplicateOrderOnlyDeps(t *testing.T) {
 	type testcase struct {
 		modules        []*moduleInfo
 		expectedPhonys []*buildDef
-		conversions    map[string][]ninjaString
+		conversions    map[string][]*ninjaString
 	}
 	testCases := []testcase{{
 		modules: []*moduleInfo{
@@ -1189,7 +1189,7 @@ func TestDeduplicateOrderOnlyDeps(t *testing.T) {
 		expectedPhonys: []*buildDef{
 			b("dedup-GKw-c0PwFokMUQ6T-TUmEWnZ4_VlQ2Qpgw-vCTT0-OQ", []string{"d"}, nil),
 		},
-		conversions: map[string][]ninjaString{
+		conversions: map[string][]*ninjaString{
 			"A": outputs("dedup-GKw-c0PwFokMUQ6T-TUmEWnZ4_VlQ2Qpgw-vCTT0-OQ"),
 			"B": outputs("dedup-GKw-c0PwFokMUQ6T-TUmEWnZ4_VlQ2Qpgw-vCTT0-OQ"),
 		},
@@ -1205,7 +1205,7 @@ func TestDeduplicateOrderOnlyDeps(t *testing.T) {
 			m(b("C", nil, []string{"a"})),
 		},
 		expectedPhonys: []*buildDef{b("dedup-ypeBEsobvcr6wjGzmiPcTaeG7_gUfE5yuYB3ha_uSLs", []string{"a"}, nil)},
-		conversions: map[string][]ninjaString{
+		conversions: map[string][]*ninjaString{
 			"A": outputs("dedup-ypeBEsobvcr6wjGzmiPcTaeG7_gUfE5yuYB3ha_uSLs"),
 			"B": outputs("b"),
 			"C": outputs("dedup-ypeBEsobvcr6wjGzmiPcTaeG7_gUfE5yuYB3ha_uSLs"),
@@ -1220,7 +1220,7 @@ func TestDeduplicateOrderOnlyDeps(t *testing.T) {
 		expectedPhonys: []*buildDef{
 			b("dedup--44g_C5MPySMYMOb1lLzwTRymLuXe4tNWQO4UFViBgM", []string{"a", "b"}, nil),
 			b("dedup-9F3lHN7zCZFVHkHogt17VAR5lkigoAdT9E_JZuYVP8E", []string{"a", "c"}, nil)},
-		conversions: map[string][]ninjaString{
+		conversions: map[string][]*ninjaString{
 			"A": outputs("dedup--44g_C5MPySMYMOb1lLzwTRymLuXe4tNWQO4UFViBgM"),
 			"B": outputs("dedup--44g_C5MPySMYMOb1lLzwTRymLuXe4tNWQO4UFViBgM"),
 			"C": outputs("dedup-9F3lHN7zCZFVHkHogt17VAR5lkigoAdT9E_JZuYVP8E"),

live_tracker.go

@@ -25,7 +25,7 @@ type liveTracker struct {
 	config interface{} // Used to evaluate variable, rule, and pool values.
 	ctx    *Context    // Used to evaluate globs
 
-	variables map[Variable]ninjaString
+	variables map[Variable]*ninjaString
 	pools     map[Pool]*poolDef
 	rules     map[Rule]*ruleDef
 }
@@ -34,7 +34,7 @@ func newLiveTracker(ctx *Context, config interface{}) *liveTracker {
 	return &liveTracker{
 		ctx:       ctx,
 		config:    config,
-		variables: make(map[Variable]ninjaString),
+		variables: make(map[Variable]*ninjaString),
 		pools:     make(map[Pool]*poolDef),
 		rules:     make(map[Rule]*ruleDef),
 	}
@@ -197,13 +197,13 @@ func (l *liveTracker) innerAddVariable(v Variable) error {
 	return nil
 }
 
-func (l *liveTracker) addNinjaStringListDeps(list []ninjaString) error {
+func (l *liveTracker) addNinjaStringListDeps(list []*ninjaString) error {
 	l.Lock()
 	defer l.Unlock()
 	return l.innerAddNinjaStringListDeps(list)
 }
 
-func (l *liveTracker) innerAddNinjaStringListDeps(list []ninjaString) error {
+func (l *liveTracker) innerAddNinjaStringListDeps(list []*ninjaString) error {
 	for _, str := range list {
 		err := l.innerAddNinjaStringDeps(str)
 		if err != nil {
@@ -213,13 +213,13 @@ func (l *liveTracker) innerAddNinjaStringListDeps(list []ninjaString) error {
 	return nil
 }
 
-func (l *liveTracker) addNinjaStringDeps(str ninjaString) error {
+func (l *liveTracker) addNinjaStringDeps(str *ninjaString) error {
 	l.Lock()
 	defer l.Unlock()
 	return l.innerAddNinjaStringDeps(str)
 }
 
-func (l *liveTracker) innerAddNinjaStringDeps(str ninjaString) error {
+func (l *liveTracker) innerAddNinjaStringDeps(str *ninjaString) error {
 	for _, v := range str.Variables() {
 		err := l.innerAddVariable(v)
 		if err != nil {

ninja_defs.go

@@ -131,11 +131,11 @@ func (p *poolDef) WriteTo(nw *ninjaWriter, name string) error {
 // A ruleDef describes a rule definition.  It does not include the name of the
 // rule.
 type ruleDef struct {
-	CommandDeps      []ninjaString
-	CommandOrderOnly []ninjaString
+	CommandDeps      []*ninjaString
+	CommandOrderOnly []*ninjaString
 	Comment          string
 	Pool             Pool
-	Variables        map[string]ninjaString
+	Variables        map[string]*ninjaString
 }
 
 func parseRuleParams(scope scope, params *RuleParams) (*ruleDef,
@@ -144,7 +144,7 @@ func parseRuleParams(scope scope, params *RuleParams) (*ruleDef,
 	r := &ruleDef{
 		Comment:   params.Comment,
 		Pool:      params.Pool,
-		Variables: make(map[string]ninjaString),
+		Variables: make(map[string]*ninjaString),
 	}
 
 	if params.Command == "" {
@@ -264,14 +264,14 @@ type buildDef struct {
 	Comment         string
 	Rule            Rule
 	RuleDef         *ruleDef
-	Outputs         []ninjaString
-	ImplicitOutputs []ninjaString
-	Inputs          []ninjaString
-	Implicits       []ninjaString
-	OrderOnly       []ninjaString
-	Validations     []ninjaString
-	Args            map[Variable]ninjaString
-	Variables       map[string]ninjaString
+	Outputs         []*ninjaString
+	ImplicitOutputs []*ninjaString
+	Inputs          []*ninjaString
+	Implicits       []*ninjaString
+	OrderOnly       []*ninjaString
+	Validations     []*ninjaString
+	Args            map[Variable]*ninjaString
+	Variables       map[string]*ninjaString
 	Optional        bool
 }
 
@@ -286,9 +286,9 @@ func parseBuildParams(scope scope, params *BuildParams) (*buildDef,
 		Rule:    rule,
 	}
 
-	setVariable := func(name string, value ninjaString) {
+	setVariable := func(name string, value *ninjaString) {
 		if b.Variables == nil {
-			b.Variables = make(map[string]ninjaString)
+			b.Variables = make(map[string]*ninjaString)
 		}
 		b.Variables[name] = value
 	}
@@ -363,7 +363,7 @@ func parseBuildParams(scope scope, params *BuildParams) (*buildDef,
 	argNameScope := rule.scope()
 
 	if len(params.Args) > 0 {
-		b.Args = make(map[Variable]ninjaString)
+		b.Args = make(map[Variable]*ninjaString)
 		for name, value := range params.Args {
 			if !rule.isArg(name) {
 				return nil, fmt.Errorf("unknown argument %q", name)
@@ -444,7 +444,7 @@ func (b *buildDef) WriteTo(nw *ninjaWriter, pkgNames map[*packageContext]string)
 	return nw.BlankLine()
 }
 
-func writeVariables(nw *ninjaWriter, variables map[string]ninjaString,
+func writeVariables(nw *ninjaWriter, variables map[string]*ninjaString,
 	pkgNames map[*packageContext]string) error {
 	var keys []string
 	for k := range variables {

ninja_strings.go

@@ -35,19 +35,28 @@ var (
 		":", "$:")
 )
 
-type ninjaString interface {
-	Value(pkgNames map[*packageContext]string) string
-	ValueWithEscaper(w io.StringWriter, pkgNames map[*packageContext]string, escaper *strings.Replacer)
-	Eval(variables map[Variable]ninjaString) (string, error)
-	Variables() []Variable
+// ninjaString contains the parsed result of a string that can contain references to variables (e.g. $cflags) that will
+// be propagated to the build.ninja file.  For literal strings with no variable references, the variables field will be
+// nil. For strings with variable references str contains the original, unparsed string, and variables contains a
+// pointer to a list of references, each with a span of bytes they should replace and a Variable interface.
+type ninjaString struct {
+	str       string
+	variables *[]variableReference
 }
 
-type varNinjaString struct {
-	strings   []string
-	variables []Variable
-}
+// variableReference contains information about a single reference to a variable (e.g. $cflags) inside a parsed
+// ninjaString.  start and end are int32 to reduce memory usage.  A nil variable is a special case of an inserted '$'
+// at the beginning of the string to handle leading whitespace that must not be stripped by ninja.
+type variableReference struct {
+	// start is the offset of the '$' character from the beginning of the unparsed string.
+	start int32
 
-type literalNinjaString string
+	// end is the offset of the character _after_ the final character of the variable name (or '}' if using the
+	//'${}'  syntax)
+	end int32
+
+	variable Variable
+}
 
 type scope interface {
 	LookupVariable(name string) (Variable, error)
@@ -55,36 +64,24 @@ type scope interface {
 	IsPoolVisible(pool Pool) bool
 }
 
-func simpleNinjaString(str string) ninjaString {
-	return literalNinjaString(str)
+func simpleNinjaString(str string) *ninjaString {
+	return &ninjaString{str: str}
 }
 
 type parseState struct {
 	scope        scope
 	str          string
-	pendingStr  string
-	stringStart int
 	varStart     int
-	result      *varNinjaString
+	varNameStart int
+	result       *ninjaString
 }
 
-func (ps *parseState) pushVariable(v Variable) {
-	if len(ps.result.variables) == len(ps.result.strings) {
-		// Last push was a variable, we need a blank string separator
-		ps.result.strings = append(ps.result.strings, "")
+func (ps *parseState) pushVariable(start, end int, v Variable) {
+	if ps.result.variables == nil {
+		ps.result.variables = &[]variableReference{{start: int32(start), end: int32(end), variable: v}}
+	} else {
+		*ps.result.variables = append(*ps.result.variables, variableReference{start: int32(start), end: int32(end), variable: v})
 	}
-	if ps.pendingStr != "" {
-		panic("oops, pushed variable with pending string")
-	}
-	ps.result.variables = append(ps.result.variables, v)
-}
-
-func (ps *parseState) pushString(s string) {
-	if len(ps.result.strings) != len(ps.result.variables) {
-		panic("oops, pushed string after string")
-	}
-	ps.result.strings = append(ps.result.strings, ps.pendingStr+s)
-	ps.pendingStr = ""
 }
 
 type stateFunc func(*parseState, int, rune) (stateFunc, error)
@@ -92,18 +89,19 @@ type stateFunc func(*parseState, int, rune) (stateFunc, error)
 // parseNinjaString parses an unescaped ninja string (i.e. all $<something>
 // occurrences are expected to be variables or $$) and returns a list of the
 // variable names that the string references.
-func parseNinjaString(scope scope, str string) (ninjaString, error) {
-	// naively pre-allocate slices by counting $ signs
+func parseNinjaString(scope scope, str string) (*ninjaString, error) {
+	// naively pre-allocate slice by counting $ signs
 	n := strings.Count(str, "$")
 	if n == 0 {
-		if strings.HasPrefix(str, " ") {
+		if len(str) > 0 && str[0] == ' ' {
 			str = "$" + str
 		}
-		return literalNinjaString(str), nil
+		return simpleNinjaString(str), nil
 	}
-	result := &varNinjaString{
-		strings:   make([]string, 0, n+1),
-		variables: make([]Variable, 0, n),
+	variableReferences := make([]variableReference, 0, n)
+	result := &ninjaString{
+		str:       str,
+		variables: &variableReferences,
 	}
 
 	parseState := &parseState{
@@ -127,12 +125,18 @@ func parseNinjaString(scope scope, str string) (ninjaString, error) {
 		return nil, err
 	}
 
+	// All the '$' characters counted initially could have been "$$" escapes, leaving no
+	// variable references.  Deallocate the variables slice if so.
+	if len(*result.variables) == 0 {
+		result.variables = nil
+	}
+
 	return result, nil
 }
 
 func parseFirstRuneState(state *parseState, i int, r rune) (stateFunc, error) {
 	if r == ' ' {
-		state.pendingStr += "$"
+		state.pushVariable(0, 1, nil)
 	}
 	return parseStringState(state, i, r)
 }
@@ -140,11 +144,10 @@ func parseFirstRuneState(state *parseState, i int, r rune) (stateFunc, error) {
 func parseStringState(state *parseState, i int, r rune) (stateFunc, error) {
 	switch {
 	case r == '$':
-		state.varStart = i + 1
+		state.varStart = i
 		return parseDollarStartState, nil
 
 	case r == eof:
-		state.pushString(state.str[state.stringStart:i])
 		return nil, nil
 
 	default:
@@ -156,21 +159,17 @@ func parseDollarStartState(state *parseState, i int, r rune) (stateFunc, error)
 	switch {
 	case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z',
 		r >= '0' && r <= '9', r == '_', r == '-':
-		// The beginning of a of the variable name.  Output the string and
-		// keep going.
-		state.pushString(state.str[state.stringStart : i-1])
+		// The beginning of a of the variable name.
+		state.varNameStart = i
 		return parseDollarState, nil
 
 	case r == '$':
-		// Just a "$$".  Go back to parseStringState without changing
-		// state.stringStart.
+		// Just a "$$".  Go back to parseStringState.
 		return parseStringState, nil
 
 	case r == '{':
-		// This is a bracketted variable name (e.g. "${blah.blah}").  Output
-		// the string and keep going.
-		state.pushString(state.str[state.stringStart : i-1])
-		state.varStart = i + 1
+		// This is a bracketted variable name (e.g. "${blah.blah}").
+		state.varNameStart = i + 1
 		return parseBracketsState, nil
 
 	case r == eof:
@@ -190,45 +189,26 @@ func parseDollarState(state *parseState, i int, r rune) (stateFunc, error) {
 		r >= '0' && r <= '9', r == '_', r == '-':
 		// A part of the variable name.  Keep going.
 		return parseDollarState, nil
+	}
 
-	case r == '$':
-		// A dollar after the variable name (e.g. "$blah$").  Output the
-		// variable we have and start a new one.
-		v, err := state.scope.LookupVariable(state.str[state.varStart:i])
+	// The variable name has ended, output what we have.
+	v, err := state.scope.LookupVariable(state.str[state.varNameStart:i])
 	if err != nil {
 		return nil, err
 	}
 
-		state.pushVariable(v)
-		state.varStart = i + 1
-		state.stringStart = i
+	state.pushVariable(state.varStart, i, v)
 
+	switch {
+	case r == '$':
+		// A dollar after the variable name (e.g. "$blah$").  Start a new one.
+		state.varStart = i
 		return parseDollarStartState, nil
 
 	case r == eof:
-		// This is the end of the variable name.
-		v, err := state.scope.LookupVariable(state.str[state.varStart:i])
-		if err != nil {
-			return nil, err
-		}
-
-		state.pushVariable(v)
-
-		// We always end with a string, even if it's an empty one.
-		state.pushString("")
-
 		return nil, nil
 
 	default:
-		// We've just gone past the end of the variable name, so record what
-		// we have.
-		v, err := state.scope.LookupVariable(state.str[state.varStart:i])
-		if err != nil {
-			return nil, err
-		}
-
-		state.pushVariable(v)
-		state.stringStart = i
 		return parseStringState, nil
 	}
 }
@@ -241,20 +221,19 @@ func parseBracketsState(state *parseState, i int, r rune) (stateFunc, error) {
 		return parseBracketsState, nil
 
 	case r == '}':
-		if state.varStart == i {
+		if state.varNameStart == i {
 			// The brackets were immediately closed.  That's no good.
 			return nil, fmt.Errorf("empty variable name at byte offset %d",
 				i)
 		}
 
 		// This is the end of the variable name.
-		v, err := state.scope.LookupVariable(state.str[state.varStart:i])
+		v, err := state.scope.LookupVariable(state.str[state.varNameStart:i])
 		if err != nil {
 			return nil, err
 		}
 
-		state.pushVariable(v)
-		state.stringStart = i + 1
+		state.pushVariable(state.varStart, i+1, v)
 		return parseStringState, nil
 
 	case r == eof:
@@ -267,13 +246,13 @@ func parseBracketsState(state *parseState, i int, r rune) (stateFunc, error) {
 	}
 }
 
-func parseNinjaStrings(scope scope, strs []string) ([]ninjaString,
+func parseNinjaStrings(scope scope, strs []string) ([]*ninjaString,
 	error) {
 
 	if len(strs) == 0 {
 		return nil, nil
 	}
-	result := make([]ninjaString, len(strs))
+	result := make([]*ninjaString, len(strs))
 	for i, str := range strs {
 		ninjaStr, err := parseNinjaString(scope, str)
 		if err != nil {
@@ -284,64 +263,80 @@ func parseNinjaStrings(scope scope, strs []string) ([]ninjaString,
 	return result, nil
 }
 
-func (n varNinjaString) Value(pkgNames map[*packageContext]string) string {
-	if len(n.strings) == 1 {
-		return defaultEscaper.Replace(n.strings[0])
+func (n *ninjaString) Value(pkgNames map[*packageContext]string) string {
+	if n.variables == nil || len(*n.variables) == 0 {
+		return defaultEscaper.Replace(n.str)
 	}
 	str := &strings.Builder{}
 	n.ValueWithEscaper(str, pkgNames, defaultEscaper)
 	return str.String()
 }
 
-func (n varNinjaString) ValueWithEscaper(w io.StringWriter, pkgNames map[*packageContext]string,
+func (n *ninjaString) ValueWithEscaper(w io.StringWriter, pkgNames map[*packageContext]string,
 	escaper *strings.Replacer) {
 
-	w.WriteString(escaper.Replace(n.strings[0]))
-	for i, v := range n.variables {
-		w.WriteString("${")
-		w.WriteString(v.fullName(pkgNames))
-		w.WriteString("}")
-		w.WriteString(escaper.Replace(n.strings[i+1]))
-	}
+	if n.variables == nil || len(*n.variables) == 0 {
+		w.WriteString(escaper.Replace(n.str))
+		return
 	}
 
-func (n varNinjaString) Eval(variables map[Variable]ninjaString) (string, error) {
-	str := n.strings[0]
-	for i, v := range n.variables {
-		variable, ok := variables[v]
+	i := 0
+	for _, v := range *n.variables {
+		w.WriteString(escaper.Replace(n.str[i:v.start]))
+		if v.variable == nil {
+			w.WriteString("$ ")
+		} else {
+			w.WriteString("${")
+			w.WriteString(v.variable.fullName(pkgNames))
+			w.WriteString("}")
+		}
+		i = int(v.end)
+	}
+	w.WriteString(escaper.Replace(n.str[i:len(n.str)]))
+}
+
+func (n *ninjaString) Eval(variables map[Variable]*ninjaString) (string, error) {
+	if n.variables == nil || len(*n.variables) == 0 {
+		return n.str, nil
+	}
+
+	w := &strings.Builder{}
+	i := 0
+	for _, v := range *n.variables {
+		w.WriteString(n.str[i:v.start])
+		if v.variable == nil {
+			w.WriteString(" ")
+		} else {
+			variable, ok := variables[v.variable]
 			if !ok {
-			return "", fmt.Errorf("no such global variable: %s", v)
+				return "", fmt.Errorf("no such global variable: %s", v.variable)
 			}
 			value, err := variable.Eval(variables)
 			if err != nil {
 				return "", err
 			}
-		str += value + n.strings[i+1]
+			w.WriteString(value)
 		}
-	return str, nil
+		i = int(v.end)
+	}
+	w.WriteString(n.str[i:len(n.str)])
+	return w.String(), nil
 }
 
-func (n varNinjaString) Variables() []Variable {
-	return n.variables
-}
-
-func (l literalNinjaString) Value(_ map[*packageContext]string) string {
-	return defaultEscaper.Replace(string(l))
-}
-
-func (l literalNinjaString) ValueWithEscaper(w io.StringWriter, _ map[*packageContext]string,
-	escaper *strings.Replacer) {
-	w.WriteString(escaper.Replace(string(l)))
-}
-
-func (l literalNinjaString) Eval(variables map[Variable]ninjaString) (string, error) {
-	return string(l), nil
-}
-
-func (l literalNinjaString) Variables() []Variable {
+func (n *ninjaString) Variables() []Variable {
+	if n.variables == nil || len(*n.variables) == 0 {
 		return nil
 	}
 
+	variables := make([]Variable, 0, len(*n.variables))
+	for _, v := range *n.variables {
+		if v.variable != nil {
+			variables = append(variables, v.variable)
+		}
+	}
+	return variables
+}
+
 func validateNinjaName(name string) error {
 	for i, r := range name {
 		valid := (r >= 'a' && r <= 'z') ||

ninja_strings_test.go

@@ -21,71 +21,110 @@ import (
 	"testing"
 )
 
-var ninjaParseTestCases = []struct {
+type testVariableRef struct {
+	start, end int
+	name       string
+}
+
+func TestParseNinjaString(t *testing.T) {
+	testCases := []struct {
 		input string
 		vars  []string
-	strs    []string
-	literal bool
+		value string
+		eval  string
 		err   string
 	}{
 		{
 			input: "abc def $ghi jkl",
 			vars:  []string{"ghi"},
-		strs:  []string{"abc def ", " jkl"},
+			value: "abc def ${namespace.ghi} jkl",
+			eval:  "abc def GHI jkl",
 		},
 		{
 			input: "abc def $ghi$jkl",
 			vars:  []string{"ghi", "jkl"},
-		strs:  []string{"abc def ", "", ""},
+			value: "abc def ${namespace.ghi}${namespace.jkl}",
+			eval:  "abc def GHIJKL",
 		},
 		{
 			input: "foo $012_-345xyz_! bar",
 			vars:  []string{"012_-345xyz_"},
-		strs:  []string{"foo ", "! bar"},
+			value: "foo ${namespace.012_-345xyz_}! bar",
+			eval:  "foo 012_-345XYZ_! bar",
 		},
 		{
 			input: "foo ${012_-345xyz_} bar",
 			vars:  []string{"012_-345xyz_"},
-		strs:  []string{"foo ", " bar"},
+			value: "foo ${namespace.012_-345xyz_} bar",
+			eval:  "foo 012_-345XYZ_ bar",
 		},
 		{
 			input: "foo ${012_-345xyz_} bar",
 			vars:  []string{"012_-345xyz_"},
-		strs:  []string{"foo ", " bar"},
+			value: "foo ${namespace.012_-345xyz_} bar",
+			eval:  "foo 012_-345XYZ_ bar",
 		},
 		{
 			input: "foo $$ bar",
 			vars:  nil,
-		strs:  []string{"foo $$ bar"},
-		// this is technically a literal, but not recognized as such due to the $$
+			value: "foo $$ bar",
+			eval:  "foo $$ bar",
 		},
 		{
 			input: "$foo${bar}",
 			vars:  []string{"foo", "bar"},
-		strs:  []string{"", "", ""},
+			value: "${namespace.foo}${namespace.bar}",
+			eval:  "FOOBAR",
 		},
 		{
 			input: "$foo$$",
 			vars:  []string{"foo"},
-		strs:  []string{"", "$$"},
+			value: "${namespace.foo}$$",
+			eval:  "FOO$$",
 		},
 		{
 			input: "foo bar",
 			vars:  nil,
-		strs:    []string{"foo bar"},
-		literal: true,
+			value: "foo bar",
+			eval:  "foo bar",
 		},
 		{
 			input: " foo ",
 			vars:  nil,
-		strs:    []string{"$ foo "},
-		literal: true,
+			value: "$ foo ",
+			eval:  "$ foo ",
+		},
+		{
+			input: "\tfoo ",
+			vars:  nil,
+			value: "\tfoo ",
+			eval:  "\tfoo ",
+		},
+		{
+			input: "\nfoo ",
+			vars:  nil,
+			value: "$\nfoo ",
+			eval:  "\nfoo ",
 		},
 		{
 			input: " $foo ",
 			vars:  []string{"foo"},
-		strs:  []string{"$ ", " "},
-	}, {
+			value: "$ ${namespace.foo} ",
+			eval:  " FOO ",
+		},
+		{
+			input: "\t$foo ",
+			vars:  []string{"foo"},
+			value: "\t${namespace.foo} ",
+			eval:  "\tFOO ",
+		},
+		{
+			input: "\n$foo ",
+			vars:  []string{"foo"},
+			value: "$\n${namespace.foo} ",
+			eval:  "\nFOO ",
+		},
+		{
 			input: "foo $ bar",
 			err:   `error parsing ninja string "foo $ bar": invalid character after '$' at byte offset 5`,
 		},
@@ -107,40 +146,33 @@ var ninjaParseTestCases = []struct {
 		},
 	}
 
-func TestParseNinjaString(t *testing.T) {
-	for _, testCase := range ninjaParseTestCases {
-		scope := newLocalScope(nil, "namespace")
-		expectedVars := []Variable{}
+	for _, testCase := range testCases {
+		t.Run(testCase.input, func(t *testing.T) {
+			scope := newLocalScope(nil, "namespace.")
+			variablesMap := map[Variable]*ninjaString{}
 			for _, varName := range testCase.vars {
-			v, err := scope.LookupVariable(varName)
+				_, err := scope.LookupVariable(varName)
 				if err != nil {
-				v, err = scope.AddLocalVariable(varName, "")
+					v, err := scope.AddLocalVariable(varName, strings.ToUpper(varName))
 					if err != nil {
 						t.Fatalf("error creating scope: %s", err)
 					}
-			}
-			expectedVars = append(expectedVars, v)
-		}
-
-		var expected ninjaString
-		if len(testCase.strs) > 0 {
-			if testCase.literal {
-				expected = literalNinjaString(testCase.strs[0])
-			} else {
-				expected = &varNinjaString{
-					strings:   testCase.strs,
-					variables: expectedVars,
-				}
+					variablesMap[v] = simpleNinjaString(strings.ToUpper(varName))
 				}
 			}
 
 			output, err := parseNinjaString(scope, testCase.input)
 			if err == nil {
-			if !reflect.DeepEqual(output, expected) {
-				t.Errorf("incorrect ninja string:")
-				t.Errorf("     input: %q", testCase.input)
-				t.Errorf("  expected: %#v", expected)
-				t.Errorf("       got: %#v", output)
+				if g, w := output.Value(nil), testCase.value; g != w {
+					t.Errorf("incorrect Value output, want %q, got %q", w, g)
+				}
+
+				eval, err := output.Eval(variablesMap)
+				if err != nil {
+					t.Errorf("unexpected error in Eval: %s", err)
+				}
+				if g, w := eval, testCase.eval; g != w {
+					t.Errorf("incorrect Eval output, want %q, got %q", w, g)
 				}
 			}
 			var errStr string
@@ -153,11 +185,12 @@ func TestParseNinjaString(t *testing.T) {
 				t.Errorf("  expected: %q", testCase.err)
 				t.Errorf("       got: %q", errStr)
 			}
+		})
 	}
 }
 
 func TestParseNinjaStringWithImportedVar(t *testing.T) {
-	ImpVar := &staticVariable{name_: "ImpVar"}
+	ImpVar := &staticVariable{name_: "ImpVar", fullName_: "g.impPkg.ImpVar"}
 	impScope := newScope(nil)
 	impScope.AddVariable(ImpVar)
 	scope := newScope(nil)
@@ -169,13 +202,59 @@ func TestParseNinjaStringWithImportedVar(t *testing.T) {
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	expect := []Variable{ImpVar}
-	if !reflect.DeepEqual(output.(*varNinjaString).variables, expect) {
+	expect := []variableReference{{8, 24, ImpVar}}
+	if !reflect.DeepEqual(*output.variables, expect) {
 		t.Errorf("incorrect output:")
 		t.Errorf("     input: %q", input)
 		t.Errorf("  expected: %#v", expect)
-		t.Errorf("       got: %#v", output)
+		t.Errorf("       got: %#v", *output.variables)
 	}
+
+	if g, w := output.Value(nil), "abc def ${g.impPkg.ImpVar} ghi"; g != w {
+		t.Errorf("incorrect Value output, want %q got %q", w, g)
+	}
+}
+
+func Benchmark_parseNinjaString(b *testing.B) {
+	b.Run("constant", func(b *testing.B) {
+		for _, l := range []int{1, 10, 100, 1000} {
+			b.Run(strconv.Itoa(l), func(b *testing.B) {
+				b.ReportAllocs()
+				for n := 0; n < b.N; n++ {
+					_ = simpleNinjaString(strings.Repeat("a", l))
+				}
+			})
+		}
+	})
+	b.Run("variable", func(b *testing.B) {
+		for _, l := range []int{1, 10, 100, 1000} {
+			scope := newLocalScope(nil, "")
+			scope.AddLocalVariable("a", strings.Repeat("b", l/3))
+			b.Run(strconv.Itoa(l), func(b *testing.B) {
+				b.ReportAllocs()
+				for n := 0; n < b.N; n++ {
+					_, _ = parseNinjaString(scope, strings.Repeat("a", l/3)+"${a}"+strings.Repeat("a", l/3))
+				}
+			})
+		}
+	})
+	b.Run("variables", func(b *testing.B) {
+		for _, l := range []int{1, 2, 3, 4, 5, 10, 100, 1000} {
+			scope := newLocalScope(nil, "")
+			str := strings.Repeat("a", 10)
+			for i := 0; i < l; i++ {
+				scope.AddLocalVariable("a"+strconv.Itoa(i), strings.Repeat("b", 10))
+				str += "${a" + strconv.Itoa(i) + "}"
+			}
+			b.Run(strconv.Itoa(l), func(b *testing.B) {
+				b.ReportAllocs()
+				for n := 0; n < b.N; n++ {
+					_, _ = parseNinjaString(scope, str)
+				}
+			})
+		}
+	})
+
 }
 
 func BenchmarkNinjaString_Value(b *testing.B) {
@@ -183,6 +262,7 @@ func BenchmarkNinjaString_Value(b *testing.B) {
 		for _, l := range []int{1, 10, 100, 1000} {
 			ns := simpleNinjaString(strings.Repeat("a", l))
 			b.Run(strconv.Itoa(l), func(b *testing.B) {
+				b.ReportAllocs()
 				for n := 0; n < b.N; n++ {
 					ns.Value(nil)
 				}
@@ -195,6 +275,7 @@ func BenchmarkNinjaString_Value(b *testing.B) {
 			scope.AddLocalVariable("a", strings.Repeat("b", l/3))
 			ns, _ := parseNinjaString(scope, strings.Repeat("a", l/3)+"${a}"+strings.Repeat("a", l/3))
 			b.Run(strconv.Itoa(l), func(b *testing.B) {
+				b.ReportAllocs()
 				for n := 0; n < b.N; n++ {
 					ns.Value(nil)
 				}
@@ -211,6 +292,7 @@ func BenchmarkNinjaString_Value(b *testing.B) {
 			}
 			ns, _ := parseNinjaString(scope, str)
 			b.Run(strconv.Itoa(l), func(b *testing.B) {
+				b.ReportAllocs()
 				for n := 0; n < b.N; n++ {
 					ns.Value(nil)
 				}

ninja_writer.go

@@ -114,7 +114,7 @@ func (n *ninjaWriter) Rule(name string) error {
 }
 
 func (n *ninjaWriter) Build(comment string, rule string, outputs, implicitOuts,
-	explicitDeps, implicitDeps, orderOnlyDeps, validations []ninjaString,
+	explicitDeps, implicitDeps, orderOnlyDeps, validations []*ninjaString,
 	pkgNames map[*packageContext]string) error {
 
 	n.justDidBlankLine = false
@@ -235,7 +235,7 @@ func (n *ninjaWriter) ScopedAssign(name, value string) error {
 	return nil
 }
 
-func (n *ninjaWriter) Default(pkgNames map[*packageContext]string, targets ...ninjaString) error {
+func (n *ninjaWriter) Default(pkgNames map[*packageContext]string, targets ...*ninjaString) error {
 	n.justDidBlankLine = false
 
 	const lineWrapLen = len(" $")

ninja_writer_test.go

@@ -139,7 +139,7 @@ func TestNinjaWriter(t *testing.T) {
 	}
 }
 
-func testNinjaStrings(s ...string) []ninjaString {
+func testNinjaStrings(s ...string) []*ninjaString {
 	ret, _ := parseNinjaStrings(nil, s)
 	return ret
 }

package_ctx.go

@@ -304,7 +304,7 @@ func (v *staticVariable) memoizeFullName(pkgNames map[*packageContext]string) {
 	v.fullName_ = v.fullName(pkgNames)
 }
 
-func (v *staticVariable) value(VariableFuncContext, interface{}) (ninjaString, error) {
+func (v *staticVariable) value(VariableFuncContext, interface{}) (*ninjaString, error) {
 	ninjaStr, err := parseNinjaString(v.pctx.scope, v.value_)
 	if err != nil {
 		err = fmt.Errorf("error parsing variable %s value: %s", v, err)
@@ -440,7 +440,7 @@ func (v *variableFunc) memoizeFullName(pkgNames map[*packageContext]string) {
 	v.fullName_ = v.fullName(pkgNames)
 }
 
-func (v *variableFunc) value(ctx VariableFuncContext, config interface{}) (ninjaString, error) {
+func (v *variableFunc) value(ctx VariableFuncContext, config interface{}) (*ninjaString, error) {
 	value, err := v.value_(ctx, config)
 	if err != nil {
 		return nil, err
@@ -504,7 +504,7 @@ func (v *argVariable) memoizeFullName(pkgNames map[*packageContext]string) {
 	// Nothing to do, full name is known at initialization.
 }
 
-func (v *argVariable) value(ctx VariableFuncContext, config interface{}) (ninjaString, error) {
+func (v *argVariable) value(ctx VariableFuncContext, config interface{}) (*ninjaString, error) {
 	return nil, errVariableIsArg
 }
 

scope.go

@@ -29,7 +29,7 @@ type Variable interface {
 	name() string                                        // "foo"
 	fullName(pkgNames map[*packageContext]string) string // "pkg.foo" or "path.to.pkg.foo"
 	memoizeFullName(pkgNames map[*packageContext]string) // precompute fullName if desired
-	value(ctx VariableFuncContext, config interface{}) (ninjaString, error)
+	value(ctx VariableFuncContext, config interface{}) (*ninjaString, error)
 	String() string
 }
 
@@ -354,7 +354,7 @@ func (s *localScope) AddLocalRule(name string, params *RuleParams,
 type localVariable struct {
 	fullName_ string
 	name_     string
-	value_    ninjaString
+	value_    *ninjaString
 }
 
 func (l *localVariable) packageContext() *packageContext {
@@ -373,7 +373,7 @@ func (l *localVariable) memoizeFullName(pkgNames map[*packageContext]string) {
 	// Nothing to do, full name is known at initialization.
 }
 
-func (l *localVariable) value(VariableFuncContext, interface{}) (ninjaString, error) {
+func (l *localVariable) value(VariableFuncContext, interface{}) (*ninjaString, error) {
 	return l.value_, nil
 }