From d1facc1ce7c985bd22a45e1f95eb61dae4594af4 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Thu, 8 Jan 2015 14:56:03 -0800 Subject: [PATCH] Return comment and position information from parser Return comment info from parser, which will be used later for pretty printing a parsed Blueprint file. Also add more position information to the AST. Change-Id: I3cf0ba8ea4ba5b931d032098ae5fdc350806ed98 --- blueprint/context.go | 10 +- blueprint/parser/parser.go | 137 ++++++++++++++------ blueprint/parser/parser_test.go | 217 +++++++++++++++++++++++--------- blueprint/unpack.go | 2 +- blueprint/unpack_test.go | 4 +- 5 files changed, 267 insertions(+), 103 deletions(-) diff --git a/blueprint/context.go b/blueprint/context.go index 005bf0d..568cfb5 100644 --- a/blueprint/context.go +++ b/blueprint/context.go @@ -355,7 +355,7 @@ func (c *Context) parse(rootDir, filename string, r io.Reader, scope = parser.NewScope(scope) scope.Remove("subdirs") - defs, errs := parser.Parse(filename, r, scope) + file, errs := parser.Parse(filename, r, scope) if len(errs) > 0 { for i, err := range errs { if parseErr, ok := err.(*parser.ParseError); ok { @@ -372,7 +372,7 @@ func (c *Context) parse(rootDir, filename string, r io.Reader, return nil, nil, errs, nil } - for _, def := range defs { + for _, def := range file.Defs { var newErrs []error var newModule *moduleInfo switch def := def.(type) { @@ -732,7 +732,7 @@ func (c *Context) convertDepsToVariant(module *moduleInfo, newSubName subName) { func (c *Context) processModuleDef(moduleDef *parser.Module, relBlueprintsFile string) (*moduleInfo, []error) { - typeName := moduleDef.Type + typeName := moduleDef.Type.Name factory, ok := c.moduleFactories[typeName] if !ok { if c.ignoreUnknownModuleTypes { @@ -742,7 +742,7 @@ func (c *Context) processModuleDef(moduleDef *parser.Module, return nil, []error{ &Error{ Err: fmt.Errorf("unrecognized module type %q", typeName), - Pos: moduleDef.Pos, + Pos: moduleDef.Type.Pos, }, } } @@ -763,7 +763,7 @@ func (c *Context) processModuleDef(moduleDef *parser.Module, return nil, errs } - group.pos = moduleDef.Pos + group.pos = moduleDef.Type.Pos group.propertyPos = make(map[string]scanner.Position) for name, propertyDef := range propertyMap { group.propertyPos[name] = propertyDef.Pos diff --git a/blueprint/parser/parser.go b/blueprint/parser/parser.go index 46d7215..6711972 100644 --- a/blueprint/parser/parser.go +++ b/blueprint/parser/parser.go @@ -23,14 +23,19 @@ func (e *ParseError) Error() string { return fmt.Sprintf("%s: %s", e.Pos, e.Err) } -func Parse(filename string, r io.Reader, scope *Scope) (defs []Definition, errs []error) { +type File struct { + Defs []Definition + Comments []Comment +} + +func Parse(filename string, r io.Reader, scope *Scope) (file *File, errs []error) { + p := newParser(r, scope) p.scanner.Filename = filename defer func() { if r := recover(); r != nil { if r == errTooManyErrors { - defs = nil errs = p.errors return } @@ -38,29 +43,39 @@ func Parse(filename string, r io.Reader, scope *Scope) (defs []Definition, errs } }() - defs = p.parseDefinitions() + defs := p.parseDefinitions() p.accept(scanner.EOF) errs = p.errors + comments := p.comments - return + return &File{ + Defs: defs, + Comments: comments, + }, errs } type parser struct { - scanner scanner.Scanner - tok rune - errors []error - scope *Scope + scanner scanner.Scanner + tok rune + errors []error + scope *Scope + parseComments bool + comments []Comment } func newParser(r io.Reader, scope *Scope) *parser { p := &parser{} p.scope = scope + p.parseComments = true p.scanner.Init(r) p.scanner.Error = func(sc *scanner.Scanner, msg string) { p.errorf(msg) } p.scanner.Mode = scanner.ScanIdents | scanner.ScanStrings | - scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments + scanner.ScanRawStrings | scanner.ScanComments + if !p.parseComments { + p.scanner.Mode |= scanner.SkipComments + } p.next() return p } @@ -87,9 +102,7 @@ func (p *parser) accept(toks ...rune) bool { scanner.TokenString(p.tok)) return false } - if p.tok != scanner.EOF { - p.tok = p.scanner.Scan() - } + p.next() } return true } @@ -97,6 +110,10 @@ func (p *parser) accept(toks ...rune) bool { func (p *parser) next() { if p.tok != scanner.EOF { p.tok = p.scanner.Scan() + for p.tok == scanner.Comment { + p.comments = append(p.comments, Comment{p.scanner.TokenText(), p.scanner.Position}) + p.tok = p.scanner.Scan() + } } return } @@ -130,16 +147,17 @@ func (p *parser) parseDefinitions() (defs []Definition) { } func (p *parser) parseAssignment(name string, - pos scanner.Position) (assignment *Assignment) { + namePos scanner.Position) (assignment *Assignment) { assignment = new(Assignment) + pos := p.scanner.Position if !p.accept('=') { return } value := p.parseExpression() - assignment.Name = name + assignment.Name = Ident{name, namePos} assignment.Value = value assignment.Pos = pos @@ -151,19 +169,22 @@ func (p *parser) parseAssignment(name string, } func (p *parser) parseModule(typ string, - pos scanner.Position) (module *Module) { + typPos scanner.Position) (module *Module) { module = new(Module) + lbracePos := p.scanner.Position if !p.accept('{') { return } properties := p.parsePropertyList() + rbracePos := p.scanner.Position p.accept('}') - module.Type = typ + module.Type = Ident{typ, typPos} module.Properties = properties - module.Pos = pos + module.LbracePos = lbracePos + module.RbracePos = rbracePos return } @@ -187,13 +208,16 @@ func (p *parser) parseProperty() (property *Property) { property = new(Property) name := p.scanner.TokenText() + namePos := p.scanner.Position + p.accept(scanner.Ident) pos := p.scanner.Position - if !p.accept(scanner.Ident, ':') { + if !p.accept(':') { return } + value := p.parseExpression() - property.Name = name + property.Name = Ident{name, namePos} property.Value = value property.Pos = pos @@ -210,8 +234,9 @@ func (p *parser) parseExpression() (value Value) { } } -func (p *parser) parseOperator(value1 Value) (value Value) { +func (p *parser) parseOperator(value1 Value) Value { operator := p.tok + pos := p.scanner.Position p.accept(operator) value2 := p.parseExpression() @@ -219,25 +244,35 @@ func (p *parser) parseOperator(value1 Value) (value Value) { if value1.Type != value2.Type { p.errorf("mismatched type in operator %c: %s != %s", operator, value1.Type, value2.Type) - return + return Value{} } + value := value1 + value.Variable = "" + switch operator { case '+': switch value1.Type { case String: - value1.StringValue += value2.StringValue + value.StringValue = value1.StringValue + value2.StringValue case List: - value1.ListValue = append(value1.ListValue, value2.ListValue...) + value.ListValue = append([]Value{}, value1.ListValue...) + value.ListValue = append(value.ListValue, value2.ListValue...) default: p.errorf("operator %c not supported on type %s", operator, value1.Type) - return + return Value{} } default: panic("unknown operator " + string(operator)) } - return value1 + value.Expression = &Expression{ + Args: [2]Value{value1, value2}, + Operator: operator, + Pos: pos, + } + + return value } func (p *parser) parseValue() (value Value) { @@ -258,7 +293,6 @@ func (p *parser) parseValue() (value Value) { } func (p *parser) parseVariable() (value Value) { - value.Pos = p.scanner.Position switch text := p.scanner.TokenText(); text { case "true": value.Type = Bool @@ -267,12 +301,15 @@ func (p *parser) parseVariable() (value Value) { value.Type = Bool value.BoolValue = false default: - assignment, err := p.scope.Get(p.scanner.TokenText()) + variable := p.scanner.TokenText() + assignment, err := p.scope.Get(variable) if err != nil { p.errorf(err.Error()) } value = assignment.Value + value.Variable = variable } + value.Pos = p.scanner.Position p.accept(scanner.Ident) return @@ -316,6 +353,7 @@ func (p *parser) parseListValue() (value Value) { } value.ListValue = elements + value.EndPos = p.scanner.Position p.accept(']') return @@ -331,10 +369,17 @@ func (p *parser) parseMapValue() (value Value) { properties := p.parsePropertyList() value.MapValue = properties + value.EndPos = p.scanner.Position p.accept('}') return } +type Expression struct { + Args [2]Value + Operator rune + Pos scanner.Position +} + type ValueType int const ( @@ -365,7 +410,7 @@ type Definition interface { } type Assignment struct { - Name string + Name Ident Value Value Pos scanner.Position } @@ -377,9 +422,10 @@ func (a *Assignment) String() string { func (a *Assignment) definitionTag() {} type Module struct { - Type string + Type Ident Properties []*Property - Pos scanner.Position + LbracePos scanner.Position + RbracePos scanner.Position } func (m *Module) String() string { @@ -387,14 +433,16 @@ func (m *Module) String() string { for i, property := range m.Properties { propertyStrings[i] = property.String() } - return fmt.Sprintf("%s@%d:%s{%s}", m.Type, m.Pos.Offset, m.Pos, + return fmt.Sprintf("%s@%d:%s-%d:%s{%s}", m.Type, + m.LbracePos.Offset, m.LbracePos, + m.RbracePos.Offset, m.RbracePos, strings.Join(propertyStrings, ", ")) } func (m *Module) definitionTag() {} type Property struct { - Name string + Name Ident Value Value Pos scanner.Position } @@ -403,13 +451,25 @@ func (p *Property) String() string { return fmt.Sprintf("%s@%d:%s: %s", p.Name, p.Pos.Offset, p.Pos, p.Value) } +type Ident struct { + Name string + Pos scanner.Position +} + +func (i Ident) String() string { + return fmt.Sprintf("%s@%d:%s", i.Name, i.Pos.Offset, i.Pos) +} + type Value struct { Type ValueType BoolValue bool StringValue string ListValue []Value MapValue []*Property + Expression *Expression + Variable string Pos scanner.Position + EndPos scanner.Position } func (p Value) String() string { @@ -423,14 +483,14 @@ func (p Value) String() string { for i, value := range p.ListValue { valueStrings[i] = value.String() } - return fmt.Sprintf("@%d:%s[%s]", p.Pos.Offset, p.Pos, + return fmt.Sprintf("@%d:%s-%d:%s[%s]", p.Pos.Offset, p.Pos, p.EndPos.Offset, p.EndPos, strings.Join(valueStrings, ", ")) case Map: propertyStrings := make([]string, len(p.MapValue)) for i, property := range p.MapValue { propertyStrings[i] = property.String() } - return fmt.Sprintf("@%d:%s{%s}", p.Pos.Offset, p.Pos, + return fmt.Sprintf("@%d:%s-%d:%s{%s}", p.Pos.Offset, p.Pos, p.EndPos.Offset, p.EndPos, strings.Join(propertyStrings, ", ")) default: panic(fmt.Errorf("bad property type: %d", p.Type)) @@ -456,11 +516,11 @@ func NewScope(s *Scope) *Scope { } func (s *Scope) Add(assignment *Assignment) error { - if old, ok := s.vars[assignment.Name]; ok { + if old, ok := s.vars[assignment.Name.Name]; ok { return fmt.Errorf("variable already set, previous assignment: %s", old) } - s.vars[assignment.Name] = assignment + s.vars[assignment.Name.Name] = assignment return nil } @@ -493,3 +553,8 @@ func (s *Scope) String() string { return strings.Join(ret, "\n") } + +type Comment struct { + Comment string + Pos scanner.Position +} diff --git a/blueprint/parser/parser_test.go b/blueprint/parser/parser_test.go index a2539ff..b5a0185 100644 --- a/blueprint/parser/parser_test.go +++ b/blueprint/parser/parser_test.go @@ -3,7 +3,6 @@ package parser import ( "bytes" "reflect" - "strings" "testing" "text/scanner" ) @@ -17,18 +16,21 @@ func mkpos(offset, line, column int) scanner.Position { } var validParseTestCases = []struct { - input string - output []Definition + input string + defs []Definition + comments []Comment }{ {` foo {} `, []Definition{ &Module{ - Type: "foo", - Pos: mkpos(3, 2, 3), + Type: Ident{"foo", mkpos(3, 2, 3)}, + LbracePos: mkpos(7, 2, 7), + RbracePos: mkpos(8, 2, 8), }, }, + nil, }, {` @@ -38,12 +40,13 @@ var validParseTestCases = []struct { `, []Definition{ &Module{ - Type: "foo", - Pos: mkpos(3, 2, 3), + Type: Ident{"foo", mkpos(3, 2, 3)}, + LbracePos: mkpos(7, 2, 7), + RbracePos: mkpos(27, 4, 3), Properties: []*Property{ { - Name: "name", - Pos: mkpos(12, 3, 4), + Name: Ident{"name", mkpos(12, 3, 4)}, + Pos: mkpos(16, 3, 8), Value: Value{ Type: String, Pos: mkpos(18, 3, 10), @@ -53,6 +56,7 @@ var validParseTestCases = []struct { }, }, }, + nil, }, {` @@ -62,12 +66,13 @@ var validParseTestCases = []struct { `, []Definition{ &Module{ - Type: "foo", - Pos: mkpos(3, 2, 3), + Type: Ident{"foo", mkpos(3, 2, 3)}, + LbracePos: mkpos(7, 2, 7), + RbracePos: mkpos(28, 4, 3), Properties: []*Property{ { - Name: "isGood", - Pos: mkpos(12, 3, 4), + Name: Ident{"isGood", mkpos(12, 3, 4)}, + Pos: mkpos(18, 3, 10), Value: Value{ Type: Bool, Pos: mkpos(20, 3, 12), @@ -77,6 +82,7 @@ var validParseTestCases = []struct { }, }, }, + nil, }, {` @@ -87,15 +93,17 @@ var validParseTestCases = []struct { `, []Definition{ &Module{ - Type: "foo", - Pos: mkpos(3, 2, 3), + Type: Ident{"foo", mkpos(3, 2, 3)}, + LbracePos: mkpos(7, 2, 7), + RbracePos: mkpos(67, 5, 3), Properties: []*Property{ { - Name: "stuff", - Pos: mkpos(12, 3, 4), + Name: Ident{"stuff", mkpos(12, 3, 4)}, + Pos: mkpos(17, 3, 9), Value: Value{ - Type: List, - Pos: mkpos(19, 3, 11), + Type: List, + Pos: mkpos(19, 3, 11), + EndPos: mkpos(63, 4, 19), ListValue: []Value{ Value{ Type: String, @@ -128,6 +136,7 @@ var validParseTestCases = []struct { }, }, }, + nil, }, {` @@ -140,19 +149,21 @@ var validParseTestCases = []struct { `, []Definition{ &Module{ - Type: "foo", - Pos: mkpos(3, 2, 3), + Type: Ident{"foo", mkpos(3, 2, 3)}, + LbracePos: mkpos(7, 2, 7), + RbracePos: mkpos(62, 7, 3), Properties: []*Property{ { - Name: "stuff", - Pos: mkpos(12, 3, 4), + Name: Ident{"stuff", mkpos(12, 3, 4)}, + Pos: mkpos(17, 3, 9), Value: Value{ - Type: Map, - Pos: mkpos(19, 3, 11), + Type: Map, + Pos: mkpos(19, 3, 11), + EndPos: mkpos(58, 6, 4), MapValue: []*Property{ { - Name: "isGood", - Pos: mkpos(25, 4, 5), + Name: Ident{"isGood", mkpos(25, 4, 5)}, + Pos: mkpos(31, 4, 11), Value: Value{ Type: Bool, Pos: mkpos(33, 4, 13), @@ -160,8 +171,8 @@ var validParseTestCases = []struct { }, }, { - Name: "name", - Pos: mkpos(43, 5, 5), + Name: Ident{"name", mkpos(43, 5, 5)}, + Pos: mkpos(47, 5, 9), Value: Value{ Type: String, Pos: mkpos(49, 5, 11), @@ -174,32 +185,48 @@ var validParseTestCases = []struct { }, }, }, + nil, }, {` - // comment + // comment1 foo { - // comment - isGood: true, // comment + // comment2 + isGood: true, // comment3 } `, []Definition{ &Module{ - Type: "foo", - Pos: mkpos(16, 3, 3), + Type: Ident{"foo", mkpos(17, 3, 3)}, + LbracePos: mkpos(21, 3, 7), + RbracePos: mkpos(70, 6, 3), Properties: []*Property{ { - Name: "isGood", - Pos: mkpos(39, 5, 4), + Name: Ident{"isGood", mkpos(41, 5, 4)}, + Pos: mkpos(47, 5, 10), Value: Value{ Type: Bool, - Pos: mkpos(47, 5, 12), + Pos: mkpos(49, 5, 12), BoolValue: true, }, }, }, }, }, + []Comment{ + Comment{ + Comment: "// comment1", + Pos: mkpos(3, 2, 3), + }, + Comment{ + Comment: "// comment2", + Pos: mkpos(26, 4, 4), + }, + Comment{ + Comment: "// comment3", + Pos: mkpos(56, 5, 19), + }, + }, }, {` @@ -213,12 +240,13 @@ var validParseTestCases = []struct { `, []Definition{ &Module{ - Type: "foo", - Pos: mkpos(3, 2, 3), + Type: Ident{"foo", mkpos(3, 2, 3)}, + LbracePos: mkpos(7, 2, 7), + RbracePos: mkpos(27, 4, 3), Properties: []*Property{ { - Name: "name", - Pos: mkpos(12, 3, 4), + Name: Ident{"name", mkpos(12, 3, 4)}, + Pos: mkpos(16, 3, 8), Value: Value{ Type: String, Pos: mkpos(18, 3, 10), @@ -228,12 +256,13 @@ var validParseTestCases = []struct { }, }, &Module{ - Type: "bar", - Pos: mkpos(32, 6, 3), + Type: Ident{"bar", mkpos(32, 6, 3)}, + LbracePos: mkpos(36, 6, 7), + RbracePos: mkpos(56, 8, 3), Properties: []*Property{ { - Name: "name", - Pos: mkpos(41, 7, 4), + Name: Ident{"name", mkpos(41, 7, 4)}, + Pos: mkpos(45, 7, 8), Value: Value{ Type: String, Pos: mkpos(47, 7, 10), @@ -243,22 +272,69 @@ var validParseTestCases = []struct { }, }, }, + nil, + }, + {` + foo = "stuff" + bar = foo + baz = foo + bar + `, + []Definition{ + &Assignment{ + Name: Ident{"foo", mkpos(3, 2, 3)}, + Pos: mkpos(7, 2, 7), + Value: Value{ + Type: String, + Pos: mkpos(9, 2, 9), + StringValue: "stuff", + }, + }, + &Assignment{ + Name: Ident{"bar", mkpos(19, 3, 3)}, + Pos: mkpos(23, 3, 7), + Value: Value{ + Type: String, + Pos: mkpos(25, 3, 9), + StringValue: "stuff", + Variable: "foo", + }, + }, + &Assignment{ + Name: Ident{"baz", mkpos(31, 4, 3)}, + Pos: mkpos(35, 4, 7), + Value: Value{ + Type: String, + Pos: mkpos(37, 4, 9), + StringValue: "stuffstuff", + Expression: &Expression{ + Args: [2]Value{ + { + Type: String, + Pos: mkpos(37, 4, 9), + StringValue: "stuff", + Variable: "foo", + }, + { + Type: String, + Pos: mkpos(43, 4, 15), + StringValue: "stuff", + Variable: "bar", + }, + }, + Operator: '+', + Pos: mkpos(41, 4, 13), + }, + }, + }, + }, + nil, }, -} - -func defListString(defs []Definition) string { - defStrings := make([]string, len(defs)) - for i, def := range defs { - defStrings[i] = def.String() - } - - return strings.Join(defStrings, ", ") } func TestParseValidInput(t *testing.T) { for _, testCase := range validParseTestCases { r := bytes.NewBufferString(testCase.input) - defs, errs := Parse("", r) + file, errs := Parse("", r, NewScope(nil)) if len(errs) != 0 { t.Errorf("test case: %s", testCase.input) t.Errorf("unexpected errors:") @@ -268,11 +344,34 @@ func TestParseValidInput(t *testing.T) { t.FailNow() } - if !reflect.DeepEqual(defs, testCase.output) { + if len(file.Defs) == len(testCase.defs) { + for i := range file.Defs { + if !reflect.DeepEqual(file.Defs[i], testCase.defs[i]) { + t.Errorf("test case: %s", testCase.input) + t.Errorf("incorrect defintion %d:", i) + t.Errorf(" expected: %s", testCase.defs[i]) + t.Errorf(" got: %s", file.Defs[i]) + } + } + } else { t.Errorf("test case: %s", testCase.input) - t.Errorf("incorrect output:") - t.Errorf(" expected: %s", defListString(testCase.output)) - t.Errorf(" got: %s", defListString(defs)) + t.Errorf("length mismatch, expected %d definitions, got %d", + len(testCase.defs), len(file.Defs)) + } + + if len(file.Comments) == len(testCase.comments) { + for i := range file.Comments { + if !reflect.DeepEqual(file.Comments, testCase.comments) { + t.Errorf("test case: %s", testCase.input) + t.Errorf("incorrect comment %d:", i) + t.Errorf(" expected: %s", testCase.comments[i]) + t.Errorf(" got: %s", file.Comments[i]) + } + } + } else { + t.Errorf("test case: %s", testCase.input) + t.Errorf("length mismatch, expected %d comments, got %d", + len(testCase.comments), len(file.Comments)) } } } diff --git a/blueprint/unpack.go b/blueprint/unpack.go index 7faa47b..018afb4 100644 --- a/blueprint/unpack.go +++ b/blueprint/unpack.go @@ -66,7 +66,7 @@ func buildPropertyMap(namePrefix string, propertyDefs []*parser.Property, propertyMap map[string]*packedProperty) (errs []error) { for _, propertyDef := range propertyDefs { - name := namePrefix + propertyDef.Name + name := namePrefix + propertyDef.Name.Name if first, present := propertyMap[name]; present { if first.property == propertyDef { // We've already added this property. diff --git a/blueprint/unpack_test.go b/blueprint/unpack_test.go index c0dd740..0b2b0e4 100644 --- a/blueprint/unpack_test.go +++ b/blueprint/unpack_test.go @@ -111,7 +111,7 @@ var validUnpackTestCases = []struct { func TestUnpackProperties(t *testing.T) { for _, testCase := range validUnpackTestCases { r := bytes.NewBufferString(testCase.input) - defs, errs := parser.Parse("", r, nil) + file, errs := parser.Parse("", r, nil) if len(errs) != 0 { t.Errorf("test case: %s", testCase.input) t.Errorf("unexpected parse errors:") @@ -121,7 +121,7 @@ func TestUnpackProperties(t *testing.T) { t.FailNow() } - module := defs[0].(*parser.Module) + module := file.Defs[0].(*parser.Module) properties := proptools.CloneProperties(reflect.ValueOf(testCase.output)) proptools.ZeroProperties(properties.Elem()) _, errs = unpackProperties(module.Properties, properties.Interface())