diff --git a/context.go b/context.go index ac4fbe6..e576331 100644 --- a/context.go +++ b/context.go @@ -430,7 +430,7 @@ func (c *Context) parse(rootDir, filename string, r io.Reader, scope = parser.NewScope(scope) scope.Remove("subdirs") - file, errs := parser.Parse(filename, r, scope) + file, errs := parser.ParseAndEval(filename, r, scope) if len(errs) > 0 { for i, err := range errs { if parseErr, ok := err.(*parser.ParseError); ok { diff --git a/parser/parser.go b/parser/parser.go index 0031547..498bdcb 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -42,11 +42,7 @@ type File struct { Comments []Comment } -func Parse(filename string, r io.Reader, scope *Scope) (file *File, errs []error) { - - p := newParser(r, scope) - p.scanner.Filename = filename - +func parse(p *parser) (file *File, errs []error) { defer func() { if r := recover(); r != nil { if r == errTooManyErrors { @@ -66,30 +62,42 @@ func Parse(filename string, r io.Reader, scope *Scope) (file *File, errs []error Defs: defs, Comments: comments, }, errs + +} + +func ParseAndEval(filename string, r io.Reader, scope *Scope) (file *File, errs []error) { + p := newParser(r, scope) + p.eval = true + p.scanner.Filename = filename + + return parse(p) +} + +func Parse(filename string, r io.Reader, scope *Scope) (file *File, errs []error) { + p := newParser(r, scope) + p.scanner.Filename = filename + + return parse(p) } type parser struct { - scanner scanner.Scanner - tok rune - errors []error - scope *Scope - parseComments bool - comments []Comment + scanner scanner.Scanner + tok rune + errors []error + scope *Scope + comments []Comment + eval bool } func newParser(r io.Reader, scope *Scope) *parser { p := &parser{} p.scope = scope - p.parseComments = true p.scanner.Init(r) p.scanner.Error = func(sc *scanner.Scanner, msg string) { p.errorf(msg) } p.scanner.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments - if !p.parseComments { - p.scanner.Mode |= scanner.SkipComments - } p.next() return p } @@ -125,7 +133,8 @@ func (p *parser) next() { if p.tok != scanner.EOF { p.tok = p.scanner.Scan() for p.tok == scanner.Comment { - p.comments = append(p.comments, Comment{p.scanner.TokenText(), p.scanner.Position}) + lines := strings.Split(p.scanner.TokenText(), "\n") + p.comments = append(p.comments, Comment{lines, p.scanner.Position}) p.tok = p.scanner.Scan() } } @@ -182,13 +191,19 @@ func (p *parser) parseAssignment(name string, if p.scope != nil { if assigner == "+=" { - p.scope.Append(assignment) - } else { - err := p.scope.Add(assignment) - if err != nil { - p.errorf("%s", err.Error()) + if old, err := p.scope.Get(assignment.Name.Name); err == nil { + if old.Referenced { + p.errorf("modified variable with += after referencing") + } + old.Value, err = p.evaluateOperator(old.Value, assignment.Value, '+', assignment.Pos) + return } } + + err := p.scope.Add(assignment) + if err != nil { + p.errorf("%s", err.Error()) + } } return @@ -279,35 +294,41 @@ func (p *parser) parseExpression() (value Value) { } } -func evaluateOperator(value1, value2 Value, operator rune, pos scanner.Position) (Value, error) { - if value1.Type != value2.Type { - return Value{}, fmt.Errorf("mismatched type in operator %c: %s != %s", operator, - value1.Type, value2.Type) - } +func (p *parser) evaluateOperator(value1, value2 Value, operator rune, + pos scanner.Position) (Value, error) { - value := value1 - value.Variable = "" + value := Value{} - switch operator { - case '+': - switch value1.Type { - case String: - value.StringValue = value1.StringValue + value2.StringValue - case List: - value.ListValue = append([]Value{}, value1.ListValue...) - value.ListValue = append(value.ListValue, value2.ListValue...) - case Map: - var err error - value.MapValue, err = addMaps(value.MapValue, value2.MapValue, pos) - if err != nil { - return Value{}, err + if p.eval { + if value1.Type != value2.Type { + return Value{}, fmt.Errorf("mismatched type in operator %c: %s != %s", operator, + value1.Type, value2.Type) + } + + value = value1 + value.Variable = "" + + switch operator { + case '+': + switch value1.Type { + case String: + value.StringValue = value1.StringValue + value2.StringValue + case List: + value.ListValue = append([]Value{}, value1.ListValue...) + value.ListValue = append(value.ListValue, value2.ListValue...) + case Map: + var err error + value.MapValue, err = p.addMaps(value.MapValue, value2.MapValue, pos) + if err != nil { + return Value{}, err + } + default: + return Value{}, fmt.Errorf("operator %c not supported on type %s", operator, + value1.Type) } default: - return Value{}, fmt.Errorf("operator %c not supported on type %s", operator, - value1.Type) + panic("unknown operator " + string(operator)) } - default: - panic("unknown operator " + string(operator)) } value.Expression = &Expression{ @@ -319,7 +340,7 @@ func evaluateOperator(value1, value2 Value, operator rune, pos scanner.Position) return value, nil } -func addMaps(map1, map2 []*Property, pos scanner.Position) ([]*Property, error) { +func (p *parser) addMaps(map1, map2 []*Property, pos scanner.Position) ([]*Property, error) { ret := make([]*Property, 0, len(map1)) inMap1 := make(map[string]*Property) @@ -341,7 +362,7 @@ func addMaps(map1, map2 []*Property, pos scanner.Position) ([]*Property, error) if prop2, ok := inBoth[prop1.Name.Name]; ok { var err error newProp := *prop1 - newProp.Value, err = evaluateOperator(prop1.Value, prop2.Value, '+', pos) + newProp.Value, err = p.evaluateOperator(prop1.Value, prop2.Value, '+', pos) if err != nil { return nil, err } @@ -367,7 +388,7 @@ func (p *parser) parseOperator(value1 Value) Value { value2 := p.parseExpression() - value, err := evaluateOperator(value1, value2, operator, pos) + value, err := p.evaluateOperator(value1, value2, operator, pos) if err != nil { p.errorf(err.Error()) return Value{} @@ -403,11 +424,14 @@ func (p *parser) parseVariable() (value Value) { value.BoolValue = false default: variable := p.scanner.TokenText() - assignment, err := p.scope.Get(variable) - if err != nil { - p.errorf(err.Error()) + if p.eval { + assignment, err := p.scope.Get(variable) + if err != nil { + p.errorf(err.Error()) + } + assignment.Referenced = true + value = assignment.Value } - value = assignment.Value value.Variable = variable } value.Pos = p.scanner.Position @@ -439,7 +463,7 @@ func (p *parser) parseListValue() (value Value) { var elements []Value for p.tok != ']' { element := p.parseExpression() - if element.Type != String { + if p.eval && element.Type != String { p.errorf("Expected string in list, found %s", element.String()) return } @@ -643,27 +667,12 @@ func (s *Scope) Add(assignment *Assignment) error { return nil } -func (s *Scope) Append(assignment *Assignment) error { - var err error - if old, ok := s.vars[assignment.Name.Name]; ok { - if old.Referenced { - return fmt.Errorf("modified variable with += after referencing") - } - old.Value, err = evaluateOperator(old.Value, assignment.Value, '+', assignment.Pos) - } else { - err = s.Add(assignment) - } - - return err -} - func (s *Scope) Remove(name string) { delete(s.vars, name) } func (s *Scope) Get(name string) (*Assignment, error) { if a, ok := s.vars[name]; ok { - a.Referenced = true return a, nil } @@ -688,6 +697,6 @@ func (s *Scope) String() string { } type Comment struct { - Comment string + Comment []string Pos scanner.Position } diff --git a/parser/parser_test.go b/parser/parser_test.go index f1e99c7..8925684 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -229,15 +229,15 @@ var validParseTestCases = []struct { }, []Comment{ Comment{ - Comment: "// comment1", + Comment: []string{"// comment1"}, Pos: mkpos(3, 2, 3), }, Comment{ - Comment: "// comment2", + Comment: []string{"// comment2"}, Pos: mkpos(26, 4, 4), }, Comment{ - Comment: "// comment3", + Comment: []string{"// comment3"}, Pos: mkpos(56, 5, 19), }, }, @@ -477,7 +477,7 @@ var validParseTestCases = []struct { func TestParseValidInput(t *testing.T) { for _, testCase := range validParseTestCases { r := bytes.NewBufferString(testCase.input) - file, errs := Parse("", r, NewScope(nil)) + file, errs := ParseAndEval("", r, NewScope(nil)) if len(errs) != 0 { t.Errorf("test case: %s", testCase.input) t.Errorf("unexpected errors:") diff --git a/parser/printer.go b/parser/printer.go index 53fbbcc..b27f5e0 100644 --- a/parser/printer.go +++ b/parser/printer.go @@ -19,36 +19,21 @@ import ( "strconv" "strings" "text/scanner" + "unicode" ) var noPos = scanner.Position{} -type whitespace int - -const ( - wsDontCare whitespace = iota - wsBoth - wsAfter - wsBefore - wsCanBreak // allow extra line breaks or comments - wsBothCanBreak // wsBoth plus allow extra line breaks or comments - wsForceBreak - wsForceDoubleBreak -) - -type tokenState struct { - ws whitespace - token string - pos scanner.Position - indent int -} - type printer struct { defs []Definition comments []Comment curComment int - prev tokenState + + pos scanner.Position + + pendingSpace bool + pendingNewline int output []byte @@ -63,8 +48,12 @@ func newPrinter(file *File) *printer { defs: file.Defs, comments: file.Comments, indentList: []int{0}, - prev: tokenState{ - ws: wsCanBreak, + + // pendingNewLine is initialized to -1 to eat initial spaces if the first token is a comment + pendingNewline: -1, + + pos: scanner.Position{ + Line: 1, }, } } @@ -75,7 +64,6 @@ func Print(file *File) ([]byte, error) { for _, def := range p.defs { p.printDef(def) } - p.prev.ws = wsDontCare p.flush() return p.output, nil } @@ -99,21 +87,23 @@ func (p *printer) printDef(def Definition) { } func (p *printer) printAssignment(assignment *Assignment) { - p.printToken(assignment.Name.Name, assignment.Name.Pos, wsDontCare) - p.printToken(assignment.Assigner, assignment.Pos, wsBoth) + p.printToken(assignment.Name.Name, assignment.Name.Pos) + p.requestSpace() + p.printToken(assignment.Assigner, assignment.Pos) + p.requestSpace() p.printValue(assignment.OrigValue) - p.prev.ws = wsForceBreak + p.requestNewline() } func (p *printer) printModule(module *Module) { - p.printToken(module.Type.Name, module.Type.Pos, wsDontCare) + p.printToken(module.Type.Name, module.Type.Pos) p.printMap(module.Properties, module.LbracePos, module.RbracePos) - p.prev.ws = wsForceDoubleBreak + p.requestDoubleNewline() } func (p *printer) printValue(value Value) { if value.Variable != "" { - p.printToken(value.Variable, value.Pos, wsDontCare) + p.printToken(value.Variable, value.Pos) } else if value.Expression != nil { p.printExpression(*value.Expression) } else { @@ -125,9 +115,9 @@ func (p *printer) printValue(value Value) { } else { s = "false" } - p.printToken(s, value.Pos, wsDontCare) + p.printToken(s, value.Pos) case String: - p.printToken(strconv.Quote(value.StringValue), value.Pos, wsDontCare) + p.printToken(strconv.Quote(value.StringValue), value.Pos) case List: p.printList(value.ListValue, value.Pos, value.EndPos) case Map: @@ -139,184 +129,219 @@ func (p *printer) printValue(value Value) { } func (p *printer) printList(list []Value, pos, endPos scanner.Position) { - p.printToken("[", pos, wsBefore) + p.requestSpace() + p.printToken("[", pos) if len(list) > 1 || pos.Line != endPos.Line { - p.prev.ws = wsForceBreak + p.requestNewline() p.indent(p.curIndent() + 4) for _, value := range list { p.printValue(value) - p.printToken(",", noPos, wsForceBreak) + p.printToken(",", noPos) + p.requestNewline() } - p.unindent() + p.unindent(endPos) } else { for _, value := range list { p.printValue(value) } } - p.printToken("]", endPos, wsDontCare) + p.printToken("]", endPos) } func (p *printer) printMap(list []*Property, pos, endPos scanner.Position) { - p.printToken("{", pos, wsBefore) + p.requestSpace() + p.printToken("{", pos) if len(list) > 0 || pos.Line != endPos.Line { - p.prev.ws = wsForceBreak + p.requestNewline() p.indent(p.curIndent() + 4) for _, prop := range list { p.printProperty(prop) - p.printToken(",", noPos, wsForceBreak) + p.printToken(",", noPos) + p.requestNewline() } - p.unindent() + p.unindent(endPos) } - p.printToken("}", endPos, wsDontCare) + p.printToken("}", endPos) } func (p *printer) printExpression(expression Expression) { p.printValue(expression.Args[0]) - p.printToken(string(expression.Operator), expression.Pos, wsBothCanBreak) + p.requestSpace() + p.printToken(string(expression.Operator), expression.Pos) + if expression.Args[0].Pos.Line == expression.Args[1].Pos.Line { + p.requestSpace() + } else { + p.requestNewline() + } p.printValue(expression.Args[1]) } func (p *printer) printProperty(property *Property) { - p.printToken(property.Name.Name, property.Name.Pos, wsDontCare) - p.printToken(":", property.Pos, wsAfter) + p.printToken(property.Name.Name, property.Name.Pos) + p.printToken(":", property.Pos) + p.requestSpace() p.printValue(property.Value) } // Print a single token, including any necessary comments or whitespace between // this token and the previously printed token -func (p *printer) printToken(s string, pos scanner.Position, ws whitespace) { - this := tokenState{ - token: s, - pos: pos, - ws: ws, - indent: p.curIndent(), +func (p *printer) printToken(s string, pos scanner.Position) { + newline := p.pendingNewline != 0 + + if pos == noPos { + pos = p.pos } - if this.pos == noPos { - this.pos = p.prev.pos + if newline { + p.printEndOfLineCommentsBefore(pos) + p.requestNewlinesForPos(pos) } - // Print the previously stored token - allowLineBreak := false - lineBreak := 0 + p.printInLineCommentsBefore(pos) - switch p.prev.ws { - case wsBothCanBreak, wsCanBreak, wsForceBreak, wsForceDoubleBreak: - allowLineBreak = true - } + p.flushSpace() - p.output = append(p.output, p.prev.token...) + p.output = append(p.output, s...) - commentIndent := max(p.prev.indent, this.indent) - p.printComments(this.pos, allowLineBreak, commentIndent) - - switch p.prev.ws { - case wsForceBreak: - lineBreak = 1 - case wsForceDoubleBreak: - lineBreak = 2 - } - - if allowLineBreak && p.prev.pos.IsValid() && - pos.Line-p.prev.pos.Line > lineBreak { - lineBreak = pos.Line - p.prev.pos.Line - } - - if lineBreak > 0 { - p.printLineBreak(lineBreak, this.indent) - } else { - p.printWhitespace(this.ws) - } - - p.prev = this + p.pos = pos } -func (p *printer) printWhitespace(ws whitespace) { - if p.prev.ws == wsBoth || ws == wsBoth || - p.prev.ws == wsBothCanBreak || ws == wsBothCanBreak || - p.prev.ws == wsAfter || ws == wsBefore { - p.output = append(p.output, ' ') - } -} - -// Pr int all comments that occur before position pos -func (p *printer) printComments(pos scanner.Position, allowLineBreak bool, indent int) { - if allowLineBreak { - for _, c := range p.skippedComments { - p.printComment(c, indent) - } - p.skippedComments = []Comment{} - } +// Print any in-line (single line /* */) comments that appear _before_ pos +func (p *printer) printInLineCommentsBefore(pos scanner.Position) { for p.curComment < len(p.comments) && p.comments[p.curComment].Pos.Offset < pos.Offset { - if !allowLineBreak && p.comments[p.curComment].Comment[0:2] == "//" { - p.skippedComments = append(p.skippedComments, p.comments[p.curComment]) + c := p.comments[p.curComment] + if c.Comment[0][0:2] == "//" || len(c.Comment) > 1 { + p.skippedComments = append(p.skippedComments, c) } else { - p.printComment(p.comments[p.curComment], indent) + p.flushSpace() + p.printComment(c) + p.requestSpace() } p.curComment++ } } -// Print a single comment, which may be a multi-line comment -func (p *printer) printComment(comment Comment, indent int) { - commentLines := strings.Split(comment.Comment, "\n") - pos := comment.Pos - for _, line := range commentLines { - if p.prev.pos.IsValid() && pos.Line > p.prev.pos.Line { - // Comment is on the next line - if p.prev.ws == wsForceDoubleBreak { - p.printLineBreak(2, indent) - p.prev.ws = wsForceBreak - } else { - p.printLineBreak(pos.Line-p.prev.pos.Line, indent) - } - } else if p.prev.pos.IsValid() { - // Comment is on the current line - p.printWhitespace(wsBoth) +// Print any comments, including end of line comments, that appear _before_ the line specified +// by pos +func (p *printer) printEndOfLineCommentsBefore(pos scanner.Position) { + for _, c := range p.skippedComments { + if !p.requestNewlinesForPos(c.Pos) { + p.requestSpace() } - p.output = append(p.output, strings.TrimSpace(line)...) - p.prev.pos = pos - pos.Line++ + p.printComment(c) + p._requestNewline() } - if comment.Comment[0:2] == "//" { - if p.prev.ws != wsForceDoubleBreak { - p.prev.ws = wsForceBreak + p.skippedComments = []Comment{} + for p.curComment < len(p.comments) && p.comments[p.curComment].Pos.Line < pos.Line { + c := p.comments[p.curComment] + if !p.requestNewlinesForPos(c.Pos) { + p.requestSpace() } - } else { - p.prev.ws = wsBothCanBreak + p.printComment(c) + p._requestNewline() + p.curComment++ } } -// Print one or two line breaks. n <= 0 is only valid if forceLineBreak is set, -// n > 2 is collapsed to a single blank line. -func (p *printer) printLineBreak(n, indent int) { - if n > 2 { - n = 2 +// Compare the line numbers of the previous and current positions to determine whether extra +// newlines should be inserted. A second newline is allowed anywhere requestNewline() is called. +func (p *printer) requestNewlinesForPos(pos scanner.Position) bool { + if pos.Line > p.pos.Line { + p._requestNewline() + if pos.Line > p.pos.Line+1 { + p.pendingNewline = 2 + } + return true } - for i := 0; i < n; i++ { + return false +} + +func (p *printer) requestSpace() { + p.pendingSpace = true +} + +// Ask for a newline to be inserted before the next token, but do not insert any comments. Used +// by the comment printers. +func (p *printer) _requestNewline() { + if p.pendingNewline == 0 { + p.pendingNewline = 1 + } +} + +// Ask for a newline to be inserted before the next token. Also inserts any end-of line comments +// for the current line +func (p *printer) requestNewline() { + pos := p.pos + pos.Line++ + p.printEndOfLineCommentsBefore(pos) + p._requestNewline() +} + +// Ask for two newlines to be inserted before the next token. Also inserts any end-of line comments +// for the current line +func (p *printer) requestDoubleNewline() { + p.requestNewline() + p.pendingNewline = 2 +} + +// Flush any pending whitespace, ignoring pending spaces if there is a pending newline +func (p *printer) flushSpace() { + if p.pendingNewline == 1 { p.output = append(p.output, '\n') + p.pad(p.curIndent()) + } else if p.pendingNewline == 2 { + p.output = append(p.output, "\n\n"...) + p.pad(p.curIndent()) + } else if p.pendingSpace == true && p.pendingNewline != -1 { + p.output = append(p.output, ' ') } - p.pad(0, indent) + p.pendingSpace = false + p.pendingNewline = 0 +} + +// Print a single comment, which may be a multi-line comment +func (p *printer) printComment(comment Comment) { + pos := comment.Pos + for i, line := range comment.Comment { + line = strings.TrimRightFunc(line, unicode.IsSpace) + p.flushSpace() + if i != 0 { + lineIndent := strings.IndexFunc(line, func(r rune) bool { return !unicode.IsSpace(r) }) + lineIndent = max(lineIndent, p.curIndent()) + p.pad(lineIndent - p.curIndent()) + pos.Line++ + } + p.output = append(p.output, strings.TrimSpace(line)...) + if i < len(comment.Comment)-1 { + p._requestNewline() + } + } + p.pos = pos } // Print any comments that occur after the last token, and a trailing newline func (p *printer) flush() { for _, c := range p.skippedComments { - p.printComment(c, p.curIndent()) + if !p.requestNewlinesForPos(c.Pos) { + p.requestSpace() + } + p.printComment(c) } - p.printToken("", noPos, wsDontCare) for p.curComment < len(p.comments) { - p.printComment(p.comments[p.curComment], p.curIndent()) + c := p.comments[p.curComment] + if !p.requestNewlinesForPos(c.Pos) { + p.requestSpace() + } + p.printComment(c) p.curComment++ } p.output = append(p.output, '\n') } // Print whitespace to pad from column l to column max -func (p *printer) pad(l, max int) { - l = max - l +func (p *printer) pad(l int) { if l > len(p.wsBuf) { p.wsBuf = make([]byte, l) for i := range p.wsBuf { @@ -330,7 +355,8 @@ func (p *printer) indent(i int) { p.indentList = append(p.indentList, i) } -func (p *printer) unindent() { +func (p *printer) unindent(pos scanner.Position) { + p.printEndOfLineCommentsBefore(pos) p.indentList = p.indentList[0 : len(p.indentList)-1] } diff --git a/parser/printer_test.go b/parser/printer_test.go index 9f26230..20fb747 100644 --- a/parser/printer_test.go +++ b/parser/printer_test.go @@ -195,7 +195,62 @@ module { // test ], //test } + //test2 +`, + }, + { + input: ` +/*test { + test: true, +}*/ + +test { +/*test: true,*/ +} + +// This +/* Is */ +// A + +// Multiline +// Comment + +test {} + +// This +/* Is */ +// A +// Trailing + +// Multiline +// Comment +`, + output: ` +/*test { + test: true, +}*/ + +test { + /*test: true,*/ +} + +// This +/* Is */ +// A + +// Multiline +// Comment + +test {} + +// This +/* Is */ +// A +// Trailing + +// Multiline +// Comment `, }, }