tequilaOS/platform_build_blueprint
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add CommentGroups

Browse source 
Determining which comments are contiguous is difficult once they have
been parsed into an out-of-band comment list, as any intervening nodes
are in a separate structure.  Group the comments into CommentGroups
during the parsing stage instead.

Change-Id: I9444c58e75333b7521b58dbfbd36ff29d139b6e3
This commit is contained in:
Colin Cross 2016-06-10 17:27:12 -07:00
parent 1ead6452b5
commit 1e73794d42
6 changed files with 154 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
parser/ast.go
View file

@ -316,6 +316,13 @@ func (x *Bool) Type() Type {
return BoolType
}
type CommentGroup struct {
Comments []*Comment
}
func (x *CommentGroup) Pos() scanner.Position { return x.Comments[0].Pos() }
func (x *CommentGroup) End() scanner.Position { return x.Comments[len(x.Comments)-1].End() }
type Comment struct {
Comment []string
Slash scanner.Position

20
parser/parser.go
View file

@ -40,7 +40,7 @@ func (e *ParseError) Error() string {
type File struct {
Name string
Defs []Definition
Comments []Comment
Comments []*CommentGroup
}
func (f *File) Pos() scanner.Position {
@ -103,7 +103,7 @@ type parser struct {
tok rune
errors []error
scope *Scope
comments []Comment
comments []*CommentGroup
eval bool
}
@ -154,10 +154,18 @@ func (p *parser) accept(toks ...rune) bool {
func (p *parser) next() {
if p.tok != scanner.EOF {
p.tok = p.scanner.Scan()
for p.tok == scanner.Comment {
lines := strings.Split(p.scanner.TokenText(), "\n")
p.comments = append(p.comments, Comment{lines, p.scanner.Position})
p.tok = p.scanner.Scan()
if p.tok == scanner.Comment {
var comments []*Comment
for p.tok == scanner.Comment {
lines := strings.Split(p.scanner.TokenText(), "\n")
if len(comments) > 0 && p.scanner.Position.Line > comments[len(comments)-1].End().Line+1 {
p.comments = append(p.comments, &CommentGroup{Comments: comments})
comments = nil
}
comments = append(comments, &Comment{lines, p.scanner.Position})
p.tok = p.scanner.Scan()
}
p.comments = append(p.comments, &CommentGroup{Comments: comments})
}
}
return

98
parser/parser_test.go
View file

@ -32,7 +32,7 @@ func mkpos(offset, line, column int) scanner.Position {
var validParseTestCases = []struct {
input string
defs []Definition
comments []Comment
comments []*CommentGroup
}{
{`
foo {}
@ -240,22 +240,38 @@ var validParseTestCases = []struct {
},
},
},
[]Comment{
Comment{
Comment: []string{"// comment1"},
Slash: mkpos(3, 2, 3),
[]*CommentGroup{
{
Comments: []*Comment{
&Comment{
Comment: []string{"// comment1"},
Slash: mkpos(3, 2, 3),
},
},
},
Comment{
Comment: []string{"/* test */"},
Slash: mkpos(21, 3, 7),
{
Comments: []*Comment{
&Comment{
Comment: []string{"/* test */"},
Slash: mkpos(21, 3, 7),
},
},
},
Comment{
Comment: []string{"// comment2"},
Slash: mkpos(37, 4, 4),
{
Comments: []*Comment{
&Comment{
Comment: []string{"// comment2"},
Slash: mkpos(37, 4, 4),
},
},
},
Comment{
Comment: []string{"// comment3"},
Slash: mkpos(67, 5, 19),
{
Comments: []*Comment{
&Comment{
Comment: []string{"// comment3"},
Slash: mkpos(67, 5, 19),
},
},
},
},
},
@ -541,6 +557,60 @@ var validParseTestCases = []struct {
},
nil,
},
{`
// comment1
// comment2
/* comment3
comment4 */
// comment5
/* comment6 */ /* comment7 */ // comment8
`,
nil,
[]*CommentGroup{
{
Comments: []*Comment{
&Comment{
Comment: []string{"// comment1"},
Slash: mkpos(3, 2, 3),
},
&Comment{
Comment: []string{"// comment2"},
Slash: mkpos(17, 3, 3),
},
},
},
{
Comments: []*Comment{
&Comment{
Comment: []string{"/* comment3", " comment4 */"},
Slash: mkpos(32, 5, 3),
},
&Comment{
Comment: []string{"// comment5"},
Slash: mkpos(63, 7, 3),
},
},
},
{
Comments: []*Comment{
&Comment{
Comment: []string{"/* comment6 */"},
Slash: mkpos(78, 9, 3),
},
&Comment{
Comment: []string{"/* comment7 */"},
Slash: mkpos(93, 9, 18),
},
&Comment{
Comment: []string{"// comment8"},
Slash: mkpos(108, 9, 33),
},
},
},
},
},
}
func TestParseValidInput(t *testing.T) {

71
parser/printer.go
View file

@ -26,7 +26,7 @@ var noPos scanner.Position
type printer struct {
defs []Definition
comments []Comment
comments []*CommentGroup
curComment int
@ -40,7 +40,7 @@ type printer struct {
indentList []int
wsBuf []byte
skippedComments []Comment
skippedComments *CommentGroup
}
func newPrinter(file *File) *printer {
@ -206,12 +206,14 @@ func (p *printer) printToken(s string, pos scanner.Position) {
// Print any in-line (single line /* */) comments that appear _before_ pos
func (p *printer) printInLineCommentsBefore(pos scanner.Position) {
for p.curComment < len(p.comments) && p.comments[p.curComment].Slash.Offset < pos.Offset {
for p.curComment < len(p.comments) && p.comments[p.curComment].Pos().Offset < pos.Offset {
c := p.comments[p.curComment]
if c.Comment[0][0:2] == "//" || len(c.Comment) > 1 {
p.skippedComments = append(p.skippedComments, c)
if c.Comments[0].Comment[0][0:2] == "//" || len(c.Comments[0].Comment) > 1 {
if p.skippedComments != nil {
panic("multiple skipped comments")
}
p.skippedComments = c
} else {
p.flushSpace()
p.printComment(c)
p.requestSpace()
}
@ -222,19 +224,13 @@ func (p *printer) printInLineCommentsBefore(pos scanner.Position) {
// Print any comments, including end of line comments, that appear _before_ the line specified
// by pos
func (p *printer) printEndOfLineCommentsBefore(pos scanner.Position) {
for _, c := range p.skippedComments {
if !p.requestNewlinesForPos(c.Slash) {
p.requestSpace()
}
p.printComment(c)
if p.skippedComments != nil {
p.printComment(p.skippedComments)
p._requestNewline()
p.skippedComments = nil
}
p.skippedComments = []Comment{}
for p.curComment < len(p.comments) && p.comments[p.curComment].Slash.Line < pos.Line {
for p.curComment < len(p.comments) && p.comments[p.curComment].Pos().Line < pos.Line {
c := p.comments[p.curComment]
if !p.requestNewlinesForPos(c.Slash) {
p.requestSpace()
}
p.printComment(c)
p._requestNewline()
p.curComment++
@ -300,39 +296,38 @@ func (p *printer) flushSpace() {
}
// Print a single comment, which may be a multi-line comment
func (p *printer) printComment(comment Comment) {
pos := comment.Slash
for i, line := range comment.Comment {
line = strings.TrimRightFunc(line, unicode.IsSpace)
p.flushSpace()
if i != 0 {
lineIndent := strings.IndexFunc(line, func(r rune) bool { return !unicode.IsSpace(r) })
lineIndent = max(lineIndent, p.curIndent())
p.pad(lineIndent - p.curIndent())
pos.Line++
func (p *printer) printComment(cg *CommentGroup) {
for _, comment := range cg.Comments {
if !p.requestNewlinesForPos(comment.Pos()) {
p.requestSpace()
}
p.output = append(p.output, strings.TrimSpace(line)...)
if i < len(comment.Comment)-1 {
p._requestNewline()
for i, line := range comment.Comment {
line = strings.TrimRightFunc(line, unicode.IsSpace)
p.flushSpace()
if i != 0 {
lineIndent := strings.IndexFunc(line, func(r rune) bool { return !unicode.IsSpace(r) })
lineIndent = max(lineIndent, p.curIndent())
p.pad(lineIndent - p.curIndent())
}
p.output = append(p.output, strings.TrimSpace(line)...)
if i < len(comment.Comment)-1 {
p._requestNewline()
}
}
p.pos = comment.End()
}
p.pos = pos
}
// Print any comments that occur after the last token, and a trailing newline
func (p *printer) flush() {
for _, c := range p.skippedComments {
if !p.requestNewlinesForPos(c.Slash) {
if p.skippedComments != nil {
if !p.requestNewlinesForPos(p.skippedComments.Pos()) {
p.requestSpace()
}
p.printComment(c)
p.printComment(p.skippedComments)
}
for p.curComment < len(p.comments) {
c := p.comments[p.curComment]
if !p.requestNewlinesForPos(c.Slash) {
p.requestSpace()
}
p.printComment(c)
p.printComment(p.comments[p.curComment])
p.curComment++
}
p.output = append(p.output, '\n')

6
parser/printer_test.go
View file

@ -205,8 +205,8 @@ test /* test */ {
deps: ["libabc"],
incs: [],
} //test
//test
test2 {
}
@ -253,7 +253,7 @@ test {
}
// This
/* Is */
/* Is *//* A */ // A
// A
// Multiline
@ -279,7 +279,7 @@ test {
}
// This
/* Is */
/* Is */ /* A */ // A
// A
// Multiline

17
parser/sort.go
View file

@ -107,7 +107,16 @@ func sortSubList(values []Expression, nextPos scanner.Position, file *File) {
sort.Sort(l)
copyValues := append([]Expression{}, values...)
copyComments := append([]Comment{}, file.Comments...)
copyComments := make([]*CommentGroup, len(file.Comments))
for i := range file.Comments {
cg := *file.Comments[i]
cg.Comments = make([]*Comment, len(cg.Comments))
for j := range file.Comments[i].Comments {
c := *file.Comments[i].Comments[j]
cg.Comments[j] = &c
}
copyComments[i] = &cg
}
curPos := values[0].Pos()
for i, e := range l {
@ -115,8 +124,8 @@ func sortSubList(values []Expression, nextPos scanner.Position, file *File) {
values[i].(*String).LiteralPos = curPos
for j, c := range copyComments {
if c.Pos().Offset > e.pos.Offset && c.Pos().Offset < e.nextPos.Offset {
file.Comments[j].Slash.Line = curPos.Line
file.Comments[j].Slash.Offset += values[i].Pos().Offset - e.pos.Offset
file.Comments[j].Comments[0].Slash.Line = curPos.Line
file.Comments[j].Comments[0].Slash.Offset += values[i].Pos().Offset - e.pos.Offset
}
}
@ -162,7 +171,7 @@ func (l elemList) Less(i, j int) bool {
return l[i].s < l[j].s
}
type commentsByOffset []Comment
type commentsByOffset []*CommentGroup
func (l commentsByOffset) Len() int {
return len(l)