Initial bpfmt tool

bpfmt is based off gofmt, and formats a blueprint file to a standard
format.

Change-Id: I060c1b6030bc937a8db217eaed237d8792c29565

Blueprints

@@ -19,7 +19,9 @@ bootstrap_go_package {
 bootstrap_go_package {
     name:    "blueprint-parser",
     pkgPath: "blueprint/parser",
-    srcs:    ["blueprint/parser/parser.go"],
+    srcs:    ["blueprint/parser/parser.go",
+              "blueprint/parser/printer.go",
+              "blueprint/parser/sort.go"],
 }
 
 bootstrap_go_package {
@@ -59,3 +61,9 @@ bootstrap_go_binary {
     deps: ["blueprint", "blueprint-bootstrap"],
     srcs: ["blueprint/bootstrap/minibp/main.go"],
 }
+
+bootstrap_go_binary {
+    name: "bpfmt",
+    deps: ["blueprint-parser"],
+    srcs: ["blueprint/bpfmt/bpfmt.go"],
+}

blueprint/bpfmt/bpfmt.go

@@ -0,0 +1,176 @@
+// Mostly copied from Go's src/cmd/gofmt:
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"blueprint/parser"
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+var (
+	// main operation modes
+	list      = flag.Bool("l", false, "list files whose formatting differs from bpfmt's")
+	write     = flag.Bool("w", false, "write result to (source) file instead of stdout")
+	doDiff    = flag.Bool("d", false, "display diffs instead of rewriting files")
+	sortLists = flag.Bool("s", false, "sort arrays")
+)
+
+var (
+	exitCode = 0
+)
+
+func report(err error) {
+	fmt.Fprintln(os.Stderr, err)
+	exitCode = 2
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "usage: bpfmt [flags] [path ...]\n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+// If in == nil, the source is the contents of the file with the given filename.
+func processFile(filename string, in io.Reader, out io.Writer) error {
+	if in == nil {
+		f, err := os.Open(filename)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		in = f
+	}
+
+	src, err := ioutil.ReadAll(in)
+	if err != nil {
+		return err
+	}
+
+	r := bytes.NewBuffer(src)
+
+	file, errs := parser.Parse(filename, r, parser.NewScope(nil))
+	if len(errs) > 0 {
+		for _, err := range errs {
+			fmt.Fprintln(os.Stderr, err)
+		}
+		return fmt.Errorf("%d parsing errors", len(errs))
+	}
+
+	if *sortLists {
+		parser.SortLists(file)
+	}
+
+	res, err := parser.Print(file)
+	if err != nil {
+		return err
+	}
+
+	if !bytes.Equal(src, res) {
+		// formatting has changed
+		if *list {
+			fmt.Fprintln(out, filename)
+		}
+		if *write {
+			err = ioutil.WriteFile(filename, res, 0644)
+			if err != nil {
+				return err
+			}
+		}
+		if *doDiff {
+			data, err := diff(src, res)
+			if err != nil {
+				return fmt.Errorf("computing diff: %s", err)
+			}
+			fmt.Printf("diff %s bpfmt/%s\n", filename, filename)
+			out.Write(data)
+		}
+	}
+
+	if !*list && !*write && !*doDiff {
+		_, err = out.Write(res)
+	}
+
+	return err
+}
+
+func visitFile(path string, f os.FileInfo, err error) error {
+	if err == nil && f.Name() == "Blueprints" {
+		err = processFile(path, nil, os.Stdout)
+	}
+	if err != nil {
+		report(err)
+	}
+	return nil
+}
+
+func walkDir(path string) {
+	filepath.Walk(path, visitFile)
+}
+
+func main() {
+	flag.Parse()
+
+	if flag.NArg() == 0 {
+		if *write {
+			fmt.Fprintln(os.Stderr, "error: cannot use -w with standard input")
+			exitCode = 2
+			return
+		}
+		if err := processFile("<standard input>", os.Stdin, os.Stdout); err != nil {
+			report(err)
+		}
+		return
+	}
+
+	for i := 0; i < flag.NArg(); i++ {
+		path := flag.Arg(i)
+		switch dir, err := os.Stat(path); {
+		case err != nil:
+			report(err)
+		case dir.IsDir():
+			walkDir(path)
+		default:
+			if err := processFile(path, nil, os.Stdout); err != nil {
+				report(err)
+			}
+		}
+	}
+}
+
+func diff(b1, b2 []byte) (data []byte, err error) {
+	f1, err := ioutil.TempFile("", "bpfmt")
+	if err != nil {
+		return
+	}
+	defer os.Remove(f1.Name())
+	defer f1.Close()
+
+	f2, err := ioutil.TempFile("", "bpfmt")
+	if err != nil {
+		return
+	}
+	defer os.Remove(f2.Name())
+	defer f2.Close()
+
+	f1.Write(b1)
+	f2.Write(b2)
+
+	data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
+	if len(data) > 0 {
+		// diff exits with a non-zero status when the files don't match.
+		// Ignore that failure as long as we get output.
+		err = nil
+	}
+	return
+
+}

blueprint/parser/printer.go

@@ -0,0 +1,267 @@
+package parser
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"text/scanner"
+)
+
+var noPos = scanner.Position{}
+
+type whitespace int
+
+const (
+	wsNone whitespace = iota
+	wsBoth
+	wsAfter
+	wsBefore
+	wsMaybe
+)
+
+type printer struct {
+	defs     []Definition
+	comments []Comment
+
+	curComment int
+	prev       scanner.Position
+
+	ws whitespace
+
+	output []byte
+
+	indentList     []int
+	wsBuf          []byte
+	forceLineBreak int
+}
+
+func newPrinter(file *File) *printer {
+	return &printer{
+		defs:       file.Defs,
+		comments:   file.Comments,
+		indentList: []int{0},
+	}
+}
+
+func Print(file *File) ([]byte, error) {
+	p := newPrinter(file)
+
+	for _, def := range p.defs {
+		p.printDef(def)
+	}
+	p.flush()
+	return p.output, nil
+}
+
+func (p *printer) Print() ([]byte, error) {
+	for _, def := range p.defs {
+		p.printDef(def)
+	}
+	p.flush()
+	return p.output, nil
+}
+
+func (p *printer) printDef(def Definition) {
+	if assignment, ok := def.(*Assignment); ok {
+		p.printAssignment(assignment)
+	} else if module, ok := def.(*Module); ok {
+		p.printModule(module)
+	} else {
+		panic("Unknown definition")
+	}
+}
+
+func (p *printer) printAssignment(assignment *Assignment) {
+	p.printToken(assignment.Name.Name, assignment.Name.Pos, wsMaybe)
+	p.printToken("=", assignment.Pos, wsBoth)
+	p.printValue(assignment.Value)
+}
+
+func (p *printer) printModule(module *Module) {
+	p.printToken(module.Type.Name, module.Type.Pos, wsBoth)
+	p.printMap(module.Properties, module.LbracePos, module.RbracePos)
+	p.forceLineBreak = 2
+}
+
+func (p *printer) printValue(value Value) {
+	if value.Variable != "" {
+		p.printToken(value.Variable, value.Pos, wsMaybe)
+	} else if value.Expression != nil {
+		p.printExpression(*value.Expression)
+	} else {
+		switch value.Type {
+		case Bool:
+			var s string
+			if value.BoolValue {
+				s = "true"
+			} else {
+				s = "false"
+			}
+			p.printToken(s, value.Pos, wsMaybe)
+		case String:
+			p.printToken(strconv.Quote(value.StringValue), value.Pos, wsMaybe)
+		case List:
+			p.printList(value.ListValue, value.Pos, value.EndPos)
+		case Map:
+			p.printMap(value.MapValue, value.Pos, value.EndPos)
+		default:
+			panic(fmt.Errorf("bad property type: %d", value.Type))
+		}
+	}
+}
+
+func (p *printer) printList(list []Value, pos, endPos scanner.Position) {
+	p.printToken("[", pos, wsBefore)
+	if len(list) > 1 || pos.Line != endPos.Line {
+		p.forceLineBreak = 1
+		p.indent(p.curIndent() + 4)
+		for _, value := range list {
+			p.printValue(value)
+			p.printToken(",", noPos, wsAfter)
+			p.forceLineBreak = 1
+		}
+		p.unindent()
+	} else {
+		for _, value := range list {
+			p.printValue(value)
+		}
+	}
+	p.printToken("]", endPos, wsAfter)
+}
+
+func (p *printer) printMap(list []*Property, pos, endPos scanner.Position) {
+	p.printToken("{", pos, wsBefore)
+	if len(list) > 0 || pos.Line != endPos.Line {
+		p.forceLineBreak = 1
+		p.indent(p.curIndent() + 4)
+		for _, prop := range list {
+			p.printProperty(prop)
+			p.printToken(",", noPos, wsAfter)
+			p.forceLineBreak = 1
+		}
+		p.unindent()
+	}
+	p.printToken("}", endPos, wsAfter)
+}
+
+func (p *printer) printExpression(expression Expression) {
+	p.printValue(expression.Args[0])
+	p.printToken(string(expression.Operator), expression.Pos, wsBoth)
+	p.printValue(expression.Args[1])
+}
+
+func (p *printer) printProperty(property *Property) {
+	p.printToken(property.Name.Name, property.Name.Pos, wsMaybe)
+	p.printToken(":", property.Pos, wsAfter)
+	p.printValue(property.Value)
+}
+
+// Print a single token, including any necessary comments or whitespace between
+// this token and the previously printed token
+func (p *printer) printToken(s string, pos scanner.Position, ws whitespace) {
+	p.printComments(pos, false)
+	if p.forceLineBreak > 0 || p.prev.Line != 0 && pos.Line > p.prev.Line {
+		p.printLineBreak(pos.Line - p.prev.Line)
+	} else {
+		p.printWhitespace(ws)
+	}
+	p.output = append(p.output, s...)
+	p.ws = ws
+	if pos != noPos {
+		p.prev = pos
+	}
+}
+
+// Print all comments that occur before position pos
+func (p *printer) printComments(pos scanner.Position, flush bool) {
+	for p.curComment < len(p.comments) && p.comments[p.curComment].Pos.Offset < pos.Offset {
+		p.printComment(p.comments[p.curComment])
+		p.curComment++
+	}
+}
+
+// Print a single comment, which may be a multi-line comment
+func (p *printer) printComment(comment Comment) {
+	commentLines := strings.Split(comment.Comment, "\n")
+	pos := comment.Pos
+	for _, line := range commentLines {
+		if p.prev.Line != 0 && pos.Line > p.prev.Line {
+			// Comment is on the next line
+			p.printLineBreak(pos.Line - p.prev.Line)
+		} else {
+			// Comment is on the current line
+			p.printWhitespace(wsBoth)
+		}
+		p.output = append(p.output, strings.TrimSpace(line)...)
+		p.prev = pos
+		pos.Line++
+	}
+	p.ws = wsBoth
+}
+
+// Print one or two line breaks.  n <= 0 is only valid if forceLineBreak is set,
+// n > 2 is collapsed to a single blank line.
+func (p *printer) printLineBreak(n int) {
+	if n > 2 {
+		n = 2
+	}
+
+	if p.forceLineBreak > n {
+		if p.forceLineBreak == 0 {
+			panic("unexpected 0 line break")
+		}
+		n = p.forceLineBreak
+	}
+
+	for i := 0; i < n; i++ {
+		p.output = append(p.output, '\n')
+	}
+
+	p.pad(0, p.curIndent())
+	p.forceLineBreak = 0
+	p.ws = wsNone
+}
+
+// Print any necessary whitespace before the next token, based on the current
+// ws value and the previous ws value.
+func (p *printer) printWhitespace(ws whitespace) {
+	if (ws == wsBefore || ws == wsBoth) && p.ws != wsNone ||
+		ws == wsMaybe && (p.ws == wsMaybe || p.ws == wsAfter || p.ws == wsBoth) {
+
+		p.output = append(p.output, ' ')
+	}
+	p.ws = ws
+}
+
+// Print any comments that occur after the last token, and a trailing newline
+func (p *printer) flush() {
+	for p.curComment < len(p.comments) {
+		p.printComment(p.comments[p.curComment])
+		p.curComment++
+	}
+	p.output = append(p.output, '\n')
+}
+
+// Print whitespace to pad from column l to column max
+func (p *printer) pad(l, max int) {
+	l = max - l
+	if l > len(p.wsBuf) {
+		p.wsBuf = make([]byte, l)
+		for i := range p.wsBuf {
+			p.wsBuf[i] = ' '
+		}
+	}
+	p.output = append(p.output, p.wsBuf[0:l]...)
+}
+
+func (p *printer) indent(i int) {
+	p.indentList = append(p.indentList, i)
+}
+
+func (p *printer) unindent() {
+	p.indentList = p.indentList[0 : len(p.indentList)-1]
+}
+
+func (p *printer) curIndent() int {
+	return p.indentList[len(p.indentList)-1]
+}

blueprint/parser/printer_test.go

@@ -0,0 +1,190 @@
+package parser
+
+import (
+	"bytes"
+	"testing"
+)
+
+var validPrinterTestCases = []struct {
+	input  string
+	output string
+}{
+	{
+		input: `
+foo {}
+`,
+		output: `
+foo {}
+`,
+	},
+	{
+		input: `
+foo{name: "abc",}
+`,
+		output: `
+foo {
+    name: "abc",
+}
+`,
+	},
+	{
+		input: `
+			foo {
+				stuff: ["asdf", "jkl;", "qwert",
+					"uiop", "bnm,"]
+			}
+			`,
+		output: `
+foo {
+    stuff: [
+        "asdf",
+        "bnm,",
+        "jkl;",
+        "qwert",
+        "uiop",
+    ],
+}
+`,
+	},
+	{
+		input: `
+		foo {
+			stuff: {
+				isGood: true,
+				name: "bar"
+			}
+		}
+		`,
+		output: `
+foo {
+    stuff: {
+        isGood: true,
+        name: "bar",
+    },
+}
+`,
+	},
+	{
+		input: `
+// comment1
+foo {
+	// comment2
+	isGood: true,  // comment3
+}
+`,
+		output: `
+// comment1
+foo {
+    // comment2
+    isGood: true, // comment3
+}
+`,
+	},
+	{
+		input: `
+foo {
+	name: "abc",
+}
+
+bar {
+	name: "def",
+}
+		`,
+		output: `
+foo {
+    name: "abc",
+}
+
+bar {
+    name: "def",
+}
+`,
+	},
+	{
+		input: `
+foo = "stuff"
+bar = foo
+baz = foo + bar
+`,
+		output: `
+foo = "stuff"
+bar = foo
+baz = foo + bar
+`,
+	},
+	{
+		input: `
+//test
+test /* test */{
+    srcs: [
+        /*"blueprint/bootstrap/bootstrap.go",
+    "blueprint/bootstrap/cleanup.go",*/
+        "blueprint/bootstrap/command.go",
+        "blueprint/bootstrap/doc.go", //doc.go
+        "blueprint/bootstrap/config.go", //config.go
+    ],
+    deps: ["libabc"],
+    incs: []
+} //test
+//test
+test2{
+}
+
+
+//test3
+`,
+		output: `
+//test
+test /* test */ {
+    srcs: [
+        /*"blueprint/bootstrap/bootstrap.go",
+        "blueprint/bootstrap/cleanup.go",*/
+        "blueprint/bootstrap/command.go",
+        "blueprint/bootstrap/config.go", //config.go
+        "blueprint/bootstrap/doc.go", //doc.go
+    ],
+    deps: ["libabc"],
+    incs: [],
+} //test
+
+//test
+test2 {
+}
+
+//test3
+`,
+	},
+}
+
+func TestPrinter(t *testing.T) {
+	for _, testCase := range validPrinterTestCases {
+		in := testCase.input[1:]
+		expected := testCase.output[1:]
+
+		r := bytes.NewBufferString(in)
+		file, errs := Parse("", r, NewScope(nil))
+		if len(errs) != 0 {
+			t.Errorf("test case: %s", in)
+			t.Errorf("unexpected errors:")
+			for _, err := range errs {
+				t.Errorf("  %s", err)
+			}
+			t.FailNow()
+		}
+
+		SortLists(file)
+
+		got, err := Print(file)
+		if err != nil {
+			t.Errorf("test case: %s", in)
+			t.Errorf("unexpected error: %s", err)
+			t.FailNow()
+		}
+
+		if string(got) != expected {
+			t.Errorf("test case: %s", in)
+			t.Errorf("  expected: %s", expected)
+			t.Errorf("       got: %s", string(got))
+		}
+	}
+}

blueprint/parser/sort.go

@@ -0,0 +1,168 @@
+package parser
+
+import (
+	"sort"
+	"text/scanner"
+)
+
+func SortLists(file *File) {
+	for _, def := range file.Defs {
+		if assignment, ok := def.(*Assignment); ok {
+			sortListsInValue(assignment.Value, file)
+		} else if module, ok := def.(*Module); ok {
+			for _, prop := range module.Properties {
+				sortListsInValue(prop.Value, file)
+			}
+		}
+	}
+	sort.Sort(commentsByOffset(file.Comments))
+}
+
+func SortList(file *File, value Value) {
+	for i := 0; i < len(value.ListValue); i++ {
+		// Find a set of values on contiguous lines
+		line := value.ListValue[i].Pos.Line
+		var j int
+		for j = i + 1; j < len(value.ListValue); j++ {
+			if value.ListValue[j].Pos.Line > line+1 {
+				break
+			}
+			line = value.ListValue[j].Pos.Line
+		}
+
+		nextPos := value.EndPos
+		if j < len(value.ListValue) {
+			nextPos = value.ListValue[j].Pos
+		}
+		sortSubList(value.ListValue[i:j], nextPos, file)
+		i = j - 1
+	}
+}
+
+func ListIsSorted(value Value) bool {
+	for i := 0; i < len(value.ListValue); i++ {
+		// Find a set of values on contiguous lines
+		line := value.ListValue[i].Pos.Line
+		var j int
+		for j = i + 1; j < len(value.ListValue); j++ {
+			if value.ListValue[j].Pos.Line > line+1 {
+				break
+			}
+			line = value.ListValue[j].Pos.Line
+		}
+
+		if !subListIsSorted(value.ListValue[i:j]) {
+			return false
+		}
+		i = j - 1
+	}
+
+	return true
+}
+
+func sortListsInValue(value Value, file *File) {
+	if value.Variable != "" {
+		return
+	}
+
+	if value.Expression != nil {
+		sortListsInValue(value.Expression.Args[0], file)
+		sortListsInValue(value.Expression.Args[1], file)
+		return
+	}
+
+	if value.Type == Map {
+		for _, p := range value.MapValue {
+			sortListsInValue(p.Value, file)
+		}
+		return
+	} else if value.Type != List {
+		return
+	}
+
+	SortList(file, value)
+}
+
+func sortSubList(values []Value, nextPos scanner.Position, file *File) {
+	l := make(elemList, len(values))
+	for i, v := range values {
+		if v.Type != String {
+			panic("list contains non-string element")
+		}
+		n := nextPos
+		if i < len(values)-1 {
+			n = values[i+1].Pos
+		}
+		l[i] = elem{v.StringValue, i, v.Pos, n}
+	}
+
+	sort.Sort(l)
+
+	copyValues := append([]Value{}, values...)
+	copyComments := append([]Comment{}, file.Comments...)
+
+	curPos := values[0].Pos
+	for i, e := range l {
+		values[i] = copyValues[e.i]
+		values[i].Pos = curPos
+		for j, c := range copyComments {
+			if c.Pos.Offset > e.pos.Offset && c.Pos.Offset < e.nextPos.Offset {
+				file.Comments[j].Pos.Line = curPos.Line
+				file.Comments[j].Pos.Offset += values[i].Pos.Offset - e.pos.Offset
+			}
+		}
+
+		curPos.Offset += e.nextPos.Offset - e.pos.Offset
+		curPos.Line++
+	}
+}
+
+func subListIsSorted(values []Value) bool {
+	prev := ""
+	for _, v := range values {
+		if v.Type != String {
+			panic("list contains non-string element")
+		}
+		if prev > v.StringValue {
+			return false
+		}
+		prev = v.StringValue
+	}
+
+	return true
+}
+
+type elem struct {
+	s       string
+	i       int
+	pos     scanner.Position
+	nextPos scanner.Position
+}
+
+type elemList []elem
+
+func (l elemList) Len() int {
+	return len(l)
+}
+
+func (l elemList) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}
+
+func (l elemList) Less(i, j int) bool {
+	return l[i].s < l[j].s
+}
+
+type commentsByOffset []Comment
+
+func (l commentsByOffset) Len() int {
+	return len(l)
+}
+
+func (l commentsByOffset) Less(i, j int) bool {
+	return l[i].Pos.Offset < l[j].Pos.Offset
+}
+
+func (l commentsByOffset) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}