634 lines
12 KiB
Go
634 lines
12 KiB
Go
// Copyright 2017 Google Inc. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package parser
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"text/scanner"
|
|
)
|
|
|
|
var errTooManyErrors = errors.New("too many errors")
|
|
|
|
const maxErrors = 100
|
|
|
|
type ParseError struct {
|
|
Err error
|
|
Pos scanner.Position
|
|
}
|
|
|
|
func (e *ParseError) Error() string {
|
|
return fmt.Sprintf("%s: %s", e.Pos, e.Err)
|
|
}
|
|
|
|
func (p *parser) Parse() ([]Node, []error) {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
if r == errTooManyErrors {
|
|
return
|
|
}
|
|
panic(r)
|
|
}
|
|
}()
|
|
|
|
p.parseLines()
|
|
p.accept(scanner.EOF)
|
|
p.nodes = append(p.nodes, p.comments...)
|
|
sort.Sort(byPosition(p.nodes))
|
|
|
|
return p.nodes, p.errors
|
|
}
|
|
|
|
type parser struct {
|
|
scanner scanner.Scanner
|
|
tok rune
|
|
errors []error
|
|
comments []Node
|
|
nodes []Node
|
|
lines []int
|
|
}
|
|
|
|
func NewParser(filename string, r io.Reader) *parser {
|
|
p := &parser{}
|
|
p.lines = []int{0}
|
|
p.scanner.Init(r)
|
|
p.scanner.Error = func(sc *scanner.Scanner, msg string) {
|
|
p.errorf(msg)
|
|
}
|
|
p.scanner.Whitespace = 0
|
|
p.scanner.IsIdentRune = func(ch rune, i int) bool {
|
|
return ch > 0 && ch != ':' && ch != '#' && ch != '=' && ch != '+' && ch != '$' &&
|
|
ch != '\\' && ch != '(' && ch != ')' && ch != '{' && ch != '}' && ch != ';' &&
|
|
ch != '|' && ch != '?' && ch != '\r' && !isWhitespace(ch)
|
|
}
|
|
p.scanner.Mode = scanner.ScanIdents
|
|
p.scanner.Filename = filename
|
|
p.next()
|
|
return p
|
|
}
|
|
|
|
func (p *parser) Unpack(pos Pos) scanner.Position {
|
|
offset := int(pos)
|
|
line := sort.Search(len(p.lines), func(i int) bool { return p.lines[i] > offset }) - 1
|
|
return scanner.Position{
|
|
Filename: p.scanner.Filename,
|
|
Line: line + 1,
|
|
Column: offset - p.lines[line] + 1,
|
|
Offset: offset,
|
|
}
|
|
}
|
|
|
|
func (p *parser) pos() Pos {
|
|
pos := p.scanner.Position
|
|
if !pos.IsValid() {
|
|
pos = p.scanner.Pos()
|
|
}
|
|
return Pos(pos.Offset)
|
|
}
|
|
|
|
func (p *parser) errorf(format string, args ...interface{}) {
|
|
err := &ParseError{
|
|
Err: fmt.Errorf(format, args...),
|
|
Pos: p.scanner.Position,
|
|
}
|
|
p.errors = append(p.errors, err)
|
|
if len(p.errors) >= maxErrors {
|
|
panic(errTooManyErrors)
|
|
}
|
|
}
|
|
|
|
func (p *parser) accept(toks ...rune) bool {
|
|
for _, tok := range toks {
|
|
if p.tok != tok {
|
|
p.errorf("expected %s, found %s", scanner.TokenString(tok),
|
|
scanner.TokenString(p.tok))
|
|
return false
|
|
}
|
|
p.next()
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (p *parser) next() {
|
|
if p.tok != scanner.EOF {
|
|
p.tok = p.scanner.Scan()
|
|
for p.tok == '\r' {
|
|
p.tok = p.scanner.Scan()
|
|
}
|
|
}
|
|
if p.tok == '\n' {
|
|
p.lines = append(p.lines, p.scanner.Position.Offset+1)
|
|
}
|
|
}
|
|
|
|
func (p *parser) parseLines() {
|
|
for {
|
|
p.ignoreWhitespace()
|
|
|
|
if p.parseDirective() {
|
|
continue
|
|
}
|
|
|
|
ident := p.parseExpression('=', '?', ':', '#', '\n')
|
|
|
|
p.ignoreSpaces()
|
|
|
|
switch p.tok {
|
|
case '?':
|
|
p.accept('?')
|
|
if p.tok == '=' {
|
|
p.parseAssignment("?=", nil, ident)
|
|
} else {
|
|
p.errorf("expected = after ?")
|
|
}
|
|
case '+':
|
|
p.accept('+')
|
|
if p.tok == '=' {
|
|
p.parseAssignment("+=", nil, ident)
|
|
} else {
|
|
p.errorf("expected = after +")
|
|
}
|
|
case ':':
|
|
p.accept(':')
|
|
switch p.tok {
|
|
case '=':
|
|
p.parseAssignment(":=", nil, ident)
|
|
default:
|
|
p.parseRule(ident)
|
|
}
|
|
case '=':
|
|
p.parseAssignment("=", nil, ident)
|
|
case '#', '\n', scanner.EOF:
|
|
ident.TrimRightSpaces()
|
|
if v, ok := toVariable(ident); ok {
|
|
p.nodes = append(p.nodes, &v)
|
|
} else if !ident.Empty() {
|
|
p.errorf("expected directive, rule, or assignment after ident " + ident.Dump())
|
|
}
|
|
switch p.tok {
|
|
case scanner.EOF:
|
|
return
|
|
case '\n':
|
|
p.accept('\n')
|
|
case '#':
|
|
p.parseComment()
|
|
}
|
|
default:
|
|
p.errorf("expected assignment or rule definition, found %s\n",
|
|
p.scanner.TokenText())
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *parser) parseDirective() bool {
|
|
if p.tok != scanner.Ident || !isDirective(p.scanner.TokenText()) {
|
|
return false
|
|
}
|
|
|
|
d := p.scanner.TokenText()
|
|
pos := p.pos()
|
|
p.accept(scanner.Ident)
|
|
endPos := NoPos
|
|
|
|
expression := SimpleMakeString("", pos)
|
|
|
|
switch d {
|
|
case "endif", "endef", "else":
|
|
// Nothing
|
|
case "define":
|
|
expression, endPos = p.parseDefine()
|
|
default:
|
|
p.ignoreSpaces()
|
|
expression = p.parseExpression()
|
|
}
|
|
|
|
p.nodes = append(p.nodes, &Directive{
|
|
NamePos: pos,
|
|
Name: d,
|
|
Args: expression,
|
|
EndPos: endPos,
|
|
})
|
|
return true
|
|
}
|
|
|
|
func (p *parser) parseDefine() (*MakeString, Pos) {
|
|
value := SimpleMakeString("", p.pos())
|
|
|
|
loop:
|
|
for {
|
|
switch p.tok {
|
|
case scanner.Ident:
|
|
value.appendString(p.scanner.TokenText())
|
|
if p.scanner.TokenText() == "endef" {
|
|
p.accept(scanner.Ident)
|
|
break loop
|
|
}
|
|
p.accept(scanner.Ident)
|
|
case '\\':
|
|
p.parseEscape()
|
|
switch p.tok {
|
|
case '\n':
|
|
value.appendString(" ")
|
|
case scanner.EOF:
|
|
p.errorf("expected escaped character, found %s",
|
|
scanner.TokenString(p.tok))
|
|
break loop
|
|
default:
|
|
value.appendString(`\` + string(p.tok))
|
|
}
|
|
p.accept(p.tok)
|
|
//TODO: handle variables inside defines? result depends if
|
|
//define is used in make or rule context
|
|
//case '$':
|
|
// variable := p.parseVariable()
|
|
// value.appendVariable(variable)
|
|
case scanner.EOF:
|
|
p.errorf("unexpected EOF while looking for endef")
|
|
break loop
|
|
default:
|
|
value.appendString(p.scanner.TokenText())
|
|
p.accept(p.tok)
|
|
}
|
|
}
|
|
|
|
return value, p.pos()
|
|
}
|
|
|
|
func (p *parser) parseEscape() {
|
|
p.scanner.Mode = 0
|
|
p.accept('\\')
|
|
p.scanner.Mode = scanner.ScanIdents
|
|
}
|
|
|
|
func (p *parser) parseExpression(end ...rune) *MakeString {
|
|
value := SimpleMakeString("", p.pos())
|
|
|
|
endParen := false
|
|
for _, r := range end {
|
|
if r == ')' {
|
|
endParen = true
|
|
}
|
|
}
|
|
parens := 0
|
|
|
|
loop:
|
|
for {
|
|
if endParen && parens > 0 && p.tok == ')' {
|
|
parens--
|
|
value.appendString(")")
|
|
p.accept(')')
|
|
continue
|
|
}
|
|
|
|
for _, r := range end {
|
|
if p.tok == r {
|
|
break loop
|
|
}
|
|
}
|
|
|
|
switch p.tok {
|
|
case '\n':
|
|
break loop
|
|
case scanner.Ident:
|
|
value.appendString(p.scanner.TokenText())
|
|
p.accept(scanner.Ident)
|
|
case '\\':
|
|
p.parseEscape()
|
|
switch p.tok {
|
|
case '\n':
|
|
value.appendString(" ")
|
|
case scanner.EOF:
|
|
p.errorf("expected escaped character, found %s",
|
|
scanner.TokenString(p.tok))
|
|
return value
|
|
default:
|
|
value.appendString(`\` + string(p.tok))
|
|
}
|
|
p.accept(p.tok)
|
|
case '#':
|
|
p.parseComment()
|
|
break loop
|
|
case '$':
|
|
var variable Variable
|
|
variable = p.parseVariable()
|
|
value.appendVariable(variable)
|
|
case scanner.EOF:
|
|
break loop
|
|
case '(':
|
|
if endParen {
|
|
parens++
|
|
}
|
|
value.appendString("(")
|
|
p.accept('(')
|
|
default:
|
|
value.appendString(p.scanner.TokenText())
|
|
p.accept(p.tok)
|
|
}
|
|
}
|
|
|
|
if parens > 0 {
|
|
p.errorf("expected closing paren %s", value.Dump())
|
|
}
|
|
return value
|
|
}
|
|
|
|
func (p *parser) parseVariable() Variable {
|
|
pos := p.pos()
|
|
p.accept('$')
|
|
var name *MakeString
|
|
switch p.tok {
|
|
case '(':
|
|
return p.parseBracketedVariable('(', ')', pos)
|
|
case '{':
|
|
return p.parseBracketedVariable('{', '}', pos)
|
|
case '$':
|
|
name = SimpleMakeString("__builtin_dollar", NoPos)
|
|
case scanner.EOF:
|
|
p.errorf("expected variable name, found %s",
|
|
scanner.TokenString(p.tok))
|
|
default:
|
|
name = p.parseExpression(variableNameEndRunes...)
|
|
}
|
|
|
|
return p.nameToVariable(name)
|
|
}
|
|
|
|
func (p *parser) parseBracketedVariable(start, end rune, pos Pos) Variable {
|
|
p.accept(start)
|
|
name := p.parseExpression(end)
|
|
p.accept(end)
|
|
return p.nameToVariable(name)
|
|
}
|
|
|
|
func (p *parser) nameToVariable(name *MakeString) Variable {
|
|
return Variable{
|
|
Name: name,
|
|
}
|
|
}
|
|
|
|
func (p *parser) parseRule(target *MakeString) {
|
|
prerequisites, newLine := p.parseRulePrerequisites(target)
|
|
|
|
recipe := ""
|
|
recipePos := p.pos()
|
|
loop:
|
|
for {
|
|
if newLine {
|
|
if p.tok == '\t' {
|
|
p.accept('\t')
|
|
newLine = false
|
|
continue loop
|
|
} else if p.parseDirective() {
|
|
newLine = false
|
|
continue
|
|
} else {
|
|
break loop
|
|
}
|
|
}
|
|
|
|
newLine = false
|
|
switch p.tok {
|
|
case '\\':
|
|
p.parseEscape()
|
|
recipe += string(p.tok)
|
|
p.accept(p.tok)
|
|
case '\n':
|
|
newLine = true
|
|
recipe += "\n"
|
|
p.accept('\n')
|
|
case scanner.EOF:
|
|
break loop
|
|
default:
|
|
recipe += p.scanner.TokenText()
|
|
p.accept(p.tok)
|
|
}
|
|
}
|
|
|
|
if prerequisites != nil {
|
|
p.nodes = append(p.nodes, &Rule{
|
|
Target: target,
|
|
Prerequisites: prerequisites,
|
|
Recipe: recipe,
|
|
RecipePos: recipePos,
|
|
})
|
|
}
|
|
}
|
|
|
|
func (p *parser) parseRulePrerequisites(target *MakeString) (*MakeString, bool) {
|
|
newLine := false
|
|
|
|
p.ignoreSpaces()
|
|
|
|
prerequisites := p.parseExpression('#', '\n', ';', ':', '=')
|
|
|
|
switch p.tok {
|
|
case '\n':
|
|
p.accept('\n')
|
|
newLine = true
|
|
case '#':
|
|
p.parseComment()
|
|
newLine = true
|
|
case ';':
|
|
p.accept(';')
|
|
case ':':
|
|
p.accept(':')
|
|
if p.tok == '=' {
|
|
p.parseAssignment(":=", target, prerequisites)
|
|
return nil, true
|
|
} else {
|
|
more := p.parseExpression('#', '\n', ';')
|
|
prerequisites.appendMakeString(more)
|
|
}
|
|
case '=':
|
|
p.parseAssignment("=", target, prerequisites)
|
|
return nil, true
|
|
default:
|
|
p.errorf("unexpected token %s after rule prerequisites", scanner.TokenString(p.tok))
|
|
}
|
|
|
|
return prerequisites, newLine
|
|
}
|
|
|
|
func (p *parser) parseComment() {
|
|
pos := p.pos()
|
|
p.accept('#')
|
|
comment := ""
|
|
loop:
|
|
for {
|
|
switch p.tok {
|
|
case '\\':
|
|
p.parseEscape()
|
|
if p.tok == '\n' {
|
|
comment += "\n"
|
|
} else {
|
|
comment += "\\" + p.scanner.TokenText()
|
|
}
|
|
p.accept(p.tok)
|
|
case '\n':
|
|
p.accept('\n')
|
|
break loop
|
|
case scanner.EOF:
|
|
break loop
|
|
default:
|
|
comment += p.scanner.TokenText()
|
|
p.accept(p.tok)
|
|
}
|
|
}
|
|
|
|
p.comments = append(p.comments, &Comment{
|
|
CommentPos: pos,
|
|
Comment: comment,
|
|
})
|
|
}
|
|
|
|
func (p *parser) parseAssignment(t string, target *MakeString, ident *MakeString) {
|
|
// The value of an assignment is everything including and after the first
|
|
// non-whitespace character after the = until the end of the logical line,
|
|
// which may included escaped newlines
|
|
p.accept('=')
|
|
value := p.parseExpression()
|
|
value.TrimLeftSpaces()
|
|
if ident.EndsWith('+') && t == "=" {
|
|
ident.TrimRightOne()
|
|
t = "+="
|
|
}
|
|
|
|
ident.TrimRightSpaces()
|
|
|
|
p.nodes = append(p.nodes, &Assignment{
|
|
Name: ident,
|
|
Value: value,
|
|
Target: target,
|
|
Type: t,
|
|
})
|
|
}
|
|
|
|
type androidMkModule struct {
|
|
assignments map[string]string
|
|
}
|
|
|
|
type androidMkFile struct {
|
|
assignments map[string]string
|
|
modules []androidMkModule
|
|
includes []string
|
|
}
|
|
|
|
var directives = [...]string{
|
|
"define",
|
|
"else",
|
|
"endef",
|
|
"endif",
|
|
"ifdef",
|
|
"ifeq",
|
|
"ifndef",
|
|
"ifneq",
|
|
"include",
|
|
"-include",
|
|
}
|
|
|
|
var functions = [...]string{
|
|
"abspath",
|
|
"addprefix",
|
|
"addsuffix",
|
|
"basename",
|
|
"dir",
|
|
"notdir",
|
|
"subst",
|
|
"suffix",
|
|
"filter",
|
|
"filter-out",
|
|
"findstring",
|
|
"firstword",
|
|
"flavor",
|
|
"join",
|
|
"lastword",
|
|
"patsubst",
|
|
"realpath",
|
|
"shell",
|
|
"sort",
|
|
"strip",
|
|
"wildcard",
|
|
"word",
|
|
"wordlist",
|
|
"words",
|
|
"origin",
|
|
"foreach",
|
|
"call",
|
|
"info",
|
|
"error",
|
|
"warning",
|
|
"if",
|
|
"or",
|
|
"and",
|
|
"value",
|
|
"eval",
|
|
"file",
|
|
}
|
|
|
|
func init() {
|
|
sort.Strings(directives[:])
|
|
sort.Strings(functions[:])
|
|
}
|
|
|
|
func isDirective(s string) bool {
|
|
for _, d := range directives {
|
|
if s == d {
|
|
return true
|
|
} else if s < d {
|
|
return false
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isFunctionName(s string) bool {
|
|
for _, f := range functions {
|
|
if s == f {
|
|
return true
|
|
} else if s < f {
|
|
return false
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isWhitespace(ch rune) bool {
|
|
return ch == ' ' || ch == '\t' || ch == '\n'
|
|
}
|
|
|
|
func isValidVariableRune(ch rune) bool {
|
|
return ch != scanner.Ident && ch != ':' && ch != '=' && ch != '#'
|
|
}
|
|
|
|
var whitespaceRunes = []rune{' ', '\t', '\n'}
|
|
var variableNameEndRunes = append([]rune{':', '=', '#', ')', '}'}, whitespaceRunes...)
|
|
|
|
func (p *parser) ignoreSpaces() int {
|
|
skipped := 0
|
|
for p.tok == ' ' || p.tok == '\t' {
|
|
p.accept(p.tok)
|
|
skipped++
|
|
}
|
|
return skipped
|
|
}
|
|
|
|
func (p *parser) ignoreWhitespace() {
|
|
for isWhitespace(p.tok) {
|
|
p.accept(p.tok)
|
|
}
|
|
}
|