Add support to extend commands bpfix and androidmk

Allows our partners to maintain partner-specific fixes in their repos. Converts most of androidmk into a library similar to bpfix. Makes some methods and types public for use by extended versions.

Bug:140727544
Test: Ran unit test cases &&
  ran test conversions with sample
Change-Id: I7e1fbf3a6a7a8bd47334f43fe3eb68cbbd3426c1

androidmk/Android.bp

@@ -19,12 +19,23 @@
 blueprint_go_binary {
     name: "androidmk",
     srcs: [
-        "cmd/androidmk/android.go",
-        "cmd/androidmk/androidmk.go",
-        "cmd/androidmk/values.go",
+        "cmd/androidmk.go",
+    ],
+    deps: [
+        "androidmk-lib",
+    ],
+}
+
+bootstrap_go_package {
+    name: "androidmk-lib",
+    pkgPath: "android/soong/androidmk/androidmk",
+    srcs: [
+        "androidmk/android.go",
+        "androidmk/androidmk.go",
+        "androidmk/values.go",
     ],
     testSrcs: [
-        "cmd/androidmk/androidmk_test.go",
+        "androidmk/androidmk_test.go",
     ],
     deps: [
         "androidmk-parser",

androidmk/androidmk/android.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package androidmk
 
 import (
 	"android/soong/android"

androidmk/androidmk/androidmk.go

@@ -12,14 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package androidmk
 
 import (
 	"bytes"
-	"flag"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"strings"
 	"text/scanner"
 
@@ -30,13 +27,6 @@ import (
 	bpparser "github.com/google/blueprint/parser"
 )
 
-var usage = func() {
-	fmt.Fprintf(os.Stderr, "usage: androidmk [flags] <inputFile>\n"+
-		"\nandroidmk parses <inputFile> as an Android.mk file and attempts to output an analogous Android.bp file (to standard out)\n")
-	flag.PrintDefaults()
-	os.Exit(1)
-}
-
 // TODO: non-expanded variables with expressions
 
 type bpFile struct {
@@ -118,31 +108,7 @@ type conditional struct {
 	eq   bool
 }
 
-func main() {
-	flag.Usage = usage
-	flag.Parse()
-	if len(flag.Args()) != 1 {
-		usage()
-	}
-	filePathToRead := flag.Arg(0)
-	b, err := ioutil.ReadFile(filePathToRead)
-	if err != nil {
-		fmt.Println(err.Error())
-		return
-	}
-
-	output, errs := convertFile(os.Args[1], bytes.NewBuffer(b))
-	if len(errs) > 0 {
-		for _, err := range errs {
-			fmt.Fprintln(os.Stderr, "ERROR: ", err)
-		}
-		os.Exit(1)
-	}
-
-	fmt.Print(output)
-}
-
-func convertFile(filename string, buffer *bytes.Buffer) (string, []error) {
+func ConvertFile(filename string, buffer *bytes.Buffer) (string, []error) {
 	p := mkparser.NewParser(filename, buffer)
 
 	nodes, errs := p.Parse()

androidmk/androidmk/androidmk_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package androidmk
 
 import (
 	"bytes"
@@ -1350,7 +1350,7 @@ func TestEndToEnd(t *testing.T) {
 			t.Error(err)
 		}
 
-		got, errs := convertFile(fmt.Sprintf("<testcase %d>", i), bytes.NewBufferString(test.in))
+		got, errs := ConvertFile(fmt.Sprintf("<testcase %d>", i), bytes.NewBufferString(test.in))
 		if len(errs) > 0 {
 			t.Errorf("Unexpected errors: %q", errs)
 			continue

androidmk/androidmk/values.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package androidmk
 
 import (
 	"fmt"

androidmk/cmd/androidmk.go

@@ -0,0 +1,56 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"android/soong/androidmk/androidmk"
+)
+
+var usage = func() {
+	fmt.Fprintf(os.Stderr, "usage: androidmk [flags] <inputFile>\n"+
+		"\nandroidmk parses <inputFile> as an Android.mk file and attempts to output an analogous Android.bp file (to standard out)\n")
+	flag.PrintDefaults()
+	os.Exit(1)
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if len(flag.Args()) != 1 {
+		usage()
+	}
+	filePathToRead := flag.Arg(0)
+	b, err := ioutil.ReadFile(filePathToRead)
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	output, errs := androidmk.ConvertFile(os.Args[1], bytes.NewBuffer(b))
+	if len(errs) > 0 {
+		for _, err := range errs {
+			fmt.Fprintln(os.Stderr, "ERROR: ", err)
+		}
+		os.Exit(1)
+	}
+
+	fmt.Print(output)
+}

androidmk/partner_androidmk/Android.bp

@@ -0,0 +1,49 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// Sample project for creating an extended androidmk
+//
+
+blueprint_go_binary {
+    name: "partner_androidmk",
+    srcs: [
+        "partner_androidmk/androidmk.go",
+    ],
+    testSrcs: [
+        "partner_androidmk/androidmk_test.go",
+    ],
+    deps: [
+        "androidmk-lib",
+        "partner_bpfix_extensions",
+    ],
+}
+
+blueprint_go_binary {
+    name: "partner_bpfix",
+    srcs: [
+        "partner_bpfix/bpfix.go",
+    ],
+    deps: [
+        "bpfix-cmd",
+        "partner_bpfix_extensions",
+    ],
+}
+
+bootstrap_go_package {
+    name: "partner_bpfix_extensions",
+    pkgPath: "partner/android/bpfix/extensions",
+    srcs: ["fixes/headers.go"],
+    deps: ["bpfix-lib"],
+}

androidmk/partner_androidmk/fixes/headers.go

@@ -0,0 +1,120 @@
+package extensions
+
+import (
+	"strings"
+
+	"github.com/google/blueprint/parser"
+
+	"android/soong/bpfix/bpfix"
+)
+
+var fixSteps = bpfix.FixStepsExtension{
+	Name: "partner-include-dirs",
+	Steps: []bpfix.FixStep{
+		{
+			Name: "fixIncludeDirs",
+			Fix:  fixIncludeDirs,
+		},
+	},
+}
+
+func init() {
+	bpfix.RegisterFixStepExtension(&fixSteps)
+}
+
+type includeDirFix struct {
+	libName  string
+	libType  string
+	variable string
+	subdir   string
+}
+
+var commonIncludeDirs = []includeDirFix{
+	{
+		libName:  "my_header_lib",
+		libType:  "header_libs",
+		variable: "TARGET_OUT_HEADERS",
+		subdir:   "/my_headers",
+	},
+}
+
+func findHeaderLib(e parser.Expression) (*includeDirFix, bool) {
+	if op, ok := e.(*parser.Operator); ok {
+		if op.Operator != '+' {
+			return nil, false
+		}
+		arg0, ok := op.Args[0].(*parser.Variable)
+		arg1, ok1 := op.Args[1].(*parser.String)
+		if !ok || !ok1 {
+			return nil, false
+		}
+		for _, lib := range commonIncludeDirs {
+			if arg0.Name == lib.variable && arg1.Value == lib.subdir {
+				return &lib, true
+			}
+		}
+	}
+	return nil, false
+}
+func searchThroughOperatorList(mod *parser.Module, e parser.Expression) {
+	if list, ok := e.(*parser.List); ok {
+		newList := make([]parser.Expression, 0, len(list.Values))
+		for _, item := range list.Values {
+			if lib, found := findHeaderLib(item); found {
+				if lib.libName != "" {
+					addLibrary(mod, lib.libType, lib.libName)
+				}
+			} else {
+				newList = append(newList, item)
+			}
+		}
+		list.Values = newList
+	}
+	if op, ok := e.(*parser.Operator); ok {
+		searchThroughOperatorList(mod, op.Args[0])
+		searchThroughOperatorList(mod, op.Args[1])
+	}
+}
+func getLiteralListProperty(mod *parser.Module, name string) (list *parser.List, found bool) {
+	prop, ok := mod.GetProperty(name)
+	if !ok {
+		return nil, false
+	}
+	list, ok = prop.Value.(*parser.List)
+	return list, ok
+}
+func addLibrary(mod *parser.Module, libType string, libName string) {
+	var list, ok = getLiteralListProperty(mod, libType)
+	if !ok {
+		list = new(parser.List)
+		prop := new(parser.Property)
+		prop.Name = libType
+		prop.Value = list
+		mod.Properties = append(mod.Properties, prop)
+	} else {
+		for _, v := range list.Values {
+			if stringValue, ok := v.(*parser.String); ok && stringValue.Value == libName {
+				return
+			}
+		}
+	}
+	lib := new(parser.String)
+	lib.Value = libName
+	list.Values = append(list.Values, lib)
+}
+func fixIncludeDirs(f *bpfix.Fixer) error {
+	tree := f.Tree()
+	for _, def := range tree.Defs {
+		mod, ok := def.(*parser.Module)
+		if !ok {
+			continue
+		}
+		if !strings.HasPrefix(mod.Type, "cc_") {
+			continue
+		}
+		if prop, ok := mod.GetProperty("include_dirs"); ok {
+			searchThroughOperatorList(mod, prop.Value)
+		}
+	}
+	return nil
+}

androidmk/partner_androidmk/partner_androidmk/androidmk.go

@@ -0,0 +1,58 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"android/soong/androidmk/androidmk"
+
+	_ "partner/android/bpfix/extensions"
+)
+
+var usage = func() {
+	fmt.Fprintf(os.Stderr, "usage: %s [flags] <inputFile>\n"+
+		"\n%s parses <inputFile> as an Android.mk file and attempts to output an analogous Android.bp file (to standard out)\n", os.Args[0], os.Args[0])
+	flag.PrintDefaults()
+	os.Exit(1)
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if len(flag.Args()) != 1 {
+		usage()
+	}
+	filePathToRead := flag.Arg(0)
+	b, err := ioutil.ReadFile(filePathToRead)
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	output, errs := androidmk.ConvertFile(os.Args[1], bytes.NewBuffer(b))
+	if len(errs) > 0 {
+		for _, err := range errs {
+			fmt.Fprintln(os.Stderr, "ERROR: ", err)
+		}
+		os.Exit(1)
+	}
+
+	fmt.Print(output)
+}

androidmk/partner_androidmk/partner_androidmk/androidmk_test.go

@@ -0,0 +1,73 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"android/soong/androidmk/androidmk"
+	"android/soong/bpfix/bpfix"
+
+	_ "partner/android/bpfix/extensions"
+)
+
+var testCases = []struct {
+	desc     string
+	in       string
+	expected string
+}{
+	{
+		desc: "headers replacement",
+		in: `
+include $(CLEAR_VARS)
+LOCAL_MODULE := test
+LOCAL_SRC_FILES := a.c
+LOCAL_C_INCLUDES := test1 $(TARGET_OUT_HEADERS)/my_headers test2
+include $(BUILD_SHARED_LIBRARY)`,
+		expected: `
+cc_library_shared {
+    name: "test",
+	srcs: ["a.c"],
+	include_dirs: [
+		"test1",
+
+		"test2",
+	],
+	header_libs: ["my_header_lib"]
+}`,
+	},
+}
+
+func TestEndToEnd(t *testing.T) {
+	for i, test := range testCases {
+		expected, err := bpfix.Reformat(test.expected)
+		if err != nil {
+			t.Error(err)
+		}
+
+		got, errs := androidmk.ConvertFile(fmt.Sprintf("<testcase %d>", i), bytes.NewBufferString(test.in))
+		if len(errs) > 0 {
+			t.Errorf("Unexpected errors: %q", errs)
+			continue
+		}
+
+		if got != expected {
+			t.Errorf("failed testcase '%s'\ninput:\n%s\n\nexpected:\n%s\ngot:\n%s\n", test.desc, strings.TrimSpace(test.in), expected, got)
+		}
+	}
+}

androidmk/partner_androidmk/partner_bpfix/bpfix.go

@@ -0,0 +1,27 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file provides a command-line interface to bpfix
+
+package main
+
+import (
+	"android/soong/bpfix/bpfix/cmd"
+
+	_ "partner/android/bpfix/extensions"
+)
+
+func main() {
+	cmd.Run()
+}

bpfix/Android.bp

@@ -19,7 +19,18 @@
 blueprint_go_binary {
     name: "bpfix",
     srcs: [
-        "cmd/bpfix.go",
+        "cmd/main.go",
+    ],
+    deps: [
+        "bpfix-cmd",
+    ],
+}
+
+bootstrap_go_package {
+    name: "bpfix-cmd",
+    pkgPath: "android/soong/bpfix/bpfix/cmd",
+    srcs: [
+        "cmd-lib/bpfix.go",
     ],
     deps: [
         "bpfix-lib",

bpfix/bpfix/bpfix.go

@@ -45,66 +45,76 @@ func Reformat(input string) (string, error) {
 // A FixRequest specifies the details of which fixes to apply to an individual file
 // A FixRequest doesn't specify whether to do a dry run or where to write the results; that's in cmd/bpfix.go
 type FixRequest struct {
-	steps []fixStep
+	steps []FixStep
+}
+type FixStepsExtension struct {
+	Name  string
+	Steps []FixStep
 }
 
-type fixStep struct {
-	name string
-	fix  func(f *Fixer) error
+type FixStep struct {
+	Name string
+	Fix  func(f *Fixer) error
 }
 
-var fixSteps = []fixStep{
+var fixStepsExtensions = []*FixStepsExtension(nil)
+
+func RegisterFixStepExtension(extension *FixStepsExtension) {
+	fixStepsExtensions = append(fixStepsExtensions, extension)
+}
+
+var fixSteps = []FixStep{
 	{
-		name: "simplifyKnownRedundantVariables",
-		fix:  runPatchListMod(simplifyKnownPropertiesDuplicatingEachOther),
+		Name: "simplifyKnownRedundantVariables",
+		Fix:  runPatchListMod(simplifyKnownPropertiesDuplicatingEachOther),
 	},
 	{
-		name: "rewriteIncorrectAndroidmkPrebuilts",
-		fix:  rewriteIncorrectAndroidmkPrebuilts,
+		Name: "rewriteIncorrectAndroidmkPrebuilts",
+		Fix:  rewriteIncorrectAndroidmkPrebuilts,
 	},
 	{
-		name: "rewriteCtsModuleTypes",
-		fix:  rewriteCtsModuleTypes,
+		Name: "rewriteCtsModuleTypes",
+		Fix:  rewriteCtsModuleTypes,
 	},
 	{
-		name: "rewriteIncorrectAndroidmkAndroidLibraries",
-		fix:  rewriteIncorrectAndroidmkAndroidLibraries,
+		Name: "rewriteIncorrectAndroidmkAndroidLibraries",
+		Fix:  rewriteIncorrectAndroidmkAndroidLibraries,
 	},
 	{
-		name: "rewriteTestModuleTypes",
-		fix:  rewriteTestModuleTypes,
+		Name: "rewriteTestModuleTypes",
+		Fix:  rewriteTestModuleTypes,
 	},
 	{
-		name: "rewriteAndroidmkJavaLibs",
-		fix:  rewriteAndroidmkJavaLibs,
+		Name: "rewriteAndroidmkJavaLibs",
+		Fix:  rewriteAndroidmkJavaLibs,
 	},
 	{
-		name: "rewriteJavaStaticLibs",
-		fix:  rewriteJavaStaticLibs,
+		Name: "rewriteJavaStaticLibs",
+		Fix:  rewriteJavaStaticLibs,
 	},
 	{
-		name: "rewritePrebuiltEtc",
-		fix:  rewriteAndroidmkPrebuiltEtc,
+		Name: "rewritePrebuiltEtc",
+		Fix:  rewriteAndroidmkPrebuiltEtc,
 	},
 	{
-		name: "mergeMatchingModuleProperties",
-		fix:  runPatchListMod(mergeMatchingModuleProperties),
+		Name: "mergeMatchingModuleProperties",
+		Fix:  runPatchListMod(mergeMatchingModuleProperties),
 	},
 	{
-		name: "reorderCommonProperties",
-		fix:  runPatchListMod(reorderCommonProperties),
+		Name: "reorderCommonProperties",
+		Fix:  runPatchListMod(reorderCommonProperties),
 	},
 	{
-		name: "removeTags",
-		fix:  runPatchListMod(removeTags),
+		Name: "removeTags",
+		Fix:  runPatchListMod(removeTags),
 	},
 	{
-		name: "rewriteAndroidTest",
-		fix:  rewriteAndroidTest,
+		Name: "rewriteAndroidTest",
+		Fix:  rewriteAndroidTest,
 	},
 	{
-		name: "rewriteAndroidAppImport",
-		fix:  rewriteAndroidAppImport,
+		Name: "rewriteAndroidAppImport",
+		Fix:  rewriteAndroidAppImport,
 	},
 }
 
@@ -113,8 +123,27 @@ func NewFixRequest() FixRequest {
 }
 
 func (r FixRequest) AddAll() (result FixRequest) {
-	result.steps = append([]fixStep(nil), r.steps...)
+	result.steps = append([]FixStep(nil), r.steps...)
 	result.steps = append(result.steps, fixSteps...)
+	for _, extension := range fixStepsExtensions {
+		result.steps = append(result.steps, extension.Steps...)
+	}
+	return result
+}
+
+func (r FixRequest) AddBase() (result FixRequest) {
+	result.steps = append([]FixStep(nil), r.steps...)
+	result.steps = append(result.steps, fixSteps...)
+	return result
+}
+
+func (r FixRequest) AddMatchingExtensions(pattern string) (result FixRequest) {
+	result.steps = append([]FixStep(nil), r.steps...)
+	for _, extension := range fixStepsExtensions {
+		if match, _ := filepath.Match(pattern, extension.Name); match {
+			result.steps = append(result.steps, extension.Steps...)
+		}
+	}
 	return result
 }
 
@@ -122,6 +151,10 @@ type Fixer struct {
 	tree *parser.File
 }
 
+func (f Fixer) Tree() *parser.File {
+	return f.tree
+}
+
 func NewFixer(tree *parser.File) *Fixer {
 	fixer := &Fixer{tree}
 
@@ -198,7 +231,7 @@ func parse(name string, r io.Reader) (*parser.File, error) {
 
 func (f *Fixer) fixTreeOnce(config FixRequest) error {
 	for _, fix := range config.steps {
-		err := fix.fix(f)
+		err := fix.Fix(f)
 		if err != nil {
 			return err
 		}

bpfix/cmd-lib/bpfix.go

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This file provides a command-line interface to bpfix
+// This file provides a bpfix command-line library
 
 // TODO(jeffrygaston) should this file be consolidated with bpfmt.go?
 
-package main
+package cmd
 
 import (
 	"bytes"
@@ -128,7 +128,7 @@ func walkDir(path string, fixRequest bpfix.FixRequest) {
 	filepath.Walk(path, makeFileVisitor(fixRequest))
 }
 
-func main() {
+func Run() {
 	flag.Parse()
 
 	fixRequest := bpfix.NewFixRequest().AddAll()

bpfix/cmd/main.go

@@ -0,0 +1,25 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file provides a wrapper to the bpfix command-line library
+
+package main
+
+import (
+	"android/soong/bpfix/bpfix/cmd"
+)
+
+func main() {
+	cmd.Run()
+}