Implement microfactory Config, Build

To make it easier to use as a package.

Change-Id: Idcb4856f34b91943a320e8281ef9e29af0faab98
This commit is contained in:
Dan Willemsen 2017-08-07 15:01:19 -07:00
parent ff092863b3
commit 5136c43117
2 changed files with 134 additions and 160 deletions

View file

@ -64,9 +64,6 @@ import (
)
var (
race = false
verbose = false
goToolDir = filepath.Join(runtime.GOROOT(), "pkg", "tool", runtime.GOOS+"_"+runtime.GOARCH)
goVersion = findGoVersion()
isGo18 = strings.Contains(goVersion, "go1.8")
@ -85,6 +82,49 @@ func findGoVersion() string {
}
}
type Config struct {
Race bool
Verbose bool
TrimPath string
pkgs []string
paths map[string]string
}
func (c *Config) Map(pkgPrefix, pathPrefix string) error {
if c.paths == nil {
c.paths = make(map[string]string)
}
if _, ok := c.paths[pkgPrefix]; ok {
return fmt.Errorf("Duplicate package prefix: %q", pkgPrefix)
}
c.pkgs = append(c.pkgs, pkgPrefix)
c.paths[pkgPrefix] = pathPrefix
return nil
}
// Path takes a package name, applies the path mappings and returns the resulting path.
//
// If the package isn't mapped, we'll return false to prevent compilation attempts.
func (c *Config) Path(pkg string) (string, bool, error) {
if c == nil || c.paths == nil {
return "", false, fmt.Errorf("No package mappings")
}
for _, pkgPrefix := range c.pkgs {
if pkg == pkgPrefix {
return c.paths[pkgPrefix], true, nil
} else if strings.HasPrefix(pkg, pkgPrefix+"/") {
return filepath.Join(c.paths[pkgPrefix], strings.TrimPrefix(pkg, pkgPrefix+"/")), true, nil
}
}
return "", false, nil
}
type GoPackage struct {
Name string
@ -133,11 +173,11 @@ func (s *linkedDepSet) ignore(name string) {
// FindDeps searches all applicable go files in `path`, parses all of them
// for import dependencies that exist in pkgMap, then recursively does the
// same for all of those dependencies.
func (p *GoPackage) FindDeps(path string, pkgMap *pkgPathMapping) error {
func (p *GoPackage) FindDeps(config *Config, path string) error {
defer un(trace("findDeps"))
depSet := newDepSet()
err := p.findDeps(path, pkgMap, depSet)
err := p.findDeps(config, path, depSet)
if err != nil {
return err
}
@ -148,7 +188,7 @@ func (p *GoPackage) FindDeps(path string, pkgMap *pkgPathMapping) error {
// findDeps is the recursive version of FindDeps. allPackages is the map of
// all locally defined packages so that the same dependency of two different
// packages is only resolved once.
func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *linkedDepSet) error {
func (p *GoPackage) findDeps(config *Config, path string, allPackages *linkedDepSet) error {
// If this ever becomes too slow, we can look at reading the files once instead of twice
// But that just complicates things today, and we're already really fast.
foundPkgs, err := parser.ParseDir(token.NewFileSet(), path, func(fi os.FileInfo) bool {
@ -201,7 +241,7 @@ func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *l
}
var pkgPath string
if path, ok, err := pkgMap.Path(name); err != nil {
if path, ok, err := config.Path(name); err != nil {
return err
} else if !ok {
// Probably in the stdlib, but if not, then the compiler will fail with a reasonable error message
@ -219,7 +259,7 @@ func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *l
allPackages.add(name, pkg)
localDeps[name] = true
if err := pkg.findDeps(pkgPath, pkgMap, allPackages); err != nil {
if err := pkg.findDeps(config, pkgPath, allPackages); err != nil {
return err
}
}
@ -227,7 +267,7 @@ func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *l
sort.Strings(p.files)
if verbose {
if config.Verbose {
fmt.Fprintf(os.Stderr, "Package %q depends on %v\n", p.Name, deps)
}
@ -239,7 +279,7 @@ func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *l
return nil
}
func (p *GoPackage) Compile(outDir, trimPath string) error {
func (p *GoPackage) Compile(config *Config, outDir string) error {
p.mutex.Lock()
defer p.mutex.Unlock()
if p.compiled {
@ -253,7 +293,7 @@ func (p *GoPackage) Compile(outDir, trimPath string) error {
wg.Add(1)
go func(dep *GoPackage) {
defer wg.Done()
dep.Compile(outDir, trimPath)
dep.Compile(config, outDir)
}(dep)
}
wg.Wait()
@ -280,13 +320,13 @@ func (p *GoPackage) Compile(outDir, trimPath string) error {
if !isGo18 {
cmd.Args = append(cmd.Args, "-c", fmt.Sprintf("%d", runtime.NumCPU()))
}
if race {
if config.Race {
cmd.Args = append(cmd.Args, "-race")
fmt.Fprintln(hash, "-race")
}
if trimPath != "" {
cmd.Args = append(cmd.Args, "-trimpath", trimPath)
fmt.Fprintln(hash, trimPath)
if config.TrimPath != "" {
cmd.Args = append(cmd.Args, "-trimpath", config.TrimPath)
fmt.Fprintln(hash, config.TrimPath)
}
for _, dep := range p.directDeps {
cmd.Args = append(cmd.Args, "-I", dep.pkgDir)
@ -350,7 +390,7 @@ func (p *GoPackage) Compile(outDir, trimPath string) error {
cmd.Stdin = nil
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if verbose {
if config.Verbose {
fmt.Fprintln(os.Stderr, cmd.Args)
}
err = cmd.Run()
@ -372,7 +412,7 @@ func (p *GoPackage) Compile(outDir, trimPath string) error {
return nil
}
func (p *GoPackage) Link(out string) error {
func (p *GoPackage) Link(config *Config, out string) error {
if p.Name != "main" {
return fmt.Errorf("Can only link main package")
}
@ -405,7 +445,7 @@ func (p *GoPackage) Link(out string) error {
}
cmd := exec.Command(filepath.Join(goToolDir, "link"), "-o", out)
if race {
if config.Race {
cmd.Args = append(cmd.Args, "-race")
}
for _, dep := range p.allDeps {
@ -415,7 +455,7 @@ func (p *GoPackage) Link(out string) error {
cmd.Stdin = nil
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if verbose {
if config.Verbose {
fmt.Fprintln(os.Stderr, cmd.Args)
}
err = cmd.Run()
@ -426,48 +466,44 @@ func (p *GoPackage) Link(out string) error {
return ioutil.WriteFile(shaFile, p.hashResult, 0666)
}
// rebuildMicrofactory checks to see if microfactory itself needs to be rebuilt,
// and if does, it will launch a new copy and return true. Otherwise it will return
// false to continue executing.
func rebuildMicrofactory(mybin string, pkgMap *pkgPathMapping) bool {
mysrc, ok, err := pkgMap.Path("github.com/google/blueprint/microfactory/main")
if err != nil {
fmt.Println(os.Stderr, "Error finding microfactory source:", err)
os.Exit(1)
}
if !ok {
fmt.Println(os.Stderr, "Could not find microfactory source")
os.Exit(1)
}
intermediates := filepath.Join(filepath.Dir(mybin), "."+filepath.Base(mybin)+"_intermediates")
err = os.MkdirAll(intermediates, 0777)
if err != nil {
fmt.Fprintln(os.Stderr, "Failed to create intermediates directory: %v", err)
os.Exit(1)
}
pkg := &GoPackage{
func Build(config *Config, out, pkg string) (*GoPackage, error) {
p := &GoPackage{
Name: "main",
}
if err := pkg.FindDeps(mysrc, pkgMap); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
path, ok, err := config.Path(pkg)
if err != nil {
return nil, fmt.Errorf("Error finding package %q for main: %v", pkg, err)
}
if !ok {
return nil, fmt.Errorf("Could not find package %q", pkg)
}
if err := pkg.Compile(intermediates, mysrc); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
intermediates := filepath.Join(filepath.Dir(out), "."+filepath.Base(out)+"_intermediates")
if err := os.MkdirAll(intermediates, 0777); err != nil {
return nil, fmt.Errorf("Failed to create intermediates directory: %v", err)
}
if err := pkg.Link(mybin); err != nil {
if err := p.FindDeps(config, path); err != nil {
return nil, fmt.Errorf("Failed to find deps: %v", err)
}
if err := p.Compile(config, intermediates); err != nil {
return nil, fmt.Errorf("Failed to compile: %v", err)
}
if err := p.Link(config, out); err != nil {
return nil, fmt.Errorf("Failed to link: %v", err)
}
return p, nil
}
// rebuildMicrofactory checks to see if microfactory itself needs to be rebuilt,
// and if does, it will launch a new copy and return true. Otherwise it will return
// false to continue executing.
func rebuildMicrofactory(config *Config, mybin string) bool {
if pkg, err := Build(config, mybin, "github.com/google/blueprint/microfactory/main"); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if !pkg.rebuilt {
} else if !pkg.rebuilt {
return false
}
@ -504,15 +540,16 @@ func un(f func()) {
// microfactory.bash will make a copy of this file renamed into the main package for use with `go run`
func main() { Main() }
func Main() {
var output, mybin, trimPath string
var pkgMap pkgPathMapping
var output, mybin string
var config Config
pkgMap := pkgPathMappingVar{&config}
flags := flag.NewFlagSet("", flag.ExitOnError)
flags.BoolVar(&race, "race", false, "enable data race detection.")
flags.BoolVar(&verbose, "v", false, "Verbose")
flags.BoolVar(&config.Race, "race", false, "enable data race detection.")
flags.BoolVar(&config.Verbose, "v", false, "Verbose")
flags.StringVar(&output, "o", "", "Output file")
flags.StringVar(&mybin, "b", "", "Microfactory binary location")
flags.StringVar(&trimPath, "trimpath", "", "remove prefix from recorded source file paths")
flags.StringVar(&config.TrimPath, "trimpath", "", "remove prefix from recorded source file paths")
flags.Var(&pkgMap, "pkg-path", "Mapping of package prefixes to file paths")
err := flags.Parse(os.Args[1:])
@ -534,61 +571,26 @@ func Main() {
}
if mybin != "" {
if rebuildMicrofactory(mybin, &pkgMap) {
if rebuildMicrofactory(&config, mybin) {
return
}
}
mainPackage := &GoPackage{
Name: "main",
}
if path, ok, err := pkgMap.Path(flags.Arg(0)); err != nil {
fmt.Fprintln(os.Stderr, "Error finding main path:", err)
os.Exit(1)
} else if !ok {
fmt.Fprintln(os.Stderr, "Cannot find path for", flags.Arg(0))
} else {
if err := mainPackage.FindDeps(path, &pkgMap); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
intermediates := filepath.Join(filepath.Dir(output), "."+filepath.Base(output)+"_intermediates")
err = os.MkdirAll(intermediates, 0777)
if err != nil {
fmt.Fprintln(os.Stderr, "Failed to create intermediates directory: %ve", err)
os.Exit(1)
}
err = mainPackage.Compile(intermediates, trimPath)
if err != nil {
fmt.Fprintln(os.Stderr, "Failed to compile:", err)
os.Exit(1)
}
err = mainPackage.Link(output)
if err != nil {
fmt.Fprintln(os.Stderr, "microfactory.go failed to link:", err)
if _, err := Build(&config, output, flags.Arg(0)); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
// pkgPathMapping can be used with flag.Var to parse -pkg-path arguments of
// <package-prefix>=<path-prefix> mappings.
type pkgPathMapping struct {
pkgs []string
type pkgPathMappingVar struct{ *Config }
paths map[string]string
}
func (pkgPathMapping) String() string {
func (pkgPathMappingVar) String() string {
return "<package-prefix>=<path-prefix>"
}
func (p *pkgPathMapping) Set(value string) error {
func (p *pkgPathMappingVar) Set(value string) error {
equalPos := strings.Index(value, "=")
if equalPos == -1 {
return fmt.Errorf("Argument must be in the form of: %q", p.String())
@ -597,34 +599,5 @@ func (p *pkgPathMapping) Set(value string) error {
pkgPrefix := strings.TrimSuffix(value[:equalPos], "/")
pathPrefix := strings.TrimSuffix(value[equalPos+1:], "/")
if p.paths == nil {
p.paths = make(map[string]string)
}
if _, ok := p.paths[pkgPrefix]; ok {
return fmt.Errorf("Duplicate package prefix: %q", pkgPrefix)
}
p.pkgs = append(p.pkgs, pkgPrefix)
p.paths[pkgPrefix] = pathPrefix
return nil
}
// Path takes a package name, applies the path mappings and returns the resulting path.
//
// If the package isn't mapped, we'll return false to prevent compilation attempts.
func (p *pkgPathMapping) Path(pkg string) (string, bool, error) {
if p.paths == nil {
return "", false, fmt.Errorf("No package mappings")
}
for _, pkgPrefix := range p.pkgs {
if pkg == pkgPrefix {
return p.paths[pkgPrefix], true, nil
} else if strings.HasPrefix(pkg, pkgPrefix+"/") {
return filepath.Join(p.paths[pkgPrefix], strings.TrimPrefix(pkg, pkgPrefix+"/")), true, nil
}
}
return "", false, nil
return p.Map(pkgPrefix, pathPrefix)
}

View file

@ -28,7 +28,7 @@ import (
func TestSimplePackagePathMap(t *testing.T) {
t.Parallel()
var pkgMap pkgPathMapping
pkgMap := pkgPathMappingVar{&Config{}}
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.Var(&pkgMap, "m", "")
err := flags.Parse([]string{
@ -73,7 +73,7 @@ func TestSimplePackagePathMap(t *testing.T) {
func TestBadPackagePathMap(t *testing.T) {
t.Parallel()
var pkgMap pkgPathMapping
pkgMap := pkgPathMappingVar{&Config{}}
if _, _, err := pkgMap.Path("testing"); err == nil {
t.Error("Expected error if no maps are specified")
}
@ -97,17 +97,17 @@ func TestBadPackagePathMap(t *testing.T) {
func TestSingleBuild(t *testing.T) {
t.Parallel()
setupDir(t, func(dir string, loadPkg loadPkgFunc) {
setupDir(t, func(config *Config, dir string, loadPkg loadPkgFunc) {
// The output binary
out := filepath.Join(dir, "out", "test")
pkg := loadPkg()
if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil {
if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil {
t.Fatalf("Got error when compiling:", err)
}
if err := pkg.Link(out); err != nil {
if err := pkg.Link(config, out); err != nil {
t.Fatal("Got error when linking:", err)
}
@ -122,22 +122,22 @@ func TestSingleBuild(t *testing.T) {
// to rebuild anything based on the shouldRebuild argument.
func testBuildAgain(t *testing.T,
shouldRecompile, shouldRelink bool,
modify func(dir string, loadPkg loadPkgFunc),
modify func(config *Config, dir string, loadPkg loadPkgFunc),
after func(pkg *GoPackage)) {
t.Parallel()
setupDir(t, func(dir string, loadPkg loadPkgFunc) {
setupDir(t, func(config *Config, dir string, loadPkg loadPkgFunc) {
// The output binary
out := filepath.Join(dir, "out", "test")
pkg := loadPkg()
if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil {
if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil {
t.Fatal("Got error when compiling:", err)
}
if err := pkg.Link(out); err != nil {
if err := pkg.Link(config, out); err != nil {
t.Fatal("Got error when linking:", err)
}
@ -155,11 +155,11 @@ func testBuildAgain(t *testing.T,
time.Sleep(1100 * time.Millisecond)
}
modify(dir, loadPkg)
modify(config, dir, loadPkg)
pkg = loadPkg()
if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil {
if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil {
t.Fatal("Got error when compiling:", err)
}
if shouldRecompile {
@ -172,7 +172,7 @@ func testBuildAgain(t *testing.T,
}
}
if err := pkg.Link(out); err != nil {
if err := pkg.Link(config, out); err != nil {
t.Fatal("Got error while linking:", err)
}
if shouldRelink {
@ -208,14 +208,14 @@ func testBuildAgain(t *testing.T,
// TestRebuildAfterNoChanges ensures that we don't rebuild if nothing
// changes
func TestRebuildAfterNoChanges(t *testing.T) {
testBuildAgain(t, false, false, func(dir string, loadPkg loadPkgFunc) {}, func(pkg *GoPackage) {})
testBuildAgain(t, false, false, func(config *Config, dir string, loadPkg loadPkgFunc) {}, func(pkg *GoPackage) {})
}
// TestRebuildAfterTimestamp ensures that we don't rebuild because
// timestamps of important files have changed. We should only rebuild if the
// content hashes are different.
func TestRebuildAfterTimestampChange(t *testing.T) {
testBuildAgain(t, false, false, func(dir string, loadPkg loadPkgFunc) {
testBuildAgain(t, false, false, func(config *Config, dir string, loadPkg loadPkgFunc) {
// Ensure that we've spent some amount of time asleep
time.Sleep(100 * time.Millisecond)
@ -231,7 +231,7 @@ func TestRebuildAfterTimestampChange(t *testing.T) {
// TestRebuildAfterGoChange ensures that we rebuild after a content change
// to a package's go file.
func TestRebuildAfterGoChange(t *testing.T) {
testBuildAgain(t, true, true, func(dir string, loadPkg loadPkgFunc) {
testBuildAgain(t, true, true, func(config *Config, dir string, loadPkg loadPkgFunc) {
if err := ioutil.WriteFile(filepath.Join(dir, "a", "a.go"), []byte(go_a_a+"\n"), 0666); err != nil {
t.Fatal("Error writing a/a.go:", err)
}
@ -248,7 +248,7 @@ func TestRebuildAfterGoChange(t *testing.T) {
// TestRebuildAfterMainChange ensures that we don't rebuild any dependencies
// if only the main package's go files are touched.
func TestRebuildAfterMainChange(t *testing.T) {
testBuildAgain(t, true, true, func(dir string, loadPkg loadPkgFunc) {
testBuildAgain(t, true, true, func(config *Config, dir string, loadPkg loadPkgFunc) {
if err := ioutil.WriteFile(filepath.Join(dir, "main", "main.go"), []byte(go_main_main+"\n"), 0666); err != nil {
t.Fatal("Error writing main/main.go:", err)
}
@ -265,7 +265,7 @@ func TestRebuildAfterMainChange(t *testing.T) {
// TestRebuildAfterRemoveOut ensures that we rebuild if the output file is
// missing, even if everything else doesn't need rebuilding.
func TestRebuildAfterRemoveOut(t *testing.T) {
testBuildAgain(t, false, true, func(dir string, loadPkg loadPkgFunc) {
testBuildAgain(t, false, true, func(config *Config, dir string, loadPkg loadPkgFunc) {
if err := os.Remove(filepath.Join(dir, "out", "test")); err != nil {
t.Fatal("Failed to remove output:", err)
}
@ -275,14 +275,14 @@ func TestRebuildAfterRemoveOut(t *testing.T) {
// TestRebuildAfterPartialBuild ensures that even if the build was interrupted
// between the recompile and relink stages, we'll still relink when we run again.
func TestRebuildAfterPartialBuild(t *testing.T) {
testBuildAgain(t, false, true, func(dir string, loadPkg loadPkgFunc) {
testBuildAgain(t, false, true, func(config *Config, dir string, loadPkg loadPkgFunc) {
if err := ioutil.WriteFile(filepath.Join(dir, "main", "main.go"), []byte(go_main_main+"\n"), 0666); err != nil {
t.Fatal("Error writing main/main.go:", err)
}
pkg := loadPkg()
if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil {
if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil {
t.Fatal("Got error when compiling:", err)
}
if !pkg.rebuilt {
@ -295,13 +295,13 @@ func TestRebuildAfterPartialBuild(t *testing.T) {
// inputs).
func BenchmarkInitialBuild(b *testing.B) {
for i := 0; i < b.N; i++ {
setupDir(b, func(dir string, loadPkg loadPkgFunc) {
setupDir(b, func(config *Config, dir string, loadPkg loadPkgFunc) {
pkg := loadPkg()
if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil {
if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil {
b.Fatal("Got error when compiling:", err)
}
if err := pkg.Link(filepath.Join(dir, "out", "test")); err != nil {
if err := pkg.Link(config, filepath.Join(dir, "out", "test")); err != nil {
b.Fatal("Got error when linking:", err)
}
})
@ -311,14 +311,14 @@ func BenchmarkInitialBuild(b *testing.B) {
// BenchmarkMinIncrementalBuild computes how long an incremental build that
// doesn't actually need to build anything takes.
func BenchmarkMinIncrementalBuild(b *testing.B) {
setupDir(b, func(dir string, loadPkg loadPkgFunc) {
setupDir(b, func(config *Config, dir string, loadPkg loadPkgFunc) {
pkg := loadPkg()
if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil {
if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil {
b.Fatal("Got error when compiling:", err)
}
if err := pkg.Link(filepath.Join(dir, "out", "test")); err != nil {
if err := pkg.Link(config, filepath.Join(dir, "out", "test")); err != nil {
b.Fatal("Got error when linking:", err)
}
@ -327,11 +327,11 @@ func BenchmarkMinIncrementalBuild(b *testing.B) {
for i := 0; i < b.N; i++ {
pkg := loadPkg()
if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil {
if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil {
b.Fatal("Got error when compiling:", err)
}
if err := pkg.Link(filepath.Join(dir, "out", "test")); err != nil {
if err := pkg.Link(config, filepath.Join(dir, "out", "test")); err != nil {
b.Fatal("Got error when linking:", err)
}
@ -381,7 +381,7 @@ type T interface {
type loadPkgFunc func() *GoPackage
func setupDir(t T, test func(dir string, loadPkg loadPkgFunc)) {
func setupDir(t T, test func(config *Config, dir string, loadPkg loadPkgFunc)) {
dir, err := ioutil.TempDir("", "test")
if err != nil {
t.Fatalf("Error creating temporary directory: %#v", err)
@ -406,17 +406,18 @@ func setupDir(t T, test func(dir string, loadPkg loadPkgFunc)) {
writeFile("a/b.go", go_a_b)
writeFile("b/a.go", go_b_a)
config := &Config{}
config.Map("android/soong", dir)
loadPkg := func() *GoPackage {
pkg := &GoPackage{
Name: "main",
}
pkgMap := &pkgPathMapping{}
pkgMap.Set("android/soong=" + dir)
if err := pkg.FindDeps(filepath.Join(dir, "main"), pkgMap); err != nil {
if err := pkg.FindDeps(config, filepath.Join(dir, "main")); err != nil {
t.Fatalf("Error finding deps: %v", err)
}
return pkg
}
test(dir, loadPkg)
test(config, dir, loadPkg)
}