diff --git a/microfactory/microfactory.go b/microfactory/microfactory.go index c1d41dc..f907669 100644 --- a/microfactory/microfactory.go +++ b/microfactory/microfactory.go @@ -64,9 +64,6 @@ import ( ) var ( - race = false - verbose = false - goToolDir = filepath.Join(runtime.GOROOT(), "pkg", "tool", runtime.GOOS+"_"+runtime.GOARCH) goVersion = findGoVersion() isGo18 = strings.Contains(goVersion, "go1.8") @@ -85,6 +82,49 @@ func findGoVersion() string { } } +type Config struct { + Race bool + Verbose bool + + TrimPath string + + pkgs []string + paths map[string]string +} + +func (c *Config) Map(pkgPrefix, pathPrefix string) error { + if c.paths == nil { + c.paths = make(map[string]string) + } + if _, ok := c.paths[pkgPrefix]; ok { + return fmt.Errorf("Duplicate package prefix: %q", pkgPrefix) + } + + c.pkgs = append(c.pkgs, pkgPrefix) + c.paths[pkgPrefix] = pathPrefix + + return nil +} + +// Path takes a package name, applies the path mappings and returns the resulting path. +// +// If the package isn't mapped, we'll return false to prevent compilation attempts. +func (c *Config) Path(pkg string) (string, bool, error) { + if c == nil || c.paths == nil { + return "", false, fmt.Errorf("No package mappings") + } + + for _, pkgPrefix := range c.pkgs { + if pkg == pkgPrefix { + return c.paths[pkgPrefix], true, nil + } else if strings.HasPrefix(pkg, pkgPrefix+"/") { + return filepath.Join(c.paths[pkgPrefix], strings.TrimPrefix(pkg, pkgPrefix+"/")), true, nil + } + } + + return "", false, nil +} + type GoPackage struct { Name string @@ -133,11 +173,11 @@ func (s *linkedDepSet) ignore(name string) { // FindDeps searches all applicable go files in `path`, parses all of them // for import dependencies that exist in pkgMap, then recursively does the // same for all of those dependencies. -func (p *GoPackage) FindDeps(path string, pkgMap *pkgPathMapping) error { +func (p *GoPackage) FindDeps(config *Config, path string) error { defer un(trace("findDeps")) depSet := newDepSet() - err := p.findDeps(path, pkgMap, depSet) + err := p.findDeps(config, path, depSet) if err != nil { return err } @@ -148,7 +188,7 @@ func (p *GoPackage) FindDeps(path string, pkgMap *pkgPathMapping) error { // findDeps is the recursive version of FindDeps. allPackages is the map of // all locally defined packages so that the same dependency of two different // packages is only resolved once. -func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *linkedDepSet) error { +func (p *GoPackage) findDeps(config *Config, path string, allPackages *linkedDepSet) error { // If this ever becomes too slow, we can look at reading the files once instead of twice // But that just complicates things today, and we're already really fast. foundPkgs, err := parser.ParseDir(token.NewFileSet(), path, func(fi os.FileInfo) bool { @@ -201,7 +241,7 @@ func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *l } var pkgPath string - if path, ok, err := pkgMap.Path(name); err != nil { + if path, ok, err := config.Path(name); err != nil { return err } else if !ok { // Probably in the stdlib, but if not, then the compiler will fail with a reasonable error message @@ -219,7 +259,7 @@ func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *l allPackages.add(name, pkg) localDeps[name] = true - if err := pkg.findDeps(pkgPath, pkgMap, allPackages); err != nil { + if err := pkg.findDeps(config, pkgPath, allPackages); err != nil { return err } } @@ -227,7 +267,7 @@ func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *l sort.Strings(p.files) - if verbose { + if config.Verbose { fmt.Fprintf(os.Stderr, "Package %q depends on %v\n", p.Name, deps) } @@ -239,7 +279,7 @@ func (p *GoPackage) findDeps(path string, pkgMap *pkgPathMapping, allPackages *l return nil } -func (p *GoPackage) Compile(outDir, trimPath string) error { +func (p *GoPackage) Compile(config *Config, outDir string) error { p.mutex.Lock() defer p.mutex.Unlock() if p.compiled { @@ -253,7 +293,7 @@ func (p *GoPackage) Compile(outDir, trimPath string) error { wg.Add(1) go func(dep *GoPackage) { defer wg.Done() - dep.Compile(outDir, trimPath) + dep.Compile(config, outDir) }(dep) } wg.Wait() @@ -280,13 +320,13 @@ func (p *GoPackage) Compile(outDir, trimPath string) error { if !isGo18 { cmd.Args = append(cmd.Args, "-c", fmt.Sprintf("%d", runtime.NumCPU())) } - if race { + if config.Race { cmd.Args = append(cmd.Args, "-race") fmt.Fprintln(hash, "-race") } - if trimPath != "" { - cmd.Args = append(cmd.Args, "-trimpath", trimPath) - fmt.Fprintln(hash, trimPath) + if config.TrimPath != "" { + cmd.Args = append(cmd.Args, "-trimpath", config.TrimPath) + fmt.Fprintln(hash, config.TrimPath) } for _, dep := range p.directDeps { cmd.Args = append(cmd.Args, "-I", dep.pkgDir) @@ -350,7 +390,7 @@ func (p *GoPackage) Compile(outDir, trimPath string) error { cmd.Stdin = nil cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - if verbose { + if config.Verbose { fmt.Fprintln(os.Stderr, cmd.Args) } err = cmd.Run() @@ -372,7 +412,7 @@ func (p *GoPackage) Compile(outDir, trimPath string) error { return nil } -func (p *GoPackage) Link(out string) error { +func (p *GoPackage) Link(config *Config, out string) error { if p.Name != "main" { return fmt.Errorf("Can only link main package") } @@ -405,7 +445,7 @@ func (p *GoPackage) Link(out string) error { } cmd := exec.Command(filepath.Join(goToolDir, "link"), "-o", out) - if race { + if config.Race { cmd.Args = append(cmd.Args, "-race") } for _, dep := range p.allDeps { @@ -415,7 +455,7 @@ func (p *GoPackage) Link(out string) error { cmd.Stdin = nil cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - if verbose { + if config.Verbose { fmt.Fprintln(os.Stderr, cmd.Args) } err = cmd.Run() @@ -426,48 +466,44 @@ func (p *GoPackage) Link(out string) error { return ioutil.WriteFile(shaFile, p.hashResult, 0666) } -// rebuildMicrofactory checks to see if microfactory itself needs to be rebuilt, -// and if does, it will launch a new copy and return true. Otherwise it will return -// false to continue executing. -func rebuildMicrofactory(mybin string, pkgMap *pkgPathMapping) bool { - mysrc, ok, err := pkgMap.Path("github.com/google/blueprint/microfactory/main") - if err != nil { - fmt.Println(os.Stderr, "Error finding microfactory source:", err) - os.Exit(1) - } - if !ok { - fmt.Println(os.Stderr, "Could not find microfactory source") - os.Exit(1) - } - - intermediates := filepath.Join(filepath.Dir(mybin), "."+filepath.Base(mybin)+"_intermediates") - - err = os.MkdirAll(intermediates, 0777) - if err != nil { - fmt.Fprintln(os.Stderr, "Failed to create intermediates directory: %v", err) - os.Exit(1) - } - - pkg := &GoPackage{ +func Build(config *Config, out, pkg string) (*GoPackage, error) { + p := &GoPackage{ Name: "main", } - if err := pkg.FindDeps(mysrc, pkgMap); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + path, ok, err := config.Path(pkg) + if err != nil { + return nil, fmt.Errorf("Error finding package %q for main: %v", pkg, err) + } + if !ok { + return nil, fmt.Errorf("Could not find package %q", pkg) } - if err := pkg.Compile(intermediates, mysrc); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + intermediates := filepath.Join(filepath.Dir(out), "."+filepath.Base(out)+"_intermediates") + if err := os.MkdirAll(intermediates, 0777); err != nil { + return nil, fmt.Errorf("Failed to create intermediates directory: %v", err) } - if err := pkg.Link(mybin); err != nil { + if err := p.FindDeps(config, path); err != nil { + return nil, fmt.Errorf("Failed to find deps: %v", err) + } + if err := p.Compile(config, intermediates); err != nil { + return nil, fmt.Errorf("Failed to compile: %v", err) + } + if err := p.Link(config, out); err != nil { + return nil, fmt.Errorf("Failed to link: %v", err) + } + return p, nil +} + +// rebuildMicrofactory checks to see if microfactory itself needs to be rebuilt, +// and if does, it will launch a new copy and return true. Otherwise it will return +// false to continue executing. +func rebuildMicrofactory(config *Config, mybin string) bool { + if pkg, err := Build(config, mybin, "github.com/google/blueprint/microfactory/main"); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) - } - - if !pkg.rebuilt { + } else if !pkg.rebuilt { return false } @@ -504,15 +540,16 @@ func un(f func()) { // microfactory.bash will make a copy of this file renamed into the main package for use with `go run` func main() { Main() } func Main() { - var output, mybin, trimPath string - var pkgMap pkgPathMapping + var output, mybin string + var config Config + pkgMap := pkgPathMappingVar{&config} flags := flag.NewFlagSet("", flag.ExitOnError) - flags.BoolVar(&race, "race", false, "enable data race detection.") - flags.BoolVar(&verbose, "v", false, "Verbose") + flags.BoolVar(&config.Race, "race", false, "enable data race detection.") + flags.BoolVar(&config.Verbose, "v", false, "Verbose") flags.StringVar(&output, "o", "", "Output file") flags.StringVar(&mybin, "b", "", "Microfactory binary location") - flags.StringVar(&trimPath, "trimpath", "", "remove prefix from recorded source file paths") + flags.StringVar(&config.TrimPath, "trimpath", "", "remove prefix from recorded source file paths") flags.Var(&pkgMap, "pkg-path", "Mapping of package prefixes to file paths") err := flags.Parse(os.Args[1:]) @@ -534,61 +571,26 @@ func Main() { } if mybin != "" { - if rebuildMicrofactory(mybin, &pkgMap) { + if rebuildMicrofactory(&config, mybin) { return } } - mainPackage := &GoPackage{ - Name: "main", - } - - if path, ok, err := pkgMap.Path(flags.Arg(0)); err != nil { - fmt.Fprintln(os.Stderr, "Error finding main path:", err) - os.Exit(1) - } else if !ok { - fmt.Fprintln(os.Stderr, "Cannot find path for", flags.Arg(0)) - } else { - if err := mainPackage.FindDeps(path, &pkgMap); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - } - - intermediates := filepath.Join(filepath.Dir(output), "."+filepath.Base(output)+"_intermediates") - - err = os.MkdirAll(intermediates, 0777) - if err != nil { - fmt.Fprintln(os.Stderr, "Failed to create intermediates directory: %ve", err) - os.Exit(1) - } - - err = mainPackage.Compile(intermediates, trimPath) - if err != nil { - fmt.Fprintln(os.Stderr, "Failed to compile:", err) - os.Exit(1) - } - - err = mainPackage.Link(output) - if err != nil { - fmt.Fprintln(os.Stderr, "microfactory.go failed to link:", err) + if _, err := Build(&config, output, flags.Arg(0)); err != nil { + fmt.Fprintln(os.Stderr, err) os.Exit(1) } } // pkgPathMapping can be used with flag.Var to parse -pkg-path arguments of // = mappings. -type pkgPathMapping struct { - pkgs []string +type pkgPathMappingVar struct{ *Config } - paths map[string]string -} - -func (pkgPathMapping) String() string { +func (pkgPathMappingVar) String() string { return "=" } -func (p *pkgPathMapping) Set(value string) error { +func (p *pkgPathMappingVar) Set(value string) error { equalPos := strings.Index(value, "=") if equalPos == -1 { return fmt.Errorf("Argument must be in the form of: %q", p.String()) @@ -597,34 +599,5 @@ func (p *pkgPathMapping) Set(value string) error { pkgPrefix := strings.TrimSuffix(value[:equalPos], "/") pathPrefix := strings.TrimSuffix(value[equalPos+1:], "/") - if p.paths == nil { - p.paths = make(map[string]string) - } - if _, ok := p.paths[pkgPrefix]; ok { - return fmt.Errorf("Duplicate package prefix: %q", pkgPrefix) - } - - p.pkgs = append(p.pkgs, pkgPrefix) - p.paths[pkgPrefix] = pathPrefix - - return nil -} - -// Path takes a package name, applies the path mappings and returns the resulting path. -// -// If the package isn't mapped, we'll return false to prevent compilation attempts. -func (p *pkgPathMapping) Path(pkg string) (string, bool, error) { - if p.paths == nil { - return "", false, fmt.Errorf("No package mappings") - } - - for _, pkgPrefix := range p.pkgs { - if pkg == pkgPrefix { - return p.paths[pkgPrefix], true, nil - } else if strings.HasPrefix(pkg, pkgPrefix+"/") { - return filepath.Join(p.paths[pkgPrefix], strings.TrimPrefix(pkg, pkgPrefix+"/")), true, nil - } - } - - return "", false, nil + return p.Map(pkgPrefix, pathPrefix) } diff --git a/microfactory/microfactory_test.go b/microfactory/microfactory_test.go index ad0967c..083c9a6 100644 --- a/microfactory/microfactory_test.go +++ b/microfactory/microfactory_test.go @@ -28,7 +28,7 @@ import ( func TestSimplePackagePathMap(t *testing.T) { t.Parallel() - var pkgMap pkgPathMapping + pkgMap := pkgPathMappingVar{&Config{}} flags := flag.NewFlagSet("", flag.ContinueOnError) flags.Var(&pkgMap, "m", "") err := flags.Parse([]string{ @@ -73,7 +73,7 @@ func TestSimplePackagePathMap(t *testing.T) { func TestBadPackagePathMap(t *testing.T) { t.Parallel() - var pkgMap pkgPathMapping + pkgMap := pkgPathMappingVar{&Config{}} if _, _, err := pkgMap.Path("testing"); err == nil { t.Error("Expected error if no maps are specified") } @@ -97,17 +97,17 @@ func TestBadPackagePathMap(t *testing.T) { func TestSingleBuild(t *testing.T) { t.Parallel() - setupDir(t, func(dir string, loadPkg loadPkgFunc) { + setupDir(t, func(config *Config, dir string, loadPkg loadPkgFunc) { // The output binary out := filepath.Join(dir, "out", "test") pkg := loadPkg() - if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil { + if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil { t.Fatalf("Got error when compiling:", err) } - if err := pkg.Link(out); err != nil { + if err := pkg.Link(config, out); err != nil { t.Fatal("Got error when linking:", err) } @@ -122,22 +122,22 @@ func TestSingleBuild(t *testing.T) { // to rebuild anything based on the shouldRebuild argument. func testBuildAgain(t *testing.T, shouldRecompile, shouldRelink bool, - modify func(dir string, loadPkg loadPkgFunc), + modify func(config *Config, dir string, loadPkg loadPkgFunc), after func(pkg *GoPackage)) { t.Parallel() - setupDir(t, func(dir string, loadPkg loadPkgFunc) { + setupDir(t, func(config *Config, dir string, loadPkg loadPkgFunc) { // The output binary out := filepath.Join(dir, "out", "test") pkg := loadPkg() - if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil { + if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil { t.Fatal("Got error when compiling:", err) } - if err := pkg.Link(out); err != nil { + if err := pkg.Link(config, out); err != nil { t.Fatal("Got error when linking:", err) } @@ -155,11 +155,11 @@ func testBuildAgain(t *testing.T, time.Sleep(1100 * time.Millisecond) } - modify(dir, loadPkg) + modify(config, dir, loadPkg) pkg = loadPkg() - if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil { + if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil { t.Fatal("Got error when compiling:", err) } if shouldRecompile { @@ -172,7 +172,7 @@ func testBuildAgain(t *testing.T, } } - if err := pkg.Link(out); err != nil { + if err := pkg.Link(config, out); err != nil { t.Fatal("Got error while linking:", err) } if shouldRelink { @@ -208,14 +208,14 @@ func testBuildAgain(t *testing.T, // TestRebuildAfterNoChanges ensures that we don't rebuild if nothing // changes func TestRebuildAfterNoChanges(t *testing.T) { - testBuildAgain(t, false, false, func(dir string, loadPkg loadPkgFunc) {}, func(pkg *GoPackage) {}) + testBuildAgain(t, false, false, func(config *Config, dir string, loadPkg loadPkgFunc) {}, func(pkg *GoPackage) {}) } // TestRebuildAfterTimestamp ensures that we don't rebuild because // timestamps of important files have changed. We should only rebuild if the // content hashes are different. func TestRebuildAfterTimestampChange(t *testing.T) { - testBuildAgain(t, false, false, func(dir string, loadPkg loadPkgFunc) { + testBuildAgain(t, false, false, func(config *Config, dir string, loadPkg loadPkgFunc) { // Ensure that we've spent some amount of time asleep time.Sleep(100 * time.Millisecond) @@ -231,7 +231,7 @@ func TestRebuildAfterTimestampChange(t *testing.T) { // TestRebuildAfterGoChange ensures that we rebuild after a content change // to a package's go file. func TestRebuildAfterGoChange(t *testing.T) { - testBuildAgain(t, true, true, func(dir string, loadPkg loadPkgFunc) { + testBuildAgain(t, true, true, func(config *Config, dir string, loadPkg loadPkgFunc) { if err := ioutil.WriteFile(filepath.Join(dir, "a", "a.go"), []byte(go_a_a+"\n"), 0666); err != nil { t.Fatal("Error writing a/a.go:", err) } @@ -248,7 +248,7 @@ func TestRebuildAfterGoChange(t *testing.T) { // TestRebuildAfterMainChange ensures that we don't rebuild any dependencies // if only the main package's go files are touched. func TestRebuildAfterMainChange(t *testing.T) { - testBuildAgain(t, true, true, func(dir string, loadPkg loadPkgFunc) { + testBuildAgain(t, true, true, func(config *Config, dir string, loadPkg loadPkgFunc) { if err := ioutil.WriteFile(filepath.Join(dir, "main", "main.go"), []byte(go_main_main+"\n"), 0666); err != nil { t.Fatal("Error writing main/main.go:", err) } @@ -265,7 +265,7 @@ func TestRebuildAfterMainChange(t *testing.T) { // TestRebuildAfterRemoveOut ensures that we rebuild if the output file is // missing, even if everything else doesn't need rebuilding. func TestRebuildAfterRemoveOut(t *testing.T) { - testBuildAgain(t, false, true, func(dir string, loadPkg loadPkgFunc) { + testBuildAgain(t, false, true, func(config *Config, dir string, loadPkg loadPkgFunc) { if err := os.Remove(filepath.Join(dir, "out", "test")); err != nil { t.Fatal("Failed to remove output:", err) } @@ -275,14 +275,14 @@ func TestRebuildAfterRemoveOut(t *testing.T) { // TestRebuildAfterPartialBuild ensures that even if the build was interrupted // between the recompile and relink stages, we'll still relink when we run again. func TestRebuildAfterPartialBuild(t *testing.T) { - testBuildAgain(t, false, true, func(dir string, loadPkg loadPkgFunc) { + testBuildAgain(t, false, true, func(config *Config, dir string, loadPkg loadPkgFunc) { if err := ioutil.WriteFile(filepath.Join(dir, "main", "main.go"), []byte(go_main_main+"\n"), 0666); err != nil { t.Fatal("Error writing main/main.go:", err) } pkg := loadPkg() - if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil { + if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil { t.Fatal("Got error when compiling:", err) } if !pkg.rebuilt { @@ -295,13 +295,13 @@ func TestRebuildAfterPartialBuild(t *testing.T) { // inputs). func BenchmarkInitialBuild(b *testing.B) { for i := 0; i < b.N; i++ { - setupDir(b, func(dir string, loadPkg loadPkgFunc) { + setupDir(b, func(config *Config, dir string, loadPkg loadPkgFunc) { pkg := loadPkg() - if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil { + if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil { b.Fatal("Got error when compiling:", err) } - if err := pkg.Link(filepath.Join(dir, "out", "test")); err != nil { + if err := pkg.Link(config, filepath.Join(dir, "out", "test")); err != nil { b.Fatal("Got error when linking:", err) } }) @@ -311,14 +311,14 @@ func BenchmarkInitialBuild(b *testing.B) { // BenchmarkMinIncrementalBuild computes how long an incremental build that // doesn't actually need to build anything takes. func BenchmarkMinIncrementalBuild(b *testing.B) { - setupDir(b, func(dir string, loadPkg loadPkgFunc) { + setupDir(b, func(config *Config, dir string, loadPkg loadPkgFunc) { pkg := loadPkg() - if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil { + if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil { b.Fatal("Got error when compiling:", err) } - if err := pkg.Link(filepath.Join(dir, "out", "test")); err != nil { + if err := pkg.Link(config, filepath.Join(dir, "out", "test")); err != nil { b.Fatal("Got error when linking:", err) } @@ -327,11 +327,11 @@ func BenchmarkMinIncrementalBuild(b *testing.B) { for i := 0; i < b.N; i++ { pkg := loadPkg() - if err := pkg.Compile(filepath.Join(dir, "out"), ""); err != nil { + if err := pkg.Compile(config, filepath.Join(dir, "out")); err != nil { b.Fatal("Got error when compiling:", err) } - if err := pkg.Link(filepath.Join(dir, "out", "test")); err != nil { + if err := pkg.Link(config, filepath.Join(dir, "out", "test")); err != nil { b.Fatal("Got error when linking:", err) } @@ -381,7 +381,7 @@ type T interface { type loadPkgFunc func() *GoPackage -func setupDir(t T, test func(dir string, loadPkg loadPkgFunc)) { +func setupDir(t T, test func(config *Config, dir string, loadPkg loadPkgFunc)) { dir, err := ioutil.TempDir("", "test") if err != nil { t.Fatalf("Error creating temporary directory: %#v", err) @@ -406,17 +406,18 @@ func setupDir(t T, test func(dir string, loadPkg loadPkgFunc)) { writeFile("a/b.go", go_a_b) writeFile("b/a.go", go_b_a) + config := &Config{} + config.Map("android/soong", dir) + loadPkg := func() *GoPackage { pkg := &GoPackage{ Name: "main", } - pkgMap := &pkgPathMapping{} - pkgMap.Set("android/soong=" + dir) - if err := pkg.FindDeps(filepath.Join(dir, "main"), pkgMap); err != nil { + if err := pkg.FindDeps(config, filepath.Join(dir, "main")); err != nil { t.Fatalf("Error finding deps: %v", err) } return pkg } - test(dir, loadPkg) + test(config, dir, loadPkg) }