Run the Finder and make its results available to Kati

The Finder runs roughly 200ms faster than findleaves.py in aosp,
and runs roughly 400ms faster in internal master.

Bug: 64363847
Test: m -j

Change-Id: I62db8dacc90871e913576fe2443021fb1749a483

cmd/multiproduct_kati/main.go

@@ -233,6 +233,9 @@ func main() {
 	var wg sync.WaitGroup
 	productConfigs := make(chan Product, len(products))
 
+	finder := build.NewSourceFinder(buildCtx, config)
+	defer finder.Shutdown()
+
 	// Run the product config for every product in parallel
 	for _, product := range products {
 		wg.Add(1)
@@ -274,6 +277,8 @@ func main() {
 				Thread:         trace.NewThread(product),
 			}}
 
+			build.FindSources(productCtx, config, finder)
+
 			productConfig := build.NewConfig(productCtx)
 			productConfig.Environment().Set("OUT_DIR", productOutDir)
 			productConfig.Lunch(productCtx, product, *buildVariant)

cmd/soong_ui/main.go

@@ -95,5 +95,9 @@ func main() {
 		}
 	}
 
+	f := build.NewSourceFinder(buildCtx, config)
+	defer f.Shutdown()
+	build.FindSources(buildCtx, config, f)
+
 	build.Build(buildCtx, config, build.BuildAll)
 }

finder/finder.go

@@ -148,10 +148,11 @@ type Finder struct {
 	filesystem          fs.FileSystem
 
 	// temporary state
-	threadPool *threadPool
-	mutex      sync.Mutex
-	fsErrs     []fsErr
-	errlock    sync.Mutex
+	threadPool        *threadPool
+	mutex             sync.Mutex
+	fsErrs            []fsErr
+	errlock           sync.Mutex
+	shutdownWaitgroup sync.WaitGroup
 
 	// non-temporary state
 	modifiedFlag int32
@@ -183,6 +184,8 @@ func New(cacheParams CacheParams, filesystem fs.FileSystem,
 
 		nodes:  *newPathMap("/"),
 		DbPath: dbPath,
+
+		shutdownWaitgroup: sync.WaitGroup{},
 	}
 
 	f.loadFromFilesystem()
@@ -195,9 +198,12 @@ func New(cacheParams CacheParams, filesystem fs.FileSystem,
 
 	// confirm that every path mentioned in the CacheConfig exists
 	for _, path := range cacheParams.RootDirs {
+		if !filepath.IsAbs(path) {
+			path = filepath.Join(f.cacheMetadata.Config.WorkingDirectory, path)
+		}
 		node := f.nodes.GetNode(filepath.Clean(path), false)
 		if node == nil || node.ModTime == 0 {
-			return nil, fmt.Errorf("%v does not exist\n", path)
+			return nil, fmt.Errorf("path %v was specified to be included in the cache but does not exist\n", path)
 		}
 	}
 
@@ -310,20 +316,32 @@ func (f *Finder) FindMatching(rootPath string, filter WalkFunc) []string {
 	return results
 }
 
-// Shutdown saves the contents of the Finder to its database file
+// Shutdown declares that the finder is no longer needed and waits for its cleanup to complete
+// Currently, that only entails waiting for the database dump to complete.
 func (f *Finder) Shutdown() {
-	f.verbosef("Shutting down\n")
+	f.waitForDbDump()
+}
+
+// End of public api
+
+func (f *Finder) goDumpDb() {
 	if f.wasModified() {
-		err := f.dumpDb()
-		if err != nil {
-			f.verbosef("%v\n", err)
-		}
+		f.shutdownWaitgroup.Add(1)
+		go func() {
+			err := f.dumpDb()
+			if err != nil {
+				f.verbosef("%v\n", err)
+			}
+			f.shutdownWaitgroup.Done()
+		}()
 	} else {
 		f.verbosef("Skipping dumping unmodified db\n")
 	}
 }
 
-// End of public api
+func (f *Finder) waitForDbDump() {
+	f.shutdownWaitgroup.Wait()
+}
 
 // joinCleanPaths is like filepath.Join but is faster because
 // joinCleanPaths doesn't have to support paths ending in "/" or containing ".."
@@ -353,6 +371,8 @@ func (f *Finder) loadFromFilesystem() {
 		f.startWithoutExternalCache()
 	}
 
+	f.goDumpDb()
+
 	f.threadPool = nil
 }
 

finder/finder_test.go

@@ -466,12 +466,13 @@ func TestRelativeFilePaths(t *testing.T) {
 	create(t, "/cwd/hi.txt", filesystem)
 	create(t, "/cwd/a/hi.txt", filesystem)
 	create(t, "/cwd/a/a/hi.txt", filesystem)
+	create(t, "/rel/a/hi.txt", filesystem)
 
 	finder := newFinder(
 		t,
 		filesystem,
 		CacheParams{
-			RootDirs:     []string{"/cwd", "/tmp/include"},
+			RootDirs:     []string{"/cwd", "../rel", "/tmp/include"},
 			IncludeFiles: []string{"hi.txt"},
 		},
 	)
@@ -491,6 +492,10 @@ func TestRelativeFilePaths(t *testing.T) {
 			"a/hi.txt",
 			"a/a/hi.txt"})
 
+	foundPaths = finder.FindNamedAt("/rel", "hi.txt")
+	assertSameResponse(t, foundPaths,
+		[]string{"/rel/a/hi.txt"})
+
 	foundPaths = finder.FindNamedAt("/tmp/include", "hi.txt")
 	assertSameResponse(t, foundPaths, []string{"/tmp/include/hi.txt"})
 }

ui/build/Android.bp

@@ -19,6 +19,7 @@ bootstrap_go_package {
         "soong-ui-logger",
         "soong-ui-tracer",
         "soong-shared",
+        "soong-finder",
     ],
     srcs: [
         "build.go",
@@ -27,6 +28,7 @@ bootstrap_go_package {
         "context.go",
         "environment.go",
         "exec.go",
+        "finder.go",
         "kati.go",
         "make.go",
         "ninja.go",

ui/build/build.go

@@ -32,6 +32,7 @@ func SetupOutDir(ctx Context, config Config) {
 	// The ninja_build file is used by our buildbots to understand that the output
 	// can be parsed as ninja output.
 	ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "ninja_build"))
+	ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), ".out-dir"))
 }
 
 var combinedBuildNinjaTemplate = template.Must(template.New("combined").Parse(`

ui/build/config.go

@@ -280,6 +280,10 @@ func (c *configImpl) TempDir() string {
 	return shared.TempDirForOutDir(c.SoongOutDir())
 }
 
+func (c *configImpl) FileListDir() string {
+	return filepath.Join(c.OutDir(), ".module_paths")
+}
+
 func (c *configImpl) KatiSuffix() string {
 	if c.katiSuffix != "" {
 		return c.katiSuffix

ui/build/finder.go

@@ -0,0 +1,102 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"android/soong/finder"
+	"android/soong/fs"
+	"android/soong/ui/logger"
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// This file provides an interface to the Finder for use in Soong UI
+// This file stores configuration information about which files to find
+
+// NewSourceFinder returns a new Finder configured to search for source files.
+// Callers of NewSourceFinder should call <f.Shutdown()> when done
+func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
+	ctx.BeginTrace("find modules")
+	defer ctx.EndTrace()
+
+	dir, err := os.Getwd()
+	if err != nil {
+		ctx.Fatalf("No working directory for module-finder: %v", err.Error())
+	}
+	cacheParams := finder.CacheParams{
+		WorkingDirectory: dir,
+		RootDirs:         []string{"."},
+		ExcludeDirs:      []string{".git", ".repo"},
+		PruneFiles:       []string{".out-dir", ".find-ignore"},
+		IncludeFiles:     []string{"Android.mk", "Android.bp", "Blueprints", "CleanSpec.mk"},
+	}
+	dumpDir := config.FileListDir()
+	f, err = finder.New(cacheParams, fs.OsFs, logger.New(ioutil.Discard),
+		filepath.Join(dumpDir, "files.db"))
+	if err != nil {
+		ctx.Fatalf("Could not create module-finder: %v", err)
+	}
+	return f
+}
+
+// FindSources searches for source files known to <f> and writes them to the filesystem for
+// use later.
+func FindSources(ctx Context, config Config, f *finder.Finder) {
+	// note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder
+	// if a caller such as multiproduct_kati wants to share one Finder among several builds
+	dumpDir := config.FileListDir()
+	os.MkdirAll(dumpDir, 0777)
+
+	androidMks := f.FindFirstNamedAt(".", "Android.mk")
+	err := dumpListToFile(androidMks, filepath.Join(dumpDir, "Android.mk.list"))
+	if err != nil {
+		ctx.Fatalf("Could not export module list: %v", err)
+	}
+
+	cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk")
+	dumpListToFile(cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list"))
+	if err != nil {
+		ctx.Fatalf("Could not export module list: %v", err)
+	}
+
+	isBlueprintFile := func(dir finder.DirEntries) (dirs []string, files []string) {
+		files = []string{}
+		for _, file := range dir.FileNames {
+			if file == "Android.bp" || file == "Blueprints" {
+				files = append(files, file)
+			}
+		}
+
+		return dir.DirNames, files
+	}
+	androidBps := f.FindMatching(".", isBlueprintFile)
+	err = dumpListToFile(androidBps, filepath.Join(dumpDir, "Android.bp.list"))
+	if err != nil {
+		ctx.Fatalf("Could not find modules: %v", err)
+	}
+}
+
+func dumpListToFile(list []string, filePath string) (err error) {
+	desiredText := strings.Join(list, "\n")
+	desiredBytes := []byte(desiredText)
+	actualBytes, readErr := ioutil.ReadFile(filePath)
+	if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) {
+		err = ioutil.WriteFile(filePath, desiredBytes, 0777)
+	}
+	return err
+}