diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 4ffe94428..532e879e2 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -176,10 +176,11 @@ func main() { // Set up files to be outputted in the log directory. logsDir := config.LogsDir() + // Common list of metric file definition. buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error") rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb") soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics") - defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, buildErrorFile, rbeMetricsFile, soongMetricsFile) + build.PrintOutDirWarning(buildCtx, config) os.MkdirAll(logsDir, 0777) @@ -195,8 +196,22 @@ func main() { buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v", config.Parallel(), config.RemoteParallel(), config.HighmemParallel()) - defer met.Dump(soongMetricsFile) - defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile) + { + // The order of the function calls is important. The last defer function call + // is the first one that is executed to save the rbe metrics to a protobuf + // file. The soong metrics file is then next. Bazel profiles are written + // before the uploadMetrics is invoked. The written files are then uploaded + // if the uploading of the metrics is enabled. + files := []string{ + buildErrorFile, // build error strings + rbeMetricsFile, // high level metrics related to remote build execution. + soongMetricsFile, // high level metrics related to this build system. + config.BazelMetricsDir(), // directory that contains a set of bazel metrics. + } + defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, files...) + defer met.Dump(soongMetricsFile) + defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile) + } // Read the time at the starting point. if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { diff --git a/ui/build/upload.go b/ui/build/upload.go index 4f30136b9..55ca800b5 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -40,13 +40,42 @@ var ( tmpDir = ioutil.TempDir ) +// pruneMetricsFiles iterates the list of paths, checking if a path exist. +// If a path is a file, it is added to the return list. If the path is a +// directory, a recursive call is made to add the children files of the +// path. +func pruneMetricsFiles(paths []string) []string { + var metricsFiles []string + for _, p := range paths { + fi, err := os.Stat(p) + // Some paths passed may not exist. For example, build errors protobuf + // file may not exist since the build was successful. + if err != nil { + continue + } + + if fi.IsDir() { + if l, err := ioutil.ReadDir(p); err == nil { + files := make([]string, 0, len(l)) + for _, fi := range l { + files = append(files, filepath.Join(p, fi.Name())) + } + metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...) + } + } else { + metricsFiles = append(metricsFiles, p) + } + } + return metricsFiles +} + // UploadMetrics uploads a set of metrics files to a server for analysis. An // uploader full path is specified in ANDROID_ENABLE_METRICS_UPLOAD environment // variable in order to upload the set of metrics files. The metrics files are // first copied to a temporary directory and the uploader is then executed in // the background to allow the user/system to continue working. Soong communicates // to the uploader through the upload_proto raw protobuf file. -func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, files ...string) { +func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) { ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics") defer ctx.EndTrace() @@ -56,15 +85,8 @@ func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted t return } - // Some files passed in to this function may not exist. For example, - // build errors protobuf file may not exist since the build was successful. - var metricsFiles []string - for _, f := range files { - if _, err := os.Stat(f); err == nil { - metricsFiles = append(metricsFiles, f) - } - } - + // Several of the files might be directories. + metricsFiles := pruneMetricsFiles(paths) if len(metricsFiles) == 0 { return } diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go index 768b03112..b740c1120 100644 --- a/ui/build/upload_test.go +++ b/ui/build/upload_test.go @@ -19,6 +19,8 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" + "sort" "strconv" "strings" "testing" @@ -27,6 +29,49 @@ import ( "android/soong/ui/logger" ) +func TestPruneMetricsFiles(t *testing.T) { + rootDir := t.TempDir() + + dirs := []string{ + filepath.Join(rootDir, "d1"), + filepath.Join(rootDir, "d1", "d2"), + filepath.Join(rootDir, "d1", "d2", "d3"), + } + + files := []string{ + filepath.Join(rootDir, "d1", "f1"), + filepath.Join(rootDir, "d1", "d2", "f1"), + filepath.Join(rootDir, "d1", "d2", "d3", "f1"), + } + + for _, d := range dirs { + if err := os.MkdirAll(d, 0777); err != nil { + t.Fatalf("got %v, expecting nil error for making directory %q", err, d) + } + } + + for _, f := range files { + if err := ioutil.WriteFile(f, []byte{}, 0777); err != nil { + t.Fatalf("got %v, expecting nil error on writing file %q", err, f) + } + } + + want := []string{ + filepath.Join(rootDir, "d1", "f1"), + filepath.Join(rootDir, "d1", "d2", "f1"), + filepath.Join(rootDir, "d1", "d2", "d3", "f1"), + } + + got := pruneMetricsFiles([]string{rootDir}) + + sort.Strings(got) + sort.Strings(want) + + if !reflect.DeepEqual(got, want) { + t.Errorf("got %q, want %q after pruning metrics files", got, want) + } +} + func TestUploadMetrics(t *testing.T) { ctx := testContext() tests := []struct {