Merge changes I394a1b62,I6906be4c

* changes:
  Add bazel metrics directory to the list of metrics files to be uploaded.
  Allow uploading a directory of metrics files.

cmd/soong_ui/main.go

@@ -176,10 +176,11 @@ func main() {
 	// Set up files to be outputted in the log directory.
 	logsDir := config.LogsDir()
 
+	// Common list of metric file definition.
 	buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
 	rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
 	soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
-	defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, buildErrorFile, rbeMetricsFile, soongMetricsFile)
+
 	build.PrintOutDirWarning(buildCtx, config)
 
 	os.MkdirAll(logsDir, 0777)
@@ -195,8 +196,22 @@ func main() {
 	buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",
 		config.Parallel(), config.RemoteParallel(), config.HighmemParallel())
 
-	defer met.Dump(soongMetricsFile)
-	defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile)
+	{
+		// The order of the function calls is important. The last defer function call
+		// is the first one that is executed to save the rbe metrics to a protobuf
+		// file. The soong metrics file is then next. Bazel profiles are written
+		// before the uploadMetrics is invoked. The written files are then uploaded
+		// if the uploading of the metrics is enabled.
+		files := []string{
+			buildErrorFile,           // build error strings
+			rbeMetricsFile,           // high level metrics related to remote build execution.
+			soongMetricsFile,         // high level metrics related to this build system.
+			config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
+		}
+		defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, files...)
+		defer met.Dump(soongMetricsFile)
+		defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile)
+	}
 
 	// Read the time at the starting point.
 	if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {

ui/build/upload.go

@@ -40,13 +40,42 @@ var (
 	tmpDir = ioutil.TempDir
 )
 
+// pruneMetricsFiles iterates the list of paths, checking if a path exist.
+// If a path is a file, it is added to the return list. If the path is a
+// directory, a recursive call is made to add the children files of the
+// path.
+func pruneMetricsFiles(paths []string) []string {
+	var metricsFiles []string
+	for _, p := range paths {
+		fi, err := os.Stat(p)
+		// Some paths passed may not exist. For example, build errors protobuf
+		// file may not exist since the build was successful.
+		if err != nil {
+			continue
+		}
+
+		if fi.IsDir() {
+			if l, err := ioutil.ReadDir(p); err == nil {
+				files := make([]string, 0, len(l))
+				for _, fi := range l {
+					files = append(files, filepath.Join(p, fi.Name()))
+				}
+				metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...)
+			}
+		} else {
+			metricsFiles = append(metricsFiles, p)
+		}
+	}
+	return metricsFiles
+}
+
 // UploadMetrics uploads a set of metrics files to a server for analysis. An
 // uploader full path is specified in ANDROID_ENABLE_METRICS_UPLOAD environment
 // variable in order to upload the set of metrics files. The metrics files are
 // first copied to a temporary directory and the uploader is then executed in
 // the background to allow the user/system to continue working. Soong communicates
 // to the uploader through the upload_proto raw protobuf file.
-func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, files ...string) {
+func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) {
 	ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics")
 	defer ctx.EndTrace()
 
@@ -56,15 +85,8 @@ func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted t
 		return
 	}
 
-	// Some files passed in to this function may not exist. For example,
-	// build errors protobuf file may not exist since the build was successful.
-	var metricsFiles []string
-	for _, f := range files {
-		if _, err := os.Stat(f); err == nil {
-			metricsFiles = append(metricsFiles, f)
-		}
-	}
-
+	// Several of the files might be directories.
+	metricsFiles := pruneMetricsFiles(paths)
 	if len(metricsFiles) == 0 {
 		return
 	}

ui/build/upload_test.go

@@ -19,6 +19,8 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"reflect"
+	"sort"
 	"strconv"
 	"strings"
 	"testing"
@@ -27,6 +29,49 @@ import (
 	"android/soong/ui/logger"
 )
 
+func TestPruneMetricsFiles(t *testing.T) {
+	rootDir := t.TempDir()
+
+	dirs := []string{
+		filepath.Join(rootDir, "d1"),
+		filepath.Join(rootDir, "d1", "d2"),
+		filepath.Join(rootDir, "d1", "d2", "d3"),
+	}
+
+	files := []string{
+		filepath.Join(rootDir, "d1", "f1"),
+		filepath.Join(rootDir, "d1", "d2", "f1"),
+		filepath.Join(rootDir, "d1", "d2", "d3", "f1"),
+	}
+
+	for _, d := range dirs {
+		if err := os.MkdirAll(d, 0777); err != nil {
+			t.Fatalf("got %v, expecting nil error for making directory %q", err, d)
+		}
+	}
+
+	for _, f := range files {
+		if err := ioutil.WriteFile(f, []byte{}, 0777); err != nil {
+			t.Fatalf("got %v, expecting nil error on writing file %q", err, f)
+		}
+	}
+
+	want := []string{
+		filepath.Join(rootDir, "d1", "f1"),
+		filepath.Join(rootDir, "d1", "d2", "f1"),
+		filepath.Join(rootDir, "d1", "d2", "d3", "f1"),
+	}
+
+	got := pruneMetricsFiles([]string{rootDir})
+
+	sort.Strings(got)
+	sort.Strings(want)
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %q, want %q after pruning metrics files", got, want)
+	}
+}
+
 func TestUploadMetrics(t *testing.T) {
 	ctx := testContext()
 	tests := []struct {