Merge "Upload build metrics after a build is completed."

cmd/soong_ui/main.go

@@ -117,6 +117,8 @@ func inList(s string, list []string) bool {
 // Command is the type of soong_ui execution. Only one type of
 // execution is specified. The args are specific to the command.
 func main() {
+	buildStartedMilli := time.Now().UnixNano() / int64(time.Millisecond)
+
 	c, args := getCommand(os.Args)
 	if c == nil {
 		fmt.Fprintf(os.Stderr, "The `soong` native UI is not yet available.\n")
@@ -166,12 +168,17 @@ func main() {
 		logsDir = filepath.Join(config.DistDir(), "logs")
 	}
 
+	buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
+	rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
+	soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
+	defer build.UploadMetrics(buildCtx, config, buildStartedMilli, buildErrorFile, rbeMetricsFile, soongMetricsFile)
+
 	os.MkdirAll(logsDir, 0777)
 	log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
 	trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
 	stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log")))
 	stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log")))
-	stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"build_error")))
+	stat.AddOutput(status.NewProtoErrorLog(log, buildErrorFile))
 	stat.AddOutput(status.NewCriticalPath(log))
 	stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, c.logsPrefix+"build_progress.pb")))
 
@@ -179,7 +186,7 @@ func main() {
 	buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",
 		config.Parallel(), config.RemoteParallel(), config.HighmemParallel())
 
-	defer met.Dump(filepath.Join(logsDir, c.logsPrefix+"soong_metrics"))
+	defer met.Dump(soongMetricsFile)
 
 	if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
 		if !strings.HasSuffix(start, "N") {

ui/build/Android.bp

@@ -56,12 +56,14 @@ bootstrap_go_package {
         "signal.go",
         "soong.go",
         "test_build.go",
+        "upload.go",
         "util.go",
     ],
     testSrcs: [
         "cleanbuild_test.go",
         "config_test.go",
         "environment_test.go",
+        "upload_test.go",
         "util_test.go",
         "proc_sync_test.go",
     ],

ui/build/config.go

@@ -961,3 +961,14 @@ func (c *configImpl) SetPdkBuild(pdk bool) {
 func (c *configImpl) IsPdkBuild() bool {
 	return c.pdkBuild
 }
+
+func (c *configImpl) BuildDateTime() string {
+	return c.buildDateTime
+}
+
+func (c *configImpl) MetricsUploaderApp() string {
+	if p, ok := c.environ.Get("ANDROID_ENABLE_METRICS_UPLOAD"); ok {
+		return p
+	}
+	return ""
+}

ui/build/config_test.go

@@ -26,6 +26,7 @@ import (
 	"testing"
 
 	"android/soong/ui/logger"
+	"android/soong/ui/status"
 )
 
 func testContext() Context {
@@ -33,6 +34,7 @@ func testContext() Context {
 		Context: context.Background(),
 		Logger:  logger.New(&bytes.Buffer{}),
 		Writer:  &bytes.Buffer{},
+		Status:  &status.Status{},
 	}}
 }
 

ui/build/upload.go

@@ -0,0 +1,80 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+// This file contains the functionality to upload data from one location to
+// another.
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+
+	upload_proto "android/soong/ui/metrics/upload_proto"
+)
+
+const (
+	uploadPbFilename = ".uploader.pb"
+)
+
+// UploadMetrics uploads a set of metrics files to a server for analysis. An
+// uploader full path is required to be specified in order to upload the set
+// of metrics files. This is accomplished by defining the ANDROID_ENABLE_METRICS_UPLOAD
+// environment variable.
+func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files ...string) {
+	uploader := config.MetricsUploaderApp()
+	// No metrics to upload if the path to the uploader was not specified.
+	if uploader == "" {
+		return
+	}
+
+	// Some files may not exist. For example, build errors protobuf file
+	// may not exist since the build was successful.
+	var metricsFiles []string
+	for _, f := range files {
+		if _, err := os.Stat(f); err == nil {
+			metricsFiles = append(metricsFiles, f)
+		}
+	}
+
+	if len(metricsFiles) == 0 {
+		return
+	}
+
+	// For platform builds, the branch and target name is hardcoded to specific
+	// values for later extraction of the metrics in the data metrics pipeline.
+	data, err := proto.Marshal(&upload_proto.Upload{
+		CreationTimestampMs:   proto.Uint64(uint64(buildStartedMilli)),
+		CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
+		BranchName:            proto.String("developer-metrics"),
+		TargetName:            proto.String("platform-build-systems-metrics"),
+		MetricsFiles:          metricsFiles,
+	})
+	if err != nil {
+		ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
+	}
+
+	pbFile := filepath.Join(config.OutDir(), uploadPbFilename)
+	if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
+		ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
+	}
+	// Remove the upload file as it's not longer needed after it has been processed by the uploader.
+	defer os.Remove(pbFile)
+
+	Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile).RunAndStreamOrFatal()
+}

ui/build/upload_test.go

@@ -0,0 +1,120 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"android/soong/ui/logger"
+)
+
+func TestUploadMetrics(t *testing.T) {
+	ctx := testContext()
+	tests := []struct {
+		description string
+		uploader    string
+		createFiles bool
+		files       []string
+	}{{
+		description: "ANDROID_ENABLE_METRICS_UPLOAD not set",
+	}, {
+		description: "no metrics files to upload",
+		uploader:    "fake",
+	}, {
+		description: "non-existent metrics files no upload",
+		uploader:    "fake",
+		files:       []string{"metrics_file_1", "metrics_file_2", "metrics_file_3"},
+	}, {
+		description: "trigger upload",
+		uploader:    "echo",
+		createFiles: true,
+		files:       []string{"metrics_file_1", "metrics_file_2"},
+	}}
+
+	for _, tt := range tests {
+		t.Run(tt.description, func(t *testing.T) {
+			defer logger.Recover(func(err error) {
+				t.Fatalf("got unexpected error: %v", err)
+			})
+
+			outDir, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("failed to create out directory: %v", outDir)
+			}
+			defer os.RemoveAll(outDir)
+
+			var metricsFiles []string
+			if tt.createFiles {
+				for _, f := range tt.files {
+					filename := filepath.Join(outDir, f)
+					metricsFiles = append(metricsFiles, filename)
+					if err := ioutil.WriteFile(filename, []byte("test file"), 0644); err != nil {
+						t.Fatalf("failed to create a fake metrics file %q for uploading: %v", filename, err)
+					}
+				}
+			}
+
+			config := Config{&configImpl{
+				environ: &Environment{
+					"OUT_DIR=" + outDir,
+					"ANDROID_ENABLE_METRICS_UPLOAD=" + tt.uploader,
+				},
+				buildDateTime: strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10),
+			}}
+
+			UploadMetrics(ctx, config, 1591031903, metricsFiles...)
+
+			if _, err := os.Stat(filepath.Join(outDir, uploadPbFilename)); err == nil {
+				t.Error("got true, want false for upload protobuf file to exist")
+			}
+		})
+	}
+}
+
+func TestUploadMetricsErrors(t *testing.T) {
+	expectedErr := "failed to write the marshaled"
+	defer logger.Recover(func(err error) {
+		got := err.Error()
+		if !strings.Contains(got, expectedErr) {
+			t.Errorf("got %q, want %q to be contained in error", got, expectedErr)
+		}
+	})
+
+	outDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("failed to create out directory: %v", outDir)
+	}
+	defer os.RemoveAll(outDir)
+
+	metricsFile := filepath.Join(outDir, "metrics_file_1")
+	if err := ioutil.WriteFile(metricsFile, []byte("test file"), 0644); err != nil {
+		t.Fatalf("failed to create a fake metrics file %q for uploading: %v", metricsFile, err)
+	}
+
+	config := Config{&configImpl{
+		environ: &Environment{
+			"ANDROID_ENABLE_METRICS_UPLOAD=fake",
+			"OUT_DIR=/bad",
+		}}}
+
+	UploadMetrics(testContext(), config, 1591031903, metricsFile)
+	t.Errorf("got nil, expecting %q as a failure", expectedErr)
+}