diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index c96510730..d3802f9ae 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -187,6 +187,7 @@ func main() { config.Parallel(), config.RemoteParallel(), config.HighmemParallel()) defer met.Dump(soongMetricsFile) + defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile) if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { if !strings.HasSuffix(start, "N") { diff --git a/ui/build/Android.bp b/ui/build/Android.bp index 0a0bb1614..4ef27212a 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -63,6 +63,7 @@ bootstrap_go_package { "cleanbuild_test.go", "config_test.go", "environment_test.go", + "rbe_test.go", "upload_test.go", "util_test.go", "proc_sync_test.go", diff --git a/ui/build/config.go b/ui/build/config.go index c4bbad76f..e567e403e 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -804,6 +804,15 @@ func (c *configImpl) StartRBE() bool { return true } +func (c *configImpl) RBEStatsOutputDir() string { + for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + return "" +} + func (c *configImpl) UseRemoteBuild() bool { return c.UseGoma() || c.UseRBE() } diff --git a/ui/build/rbe.go b/ui/build/rbe.go index ceea4bf2b..fcdab3b03 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -15,14 +15,39 @@ package build import ( + "os" "path/filepath" "android/soong/ui/metrics" ) -const bootstrapCmd = "bootstrap" -const rbeLeastNProcs = 2500 -const rbeLeastNFiles = 16000 +const ( + rbeLeastNProcs = 2500 + rbeLeastNFiles = 16000 + + // prebuilt RBE binaries + bootstrapCmd = "bootstrap" + + // RBE metrics proto buffer file + rbeMetricsPBFilename = "rbe_metrics.pb" +) + +func rbeCommand(ctx Context, config Config, rbeCmd string) string { + var cmdPath string + if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok { + cmdPath = filepath.Join(rbeDir, rbeCmd) + } else if home, ok := config.Environment().Get("HOME"); ok { + cmdPath = filepath.Join(home, "rbe", rbeCmd) + } else { + ctx.Fatalf("rbe command path not found") + } + + if _, err := os.Stat(cmdPath); err != nil && os.IsNotExist(err) { + ctx.Fatalf("rbe command %q not found", rbeCmd) + } + + return cmdPath +} func startRBE(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSetupTool, "rbe_bootstrap") @@ -35,18 +60,50 @@ func startRBE(ctx Context, config Config) { ctx.Fatalf("max open files is insufficient: %d; want >= %d.\n", n, rbeLeastNFiles) } - var rbeBootstrap string - if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok { - rbeBootstrap = filepath.Join(rbeDir, bootstrapCmd) - } else if home, ok := config.Environment().Get("HOME"); ok { - rbeBootstrap = filepath.Join(home, "rbe", bootstrapCmd) - } else { - ctx.Fatalln("rbe bootstrap not found") - } - - cmd := Command(ctx, config, "boostrap", rbeBootstrap) + cmd := Command(ctx, config, "startRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd)) if output, err := cmd.CombinedOutput(); err != nil { ctx.Fatalf("rbe bootstrap failed with: %v\n%s\n", err, output) } } + +func stopRBE(ctx Context, config Config) { + cmd := Command(ctx, config, "stopRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd), "-shutdown") + if output, err := cmd.CombinedOutput(); err != nil { + ctx.Fatalf("rbe bootstrap with shutdown failed with: %v\n%s\n", err, output) + } +} + +// DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics. +// The protobuf file is created if RBE is enabled and the proxy service has +// started. The proxy service is shutdown in order to dump the RBE metrics to the +// protobuf file. +func DumpRBEMetrics(ctx Context, config Config, filename string) { + ctx.BeginTrace(metrics.RunShutdownTool, "dump_rbe_metrics") + defer ctx.EndTrace() + + // Remove the previous metrics file in case there is a failure or RBE has been + // disable for this run. + os.Remove(filename) + + // If RBE is not enabled then there are no metrics to generate. + // If RBE does not require to start, the RBE proxy maybe started + // manually for debugging purpose and can generate the metrics + // afterwards. + if !config.StartRBE() { + return + } + + outputDir := config.RBEStatsOutputDir() + if outputDir == "" { + ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.") + } + metricsFile := filepath.Join(outputDir, rbeMetricsPBFilename) + + // Stop the proxy first in order to generate the RBE metrics protobuf file. + stopRBE(ctx, config) + + if _, err := copyFile(metricsFile, filename); err != nil { + ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err) + } +} diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go new file mode 100644 index 000000000..2c4995b1d --- /dev/null +++ b/ui/build/rbe_test.go @@ -0,0 +1,142 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "android/soong/ui/logger" +) + +func TestDumpRBEMetrics(t *testing.T) { + ctx := testContext() + tests := []struct { + description string + env []string + generated bool + }{{ + description: "RBE disabled", + env: []string{ + "NOSTART_RBE=true", + }, + }, { + description: "rbe metrics generated", + env: []string{ + "USE_RBE=true", + }, + generated: true, + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + tmpDir := t.TempDir() + + rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd) + if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(rbeBootstrapProgram), 0755); err != nil { + t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err) + } + + env := Environment(tt.env) + env.Set("OUT_DIR", tmpDir) + env.Set("RBE_DIR", tmpDir) + env.Set("RBE_output_dir", t.TempDir()) + config := Config{&configImpl{ + environ: &env, + }} + + rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename) + DumpRBEMetrics(ctx, config, rbeMetricsFilename) + + // Validate that the rbe metrics file exists if RBE is enabled. + if _, err := os.Stat(rbeMetricsFilename); err == nil { + if !tt.generated { + t.Errorf("got true, want false for rbe metrics file %s to exist.", rbeMetricsFilename) + } + } else if os.IsNotExist(err) { + if tt.generated { + t.Errorf("got false, want true for rbe metrics file %s to exist.", rbeMetricsFilename) + } + } else { + t.Errorf("unknown error found on checking %s exists: %v", rbeMetricsFilename, err) + } + }) + } +} + +func TestDumpRBEMetricsErrors(t *testing.T) { + ctx := testContext() + tests := []struct { + description string + rbeOutputDirDefined bool + bootstrapProgram string + expectedErr string + }{{ + description: "output_dir not defined", + bootstrapProgram: rbeBootstrapProgram, + expectedErr: "RBE output dir variable not defined", + }, { + description: "stopRBE failed", + rbeOutputDirDefined: true, + bootstrapProgram: "#!/bin/bash\nexit 1", + expectedErr: "shutdown failed", + }, { + description: "failed to copy metrics file", + rbeOutputDirDefined: true, + bootstrapProgram: "#!/bin/bash", + expectedErr: "failed to copy", + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + got := err.Error() + if !strings.Contains(got, tt.expectedErr) { + t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr) + } + }) + + tmpDir := t.TempDir() + + rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd) + if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(tt.bootstrapProgram), 0755); err != nil { + t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err) + } + + env := &Environment{} + env.Set("USE_RBE", "true") + env.Set("OUT_DIR", tmpDir) + env.Set("RBE_DIR", tmpDir) + + if tt.rbeOutputDirDefined { + env.Set("RBE_output_dir", t.TempDir()) + } + + config := Config{&configImpl{ + environ: env, + }} + + rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename) + DumpRBEMetrics(ctx, config, rbeMetricsFilename) + t.Errorf("got nil, expecting %q as a failure", tt.expectedErr) + }) + } +} + +var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s", rbeMetricsPBFilename) diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 3e76d3771..e055b769c 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -25,12 +25,13 @@ import ( ) const ( - RunSetupTool = "setup" - RunKati = "kati" - RunSoong = "soong" - PrimaryNinja = "ninja" - TestRun = "test" - Total = "total" + PrimaryNinja = "ninja" + RunKati = "kati" + RunSetupTool = "setup" + RunShutdownTool = "shutdown" + RunSoong = "soong" + TestRun = "test" + Total = "total" ) type Metrics struct {