Cleanup RBE logs directory

RBE logs directory is currently messy:
1. We use RBE_output_dir variable to specify where rbe_metrics.txt /
   rbe_metrics.pb file should go to.
2. We use proxy_log_dir to specify where *.rpl / *.rpi (detailed per
   action info log file) should go to.
3. We use RBE_log_dir to specify where reproxy.* / bootstrap.* log files
   should go to.

Ideally, all RBE related logs should go to one single directory. In this
CL, I'm creating a temporary log directory under out/soong/.temp/rbe/
where all RBE related log files per build would go to. The log dir
prefix is also being set to the same prefix as the socket address file.

Test: Ran a sample build with `m libc` and ensured that logs are getting
cleared across rebuilds and that `rbe_metrics.pb` file is properly
generated and being copied to the right location (i.e., from
out/soong/.temp/rbe/<rand> to out/ dir)

Bug: b/233382420
Change-Id: I46bd38d50419cb9e54e8202d24222979e47ff5ca

ui/build/build.go

@@ -266,6 +266,7 @@ func Build(ctx Context, config Config) {
 	}
 
 	if config.StartRBE() {
+		cleanupRBELogsDir(ctx, config)
 		startRBE(ctx, config)
 		defer DumpRBEMetrics(ctx, config, filepath.Join(config.LogsDir(), "rbe_metrics.pb"))
 	}

ui/build/config.go

@@ -19,12 +19,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"android/soong/shared"
@@ -42,6 +44,15 @@ const (
 	envConfigFetchTimeout = 10 * time.Second
 )
 
+var (
+	rbeRandPrefix int
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+	rbeRandPrefix = rand.Intn(1000)
+}
+
 type Config struct{ *configImpl }
 
 type configImpl struct {
@@ -1144,34 +1155,25 @@ func (c *configImpl) StartRBE() bool {
 	return true
 }
 
-func (c *configImpl) rbeLogDir() string {
-	for _, f := range []string{"RBE_log_dir", "FLAG_log_dir"} {
+func (c *configImpl) rbeProxyLogsDir() string {
+	for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
 		if v, ok := c.environ.Get(f); ok {
 			return v
 		}
 	}
-	if c.Dist() {
-		return c.LogsDir()
-	}
-	return c.OutDir()
+	buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
+	return filepath.Join(buildTmpDir, "rbe")
 }
 
-func (c *configImpl) rbeStatsOutputDir() string {
-	for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} {
-		if v, ok := c.environ.Get(f); ok {
-			return v
+func (c *configImpl) shouldCleanupRBELogsDir() bool {
+	// Perform a log directory cleanup only when the log directory
+	// is auto created by the build rather than user-specified.
+	for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
+		if _, ok := c.environ.Get(f); ok {
+			return false
 		}
 	}
-	return c.rbeLogDir()
-}
-
-func (c *configImpl) rbeLogPath() string {
-	for _, f := range []string{"RBE_log_path", "FLAG_log_path"} {
-		if v, ok := c.environ.Get(f); ok {
-			return v
-		}
-	}
-	return fmt.Sprintf("text://%v/reproxy_log.txt", c.rbeLogDir())
+	return true
 }
 
 func (c *configImpl) rbeExecRoot() string {
@@ -1223,6 +1225,23 @@ func (c *configImpl) rbeAuth() (string, string) {
 	return "RBE_use_application_default_credentials", "true"
 }
 
+func (c *configImpl) rbeSockAddr(dir string) (string, error) {
+	maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
+	base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
+
+	name := filepath.Join(dir, base)
+	if len(name) < maxNameLen {
+		return name, nil
+	}
+
+	name = filepath.Join("/tmp", base)
+	if len(name) < maxNameLen {
+		return name, nil
+	}
+
+	return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
+}
+
 func (c *configImpl) UseRemoteBuild() bool {
 	return c.UseGoma() || c.UseRBE()
 }

ui/build/rbe.go

@@ -16,12 +16,9 @@ package build
 
 import (
 	"fmt"
-	"math/rand"
 	"os"
 	"path/filepath"
 	"runtime"
-	"syscall"
-	"time"
 
 	"android/soong/ui/metrics"
 )
@@ -54,34 +51,16 @@ func rbeCommand(ctx Context, config Config, rbeCmd string) string {
 	return cmdPath
 }
 
-func sockAddr(dir string) (string, error) {
-	maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
-	rand.Seed(time.Now().UnixNano())
-	base := fmt.Sprintf("reproxy_%v.sock", rand.Intn(1000))
-
-	name := filepath.Join(dir, base)
-	if len(name) < maxNameLen {
-		return name, nil
-	}
-
-	name = filepath.Join("/tmp", base)
-	if len(name) < maxNameLen {
-		return name, nil
-	}
-
-	return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
-}
-
 func getRBEVars(ctx Context, config Config) map[string]string {
 	vars := map[string]string{
-		"RBE_log_path":   config.rbeLogPath(),
-		"RBE_log_dir":    config.rbeLogDir(),
+		"RBE_log_dir":    config.rbeProxyLogsDir(),
 		"RBE_re_proxy":   config.rbeReproxy(),
 		"RBE_exec_root":  config.rbeExecRoot(),
-		"RBE_output_dir": config.rbeStatsOutputDir(),
+		"RBE_output_dir": config.rbeProxyLogsDir(),
+		"RBE_proxy_log_dir": config.rbeProxyLogsDir(),
 	}
 	if config.StartRBE() {
-		name, err := sockAddr(absPath(ctx, config.TempDir()))
+		name, err := config.rbeSockAddr(absPath(ctx, config.TempDir()))
 		if err != nil {
 			ctx.Fatalf("Error retrieving socket address: %v", err)
 			return nil
@@ -100,6 +79,17 @@ func getRBEVars(ctx Context, config Config) map[string]string {
 	return vars
 }
 
+func cleanupRBELogsDir(ctx Context, config Config) {
+	if !config.shouldCleanupRBELogsDir() {
+		return
+	}
+
+	rbeTmpDir := config.rbeProxyLogsDir()
+	if err := os.RemoveAll(rbeTmpDir); err != nil {
+		fmt.Fprintln(ctx.Writer, "\033[33mUnable to remove RBE log directory: ", err, "\033[0m")
+	}
+}
+
 func startRBE(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSetupTool, "rbe_bootstrap")
 	defer ctx.EndTrace()
@@ -110,6 +100,11 @@ func startRBE(ctx Context, config Config) {
 	if n := ulimitOrFatal(ctx, config, "-n"); n < rbeLeastNFiles {
 		ctx.Fatalf("max open files is insufficient: %d; want >= %d.\n", n, rbeLeastNFiles)
 	}
+	if _, err := os.Stat(config.rbeProxyLogsDir()); os.IsNotExist(err) {
+		if err := os.MkdirAll(config.rbeProxyLogsDir(), 0744); err != nil {
+			ctx.Fatalf("Unable to create logs dir (%v) for RBE: %v", config.rbeProxyLogsDir, err)
+		}
+	}
 
 	cmd := Command(ctx, config, "startRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd))
 
@@ -151,7 +146,7 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) {
 		return
 	}
 
-	outputDir := config.rbeStatsOutputDir()
+	outputDir := config.rbeProxyLogsDir()
 	if outputDir == "" {
 		ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.")
 	}

ui/build/rbe_test.go

@@ -56,7 +56,8 @@ func TestDumpRBEMetrics(t *testing.T) {
 			env := Environment(tt.env)
 			env.Set("OUT_DIR", tmpDir)
 			env.Set("RBE_DIR", tmpDir)
-			env.Set("RBE_output_dir", t.TempDir())
+			env.Set("RBE_output_dir", tmpDir)
+			env.Set("RBE_proxy_log_dir", tmpDir)
 			config := Config{&configImpl{
 				environ: &env,
 			}}