Allow Bazel to write to an external DIST_DIR (outside of OUT_DIR).

Also get Bazel to write real files there (not symlinks) so that the DIST_DIR can be independent.

Test: Manually using e.g. DIST_DIR=/tmp/foo USE_BAZEL=1 m dist
Change-Id: I39d5219500864c9ecc85f356a028e9b5bf2607f4

cmd/multiproduct_kati/main.go

@@ -185,7 +185,11 @@ func main() {
 		Status:  stat,
 	}}
 
-	config := build.NewConfig(buildCtx)
+	args := ""
+	if *alternateResultDir {
+		args = "dist"
+	}
+	config := build.NewConfig(buildCtx, args)
 	if *outDir == "" {
 		name := "multiproduct"
 		if !*incremental {
@@ -212,15 +216,10 @@ func main() {
 	os.MkdirAll(logsDir, 0777)
 
 	build.SetupOutDir(buildCtx, config)
-	if *alternateResultDir {
+
-		distLogsDir := filepath.Join(config.DistDir(), "logs")
+	os.MkdirAll(config.LogsDir(), 0777)
-		os.MkdirAll(distLogsDir, 0777)
+	log.SetOutput(filepath.Join(config.LogsDir(), "soong.log"))
-		log.SetOutput(filepath.Join(distLogsDir, "soong.log"))
+	trace.SetOutput(filepath.Join(config.LogsDir(), "build.trace"))
-		trace.SetOutput(filepath.Join(distLogsDir, "build.trace"))
-	} else {
-		log.SetOutput(filepath.Join(config.OutDir(), "soong.log"))
-		trace.SetOutput(filepath.Join(config.OutDir(), "build.trace"))
-	}
 
 	var jobs = *numJobs
 	if jobs < 1 {
@@ -344,7 +343,7 @@ func main() {
 			FileArgs: []zip.FileArg{
 				{GlobDir: logsDir, SourcePrefixToStrip: logsDir},
 			},
-			OutputFilePath:   filepath.Join(config.DistDir(), "logs.zip"),
+			OutputFilePath:   filepath.Join(config.RealDistDir(), "logs.zip"),
 			NumParallelJobs:  runtime.NumCPU(),
 			CompressionLevel: 5,
 		}

cmd/soong_ui/main.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -173,6 +174,10 @@ func main() {
 
 	build.SetupOutDir(buildCtx, config)
 
+	if config.UseBazel() {
+		defer populateExternalDistDir(buildCtx, config)
+	}
+
 	// Set up files to be outputted in the log directory.
 	logsDir := config.LogsDir()
 
@@ -510,3 +515,72 @@ func getCommand(args []string) (*command, []string, error) {
 	// command not found
 	return nil, nil, fmt.Errorf("Command not found: %q", args)
 }
+
+// For Bazel support, this moves files and directories from e.g. out/dist/$f to DIST_DIR/$f if necessary.
+func populateExternalDistDir(ctx build.Context, config build.Config) {
+	// Make sure that internalDistDirPath and externalDistDirPath are both absolute paths, so we can compare them
+	var err error
+	var internalDistDirPath string
+	var externalDistDirPath string
+	if internalDistDirPath, err = filepath.Abs(config.DistDir()); err != nil {
+		ctx.Fatalf("Unable to find absolute path of %s: %s", internalDistDirPath, err)
+	}
+	if externalDistDirPath, err = filepath.Abs(config.RealDistDir()); err != nil {
+		ctx.Fatalf("Unable to find absolute path of %s: %s", externalDistDirPath, err)
+	}
+	if externalDistDirPath == internalDistDirPath {
+		return
+	}
+
+	// Make sure the external DIST_DIR actually exists before trying to write to it
+	if err = os.MkdirAll(externalDistDirPath, 0755); err != nil {
+		ctx.Fatalf("Unable to make directory %s: %s", externalDistDirPath, err)
+	}
+
+	ctx.Println("Populating external DIST_DIR...")
+
+	populateExternalDistDirHelper(ctx, config, internalDistDirPath, externalDistDirPath)
+}
+
+func populateExternalDistDirHelper(ctx build.Context, config build.Config, internalDistDirPath string, externalDistDirPath string) {
+	files, err := ioutil.ReadDir(internalDistDirPath)
+	if err != nil {
+		ctx.Fatalf("Can't read internal distdir %s: %s", internalDistDirPath, err)
+	}
+	for _, f := range files {
+		internalFilePath := filepath.Join(internalDistDirPath, f.Name())
+		externalFilePath := filepath.Join(externalDistDirPath, f.Name())
+
+		if f.IsDir() {
+			// Moving a directory - check if there is an existing directory to merge with
+			externalLstat, err := os.Lstat(externalFilePath)
+			if err != nil {
+				if !os.IsNotExist(err) {
+					ctx.Fatalf("Can't lstat external %s: %s", externalDistDirPath, err)
+				}
+				// Otherwise, if the error was os.IsNotExist, that's fine and we fall through to the rename at the bottom
+			} else {
+				if externalLstat.IsDir() {
+					// Existing dir - try to merge the directories?
+					populateExternalDistDirHelper(ctx, config, internalFilePath, externalFilePath)
+					continue
+				} else {
+					// Existing file being replaced with a directory. Delete the existing file...
+					if err := os.RemoveAll(externalFilePath); err != nil {
+						ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
+					}
+				}
+			}
+		} else {
+			// Moving a file (not a dir) - delete any existing file or directory
+			if err := os.RemoveAll(externalFilePath); err != nil {
+				ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
+			}
+		}
+
+		// The actual move - do a rename instead of a copy in order to save disk space.
+		if err := os.Rename(internalFilePath, externalFilePath); err != nil {
+			ctx.Fatalf("Unable to rename %s -> %s due to error %s", internalFilePath, externalFilePath, err)
+		}
+	}
+}

ui/build/bazel.go

@@ -78,7 +78,10 @@ func runBazel(ctx Context, config Config) {
 	bazelEnv["PACKAGE_NINJA"] = config.KatiPackageNinjaFile()
 	bazelEnv["SOONG_NINJA"] = config.SoongNinjaFile()
 
+	// NOTE: When Bazel is used, config.DistDir() is rigged to return a fake distdir under config.OutDir()
+	// This is to ensure that Bazel can actually write there. See config.go for more details.
 	bazelEnv["DIST_DIR"] = config.DistDir()
+
 	bazelEnv["SHELL"] = "/bin/bash"
 
 	// `tools/bazel` is the default entry point for executing Bazel in the AOSP
@@ -189,13 +192,14 @@ func runBazel(ctx Context, config Config) {
 	// currently hardcoded as ninja_build.output_root.
 	bazelNinjaBuildOutputRoot := filepath.Join(outputBasePath, "..", "out")
 
-	ctx.Println("Creating output symlinks..")
+	ctx.Println("Populating output directory...")
-	symlinkOutdir(ctx, config, bazelNinjaBuildOutputRoot, ".")
+	populateOutdir(ctx, config, bazelNinjaBuildOutputRoot, ".")
 }
 
 // For all files F recursively under rootPath/relativePath, creates symlinks
 // such that OutDir/F resolves to rootPath/F via symlinks.
-func symlinkOutdir(ctx Context, config Config, rootPath string, relativePath string) {
+// NOTE: For distdir paths we rename files instead of creating symlinks, so that the distdir is independent.
+func populateOutdir(ctx Context, config Config, rootPath string, relativePath string) {
 	destDir := filepath.Join(rootPath, relativePath)
 	os.MkdirAll(destDir, 0755)
 	files, err := ioutil.ReadDir(destDir)
@@ -220,7 +224,7 @@ func symlinkOutdir(ctx Context, config Config, rootPath string, relativePath str
 		if srcLstatErr == nil {
 			if srcLstatResult.IsDir() && destLstatResult.IsDir() {
 				// src and dest are both existing dirs - recurse on the dest dir contents...
-				symlinkOutdir(ctx, config, rootPath, filepath.Join(relativePath, f.Name()))
+				populateOutdir(ctx, config, rootPath, filepath.Join(relativePath, f.Name()))
 			} else {
 				// Ignore other pre-existing src files (could be pre-existing files, directories, symlinks, ...)
 				// This can arise for files which are generated under OutDir outside of soong_build, such as .bootstrap files.
@@ -231,9 +235,17 @@ func symlinkOutdir(ctx Context, config Config, rootPath string, relativePath str
 				ctx.Fatalf("Unable to Lstat src %s: %s", srcPath, srcLstatErr)
 			}
 
+			if strings.Contains(destDir, config.DistDir()) {
+				// We need to make a "real" file/dir instead of making a symlink (because the distdir can't have symlinks)
+				// Rename instead of copy in order to save disk space.
+				if err := os.Rename(destPath, srcPath); err != nil {
+					ctx.Fatalf("Unable to rename %s -> %s due to error %s", srcPath, destPath, err)
+				}
+			} else {
 				// src does not exist, so try to create a src -> dest symlink (i.e. a Soong path -> Bazel path symlink)
-			if symlinkErr := os.Symlink(destPath, srcPath); symlinkErr != nil {
+				if err := os.Symlink(destPath, srcPath); err != nil {
-				ctx.Fatalf("Unable to create symlink %s -> %s due to error %s", srcPath, destPath, symlinkErr)
+					ctx.Fatalf("Unable to create symlink %s -> %s due to error %s", srcPath, destPath, err)
+				}
 			}
 		}
 	}

ui/build/build.go

@@ -302,7 +302,7 @@ func distGzipFile(ctx Context, config Config, src string, subDirs ...string) {
 	}
 
 	subDir := filepath.Join(subDirs...)
-	destDir := filepath.Join(config.DistDir(), "soong_ui", subDir)
+	destDir := filepath.Join(config.RealDistDir(), "soong_ui", subDir)
 
 	if err := os.MkdirAll(destDir, 0777); err != nil { // a+rwx
 		ctx.Printf("failed to mkdir %s: %s", destDir, err.Error())
@@ -321,7 +321,7 @@ func distFile(ctx Context, config Config, src string, subDirs ...string) {
 	}
 
 	subDir := filepath.Join(subDirs...)
-	destDir := filepath.Join(config.DistDir(), "soong_ui", subDir)
+	destDir := filepath.Join(config.RealDistDir(), "soong_ui", subDir)
 
 	if err := os.MkdirAll(destDir, 0777); err != nil { // a+rwx
 		ctx.Printf("failed to mkdir %s: %s", destDir, err.Error())

ui/build/config.go

@@ -65,6 +65,12 @@ type configImpl struct {
 	brokenNinjaEnvVars []string
 
 	pathReplaced bool
+
+	useBazel bool
+
+	// During Bazel execution, Bazel cannot write outside OUT_DIR.
+	// So if DIST_DIR is set to an external dir (outside of OUT_DIR), we need to rig it temporarily and then migrate files at the end of the build.
+	riggedDistDirForBazel string
 }
 
 const srcDirFileCheck = "build/soong/root.bp"
@@ -221,7 +227,7 @@ func NewConfig(ctx Context, args ...string) Config {
 		ctx.Fatalln("Directory names containing spaces are not supported")
 	}
 
-	if distDir := ret.DistDir(); strings.ContainsRune(distDir, ' ') {
+	if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
 		ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
 		ctx.Println()
 		ctx.Printf("%q\n", distDir)
@@ -279,12 +285,22 @@ func NewConfig(ctx Context, args ...string) Config {
 	if err := os.RemoveAll(bpd); err != nil {
 		ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
 	}
+
+	ret.useBazel = ret.environ.IsEnvTrue("USE_BAZEL")
+
 	if ret.UseBazel() {
 		if err := os.MkdirAll(bpd, 0777); err != nil {
 			ctx.Fatalf("Failed to create bazel profile directory %q: %v", bpd, err)
 		}
 	}
 
+	if ret.UseBazel() {
+		ret.riggedDistDirForBazel = filepath.Join(ret.OutDir(), "dist")
+	} else {
+		// Not rigged
+		ret.riggedDistDirForBazel = ret.distDir
+	}
+
 	c := Config{ret}
 	storeConfigMetrics(ctx, c)
 	return c
@@ -697,6 +713,14 @@ func (c *configImpl) OutDir() string {
 }
 
 func (c *configImpl) DistDir() string {
+	if c.UseBazel() {
+		return c.riggedDistDirForBazel
+	} else {
+		return c.distDir
+	}
+}
+
+func (c *configImpl) RealDistDir() string {
 	return c.distDir
 }
 
@@ -863,13 +887,7 @@ func (c *configImpl) UseRBE() bool {
 }
 
 func (c *configImpl) UseBazel() bool {
-	if v, ok := c.environ.Get("USE_BAZEL"); ok {
+	return c.useBazel
-		v = strings.TrimSpace(v)
-		if v != "" && v != "false" {
-			return true
-		}
-	}
-	return false
 }
 
 func (c *configImpl) StartRBE() bool {
@@ -886,14 +904,14 @@ func (c *configImpl) StartRBE() bool {
 	return true
 }
 
-func (c *configImpl) logDir() string {
+func (c *configImpl) rbeLogDir() string {
 	for _, f := range []string{"RBE_log_dir", "FLAG_log_dir"} {
 		if v, ok := c.environ.Get(f); ok {
 			return v
 		}
 	}
 	if c.Dist() {
-		return filepath.Join(c.DistDir(), "logs")
+		return c.LogsDir()
 	}
 	return c.OutDir()
 }
@@ -904,7 +922,7 @@ func (c *configImpl) rbeStatsOutputDir() string {
 			return v
 		}
 	}
-	return c.logDir()
+	return c.rbeLogDir()
 }
 
 func (c *configImpl) rbeLogPath() string {
@@ -913,7 +931,7 @@ func (c *configImpl) rbeLogPath() string {
 			return v
 		}
 	}
-	return fmt.Sprintf("text://%v/reproxy_log.txt", c.logDir())
+	return fmt.Sprintf("text://%v/reproxy_log.txt", c.rbeLogDir())
 }
 
 func (c *configImpl) rbeExecRoot() string {
@@ -1128,7 +1146,8 @@ func (c *configImpl) MetricsUploaderApp() string {
 // is <dist_dir>/logs.
 func (c *configImpl) LogsDir() string {
 	if c.Dist() {
-		return filepath.Join(c.DistDir(), "logs")
+		// Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
+		return filepath.Join(c.RealDistDir(), "logs")
 	}
 	return c.OutDir()
 }

ui/build/rbe.go

@@ -74,7 +74,7 @@ func sockAddr(dir string) (string, error) {
 func getRBEVars(ctx Context, config Config) map[string]string {
 	vars := map[string]string{
 		"RBE_log_path":   config.rbeLogPath(),
-		"RBE_log_dir":    config.logDir(),
+		"RBE_log_dir":    config.rbeLogDir(),
 		"RBE_re_proxy":   config.rbeReproxy(),
 		"RBE_exec_root":  config.rbeExecRoot(),
 		"RBE_output_dir": config.rbeStatsOutputDir(),