Merge "cd to / before running soong_build ."

android/env.go

@@ -17,7 +17,6 @@ package android
 import (
 	"fmt"
 	"os"
-	"os/exec"
 	"strings"
 	"syscall"
 
@@ -34,37 +33,27 @@ import (
 var originalEnv map[string]string
 var soongDelveListen string
 var soongDelvePath string
-var soongDelveEnv []string
+var isDebugging bool
 
-func init() {
+func InitEnvironment(envFile string) {
-	// Delve support needs to read this environment variable very early, before NewConfig has created a way to
+	var err error
-	// access originalEnv with dependencies.  Store the value where soong_build can find it, it will manually
+	originalEnv, err = shared.EnvFromFile(envFile)
-	// ensure the dependencies are created.
+	if err != nil {
-	soongDelveListen = os.Getenv("SOONG_DELVE")
+		panic(err)
-	soongDelvePath = os.Getenv("SOONG_DELVE_PATH")
-	if soongDelvePath == "" {
-		soongDelvePath, _ = exec.LookPath("dlv")
 	}
 
-	originalEnv = make(map[string]string)
+	soongDelveListen = originalEnv["SOONG_DELVE"]
-	soongDelveEnv = []string{}
+	soongDelvePath = originalEnv["SOONG_DELVE_PATH"]
-	for _, env := range os.Environ() {
-		idx := strings.IndexRune(env, '=')
-		if idx != -1 {
-			originalEnv[env[:idx]] = env[idx+1:]
-			if env[:idx] != "SOONG_DELVE" && env[:idx] != "SOONG_DELVE_PATH" {
-				soongDelveEnv = append(soongDelveEnv, env)
-			}
-		}
-	}
-
-	// Clear the environment to prevent use of os.Getenv(), which would not provide dependencies on environment
-	// variable values.  The environment is available through ctx.Config().Getenv, ctx.Config().IsEnvTrue, etc.
-	os.Clearenv()
 }
 
+// Returns whether the current process is running under Delve due to
+// ReexecWithDelveMaybe().
+func IsDebugging() bool {
+	return isDebugging
+}
 func ReexecWithDelveMaybe() {
-	if soongDelveListen == "" {
+	isDebugging = os.Getenv("SOONG_DELVE_REEXECUTED") == "true"
+	if isDebugging || soongDelveListen == "" {
 		return
 	}
 
@@ -72,6 +61,17 @@ func ReexecWithDelveMaybe() {
 		fmt.Fprintln(os.Stderr, "SOONG_DELVE is set but failed to find dlv")
 		os.Exit(1)
 	}
+
+	soongDelveEnv := []string{}
+	for _, env := range os.Environ() {
+		idx := strings.IndexRune(env, '=')
+		if idx != -1 {
+			soongDelveEnv = append(soongDelveEnv, env)
+		}
+	}
+
+	soongDelveEnv = append(soongDelveEnv, "SOONG_DELVE_REEXECUTED=true")
+
 	dlvArgv := []string{
 		soongDelvePath,
 		"--listen=:" + soongDelveListen,
@@ -88,17 +88,6 @@ func ReexecWithDelveMaybe() {
 	os.Exit(1)
 }
 
-// getenv checks either os.Getenv or originalEnv so that it works before or after the init()
-// function above.  It doesn't add any dependencies on the environment variable, so it should
-// only be used for values that won't change.  For values that might change use ctx.Config().Getenv.
-func getenv(key string) string {
-	if originalEnv == nil {
-		return os.Getenv(key)
-	} else {
-		return originalEnv[key]
-	}
-}
-
 func EnvSingleton() Singleton {
 	return &envSingleton{}
 }
@@ -108,7 +97,7 @@ type envSingleton struct{}
 func (c *envSingleton) GenerateBuildActions(ctx SingletonContext) {
 	envDeps := ctx.Config().EnvDeps()
 
-	envFile := PathForOutput(ctx, ".soong.environment")
+	envFile := PathForOutput(ctx, "soong.environment.used")
 	if ctx.Failed() {
 		return
 	}

android/sandbox.go

@@ -14,29 +14,8 @@
 
 package android
 
-import (
+func InitSandbox(topDir string) {
-	"fmt"
+	absSrcDir = topDir
-	"os"
-)
-
-func init() {
-	// Stash the working directory in a private variable and then change the working directory
-	// to "/", which will prevent untracked accesses to files by Go Soong plugins. The
-	// SOONG_SANDBOX_SOONG_BUILD environment variable is set by soong_ui, and is not
-	// overrideable on the command line.
-
-	orig, err := os.Getwd()
-	if err != nil {
-		panic(fmt.Errorf("failed to get working directory: %s", err))
-	}
-	absSrcDir = orig
-
-	if getenv("SOONG_SANDBOX_SOONG_BUILD") == "true" {
-		err = os.Chdir("/")
-		if err != nil {
-			panic(fmt.Errorf("failed to change working directory to '/': %s", err))
-		}
-	}
 }
 
 // DO NOT USE THIS FUNCTION IN NEW CODE.

android/writedocs.go

@@ -34,7 +34,8 @@ func DocsSingleton() Singleton {
 type docsSingleton struct{}
 
 func primaryBuilderPath(ctx SingletonContext) Path {
-	primaryBuilder, err := filepath.Rel(ctx.Config().BuildDir(), os.Args[0])
+	buildDir := absolutePath(ctx.Config().BuildDir())
+	primaryBuilder, err := filepath.Rel(buildDir, os.Args[0])
 	if err != nil {
 		ctx.Errorf("path to primary builder %q is not in build dir %q",
 			os.Args[0], ctx.Config().BuildDir())

apex/apex_test.go

@@ -2534,7 +2534,8 @@ func TestVendorApex(t *testing.T) {
 	var builder strings.Builder
 	data.Custom(&builder, name, prefix, "", data)
 	androidMk := builder.String()
-	ensureContains(t, androidMk, `LOCAL_MODULE_PATH := /tmp/target/product/test_device/vendor/apex`)
+	installPath := path.Join(buildDir, "../target/product/test_device/vendor/apex")
+	ensureContains(t, androidMk, "LOCAL_MODULE_PATH := "+installPath)
 
 	apexManifestRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexManifestRule")
 	requireNativeLibs := names(apexManifestRule.Args["requireNativeLibs"])

cmd/soong_build/main.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"android/soong/shared"
 	"github.com/google/blueprint/bootstrap"
 
 	"android/soong/android"
@@ -28,11 +29,15 @@ import (
 )
 
 var (
+	topDir            string
+	outDir            string
 	docFile           string
 	bazelQueryViewDir string
 )
 
 func init() {
+	flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
+	flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)")
 	flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
 	flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory")
 }
@@ -80,15 +85,22 @@ func newConfig(srcDir string) android.Config {
 }
 
 func main() {
-	android.ReexecWithDelveMaybe()
 	flag.Parse()
 
+	android.InitSandbox(topDir)
+	android.InitEnvironment(shared.JoinPath(topDir, outDir, "soong.environment.available"))
+	android.ReexecWithDelveMaybe()
+
 	// The top-level Blueprints file is passed as the first argument.
 	srcDir := filepath.Dir(flag.Arg(0))
 	var ctx *android.Context
 	configuration := newConfig(srcDir)
 	extraNinjaDeps := []string{configuration.ProductVariablesFileName}
 
+	// These two are here so that we restart a non-debugged soong_build when the
+	// user sets SOONG_DELVE the first time.
+	configuration.Getenv("SOONG_DELVE")
+	configuration.Getenv("SOONG_DELVE_PATH")
 	// Read the SOONG_DELVE again through configuration so that there is a dependency on the environment variable
 	// and soong_build will rerun when it is set for the first time.
 	if listen := configuration.Getenv("SOONG_DELVE"); listen != "" {

java/rro_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"android/soong/android"
+	"android/soong/shared"
 )
 
 func TestRuntimeResourceOverlay(t *testing.T) {
@@ -105,7 +106,7 @@ func TestRuntimeResourceOverlay(t *testing.T) {
 
 	// Check device location.
 	path = androidMkEntries.EntryMap["LOCAL_MODULE_PATH"]
-	expectedPath = []string{"/tmp/target/product/test_device/product/overlay"}
+	expectedPath = []string{shared.JoinPath(buildDir, "../target/product/test_device/product/overlay")}
 	if !reflect.DeepEqual(path, expectedPath) {
 		t.Errorf("Unexpected LOCAL_MODULE_PATH value: %v, expected: %v", path, expectedPath)
 	}
@@ -114,7 +115,7 @@ func TestRuntimeResourceOverlay(t *testing.T) {
 	m = ctx.ModuleForTests("foo_themed", "android_common")
 	androidMkEntries = android.AndroidMkEntriesForTest(t, ctx, m.Module())[0]
 	path = androidMkEntries.EntryMap["LOCAL_MODULE_PATH"]
-	expectedPath = []string{"/tmp/target/product/test_device/product/overlay/faza"}
+	expectedPath = []string{shared.JoinPath(buildDir, "../target/product/test_device/product/overlay/faza")}
 	if !reflect.DeepEqual(path, expectedPath) {
 		t.Errorf("Unexpected LOCAL_MODULE_PATH value: %v, expected: %v", path, expectedPath)
 	}
@@ -160,7 +161,7 @@ func TestRuntimeResourceOverlay_JavaDefaults(t *testing.T) {
 
 	// Check device location.
 	path := android.AndroidMkEntriesForTest(t, ctx, m.Module())[0].EntryMap["LOCAL_MODULE_PATH"]
-	expectedPath := []string{"/tmp/target/product/test_device/product/overlay/default_theme"}
+	expectedPath := []string{shared.JoinPath(buildDir, "../target/product/test_device/product/overlay/default_theme")}
 	if !reflect.DeepEqual(path, expectedPath) {
 		t.Errorf("Unexpected LOCAL_MODULE_PATH value: %q, expected: %q", path, expectedPath)
 	}
@@ -179,7 +180,7 @@ func TestRuntimeResourceOverlay_JavaDefaults(t *testing.T) {
 
 	// Check device location.
 	path = android.AndroidMkEntriesForTest(t, ctx, m.Module())[0].EntryMap["LOCAL_MODULE_PATH"]
-	expectedPath = []string{"/tmp/target/product/test_device/system/overlay"}
+	expectedPath = []string{shared.JoinPath(buildDir, "../target/product/test_device/system/overlay")}
 	if !reflect.DeepEqual(path, expectedPath) {
 		t.Errorf("Unexpected LOCAL_MODULE_PATH value: %v, expected: %v", path, expectedPath)
 	}

sh/sh_binary_test.go

@@ -3,6 +3,7 @@ package sh
 import (
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"reflect"
 	"testing"
@@ -73,7 +74,8 @@ func TestShTestSubDir(t *testing.T) {
 
 	entries := android.AndroidMkEntriesForTest(t, ctx, mod)[0]
 
-	expectedPath := "/tmp/target/product/test_device/data/nativetest64/foo_test"
+	expectedPath := path.Join(buildDir,
+		"../target/product/test_device/data/nativetest64/foo_test")
 	actualPath := entries.EntryMap["LOCAL_MODULE_PATH"][0]
 	if expectedPath != actualPath {
 		t.Errorf("Unexpected LOCAL_MODULE_PATH expected: %q, actual: %q", expectedPath, actualPath)
@@ -97,7 +99,8 @@ func TestShTest(t *testing.T) {
 
 	entries := android.AndroidMkEntriesForTest(t, ctx, mod)[0]
 
-	expectedPath := "/tmp/target/product/test_device/data/nativetest64/foo"
+	expectedPath := path.Join(buildDir,
+		"../target/product/test_device/data/nativetest64/foo")
 	actualPath := entries.EntryMap["LOCAL_MODULE_PATH"][0]
 	if expectedPath != actualPath {
 		t.Errorf("Unexpected LOCAL_MODULE_PATH expected: %q, actual: %q", expectedPath, actualPath)

shared/Android.bp

@@ -8,6 +8,10 @@ bootstrap_go_package {
     srcs: [
         "env.go",
         "paths.go",
+        "debug.go",
+    ],
+    testSrcs: [
+        "paths_test.go",
     ],
     deps: [
         "soong-bazel",

shared/debug.go

@@ -0,0 +1,17 @@
+package shared
+
+import (
+	"os"
+	"os/exec"
+)
+
+// Finds the Delve binary to use. Either uses the SOONG_DELVE_PATH environment
+// variable or if that is unset, looks at $PATH.
+func ResolveDelveBinary() string {
+	result := os.Getenv("SOONG_DELVE_PATH")
+	if result == "" {
+		result, _ = exec.LookPath("dlv")
+	}
+
+	return result
+}

shared/env.go

@@ -91,6 +91,28 @@ func StaleEnvFile(filepath string, getenv func(string) string) (bool, error) {
 	return false, nil
 }
 
+// Deserializes and environment serialized by EnvFileContents() and returns it
+// as a map[string]string.
+func EnvFromFile(envFile string) (map[string]string, error) {
+	result := make(map[string]string)
+	data, err := ioutil.ReadFile(envFile)
+	if err != nil {
+		return result, err
+	}
+
+	var contents envFileData
+	err = json.Unmarshal(data, &contents)
+	if err != nil {
+		return result, err
+	}
+
+	for _, entry := range contents {
+		result[entry.Key] = entry.Value
+	}
+
+	return result, nil
+}
+
 // Implements sort.Interface so that we can use sort.Sort on envFileData arrays.
 func (e envFileData) Len() int {
 	return len(e)

shared/paths.go

@@ -30,6 +30,21 @@ type SharedPaths interface {
 	BazelMetricsDir() string
 }
 
+// Joins the path strings in the argument list, taking absolute paths into
+// account. That is, if one of the strings is an absolute path, the ones before
+// are ignored.
+func JoinPath(base string, rest ...string) string {
+	result := base
+	for _, next := range rest {
+		if filepath.IsAbs(next) {
+			result = next
+		} else {
+			result = filepath.Join(result, next)
+		}
+	}
+	return result
+}
+
 // Given the out directory, returns the root of the temp directory (to be cleared at the start of each execution of Soong)
 func TempDirForOutDir(outDir string) (tempPath string) {
 	return filepath.Join(outDir, ".temp")

shared/paths_test.go

@@ -0,0 +1,32 @@
+// Copyright 2021 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared
+
+import (
+	"testing"
+)
+
+func assertEqual(t *testing.T, expected, actual string) {
+	t.Helper()
+	if expected != actual {
+		t.Errorf("expected %q != got %q", expected, actual)
+	}
+}
+
+func TestJoinPath(t *testing.T) {
+	assertEqual(t, "/a/b", JoinPath("c/d", "/a/b"))
+	assertEqual(t, "a/b", JoinPath("a", "b"))
+	assertEqual(t, "/a/b", JoinPath("x", "/a", "b"))
+}

ui/build/environment.go

@@ -33,6 +33,19 @@ func OsEnvironment() *Environment {
 	return &env
 }
 
+// Returns a copy of the environment as a map[string]string.
+func (e *Environment) AsMap() map[string]string {
+	result := make(map[string]string)
+
+	for _, envVar := range *e {
+		if k, v, ok := decodeKeyValue(envVar); ok {
+			result[k] = v
+		}
+	}
+
+	return result
+}
+
 // Get returns the value associated with the key, and whether it exists.
 // It's equivalent to the os.LookupEnv function, but with this copy of the
 // Environment.

ui/build/soong.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 
 	"android/soong/shared"
+
 	soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
 
 	"github.com/golang/protobuf/proto"
@@ -30,6 +31,15 @@ import (
 	"android/soong/ui/status"
 )
 
+func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
+	data, err := shared.EnvFileContents(envDeps)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(envFile, data, 0644)
+}
+
 // This uses Android.bp files and various tools to generate <builddir>/build.ninja.
 //
 // However, the execution of <builddir>/build.ninja happens later in build/soong/ui/build/build.go#Build()
@@ -47,6 +57,12 @@ func runSoong(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSoong, "soong")
 	defer ctx.EndTrace()
 
+	// We have two environment files: .available is the one with every variable,
+	// .used with the ones that were actually used. The latter is used to
+	// determine whether Soong needs to be re-run since why re-run it if only
+	// unused variables were changed?
+	envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available")
+
 	// Use an anonymous inline function for tracing purposes (this pattern is used several times below).
 	func() {
 		ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
@@ -61,6 +77,7 @@ func runSoong(ctx Context, config Config) {
 		}
 
 		cmd := Command(ctx, config, "blueprint bootstrap", "build/blueprint/bootstrap.bash", args...)
+
 		cmd.Environment.Set("BLUEPRINTDIR", "./build/blueprint")
 		cmd.Environment.Set("BOOTSTRAP", "./build/blueprint/bootstrap.bash")
 		cmd.Environment.Set("BUILDDIR", config.SoongOutDir())
@@ -74,11 +91,32 @@ func runSoong(ctx Context, config Config) {
 		cmd.RunAndPrintOrFatal()
 	}()
 
+	soongBuildEnv := config.Environment().Copy()
+	soongBuildEnv.Set("TOP", os.Getenv("TOP"))
+	// These two dependencies are read from bootstrap.go, but also need to be here
+	// so that soong_build can declare a dependency on them
+	soongBuildEnv.Set("SOONG_DELVE", os.Getenv("SOONG_DELVE"))
+	soongBuildEnv.Set("SOONG_DELVE_PATH", os.Getenv("SOONG_DELVE_PATH"))
+	soongBuildEnv.Set("SOONG_OUTDIR", config.SoongOutDir())
+	// For Bazel mixed builds.
+	soongBuildEnv.Set("BAZEL_PATH", "./tools/bazel")
+	soongBuildEnv.Set("BAZEL_HOME", filepath.Join(config.BazelOutDir(), "bazelhome"))
+	soongBuildEnv.Set("BAZEL_OUTPUT_BASE", filepath.Join(config.BazelOutDir(), "output"))
+	soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
+	soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
+
+	if os.Getenv("SOONG_DELVE") != "" {
+		// SOONG_DELVE is already in cmd.Environment
+		soongBuildEnv.Set("SOONG_DELVE_PATH", shared.ResolveDelveBinary())
+	}
+
+	writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
+
 	func() {
 		ctx.BeginTrace(metrics.RunSoong, "environment check")
 		defer ctx.EndTrace()
 
-		envFile := filepath.Join(config.SoongOutDir(), ".soong.environment")
+		envFile := filepath.Join(config.SoongOutDir(), "soong.environment.used")
 		getenv := func(k string) string {
 			v, _ := config.Environment().Get(k)
 			return v
@@ -134,14 +172,7 @@ func runSoong(ctx Context, config Config) {
 			"--frontend_file", fifo,
 			"-f", filepath.Join(config.SoongOutDir(), file))
 
-		// For Bazel mixed builds.
+		cmd.Environment.Set("SOONG_OUTDIR", config.SoongOutDir())
-		cmd.Environment.Set("BAZEL_PATH", "./tools/bazel")
-		cmd.Environment.Set("BAZEL_HOME", filepath.Join(config.BazelOutDir(), "bazelhome"))
-		cmd.Environment.Set("BAZEL_OUTPUT_BASE", filepath.Join(config.BazelOutDir(), "output"))
-		cmd.Environment.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
-		cmd.Environment.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
-
-		cmd.Environment.Set("SOONG_SANDBOX_SOONG_BUILD", "true")
 		cmd.Sandbox = soongSandbox
 		cmd.RunAndStreamOrFatal()
 	}