diff --git a/android/env.go b/android/env.go index a8c7777ad..58ad0b615 100644 --- a/android/env.go +++ b/android/env.go @@ -17,7 +17,6 @@ package android import ( "fmt" "os" - "os/exec" "strings" "syscall" @@ -34,37 +33,27 @@ import ( var originalEnv map[string]string var soongDelveListen string var soongDelvePath string -var soongDelveEnv []string +var isDebugging bool -func init() { - // Delve support needs to read this environment variable very early, before NewConfig has created a way to - // access originalEnv with dependencies. Store the value where soong_build can find it, it will manually - // ensure the dependencies are created. - soongDelveListen = os.Getenv("SOONG_DELVE") - soongDelvePath = os.Getenv("SOONG_DELVE_PATH") - if soongDelvePath == "" { - soongDelvePath, _ = exec.LookPath("dlv") +func InitEnvironment(envFile string) { + var err error + originalEnv, err = shared.EnvFromFile(envFile) + if err != nil { + panic(err) } - originalEnv = make(map[string]string) - soongDelveEnv = []string{} - for _, env := range os.Environ() { - idx := strings.IndexRune(env, '=') - if idx != -1 { - originalEnv[env[:idx]] = env[idx+1:] - if env[:idx] != "SOONG_DELVE" && env[:idx] != "SOONG_DELVE_PATH" { - soongDelveEnv = append(soongDelveEnv, env) - } - } - } - - // Clear the environment to prevent use of os.Getenv(), which would not provide dependencies on environment - // variable values. The environment is available through ctx.Config().Getenv, ctx.Config().IsEnvTrue, etc. - os.Clearenv() + soongDelveListen = originalEnv["SOONG_DELVE"] + soongDelvePath = originalEnv["SOONG_DELVE_PATH"] } +// Returns whether the current process is running under Delve due to +// ReexecWithDelveMaybe(). +func IsDebugging() bool { + return isDebugging +} func ReexecWithDelveMaybe() { - if soongDelveListen == "" { + isDebugging = os.Getenv("SOONG_DELVE_REEXECUTED") == "true" + if isDebugging || soongDelveListen == "" { return } @@ -72,6 +61,17 @@ func ReexecWithDelveMaybe() { fmt.Fprintln(os.Stderr, "SOONG_DELVE is set but failed to find dlv") os.Exit(1) } + + soongDelveEnv := []string{} + for _, env := range os.Environ() { + idx := strings.IndexRune(env, '=') + if idx != -1 { + soongDelveEnv = append(soongDelveEnv, env) + } + } + + soongDelveEnv = append(soongDelveEnv, "SOONG_DELVE_REEXECUTED=true") + dlvArgv := []string{ soongDelvePath, "--listen=:" + soongDelveListen, @@ -88,17 +88,6 @@ func ReexecWithDelveMaybe() { os.Exit(1) } -// getenv checks either os.Getenv or originalEnv so that it works before or after the init() -// function above. It doesn't add any dependencies on the environment variable, so it should -// only be used for values that won't change. For values that might change use ctx.Config().Getenv. -func getenv(key string) string { - if originalEnv == nil { - return os.Getenv(key) - } else { - return originalEnv[key] - } -} - func EnvSingleton() Singleton { return &envSingleton{} } @@ -108,7 +97,7 @@ type envSingleton struct{} func (c *envSingleton) GenerateBuildActions(ctx SingletonContext) { envDeps := ctx.Config().EnvDeps() - envFile := PathForOutput(ctx, ".soong.environment") + envFile := PathForOutput(ctx, "soong.environment.used") if ctx.Failed() { return } diff --git a/android/sandbox.go b/android/sandbox.go index ed022fb87..28e903ad4 100644 --- a/android/sandbox.go +++ b/android/sandbox.go @@ -14,29 +14,8 @@ package android -import ( - "fmt" - "os" -) - -func init() { - // Stash the working directory in a private variable and then change the working directory - // to "/", which will prevent untracked accesses to files by Go Soong plugins. The - // SOONG_SANDBOX_SOONG_BUILD environment variable is set by soong_ui, and is not - // overrideable on the command line. - - orig, err := os.Getwd() - if err != nil { - panic(fmt.Errorf("failed to get working directory: %s", err)) - } - absSrcDir = orig - - if getenv("SOONG_SANDBOX_SOONG_BUILD") == "true" { - err = os.Chdir("/") - if err != nil { - panic(fmt.Errorf("failed to change working directory to '/': %s", err)) - } - } +func InitSandbox(topDir string) { + absSrcDir = topDir } // DO NOT USE THIS FUNCTION IN NEW CODE. diff --git a/android/writedocs.go b/android/writedocs.go index 91c2318f1..6417690db 100644 --- a/android/writedocs.go +++ b/android/writedocs.go @@ -34,7 +34,8 @@ func DocsSingleton() Singleton { type docsSingleton struct{} func primaryBuilderPath(ctx SingletonContext) Path { - primaryBuilder, err := filepath.Rel(ctx.Config().BuildDir(), os.Args[0]) + buildDir := absolutePath(ctx.Config().BuildDir()) + primaryBuilder, err := filepath.Rel(buildDir, os.Args[0]) if err != nil { ctx.Errorf("path to primary builder %q is not in build dir %q", os.Args[0], ctx.Config().BuildDir()) diff --git a/apex/apex_test.go b/apex/apex_test.go index 8ae663470..aa68378bb 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -2534,7 +2534,8 @@ func TestVendorApex(t *testing.T) { var builder strings.Builder data.Custom(&builder, name, prefix, "", data) androidMk := builder.String() - ensureContains(t, androidMk, `LOCAL_MODULE_PATH := /tmp/target/product/test_device/vendor/apex`) + installPath := path.Join(buildDir, "../target/product/test_device/vendor/apex") + ensureContains(t, androidMk, "LOCAL_MODULE_PATH := "+installPath) apexManifestRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexManifestRule") requireNativeLibs := names(apexManifestRule.Args["requireNativeLibs"]) diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 3a6fecaf0..e7f995fda 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -21,6 +21,7 @@ import ( "path/filepath" "strings" + "android/soong/shared" "github.com/google/blueprint/bootstrap" "android/soong/android" @@ -28,11 +29,15 @@ import ( ) var ( + topDir string + outDir string docFile string bazelQueryViewDir string ) func init() { + flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree") + flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)") flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output") flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory") } @@ -80,15 +85,22 @@ func newConfig(srcDir string) android.Config { } func main() { - android.ReexecWithDelveMaybe() flag.Parse() + android.InitSandbox(topDir) + android.InitEnvironment(shared.JoinPath(topDir, outDir, "soong.environment.available")) + android.ReexecWithDelveMaybe() + // The top-level Blueprints file is passed as the first argument. srcDir := filepath.Dir(flag.Arg(0)) var ctx *android.Context configuration := newConfig(srcDir) extraNinjaDeps := []string{configuration.ProductVariablesFileName} + // These two are here so that we restart a non-debugged soong_build when the + // user sets SOONG_DELVE the first time. + configuration.Getenv("SOONG_DELVE") + configuration.Getenv("SOONG_DELVE_PATH") // Read the SOONG_DELVE again through configuration so that there is a dependency on the environment variable // and soong_build will rerun when it is set for the first time. if listen := configuration.Getenv("SOONG_DELVE"); listen != "" { diff --git a/java/rro_test.go b/java/rro_test.go index edbf1708b..061d9d348 100644 --- a/java/rro_test.go +++ b/java/rro_test.go @@ -20,6 +20,7 @@ import ( "testing" "android/soong/android" + "android/soong/shared" ) func TestRuntimeResourceOverlay(t *testing.T) { @@ -105,7 +106,7 @@ func TestRuntimeResourceOverlay(t *testing.T) { // Check device location. path = androidMkEntries.EntryMap["LOCAL_MODULE_PATH"] - expectedPath = []string{"/tmp/target/product/test_device/product/overlay"} + expectedPath = []string{shared.JoinPath(buildDir, "../target/product/test_device/product/overlay")} if !reflect.DeepEqual(path, expectedPath) { t.Errorf("Unexpected LOCAL_MODULE_PATH value: %v, expected: %v", path, expectedPath) } @@ -114,7 +115,7 @@ func TestRuntimeResourceOverlay(t *testing.T) { m = ctx.ModuleForTests("foo_themed", "android_common") androidMkEntries = android.AndroidMkEntriesForTest(t, ctx, m.Module())[0] path = androidMkEntries.EntryMap["LOCAL_MODULE_PATH"] - expectedPath = []string{"/tmp/target/product/test_device/product/overlay/faza"} + expectedPath = []string{shared.JoinPath(buildDir, "../target/product/test_device/product/overlay/faza")} if !reflect.DeepEqual(path, expectedPath) { t.Errorf("Unexpected LOCAL_MODULE_PATH value: %v, expected: %v", path, expectedPath) } @@ -160,7 +161,7 @@ func TestRuntimeResourceOverlay_JavaDefaults(t *testing.T) { // Check device location. path := android.AndroidMkEntriesForTest(t, ctx, m.Module())[0].EntryMap["LOCAL_MODULE_PATH"] - expectedPath := []string{"/tmp/target/product/test_device/product/overlay/default_theme"} + expectedPath := []string{shared.JoinPath(buildDir, "../target/product/test_device/product/overlay/default_theme")} if !reflect.DeepEqual(path, expectedPath) { t.Errorf("Unexpected LOCAL_MODULE_PATH value: %q, expected: %q", path, expectedPath) } @@ -179,7 +180,7 @@ func TestRuntimeResourceOverlay_JavaDefaults(t *testing.T) { // Check device location. path = android.AndroidMkEntriesForTest(t, ctx, m.Module())[0].EntryMap["LOCAL_MODULE_PATH"] - expectedPath = []string{"/tmp/target/product/test_device/system/overlay"} + expectedPath = []string{shared.JoinPath(buildDir, "../target/product/test_device/system/overlay")} if !reflect.DeepEqual(path, expectedPath) { t.Errorf("Unexpected LOCAL_MODULE_PATH value: %v, expected: %v", path, expectedPath) } diff --git a/sh/sh_binary_test.go b/sh/sh_binary_test.go index fb7ab1358..f48f7fb29 100644 --- a/sh/sh_binary_test.go +++ b/sh/sh_binary_test.go @@ -3,6 +3,7 @@ package sh import ( "io/ioutil" "os" + "path" "path/filepath" "reflect" "testing" @@ -73,7 +74,8 @@ func TestShTestSubDir(t *testing.T) { entries := android.AndroidMkEntriesForTest(t, ctx, mod)[0] - expectedPath := "/tmp/target/product/test_device/data/nativetest64/foo_test" + expectedPath := path.Join(buildDir, + "../target/product/test_device/data/nativetest64/foo_test") actualPath := entries.EntryMap["LOCAL_MODULE_PATH"][0] if expectedPath != actualPath { t.Errorf("Unexpected LOCAL_MODULE_PATH expected: %q, actual: %q", expectedPath, actualPath) @@ -97,7 +99,8 @@ func TestShTest(t *testing.T) { entries := android.AndroidMkEntriesForTest(t, ctx, mod)[0] - expectedPath := "/tmp/target/product/test_device/data/nativetest64/foo" + expectedPath := path.Join(buildDir, + "../target/product/test_device/data/nativetest64/foo") actualPath := entries.EntryMap["LOCAL_MODULE_PATH"][0] if expectedPath != actualPath { t.Errorf("Unexpected LOCAL_MODULE_PATH expected: %q, actual: %q", expectedPath, actualPath) diff --git a/shared/Android.bp b/shared/Android.bp index c79bc2b39..deb17f8f6 100644 --- a/shared/Android.bp +++ b/shared/Android.bp @@ -8,6 +8,10 @@ bootstrap_go_package { srcs: [ "env.go", "paths.go", + "debug.go", + ], + testSrcs: [ + "paths_test.go", ], deps: [ "soong-bazel", diff --git a/shared/debug.go b/shared/debug.go new file mode 100644 index 000000000..0c9ba4f95 --- /dev/null +++ b/shared/debug.go @@ -0,0 +1,17 @@ +package shared + +import ( + "os" + "os/exec" +) + +// Finds the Delve binary to use. Either uses the SOONG_DELVE_PATH environment +// variable or if that is unset, looks at $PATH. +func ResolveDelveBinary() string { + result := os.Getenv("SOONG_DELVE_PATH") + if result == "" { + result, _ = exec.LookPath("dlv") + } + + return result +} diff --git a/shared/env.go b/shared/env.go index 7900daa88..152729ba0 100644 --- a/shared/env.go +++ b/shared/env.go @@ -91,6 +91,28 @@ func StaleEnvFile(filepath string, getenv func(string) string) (bool, error) { return false, nil } +// Deserializes and environment serialized by EnvFileContents() and returns it +// as a map[string]string. +func EnvFromFile(envFile string) (map[string]string, error) { + result := make(map[string]string) + data, err := ioutil.ReadFile(envFile) + if err != nil { + return result, err + } + + var contents envFileData + err = json.Unmarshal(data, &contents) + if err != nil { + return result, err + } + + for _, entry := range contents { + result[entry.Key] = entry.Value + } + + return result, nil +} + // Implements sort.Interface so that we can use sort.Sort on envFileData arrays. func (e envFileData) Len() int { return len(e) diff --git a/shared/paths.go b/shared/paths.go index 1b9ff6098..fca8b4c15 100644 --- a/shared/paths.go +++ b/shared/paths.go @@ -30,6 +30,21 @@ type SharedPaths interface { BazelMetricsDir() string } +// Joins the path strings in the argument list, taking absolute paths into +// account. That is, if one of the strings is an absolute path, the ones before +// are ignored. +func JoinPath(base string, rest ...string) string { + result := base + for _, next := range rest { + if filepath.IsAbs(next) { + result = next + } else { + result = filepath.Join(result, next) + } + } + return result +} + // Given the out directory, returns the root of the temp directory (to be cleared at the start of each execution of Soong) func TempDirForOutDir(outDir string) (tempPath string) { return filepath.Join(outDir, ".temp") diff --git a/shared/paths_test.go b/shared/paths_test.go new file mode 100644 index 000000000..018d55f34 --- /dev/null +++ b/shared/paths_test.go @@ -0,0 +1,32 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared + +import ( + "testing" +) + +func assertEqual(t *testing.T, expected, actual string) { + t.Helper() + if expected != actual { + t.Errorf("expected %q != got %q", expected, actual) + } +} + +func TestJoinPath(t *testing.T) { + assertEqual(t, "/a/b", JoinPath("c/d", "/a/b")) + assertEqual(t, "a/b", JoinPath("a", "b")) + assertEqual(t, "/a/b", JoinPath("x", "/a", "b")) +} diff --git a/ui/build/environment.go b/ui/build/environment.go index 6d8a28fdf..50d059f8d 100644 --- a/ui/build/environment.go +++ b/ui/build/environment.go @@ -33,6 +33,19 @@ func OsEnvironment() *Environment { return &env } +// Returns a copy of the environment as a map[string]string. +func (e *Environment) AsMap() map[string]string { + result := make(map[string]string) + + for _, envVar := range *e { + if k, v, ok := decodeKeyValue(envVar); ok { + result[k] = v + } + } + + return result +} + // Get returns the value associated with the key, and whether it exists. // It's equivalent to the os.LookupEnv function, but with this copy of the // Environment. diff --git a/ui/build/soong.go b/ui/build/soong.go index 125dbcc11..fc4366397 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -21,6 +21,7 @@ import ( "strconv" "android/soong/shared" + soong_metrics_proto "android/soong/ui/metrics/metrics_proto" "github.com/golang/protobuf/proto" @@ -30,6 +31,15 @@ import ( "android/soong/ui/status" ) +func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error { + data, err := shared.EnvFileContents(envDeps) + if err != nil { + return err + } + + return ioutil.WriteFile(envFile, data, 0644) +} + // This uses Android.bp files and various tools to generate /build.ninja. // // However, the execution of /build.ninja happens later in build/soong/ui/build/build.go#Build() @@ -47,6 +57,12 @@ func runSoong(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSoong, "soong") defer ctx.EndTrace() + // We have two environment files: .available is the one with every variable, + // .used with the ones that were actually used. The latter is used to + // determine whether Soong needs to be re-run since why re-run it if only + // unused variables were changed? + envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available") + // Use an anonymous inline function for tracing purposes (this pattern is used several times below). func() { ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap") @@ -61,6 +77,7 @@ func runSoong(ctx Context, config Config) { } cmd := Command(ctx, config, "blueprint bootstrap", "build/blueprint/bootstrap.bash", args...) + cmd.Environment.Set("BLUEPRINTDIR", "./build/blueprint") cmd.Environment.Set("BOOTSTRAP", "./build/blueprint/bootstrap.bash") cmd.Environment.Set("BUILDDIR", config.SoongOutDir()) @@ -74,11 +91,32 @@ func runSoong(ctx Context, config Config) { cmd.RunAndPrintOrFatal() }() + soongBuildEnv := config.Environment().Copy() + soongBuildEnv.Set("TOP", os.Getenv("TOP")) + // These two dependencies are read from bootstrap.go, but also need to be here + // so that soong_build can declare a dependency on them + soongBuildEnv.Set("SOONG_DELVE", os.Getenv("SOONG_DELVE")) + soongBuildEnv.Set("SOONG_DELVE_PATH", os.Getenv("SOONG_DELVE_PATH")) + soongBuildEnv.Set("SOONG_OUTDIR", config.SoongOutDir()) + // For Bazel mixed builds. + soongBuildEnv.Set("BAZEL_PATH", "./tools/bazel") + soongBuildEnv.Set("BAZEL_HOME", filepath.Join(config.BazelOutDir(), "bazelhome")) + soongBuildEnv.Set("BAZEL_OUTPUT_BASE", filepath.Join(config.BazelOutDir(), "output")) + soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, ".")) + soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir()) + + if os.Getenv("SOONG_DELVE") != "" { + // SOONG_DELVE is already in cmd.Environment + soongBuildEnv.Set("SOONG_DELVE_PATH", shared.ResolveDelveBinary()) + } + + writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap()) + func() { ctx.BeginTrace(metrics.RunSoong, "environment check") defer ctx.EndTrace() - envFile := filepath.Join(config.SoongOutDir(), ".soong.environment") + envFile := filepath.Join(config.SoongOutDir(), "soong.environment.used") getenv := func(k string) string { v, _ := config.Environment().Get(k) return v @@ -134,14 +172,7 @@ func runSoong(ctx Context, config Config) { "--frontend_file", fifo, "-f", filepath.Join(config.SoongOutDir(), file)) - // For Bazel mixed builds. - cmd.Environment.Set("BAZEL_PATH", "./tools/bazel") - cmd.Environment.Set("BAZEL_HOME", filepath.Join(config.BazelOutDir(), "bazelhome")) - cmd.Environment.Set("BAZEL_OUTPUT_BASE", filepath.Join(config.BazelOutDir(), "output")) - cmd.Environment.Set("BAZEL_WORKSPACE", absPath(ctx, ".")) - cmd.Environment.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir()) - - cmd.Environment.Set("SOONG_SANDBOX_SOONG_BUILD", "true") + cmd.Environment.Set("SOONG_OUTDIR", config.SoongOutDir()) cmd.Sandbox = soongSandbox cmd.RunAndStreamOrFatal() }