diff --git a/tests/lib.sh b/tests/lib.sh index 0766d85bd..50b0ef8ab 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -8,6 +8,11 @@ HARDWIRED_MOCK_TOP= REAL_TOP="$(readlink -f "$(dirname "$0")"/../../..)" +function make_mock_top { + mock=$(mktemp -t -d st.XXXXX) + echo "$mock" +} + if [[ -n "$HARDWIRED_MOCK_TOP" ]]; then MOCK_TOP="$HARDWIRED_MOCK_TOP" else @@ -197,3 +202,11 @@ function scan_and_run_tests { info "Completed test case \e[96;1m$f\e[0m" done } + +function move_mock_top { + MOCK_TOP2=$(make_mock_top) + rm -rf $MOCK_TOP2 + mv $MOCK_TOP $MOCK_TOP2 + MOCK_TOP=$MOCK_TOP2 + trap cleanup_mock_top EXIT +} diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh index 6b9ff8b93..223baa4ac 100755 --- a/tests/run_integration_tests.sh +++ b/tests/run_integration_tests.sh @@ -23,4 +23,4 @@ TEST_BAZEL=true extra_build_params=--bazel-mode-staging "$TOP/build/soong/tests/ "$TOP/build/soong/tests/apex_cc_module_arch_variant_tests.sh" "aosp_cf_arm64_phone" "armv8-a" "cortex-a53" "$TOP/build/bazel/ci/b_test.sh" - +"$TOP/build/soong/tests/symlinks_path_test.sh" diff --git a/tests/symlinks_path_test.sh b/tests/symlinks_path_test.sh new file mode 100755 index 000000000..ed429119c --- /dev/null +++ b/tests/symlinks_path_test.sh @@ -0,0 +1,51 @@ +#!/bin/bash -eu + +set -o pipefail + +# Test that relative symlinks work by recreating the bug in b/259191764 +# In some cases, developers prefer to move their checkouts. This causes +# issues in that symlinked files (namely, the bazel wrapper script) +# cannot be found. As such, we implemented relative symlinks so that a +# moved checkout doesn't need a full clean before rebuilding. +# The bazel output base will still need to be removed, as Starlark +# doesn't seem to support relative symlinks yet. + +source "$(dirname "$0")/lib.sh" + +function check_link_has_mock_top_prefix { + input_link=$1 + link_target=`readlink $input_link` + if [[ $link_target != "$MOCK_TOP"* ]]; then + echo "Symlink for file $input_link -> $link_target doesn't start with $MOCK_TOP" + exit 1 + fi +} + +function test_symlinks_updated_when_top_dir_changed { + setup + + mkdir -p a + touch a/g.txt + cat > a/Android.bp <<'EOF' +filegroup { + name: "g", + srcs: ["g.txt"], + bazel_module: {bp2build_available: true}, +} +EOF + # A directory under $MOCK_TOP + outdir=out2 + + # Modify OUT_DIR in a subshell so it doesn't affect the top level one. + (export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g) + + g_txt="out2/soong/workspace/a/g.txt" + check_link_has_mock_top_prefix "$g_txt" + + move_mock_top + + (export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g) + check_link_has_mock_top_prefix "$g_txt" +} + +scan_and_run_tests \ No newline at end of file diff --git a/ui/build/soong.go b/ui/build/soong.go index 44c20a013..ac9bf3a60 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -16,10 +16,13 @@ package build import ( "fmt" + "io/fs" "os" "path/filepath" "strconv" "strings" + "sync" + "sync/atomic" "android/soong/bazel" "android/soong/ui/metrics" @@ -50,6 +53,13 @@ const ( bootstrapEpoch = 1 ) +var ( + // Used during parallel update of symlinks in out directory to reflect new + // TOP dir. + symlinkWg sync.WaitGroup + numFound, numUpdated uint32 +) + func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error { data, err := shared.EnvFileContents(envDeps) if err != nil { @@ -465,10 +475,118 @@ func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string) } } +func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error { + defer symlinkWg.Done() + + visit := func(path string, d fs.DirEntry, err error) error { + if d.IsDir() && path != dir { + symlinkWg.Add(1) + go updateSymlinks(ctx, path, prevCWD, cwd) + return filepath.SkipDir + } + f, err := d.Info() + if err != nil { + return err + } + // If the file is not a symlink, we don't have to update it. + if f.Mode()&os.ModeSymlink != os.ModeSymlink { + return nil + } + + atomic.AddUint32(&numFound, 1) + target, err := os.Readlink(path) + if err != nil { + return err + } + if strings.HasPrefix(target, prevCWD) && + (len(target) == len(prevCWD) || target[len(prevCWD)] == '/') { + target = filepath.Join(cwd, target[len(prevCWD):]) + if err := os.Remove(path); err != nil { + return err + } + if err := os.Symlink(target, path); err != nil { + return err + } + atomic.AddUint32(&numUpdated, 1) + } + return nil + } + + if err := filepath.WalkDir(dir, visit); err != nil { + return err + } + return nil +} + +func fixOutDirSymlinks(ctx Context, config Config, outDir string) error { + cwd, err := os.Getwd() + if err != nil { + return err + } + + // Record the .top as the very last thing in the function. + tf := filepath.Join(outDir, ".top") + defer func() { + if err := os.WriteFile(tf, []byte(cwd), 0644); err != nil { + fmt.Fprintf(os.Stderr, fmt.Sprintf("Unable to log CWD: %v", err)) + } + }() + + // Find the previous working directory if it was recorded. + var prevCWD string + pcwd, err := os.ReadFile(tf) + if err != nil { + if os.IsNotExist(err) { + // No previous working directory recorded, nothing to do. + return nil + } + return err + } + prevCWD = strings.Trim(string(pcwd), "\n") + + if prevCWD == cwd { + // We are in the same source dir, nothing to update. + return nil + } + + symlinkWg.Add(1) + if err := updateSymlinks(ctx, outDir, prevCWD, cwd); err != nil { + return err + } + symlinkWg.Wait() + ctx.Println(fmt.Sprintf("Updated %d/%d symlinks in dir %v", numUpdated, numFound, outDir)) + return nil +} + +func migrateOutputSymlinks(ctx Context, config Config) error { + // Figure out the real out directory ("out" could be a symlink). + outDir := config.OutDir() + s, err := os.Lstat(outDir) + if err != nil { + if os.IsNotExist(err) { + // No out dir exists, no symlinks to migrate. + return nil + } + return err + } + if s.Mode()&os.ModeSymlink == os.ModeSymlink { + target, err := filepath.EvalSymlinks(outDir) + if err != nil { + return err + } + outDir = target + } + return fixOutDirSymlinks(ctx, config, outDir) +} + func runSoong(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSoong, "soong") defer ctx.EndTrace() + if err := migrateOutputSymlinks(ctx, config); err != nil { + ctx.Fatalf("failed to migrate output directory to current TOP dir: %v", err) + } + // We have two environment files: .available is the one with every variable, // .used with the ones that were actually used. The latter is used to // determine whether Soong needs to be re-run since why re-run it if only