Update symlinks in output directory when TOP dir changes

When TOP dir changes and we reuse the same output between the old and new checkouts, we need to rewrite all the symlinks used by Bazel and other tools in the out-directory to point to the new top directory. Otherwise, if the old source dir is deleted, the build will fail.

I used the OUT_DIR/soong/soong.environment.available file to find out the previous PWD.

Tested:
1. Create source dir 1, run build, create source dir 2, remote source dir 1, reuse out dir and rerun build => build succeeded with this change.
2. m libc after moving build TOP. Only the analysis phase was rerun, the actual build was not rerun.

Bug: b/300498226

Change-Id: I196625baa1f4efe7a4734accfa1f0be7c98a7920

tests/lib.sh

@@ -8,6 +8,11 @@ HARDWIRED_MOCK_TOP=
 
 REAL_TOP="$(readlink -f "$(dirname "$0")"/../../..)"
 
+function make_mock_top {
+  mock=$(mktemp -t -d st.XXXXX)
+  echo "$mock"
+}
+
 if [[ -n "$HARDWIRED_MOCK_TOP" ]]; then
   MOCK_TOP="$HARDWIRED_MOCK_TOP"
 else
@@ -197,3 +202,11 @@ function scan_and_run_tests {
     info "Completed test case \e[96;1m$f\e[0m"
   done
 }
+
+function move_mock_top {
+  MOCK_TOP2=$(make_mock_top)
+  rm -rf $MOCK_TOP2
+  mv $MOCK_TOP $MOCK_TOP2
+  MOCK_TOP=$MOCK_TOP2
+  trap cleanup_mock_top EXIT
+}

tests/run_integration_tests.sh

@@ -23,4 +23,4 @@ TEST_BAZEL=true extra_build_params=--bazel-mode-staging "$TOP/build/soong/tests/
 "$TOP/build/soong/tests/apex_cc_module_arch_variant_tests.sh" "aosp_cf_arm64_phone" "armv8-a" "cortex-a53"
 
 "$TOP/build/bazel/ci/b_test.sh"
-
+"$TOP/build/soong/tests/symlinks_path_test.sh"

tests/symlinks_path_test.sh

@@ -0,0 +1,51 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+# Test that relative symlinks work by recreating the bug in b/259191764
+# In some cases, developers prefer to move their checkouts. This causes
+# issues in that symlinked files (namely, the bazel wrapper script)
+# cannot be found. As such, we implemented relative symlinks so that a
+# moved checkout doesn't need a full clean before rebuilding.
+# The bazel output base will still need to be removed, as Starlark
+# doesn't seem to support relative symlinks yet.
+
+source "$(dirname "$0")/lib.sh"
+
+function check_link_has_mock_top_prefix {
+  input_link=$1
+  link_target=`readlink $input_link`
+  if [[ $link_target != "$MOCK_TOP"* ]]; then
+    echo "Symlink for file $input_link -> $link_target doesn't start with $MOCK_TOP"
+    exit 1
+  fi
+}
+
+function test_symlinks_updated_when_top_dir_changed {
+  setup
+
+  mkdir -p a
+  touch a/g.txt
+  cat > a/Android.bp <<'EOF'
+filegroup {
+    name: "g",
+    srcs: ["g.txt"],
+    bazel_module: {bp2build_available: true},
+}
+EOF
+  # A directory under $MOCK_TOP
+  outdir=out2
+
+  # Modify OUT_DIR in a subshell so it doesn't affect the top level one.
+  (export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g)
+
+  g_txt="out2/soong/workspace/a/g.txt"
+  check_link_has_mock_top_prefix "$g_txt"
+
+  move_mock_top
+
+  (export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g)
+  check_link_has_mock_top_prefix "$g_txt"
+}
+
+scan_and_run_tests

ui/build/soong.go

@@ -16,10 +16,13 @@ package build
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 
 	"android/soong/bazel"
 	"android/soong/ui/metrics"
@@ -50,6 +53,13 @@ const (
 	bootstrapEpoch = 1
 )
 
+var (
+	// Used during parallel update of symlinks in out directory to reflect new
+	// TOP dir.
+	symlinkWg            sync.WaitGroup
+	numFound, numUpdated uint32
+)
+
 func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
 	data, err := shared.EnvFileContents(envDeps)
 	if err != nil {
@@ -465,10 +475,118 @@ func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string)
 	}
 }
 
+func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error {
+	defer symlinkWg.Done()
+
+	visit := func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() && path != dir {
+			symlinkWg.Add(1)
+			go updateSymlinks(ctx, path, prevCWD, cwd)
+			return filepath.SkipDir
+		}
+		f, err := d.Info()
+		if err != nil {
+			return err
+		}
+		// If the file is not a symlink, we don't have to update it.
+		if f.Mode()&os.ModeSymlink != os.ModeSymlink {
+			return nil
+		}
+
+		atomic.AddUint32(&numFound, 1)
+		target, err := os.Readlink(path)
+		if err != nil {
+			return err
+		}
+		if strings.HasPrefix(target, prevCWD) &&
+			(len(target) == len(prevCWD) || target[len(prevCWD)] == '/') {
+			target = filepath.Join(cwd, target[len(prevCWD):])
+			if err := os.Remove(path); err != nil {
+				return err
+			}
+			if err := os.Symlink(target, path); err != nil {
+				return err
+			}
+			atomic.AddUint32(&numUpdated, 1)
+		}
+		return nil
+	}
+
+	if err := filepath.WalkDir(dir, visit); err != nil {
+		return err
+	}
+	return nil
+}
+
+func fixOutDirSymlinks(ctx Context, config Config, outDir string) error {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	// Record the .top as the very last thing in the function.
+	tf := filepath.Join(outDir, ".top")
+	defer func() {
+		if err := os.WriteFile(tf, []byte(cwd), 0644); err != nil {
+			fmt.Fprintf(os.Stderr, fmt.Sprintf("Unable to log CWD: %v", err))
+		}
+	}()
+
+	// Find the previous working directory if it was recorded.
+	var prevCWD string
+	pcwd, err := os.ReadFile(tf)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// No previous working directory recorded, nothing to do.
+			return nil
+		}
+		return err
+	}
+	prevCWD = strings.Trim(string(pcwd), "\n")
+
+	if prevCWD == cwd {
+		// We are in the same source dir, nothing to update.
+		return nil
+	}
+
+	symlinkWg.Add(1)
+	if err := updateSymlinks(ctx, outDir, prevCWD, cwd); err != nil {
+		return err
+	}
+	symlinkWg.Wait()
+	ctx.Println(fmt.Sprintf("Updated %d/%d symlinks in dir %v", numUpdated, numFound, outDir))
+	return nil
+}
+
+func migrateOutputSymlinks(ctx Context, config Config) error {
+	// Figure out the real out directory ("out" could be a symlink).
+	outDir := config.OutDir()
+	s, err := os.Lstat(outDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// No out dir exists, no symlinks to migrate.
+			return nil
+		}
+		return err
+	}
+	if s.Mode()&os.ModeSymlink == os.ModeSymlink {
+		target, err := filepath.EvalSymlinks(outDir)
+		if err != nil {
+			return err
+		}
+		outDir = target
+	}
+	return fixOutDirSymlinks(ctx, config, outDir)
+}
+
 func runSoong(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSoong, "soong")
 	defer ctx.EndTrace()
 
+	if err := migrateOutputSymlinks(ctx, config); err != nil {
+		ctx.Fatalf("failed to migrate output directory to current TOP dir: %v", err)
+	}
+
 	// We have two environment files: .available is the one with every variable,
 	// .used with the ones that were actually used. The latter is used to
 	// determine whether Soong needs to be re-run since why re-run it if only