Merge "Update symlinks in output directory when TOP dir changes" into main

tests/lib.sh

@@ -8,6 +8,11 @@ HARDWIRED_MOCK_TOP=
 
 REAL_TOP="$(readlink -f "$(dirname "$0")"/../../..)"
 
+function make_mock_top {
+  mock=$(mktemp -t -d st.XXXXX)
+  echo "$mock"
+}
+
 if [[ -n "$HARDWIRED_MOCK_TOP" ]]; then
   MOCK_TOP="$HARDWIRED_MOCK_TOP"
 else
@@ -198,3 +203,11 @@ function scan_and_run_tests {
     info "Completed test case \e[96;1m$f\e[0m"
   done
 }
+
+function move_mock_top {
+  MOCK_TOP2=$(make_mock_top)
+  rm -rf $MOCK_TOP2
+  mv $MOCK_TOP $MOCK_TOP2
+  MOCK_TOP=$MOCK_TOP2
+  trap cleanup_mock_top EXIT
+}

tests/run_integration_tests.sh

@@ -23,4 +23,4 @@ TEST_BAZEL=true extra_build_params=--bazel-mode-staging "$TOP/build/soong/tests/
 "$TOP/build/soong/tests/apex_cc_module_arch_variant_tests.sh" "aosp_cf_arm64_phone" "armv8-a" "cortex-a53"
 
 "$TOP/build/bazel/ci/b_test.sh"
-
+"$TOP/build/soong/tests/symlinks_path_test.sh"

tests/symlinks_path_test.sh

@@ -0,0 +1,51 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+# Test that relative symlinks work by recreating the bug in b/259191764
+# In some cases, developers prefer to move their checkouts. This causes
+# issues in that symlinked files (namely, the bazel wrapper script)
+# cannot be found. As such, we implemented relative symlinks so that a
+# moved checkout doesn't need a full clean before rebuilding.
+# The bazel output base will still need to be removed, as Starlark
+# doesn't seem to support relative symlinks yet.
+
+source "$(dirname "$0")/lib.sh"
+
+function check_link_has_mock_top_prefix {
+  input_link=$1
+  link_target=`readlink $input_link`
+  if [[ $link_target != "$MOCK_TOP"* ]]; then
+    echo "Symlink for file $input_link -> $link_target doesn't start with $MOCK_TOP"
+    exit 1
+  fi
+}
+
+function test_symlinks_updated_when_top_dir_changed {
+  setup
+
+  mkdir -p a
+  touch a/g.txt
+  cat > a/Android.bp <<'EOF'
+filegroup {
+    name: "g",
+    srcs: ["g.txt"],
+    bazel_module: {bp2build_available: true},
+}
+EOF
+  # A directory under $MOCK_TOP
+  outdir=out2
+
+  # Modify OUT_DIR in a subshell so it doesn't affect the top level one.
+  (export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g)
+
+  g_txt="out2/soong/workspace/a/g.txt"
+  check_link_has_mock_top_prefix "$g_txt"
+
+  move_mock_top
+
+  (export OUT_DIR=$MOCK_TOP/$outdir; run_soong bp2build && run_bazel build --config=bp2build --config=ci //a:g)
+  check_link_has_mock_top_prefix "$g_txt"
+}
+
+scan_and_run_tests

ui/build/soong.go

@@ -16,10 +16,13 @@ package build
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 
 	"android/soong/bazel"
 	"android/soong/ui/metrics"
@@ -50,6 +53,13 @@ const (
 	bootstrapEpoch = 1
 )
 
+var (
+	// Used during parallel update of symlinks in out directory to reflect new
+	// TOP dir.
+	symlinkWg            sync.WaitGroup
+	numFound, numUpdated uint32
+)
+
 func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
 	data, err := shared.EnvFileContents(envDeps)
 	if err != nil {
@@ -465,10 +475,118 @@ func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string)
 	}
 }
 
+func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error {
+	defer symlinkWg.Done()
+
+	visit := func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() && path != dir {
+			symlinkWg.Add(1)
+			go updateSymlinks(ctx, path, prevCWD, cwd)
+			return filepath.SkipDir
+		}
+		f, err := d.Info()
+		if err != nil {
+			return err
+		}
+		// If the file is not a symlink, we don't have to update it.
+		if f.Mode()&os.ModeSymlink != os.ModeSymlink {
+			return nil
+		}
+
+		atomic.AddUint32(&numFound, 1)
+		target, err := os.Readlink(path)
+		if err != nil {
+			return err
+		}
+		if strings.HasPrefix(target, prevCWD) &&
+			(len(target) == len(prevCWD) || target[len(prevCWD)] == '/') {
+			target = filepath.Join(cwd, target[len(prevCWD):])
+			if err := os.Remove(path); err != nil {
+				return err
+			}
+			if err := os.Symlink(target, path); err != nil {
+				return err
+			}
+			atomic.AddUint32(&numUpdated, 1)
+		}
+		return nil
+	}
+
+	if err := filepath.WalkDir(dir, visit); err != nil {
+		return err
+	}
+	return nil
+}
+
+func fixOutDirSymlinks(ctx Context, config Config, outDir string) error {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	// Record the .top as the very last thing in the function.
+	tf := filepath.Join(outDir, ".top")
+	defer func() {
+		if err := os.WriteFile(tf, []byte(cwd), 0644); err != nil {
+			fmt.Fprintf(os.Stderr, fmt.Sprintf("Unable to log CWD: %v", err))
+		}
+	}()
+
+	// Find the previous working directory if it was recorded.
+	var prevCWD string
+	pcwd, err := os.ReadFile(tf)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// No previous working directory recorded, nothing to do.
+			return nil
+		}
+		return err
+	}
+	prevCWD = strings.Trim(string(pcwd), "\n")
+
+	if prevCWD == cwd {
+		// We are in the same source dir, nothing to update.
+		return nil
+	}
+
+	symlinkWg.Add(1)
+	if err := updateSymlinks(ctx, outDir, prevCWD, cwd); err != nil {
+		return err
+	}
+	symlinkWg.Wait()
+	ctx.Println(fmt.Sprintf("Updated %d/%d symlinks in dir %v", numUpdated, numFound, outDir))
+	return nil
+}
+
+func migrateOutputSymlinks(ctx Context, config Config) error {
+	// Figure out the real out directory ("out" could be a symlink).
+	outDir := config.OutDir()
+	s, err := os.Lstat(outDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// No out dir exists, no symlinks to migrate.
+			return nil
+		}
+		return err
+	}
+	if s.Mode()&os.ModeSymlink == os.ModeSymlink {
+		target, err := filepath.EvalSymlinks(outDir)
+		if err != nil {
+			return err
+		}
+		outDir = target
+	}
+	return fixOutDirSymlinks(ctx, config, outDir)
+}
+
 func runSoong(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSoong, "soong")
 	defer ctx.EndTrace()
 
+	if err := migrateOutputSymlinks(ctx, config); err != nil {
+		ctx.Fatalf("failed to migrate output directory to current TOP dir: %v", err)
+	}
+
 	// We have two environment files: .available is the one with every variable,
 	// .used with the ones that were actually used. The latter is used to
 	// determine whether Soong needs to be re-run since why re-run it if only