Add include_make_built_files

If `include_make_built_files` is set to the name of a partition, the
make-built files from that partition will be incorperated into this
soong module. This is to ease the transition to soong built filesystems.

If any files are present in both the soong-built file list and the
make-built one, the soong ones will be preferred.

Bug: 329146343
Test: go test
Change-Id: I456b283e1189116e699ed75357cc056f5d217688

filesystem/filesystem.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"android/soong/android"
@@ -109,6 +110,12 @@ type filesystemProperties struct {
 
 	// Mount point for this image. Default is "/"
 	Mount_point *string
+
+	// If set to the name of a partition ("system", "vendor", etc), this filesystem module
+	// will also include the contents of the make-built staging directories. If any soong
+	// modules would be installed to the same location as a make module, they will overwrite
+	// the make version.
+	Include_make_built_files string
 }
 
 // android_filesystem packages a set of modules and their transitive dependencies into a filesystem
@@ -214,22 +221,21 @@ func (f *filesystem) buildNonDepsFiles(ctx android.ModuleContext, builder *andro
 	}
 
 	// create extra files if there's any
-	rootForExtraFiles := android.PathForModuleGen(ctx, "root-extra").OutputPath
-	var extraFiles android.OutputPaths
 	if f.buildExtraFiles != nil {
-		extraFiles = f.buildExtraFiles(ctx, rootForExtraFiles)
+		rootForExtraFiles := android.PathForModuleGen(ctx, "root-extra").OutputPath
-	}
+		extraFiles := f.buildExtraFiles(ctx, rootForExtraFiles)
-
+		for _, f := range extraFiles {
-	for _, f := range extraFiles {
+			rel, err := filepath.Rel(rootForExtraFiles.String(), f.String())
-		rel, err := filepath.Rel(rootForExtraFiles.String(), f.String())
+			if err != nil || strings.HasPrefix(rel, "..") {
-		if err != nil || strings.HasPrefix(rel, "..") {
+				ctx.ModuleErrorf("can't make %q relative to %q", f, rootForExtraFiles)
-			ctx.ModuleErrorf("can't make %q relative to %q", f, rootForExtraFiles)
+			}
-			continue
+		}
+		if len(extraFiles) > 0 {
+			builder.Command().BuiltTool("merge_directories").
+				Implicits(extraFiles.Paths()).
+				Text(rootDir.String()).
+				Text(rootForExtraFiles.String())
 		}
-		dst := rootDir.Join(ctx, rel)
-		builder.Command().Textf("(! [ -e %s -o -L %s ] || (echo \"%s already exists from an earlier stage of the build\" && exit 1))", dst, dst, dst)
-		builder.Command().Text("mkdir -p").Text(filepath.Dir(dst.String()))
-		builder.Command().Text("cp -P").Input(f).Output(dst)
 	}
 }
 
@@ -245,6 +251,7 @@ func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) androi
 	f.entries = f.CopySpecsToDir(ctx, builder, f.gatherFilteredPackagingSpecs(ctx), rebasedDir)
 
 	f.buildNonDepsFiles(ctx, builder, rootDir)
+	f.addMakeBuiltFiles(ctx, builder, rootDir)
 
 	// run host_init_verifier
 	// Ideally we should have a concept of pluggable linters that verify the generated image.
@@ -362,6 +369,10 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool)
 		ctx.PropertyErrorf("file_contexts", "file_contexts is not supported for compressed cpio image.")
 	}
 
+	if f.properties.Include_make_built_files != "" {
+		ctx.PropertyErrorf("include_make_built_files", "include_make_built_files is not supported for compressed cpio image.")
+	}
+
 	rootDir := android.PathForModuleOut(ctx, "root").OutputPath
 	rebasedDir := rootDir
 	if f.properties.Base_dir != nil {
@@ -395,6 +406,41 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool)
 	return output
 }
 
+var validPartitions = []string{
+	"system",
+	"userdata",
+	"cache",
+	"system_other",
+	"vendor",
+	"product",
+	"system_ext",
+	"odm",
+	"vendor_dlkm",
+	"odm_dlkm",
+	"system_dlkm",
+}
+
+func (f *filesystem) addMakeBuiltFiles(ctx android.ModuleContext, builder *android.RuleBuilder, rootDir android.Path) {
+	partition := f.properties.Include_make_built_files
+	if partition == "" {
+		return
+	}
+	if !slices.Contains(validPartitions, partition) {
+		ctx.PropertyErrorf("include_make_built_files", "Expected one of %#v, found %q", validPartitions, partition)
+		return
+	}
+	stampFile := fmt.Sprintf("target/product/%s/obj/PACKAGING/%s_intermediates/staging_dir.stamp", ctx.Config().DeviceName(), partition)
+	fileListFile := fmt.Sprintf("target/product/%s/obj/PACKAGING/%s_intermediates/file_list.txt", ctx.Config().DeviceName(), partition)
+	stagingDir := fmt.Sprintf("target/product/%s/%s", ctx.Config().DeviceName(), partition)
+
+	builder.Command().BuiltTool("merge_directories").
+		Implicit(android.PathForArbitraryOutput(ctx, stampFile)).
+		Text("--ignore-duplicates").
+		FlagWithInput("--file-list", android.PathForArbitraryOutput(ctx, fileListFile)).
+		Text(rootDir.String()).
+		Text(android.PathForArbitraryOutput(ctx, stagingDir).String())
+}
+
 var _ android.AndroidMkEntriesProvider = (*filesystem)(nil)
 
 // Implements android.AndroidMkEntriesProvider

filesystem/filesystem_test.go

@@ -16,6 +16,7 @@ package filesystem
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	"android/soong/android"
@@ -93,6 +94,22 @@ func TestFileSystemDeps(t *testing.T) {
 	}
 }
 
+func TestIncludeMakeBuiltFiles(t *testing.T) {
+	result := fixture.RunTestWithBp(t, `
+		android_filesystem {
+			name: "myfilesystem",
+			include_make_built_files: "system",
+		}
+	`)
+
+	output := result.ModuleForTests("myfilesystem", "android_common").Output("myfilesystem.img")
+
+	stampFile := filepath.Join(result.Config.OutDir(), "target/product/test_device/obj/PACKAGING/system_intermediates/staging_dir.stamp")
+	fileListFile := filepath.Join(result.Config.OutDir(), "target/product/test_device/obj/PACKAGING/system_intermediates/file_list.txt")
+	android.AssertStringListContains(t, "deps of filesystem must include the staging dir stamp file", output.Implicits.Strings(), stampFile)
+	android.AssertStringListContains(t, "deps of filesystem must include the staging dir file list", output.Implicits.Strings(), fileListFile)
+}
+
 func TestFileSystemFillsLinkerConfigWithStubLibs(t *testing.T) {
 	result := fixture.RunTestWithBp(t, `
 		android_system_image {

scripts/Android.bp

@@ -292,3 +292,16 @@ sh_binary_host {
     name: "keep-flagged-apis",
     src: "keep-flagged-apis.sh",
 }
+
+python_binary_host {
+    name: "merge_directories",
+    main: "merge_directories.py",
+    srcs: [
+        "merge_directories.py",
+    ],
+    version: {
+        py3: {
+            embedded_launcher: true,
+        },
+    },
+}

scripts/merge_directories.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import shutil
+import sys
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Given a list of directories, this script will copy the contents of all of "
+        "them into the first directory, erroring out if any duplicate files are found."
+    )
+    parser.add_argument(
+        "--ignore-duplicates",
+        action="store_true",
+        help="Don't error out on duplicate files, just skip them. The file from the earliest "
+        "directory listed on the command line will be the winner."
+    )
+    parser.add_argument(
+        "--file-list",
+        help="Path to a text file containing paths relative to in_dir. Only these paths will be "
+        "copied out of in_dir."
+    )
+    parser.add_argument("out_dir")
+    parser.add_argument("in_dir")
+    args = parser.parse_args()
+
+    if not os.path.isdir(args.out_dir):
+        sys.exit(f"error: {args.out_dir} must be a directory")
+    if not os.path.isdir(args.in_dir):
+        sys.exit(f"error: {args.in_dir} must be a directory")
+
+    file_list = None
+    if args.file_list:
+        with open(file_list_file, "r") as f:
+            file_list = f.read().strip().splitlines()
+
+    in_dir = args.in_dir
+    for root, dirs, files in os.walk(in_dir):
+        rel_root = os.path.relpath(root, in_dir)
+        dst_root = os.path.join(args.out_dir, rel_root)
+        made_parent_dirs = False
+        for f in files:
+            src = os.path.join(root, f)
+            dst = os.path.join(dst_root, f)
+            p = os.path.normpath(os.path.join(rel_root, f))
+            if file_list is not None and p not in file_list:
+                continue
+            if os.path.lexists(dst):
+                if args.ignore_duplicates:
+                    continue
+                sys.exit(f"error: {p} exists in both {args.out_dir} and {in_dir}")
+
+            if not made_parent_dirs:
+                os.makedirs(dst_root, exist_ok=True)
+                made_parent_dirs = True
+
+            shutil.copy2(src, dst, follow_symlinks=False)
+
+if __name__ == "__main__":
+    main()