diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index abc656f49..64a2e235f 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "path/filepath" + "slices" "strings" "android/soong/android" @@ -109,6 +110,12 @@ type filesystemProperties struct { // Mount point for this image. Default is "/" Mount_point *string + + // If set to the name of a partition ("system", "vendor", etc), this filesystem module + // will also include the contents of the make-built staging directories. If any soong + // modules would be installed to the same location as a make module, they will overwrite + // the make version. + Include_make_built_files string } // android_filesystem packages a set of modules and their transitive dependencies into a filesystem @@ -214,22 +221,21 @@ func (f *filesystem) buildNonDepsFiles(ctx android.ModuleContext, builder *andro } // create extra files if there's any - rootForExtraFiles := android.PathForModuleGen(ctx, "root-extra").OutputPath - var extraFiles android.OutputPaths if f.buildExtraFiles != nil { - extraFiles = f.buildExtraFiles(ctx, rootForExtraFiles) - } - - for _, f := range extraFiles { - rel, err := filepath.Rel(rootForExtraFiles.String(), f.String()) - if err != nil || strings.HasPrefix(rel, "..") { - ctx.ModuleErrorf("can't make %q relative to %q", f, rootForExtraFiles) - continue + rootForExtraFiles := android.PathForModuleGen(ctx, "root-extra").OutputPath + extraFiles := f.buildExtraFiles(ctx, rootForExtraFiles) + for _, f := range extraFiles { + rel, err := filepath.Rel(rootForExtraFiles.String(), f.String()) + if err != nil || strings.HasPrefix(rel, "..") { + ctx.ModuleErrorf("can't make %q relative to %q", f, rootForExtraFiles) + } + } + if len(extraFiles) > 0 { + builder.Command().BuiltTool("merge_directories"). + Implicits(extraFiles.Paths()). + Text(rootDir.String()). + Text(rootForExtraFiles.String()) } - dst := rootDir.Join(ctx, rel) - builder.Command().Textf("(! [ -e %s -o -L %s ] || (echo \"%s already exists from an earlier stage of the build\" && exit 1))", dst, dst, dst) - builder.Command().Text("mkdir -p").Text(filepath.Dir(dst.String())) - builder.Command().Text("cp -P").Input(f).Output(dst) } } @@ -245,6 +251,7 @@ func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) androi f.entries = f.CopySpecsToDir(ctx, builder, f.gatherFilteredPackagingSpecs(ctx), rebasedDir) f.buildNonDepsFiles(ctx, builder, rootDir) + f.addMakeBuiltFiles(ctx, builder, rootDir) // run host_init_verifier // Ideally we should have a concept of pluggable linters that verify the generated image. @@ -362,6 +369,10 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) ctx.PropertyErrorf("file_contexts", "file_contexts is not supported for compressed cpio image.") } + if f.properties.Include_make_built_files != "" { + ctx.PropertyErrorf("include_make_built_files", "include_make_built_files is not supported for compressed cpio image.") + } + rootDir := android.PathForModuleOut(ctx, "root").OutputPath rebasedDir := rootDir if f.properties.Base_dir != nil { @@ -395,6 +406,41 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) return output } +var validPartitions = []string{ + "system", + "userdata", + "cache", + "system_other", + "vendor", + "product", + "system_ext", + "odm", + "vendor_dlkm", + "odm_dlkm", + "system_dlkm", +} + +func (f *filesystem) addMakeBuiltFiles(ctx android.ModuleContext, builder *android.RuleBuilder, rootDir android.Path) { + partition := f.properties.Include_make_built_files + if partition == "" { + return + } + if !slices.Contains(validPartitions, partition) { + ctx.PropertyErrorf("include_make_built_files", "Expected one of %#v, found %q", validPartitions, partition) + return + } + stampFile := fmt.Sprintf("target/product/%s/obj/PACKAGING/%s_intermediates/staging_dir.stamp", ctx.Config().DeviceName(), partition) + fileListFile := fmt.Sprintf("target/product/%s/obj/PACKAGING/%s_intermediates/file_list.txt", ctx.Config().DeviceName(), partition) + stagingDir := fmt.Sprintf("target/product/%s/%s", ctx.Config().DeviceName(), partition) + + builder.Command().BuiltTool("merge_directories"). + Implicit(android.PathForArbitraryOutput(ctx, stampFile)). + Text("--ignore-duplicates"). + FlagWithInput("--file-list", android.PathForArbitraryOutput(ctx, fileListFile)). + Text(rootDir.String()). + Text(android.PathForArbitraryOutput(ctx, stagingDir).String()) +} + var _ android.AndroidMkEntriesProvider = (*filesystem)(nil) // Implements android.AndroidMkEntriesProvider diff --git a/filesystem/filesystem_test.go b/filesystem/filesystem_test.go index f09ed0838..6d62746ec 100644 --- a/filesystem/filesystem_test.go +++ b/filesystem/filesystem_test.go @@ -16,6 +16,7 @@ package filesystem import ( "os" + "path/filepath" "testing" "android/soong/android" @@ -93,6 +94,22 @@ func TestFileSystemDeps(t *testing.T) { } } +func TestIncludeMakeBuiltFiles(t *testing.T) { + result := fixture.RunTestWithBp(t, ` + android_filesystem { + name: "myfilesystem", + include_make_built_files: "system", + } + `) + + output := result.ModuleForTests("myfilesystem", "android_common").Output("myfilesystem.img") + + stampFile := filepath.Join(result.Config.OutDir(), "target/product/test_device/obj/PACKAGING/system_intermediates/staging_dir.stamp") + fileListFile := filepath.Join(result.Config.OutDir(), "target/product/test_device/obj/PACKAGING/system_intermediates/file_list.txt") + android.AssertStringListContains(t, "deps of filesystem must include the staging dir stamp file", output.Implicits.Strings(), stampFile) + android.AssertStringListContains(t, "deps of filesystem must include the staging dir file list", output.Implicits.Strings(), fileListFile) +} + func TestFileSystemFillsLinkerConfigWithStubLibs(t *testing.T) { result := fixture.RunTestWithBp(t, ` android_system_image { diff --git a/scripts/Android.bp b/scripts/Android.bp index e2fd59f17..f36329b1d 100644 --- a/scripts/Android.bp +++ b/scripts/Android.bp @@ -292,3 +292,16 @@ sh_binary_host { name: "keep-flagged-apis", src: "keep-flagged-apis.sh", } + +python_binary_host { + name: "merge_directories", + main: "merge_directories.py", + srcs: [ + "merge_directories.py", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, +} diff --git a/scripts/merge_directories.py b/scripts/merge_directories.py new file mode 100755 index 000000000..3f8631bab --- /dev/null +++ b/scripts/merge_directories.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import argparse +import os +import shutil +import sys + +def main(): + parser = argparse.ArgumentParser( + description="Given a list of directories, this script will copy the contents of all of " + "them into the first directory, erroring out if any duplicate files are found." + ) + parser.add_argument( + "--ignore-duplicates", + action="store_true", + help="Don't error out on duplicate files, just skip them. The file from the earliest " + "directory listed on the command line will be the winner." + ) + parser.add_argument( + "--file-list", + help="Path to a text file containing paths relative to in_dir. Only these paths will be " + "copied out of in_dir." + ) + parser.add_argument("out_dir") + parser.add_argument("in_dir") + args = parser.parse_args() + + if not os.path.isdir(args.out_dir): + sys.exit(f"error: {args.out_dir} must be a directory") + if not os.path.isdir(args.in_dir): + sys.exit(f"error: {args.in_dir} must be a directory") + + file_list = None + if args.file_list: + with open(file_list_file, "r") as f: + file_list = f.read().strip().splitlines() + + in_dir = args.in_dir + for root, dirs, files in os.walk(in_dir): + rel_root = os.path.relpath(root, in_dir) + dst_root = os.path.join(args.out_dir, rel_root) + made_parent_dirs = False + for f in files: + src = os.path.join(root, f) + dst = os.path.join(dst_root, f) + p = os.path.normpath(os.path.join(rel_root, f)) + if file_list is not None and p not in file_list: + continue + if os.path.lexists(dst): + if args.ignore_duplicates: + continue + sys.exit(f"error: {p} exists in both {args.out_dir} and {in_dir}") + + if not made_parent_dirs: + os.makedirs(dst_root, exist_ok=True) + made_parent_dirs = True + + shutil.copy2(src, dst, follow_symlinks=False) + +if __name__ == "__main__": + main()