Reland: Use depsets for transitive manifests and assets

Instead of rolling assets and manifests up through each static lib,
provide them as a DepSet.  This will help with the next patch, which
needs to pass all the transitive manifests and R.txt files together.

This relands Id8b3aa2bed3771e82ab6bde192c9b43baa38b54c with a fix
to include additional manifests from dependencies in the final
manifest merger.

Test: app_test.go
Test: TestManifestMerger
Change-Id: Ied0a6cfee2f18e87188db145b9411d4a903ab6c9
This commit is contained in:
Colin Cross 2023-07-14 17:23:41 +00:00
parent 02bccdf37c
commit ab8d138d42
4 changed files with 190 additions and 116 deletions

View file

@ -31,10 +31,9 @@ import (
type AndroidLibraryDependency interface {
LibraryDependency
ExportPackage() android.Path
ExportedRRODirs() []rroDir
ExportedStaticPackages() android.Paths
ExportedManifests() android.Paths
ExportedAssets() android.OptionalPath
ResourcesNodeDepSet() *android.DepSet[*resourcesNode]
RRODirsDepSet() *android.DepSet[rroDir]
ManifestsDepSet() *android.DepSet[android.Path]
SetRROEnforcedForDependent(enforce bool)
IsRROEnforced(ctx android.BaseModuleContext) bool
}
@ -97,9 +96,7 @@ type aapt struct {
aaptSrcJar android.Path
exportPackage android.Path
manifestPath android.Path
transitiveManifestPaths android.Paths
proguardOptionsFile android.Path
rroDirs []rroDir
rTxt android.Path
extraAaptPackagesFile android.Path
mergedManifestFile android.Path
@ -118,6 +115,10 @@ type aapt struct {
splits []split
aaptProperties aaptProperties
resourcesNodesDepSet *android.DepSet[*resourcesNode]
rroDirsDepSet *android.DepSet[rroDir]
manifestsDepSet *android.DepSet[android.Path]
}
type split struct {
@ -141,17 +142,16 @@ func propagateRROEnforcementMutator(ctx android.TopDownMutatorContext) {
func (a *aapt) ExportPackage() android.Path {
return a.exportPackage
}
func (a *aapt) ExportedRRODirs() []rroDir {
return a.rroDirs
func (a *aapt) ResourcesNodeDepSet() *android.DepSet[*resourcesNode] {
return a.resourcesNodesDepSet
}
func (a *aapt) ExportedManifests() android.Paths {
return a.transitiveManifestPaths
func (a *aapt) RRODirsDepSet() *android.DepSet[rroDir] {
return a.rroDirsDepSet
}
func (a *aapt) ExportedAssets() android.OptionalPath {
return a.assetPackage
func (a *aapt) ManifestsDepSet() *android.DepSet[android.Path] {
return a.manifestsDepSet
}
func (a *aapt) SetRROEnforcedForDependent(enforce bool) {
@ -291,7 +291,7 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
classLoaderContexts dexpreopt.ClassLoaderContextMap, excludedLibs []string,
enforceDefaultTargetSdkVersion bool, extraLinkFlags ...string) {
transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, assetPackages, libDeps, libFlags :=
staticResourcesNodesDepSet, staticRRODirsDepSet, staticManifestsDepSet, sharedDeps, libFlags :=
aaptLibs(ctx, sdkContext, classLoaderContexts)
// Exclude any libraries from the supplied list.
@ -314,13 +314,20 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
EnforceDefaultTargetSdkVersion: enforceDefaultTargetSdkVersion,
})
staticDeps := transitiveAarDeps(staticResourcesNodesDepSet.ToList())
// Add additional manifest files to transitive manifests.
additionalManifests := android.PathsForModuleSrc(ctx, a.aaptProperties.Additional_manifests)
a.transitiveManifestPaths = append(android.Paths{manifestPath}, additionalManifests...)
a.transitiveManifestPaths = append(a.transitiveManifestPaths, transitiveStaticLibManifests...)
transitiveManifestPaths := append(android.Paths{manifestPath}, additionalManifests...)
// TODO(b/288358614): Soong has historically not merged manifests from dependencies of android_library_import
// modules. Merging manifests from dependencies could remove the need for pom2bp to generate the "-nodeps" copies
// of androidx libraries, but doing so triggers errors due to errors introduced by existing dependencies of
// android_library_import modules. If this is fixed, staticManifestsDepSet can be dropped completely in favor of
// staticResourcesNodesDepSet.manifests()
transitiveManifestPaths = append(transitiveManifestPaths, staticManifestsDepSet.ToList()...)
if len(a.transitiveManifestPaths) > 1 && !Bool(a.aaptProperties.Dont_merge_manifests) {
a.mergedManifestFile = manifestMerger(ctx, a.transitiveManifestPaths[0], a.transitiveManifestPaths[1:], a.isLibrary)
if len(transitiveManifestPaths) > 1 && !Bool(a.aaptProperties.Dont_merge_manifests) {
a.mergedManifestFile = manifestMerger(ctx, transitiveManifestPaths[0], transitiveManifestPaths[1:], a.isLibrary)
if !a.isLibrary {
// Only use the merged manifest for applications. For libraries, the transitive closure of manifests
// will be propagated to the final application and merged there. The merged manifest for libraries is
@ -333,9 +340,9 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
compileFlags, linkFlags, linkDeps, resDirs, overlayDirs, rroDirs, resZips := a.aapt2Flags(ctx, sdkContext, manifestPath)
rroDirs = append(rroDirs, staticRRODirs...)
linkFlags = append(linkFlags, libFlags...)
linkDeps = append(linkDeps, libDeps...)
linkDeps = append(linkDeps, sharedDeps...)
linkDeps = append(linkDeps, staticDeps.resPackages()...)
linkFlags = append(linkFlags, extraLinkFlags...)
if a.isLibrary {
linkFlags = append(linkFlags, "--static-lib")
@ -363,6 +370,10 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
var compiledRes, compiledOverlay android.Paths
// AAPT2 overlays are in lowest to highest priority order, reverse the topological order
// of transitiveStaticLibs.
transitiveStaticLibs := android.ReversePaths(staticDeps.resPackages())
compiledOverlay = append(compiledOverlay, transitiveStaticLibs...)
if len(transitiveStaticLibs) > 0 {
@ -404,12 +415,18 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
})
}
// No need to specify assets from dependencies to aapt2Link for libraries, all transitive assets will be
// provided to the final app aapt2Link step.
var transitiveAssets android.Paths
if !a.isLibrary {
transitiveAssets = android.ReverseSliceInPlace(staticDeps.assets())
}
aapt2Link(ctx, packageRes, srcJar, proguardOptionsFile, rTxt, extraPackages,
linkFlags, linkDeps, compiledRes, compiledOverlay, assetPackages, splitPackages)
linkFlags, linkDeps, compiledRes, compiledOverlay, transitiveAssets, splitPackages)
// Extract assets from the resource package output so that they can be used later in aapt2link
// for modules that depend on this one.
if android.PrefixInList(linkFlags, "-A ") || len(assetPackages) > 0 {
if android.PrefixInList(linkFlags, "-A ") {
assets := android.PathForModuleOut(ctx, "assets.zip")
ctx.Build(pctx, android.BuildParams{
Rule: extractAssetsRule,
@ -424,17 +441,66 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
a.exportPackage = packageRes
a.manifestPath = manifestPath
a.proguardOptionsFile = proguardOptionsFile
a.rroDirs = rroDirs
a.extraAaptPackagesFile = extraPackages
a.rTxt = rTxt
a.splits = splits
a.resourcesNodesDepSet = android.NewDepSetBuilder[*resourcesNode](android.TOPOLOGICAL).
Direct(&resourcesNode{
resPackage: a.exportPackage,
manifest: a.manifestPath,
additionalManifests: additionalManifests,
assets: a.assetPackage,
}).
Transitive(staticResourcesNodesDepSet).Build()
a.rroDirsDepSet = android.NewDepSetBuilder[rroDir](android.TOPOLOGICAL).
Direct(rroDirs...).
Transitive(staticRRODirsDepSet).Build()
a.manifestsDepSet = android.NewDepSetBuilder[android.Path](android.TOPOLOGICAL).
Direct(a.manifestPath).
DirectSlice(additionalManifests).
Transitive(staticManifestsDepSet).Build()
}
type resourcesNode struct {
resPackage android.Path
manifest android.Path
additionalManifests android.Paths
assets android.OptionalPath
}
type transitiveAarDeps []*resourcesNode
func (t transitiveAarDeps) resPackages() android.Paths {
var paths android.Paths
for _, dep := range t {
paths = append(paths, dep.resPackage)
}
return android.FirstUniquePaths(paths)
}
func (t transitiveAarDeps) manifests() android.Paths {
var paths android.Paths
for _, dep := range t {
paths = append(paths, dep.manifest)
paths = append(paths, dep.additionalManifests...)
}
return android.FirstUniquePaths(paths)
}
func (t transitiveAarDeps) assets() android.Paths {
var paths android.Paths
for _, dep := range t {
if dep.assets.Valid() {
paths = append(paths, dep.assets.Path())
}
}
return paths
}
// aaptLibs collects libraries from dependencies and sdk_version and converts them into paths
func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoaderContexts dexpreopt.ClassLoaderContextMap) (
transitiveStaticLibs, transitiveStaticLibManifests android.Paths, staticRRODirs []rroDir, assets, deps android.Paths, flags []string) {
var sharedLibs android.Paths
staticResourcesNodes *android.DepSet[*resourcesNode], staticRRODirs *android.DepSet[rroDir],
staticManifests *android.DepSet[android.Path], sharedLibs android.Paths, flags []string) {
if classLoaderContexts == nil {
// Not all callers need to compute class loader context, those who don't just pass nil.
@ -447,6 +513,10 @@ func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoa
sharedLibs = append(sharedLibs, sdkDep.jars...)
}
var resourcesNodeDepSets []*android.DepSet[*resourcesNode]
rroDirsDepSetBuilder := android.NewDepSetBuilder[rroDir](android.TOPOLOGICAL)
manifestsDepSetBuilder := android.NewDepSetBuilder[android.Path](android.TOPOLOGICAL)
ctx.VisitDirectDeps(func(module android.Module) {
depTag := ctx.OtherModuleDependencyTag(module)
@ -469,32 +539,28 @@ func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoa
}
case staticLibTag:
if exportPackage != nil {
transitiveStaticLibs = append(transitiveStaticLibs, aarDep.ExportedStaticPackages()...)
transitiveStaticLibs = append(transitiveStaticLibs, exportPackage)
transitiveStaticLibManifests = append(transitiveStaticLibManifests, aarDep.ExportedManifests()...)
if aarDep.ExportedAssets().Valid() {
assets = append(assets, aarDep.ExportedAssets().Path())
}
outer:
for _, d := range aarDep.ExportedRRODirs() {
for _, e := range staticRRODirs {
if d.path == e.path {
continue outer
}
}
staticRRODirs = append(staticRRODirs, d)
}
resourcesNodeDepSets = append(resourcesNodeDepSets, aarDep.ResourcesNodeDepSet())
rroDirsDepSetBuilder.Transitive(aarDep.RRODirsDepSet())
manifestsDepSetBuilder.Transitive(aarDep.ManifestsDepSet())
}
}
addCLCFromDep(ctx, module, classLoaderContexts)
})
deps = append(deps, sharedLibs...)
deps = append(deps, transitiveStaticLibs...)
// AAPT2 overlays are in lowest to highest priority order, the topological order will be reversed later.
// Reverse the dependency order now going into the depset so that it comes out in order after the second
// reverse later.
// NOTE: this is legacy and probably incorrect behavior, for most other cases (e.g. conflicting classes in
// dependencies) the highest priority dependency is listed first, but for resources the highest priority
// dependency has to be listed last.
staticResourcesNodes = android.NewDepSet(android.TOPOLOGICAL, nil,
android.ReverseSliceInPlace(resourcesNodeDepSets))
if len(transitiveStaticLibs) > 0 {
staticRRODirs = rroDirsDepSetBuilder.Build()
staticManifests = manifestsDepSetBuilder.Build()
if len(staticResourcesNodes.ToList()) > 0 {
flags = append(flags, "--auto-add-overlay")
}
@ -502,10 +568,7 @@ func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoa
flags = append(flags, "-I "+sharedLib.String())
}
transitiveStaticLibs = android.FirstUniquePaths(transitiveStaticLibs)
transitiveStaticLibManifests = android.FirstUniquePaths(transitiveStaticLibManifests)
return transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, assets, deps, flags
return staticResourcesNodes, staticRRODirs, staticManifests, sharedLibs, flags
}
type AndroidLibrary struct {
@ -516,8 +579,6 @@ type AndroidLibrary struct {
androidLibraryProperties androidLibraryProperties
aarFile android.WritablePath
exportedStaticPackages android.Paths
}
var _ android.OutputFileProducer = (*AndroidLibrary)(nil)
@ -532,10 +593,6 @@ func (a *AndroidLibrary) OutputFiles(tag string) (android.Paths, error) {
}
}
func (a *AndroidLibrary) ExportedStaticPackages() android.Paths {
return a.exportedStaticPackages
}
var _ AndroidLibraryDependency = (*AndroidLibrary)(nil)
func (a *AndroidLibrary) DepsMutator(ctx android.BottomUpMutatorContext) {
@ -579,19 +636,15 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext)
a.exportedProguardFlagFiles = append(a.exportedProguardFlagFiles,
android.PathsForModuleSrc(ctx, a.dexProperties.Optimize.Proguard_flags_files)...)
ctx.VisitDirectDeps(func(m android.Module) {
if ctx.OtherModuleDependencyTag(m) == staticLibTag {
if lib, ok := m.(LibraryDependency); ok {
a.exportedProguardFlagFiles = append(a.exportedProguardFlagFiles, lib.ExportedProguardFlagFiles()...)
}
if alib, ok := m.(AndroidLibraryDependency); ok {
a.exportedStaticPackages = append(a.exportedStaticPackages, alib.ExportPackage())
a.exportedStaticPackages = append(a.exportedStaticPackages, alib.ExportedStaticPackages()...)
}
}
})
a.exportedProguardFlagFiles = android.FirstUniquePaths(a.exportedProguardFlagFiles)
a.exportedStaticPackages = android.FirstUniquePaths(a.exportedStaticPackages)
prebuiltJniPackages := android.Paths{}
ctx.VisitDirectDeps(func(module android.Module) {
@ -681,7 +734,8 @@ type AARImport struct {
manifest android.WritablePath
assetsPackage android.WritablePath
exportedStaticPackages android.Paths
resourcesNodesDepSet *android.DepSet[*resourcesNode]
manifestsDepSet *android.DepSet[android.Path]
hideApexVariantFromMake bool
@ -738,25 +792,20 @@ var _ AndroidLibraryDependency = (*AARImport)(nil)
func (a *AARImport) ExportPackage() android.Path {
return a.exportPackage
}
func (a *AARImport) ExportedProguardFlagFiles() android.Paths {
return android.Paths{a.proguardFlags}
}
func (a *AARImport) ExportedRRODirs() []rroDir {
return nil
func (a *AARImport) ResourcesNodeDepSet() *android.DepSet[*resourcesNode] {
return a.resourcesNodesDepSet
}
func (a *AARImport) ExportedStaticPackages() android.Paths {
return a.exportedStaticPackages
func (a *AARImport) RRODirsDepSet() *android.DepSet[rroDir] {
return android.NewDepSet[rroDir](android.TOPOLOGICAL, nil, nil)
}
func (a *AARImport) ExportedManifests() android.Paths {
return android.Paths{a.manifest}
}
func (a *AARImport) ExportedAssets() android.OptionalPath {
return android.OptionalPathForPath(a.assetsPackage)
func (a *AARImport) ManifestsDepSet() *android.DepSet[android.Path] {
return a.manifestsDepSet
}
// RRO enforcement is not available on aar_import since its RRO dirs are not
@ -891,32 +940,44 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) {
linkFlags = append(linkFlags, "--manifest "+a.manifest.String())
linkDeps = append(linkDeps, a.manifest)
transitiveStaticLibs, staticLibManifests, staticRRODirs, transitiveAssets, libDeps, libFlags :=
staticResourcesNodesDepSet, staticRRODirsDepSet, staticManifestsDepSet, sharedLibs, libFlags :=
aaptLibs(ctx, android.SdkContext(a), nil)
_ = staticLibManifests
_ = staticRRODirs
_ = staticRRODirsDepSet
staticDeps := transitiveAarDeps(staticResourcesNodesDepSet.ToList())
linkDeps = append(linkDeps, libDeps...)
// AAPT2 overlays are in lowest to highest priority order, reverse the topological order
// of transitiveStaticLibs.
transitiveStaticLibs := android.ReversePaths(staticDeps.resPackages())
linkDeps = append(linkDeps, sharedLibs...)
linkDeps = append(linkDeps, transitiveStaticLibs...)
linkFlags = append(linkFlags, libFlags...)
overlayRes := append(android.Paths{flata}, transitiveStaticLibs...)
transitiveAssets := android.ReverseSliceInPlace(staticDeps.assets())
aapt2Link(ctx, a.exportPackage, srcJar, proguardOptionsFile, rTxt, a.extraAaptPackagesFile,
linkFlags, linkDeps, nil, overlayRes, transitiveAssets, nil)
// Merge this import's assets with its dependencies' assets (if there are any).
if len(transitiveAssets) > 0 {
mergedAssets := android.PathForModuleOut(ctx, "merged-assets.zip")
inputZips := append(android.Paths{a.assetsPackage}, transitiveAssets...)
ctx.Build(pctx, android.BuildParams{
Rule: mergeAssetsRule,
Inputs: inputZips,
Output: mergedAssets,
Description: "merge assets from dependencies and self",
resourcesNodesDepSetBuilder := android.NewDepSetBuilder[*resourcesNode](android.TOPOLOGICAL)
resourcesNodesDepSetBuilder.Direct(&resourcesNode{
resPackage: a.exportPackage,
manifest: a.manifest,
assets: android.OptionalPathForPath(a.assetsPackage),
})
a.assetsPackage = mergedAssets
}
resourcesNodesDepSetBuilder.Transitive(staticResourcesNodesDepSet)
a.resourcesNodesDepSet = resourcesNodesDepSetBuilder.Build()
manifestDepSetBuilder := android.NewDepSetBuilder[android.Path](android.TOPOLOGICAL).Direct(a.manifest)
// TODO(b/288358614): Soong has historically not merged manifests from dependencies of android_library_import
// modules. Merging manifests from dependencies could remove the need for pom2bp to generate the "-nodeps" copies
// of androidx libraries, but doing so triggers errors due to errors introduced by existing dependencies of
// android_library_import modules. If this is fixed, AndroidLibraryDependency.ManifestsDepSet can be dropped
// completely in favor of AndroidLibraryDependency.ResourceNodesDepSet.manifest
//manifestDepSetBuilder.Transitive(transitiveStaticDeps.manifests)
_ = staticManifestsDepSet
a.manifestsDepSet = manifestDepSetBuilder.Build()
a.collectTransitiveHeaderJars(ctx)
ctx.SetProvider(JavaInfoProvider, JavaInfo{

View file

@ -368,8 +368,13 @@ func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries {
filterRRO := func(filter overlayType) android.Paths {
var paths android.Paths
for _, d := range app.rroDirs {
seen := make(map[android.Path]bool)
for _, d := range app.rroDirsDepSet.ToList() {
if d.overlayType == filter {
if seen[d.path] {
continue
}
seen[d.path] = true
paths = append(paths, d.path)
}
}

View file

@ -204,8 +204,8 @@ func (a *AndroidApp) ExportedProguardFlagFiles() android.Paths {
return nil
}
func (a *AndroidApp) ExportedStaticPackages() android.Paths {
return nil
func (a *AndroidApp) ResourcesNodeDepSet() *android.DepSet[*resourcesNode] {
return a.aapt.resourcesNodesDepSet
}
func (a *AndroidApp) OutputFile() android.Path {

View file

@ -599,7 +599,7 @@ func TestLibraryAssets(t *testing.T) {
android_library {
name: "lib3",
sdk_version: "current",
static_libs: ["lib4"],
static_libs: ["lib4", "import"],
}
android_library {
@ -607,6 +607,12 @@ func TestLibraryAssets(t *testing.T) {
sdk_version: "current",
asset_dirs: ["assets_b"],
}
android_library_import {
name: "import",
sdk_version: "current",
aars: ["import.aar"],
}
`
testCases := []struct {
@ -616,11 +622,12 @@ func TestLibraryAssets(t *testing.T) {
}{
{
name: "foo",
// lib1 has its own asset. lib3 doesn't have any, but provides lib4's transitively.
// lib1 has its own assets. lib3 doesn't have any, but lib4 and import have assets.
assetPackages: []string{
"out/soong/.intermediates/foo/android_common/aapt2/package-res.apk",
"out/soong/.intermediates/lib1/android_common/assets.zip",
"out/soong/.intermediates/lib3/android_common/assets.zip",
"out/soong/.intermediates/lib4/android_common/assets.zip",
"out/soong/.intermediates/import/android_common/assets.zip",
},
},
{
@ -632,10 +639,6 @@ func TestLibraryAssets(t *testing.T) {
},
{
name: "lib3",
assetPackages: []string{
"out/soong/.intermediates/lib3/android_common/aapt2/package-res.apk",
"out/soong/.intermediates/lib4/android_common/assets.zip",
},
},
{
name: "lib4",
@ -761,11 +764,14 @@ func TestAndroidResourceProcessor(t *testing.T) {
appResources: nil,
appOverlays: []string{
"out/soong/.intermediates/transitive/android_common/package-res.apk",
"out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk",
"out/soong/.intermediates/transitive_import/android_common/package-res.apk",
"out/soong/.intermediates/direct/android_common/package-res.apk",
"out/soong/.intermediates/direct_import_dep/android_common/package-res.apk",
"out/soong/.intermediates/direct_import/android_common/package-res.apk",
"out/soong/.intermediates/app/android_common/aapt2/app/res/values_strings.arsc.flat",
},
appImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"},
appSrcJars: []string{"out/soong/.intermediates/app/android_common/gen/android/R.srcjar"},
appClasspath: []string{
@ -782,9 +788,11 @@ func TestAndroidResourceProcessor(t *testing.T) {
directResources: nil,
directOverlays: []string{
"out/soong/.intermediates/transitive/android_common/package-res.apk",
"out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk",
"out/soong/.intermediates/transitive_import/android_common/package-res.apk",
"out/soong/.intermediates/direct/android_common/aapt2/direct/res/values_strings.arsc.flat",
},
directImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"},
directSrcJars: []string{"out/soong/.intermediates/direct/android_common/gen/android/R.srcjar"},
directClasspath: []string{
@ -1198,7 +1206,7 @@ func TestAndroidResourceOverlays(t *testing.T) {
overlayFiles = resourceListToFiles(module, android.PathsRelativeToTop(overlayList.Inputs))
}
for _, d := range module.Module().(AndroidLibraryDependency).ExportedRRODirs() {
for _, d := range module.Module().(AndroidLibraryDependency).RRODirsDepSet().ToList() {
var prefix string
if d.overlayType == device {
prefix = "device:"