Merge "Use depsets for transitive manifests and assets" into main

This commit is contained in:
Colin Cross 2023-07-12 21:49:13 +00:00 committed by Gerrit Code Review
commit 654f176379
4 changed files with 186 additions and 116 deletions

View file

@ -31,10 +31,9 @@ import (
type AndroidLibraryDependency interface {
LibraryDependency
ExportPackage() android.Path
ExportedRRODirs() []rroDir
ExportedStaticPackages() android.Paths
ExportedManifests() android.Paths
ExportedAssets() android.OptionalPath
ResourcesNodeDepSet() *android.DepSet[resourcesNode]
RRODirsDepSet() *android.DepSet[rroDir]
ManifestsDepSet() *android.DepSet[android.Path]
SetRROEnforcedForDependent(enforce bool)
IsRROEnforced(ctx android.BaseModuleContext) bool
}
@ -94,30 +93,32 @@ type aaptProperties struct {
}
type aapt struct {
aaptSrcJar android.Path
exportPackage android.Path
manifestPath android.Path
transitiveManifestPaths android.Paths
proguardOptionsFile android.Path
rroDirs []rroDir
rTxt android.Path
extraAaptPackagesFile android.Path
mergedManifestFile android.Path
noticeFile android.OptionalPath
assetPackage android.OptionalPath
isLibrary bool
defaultManifestVersion string
useEmbeddedNativeLibs bool
useEmbeddedDex bool
usesNonSdkApis bool
hasNoCode bool
LoggingParent string
resourceFiles android.Paths
aaptSrcJar android.Path
exportPackage android.Path
manifestPath android.Path
proguardOptionsFile android.Path
rTxt android.Path
extraAaptPackagesFile android.Path
mergedManifestFile android.Path
noticeFile android.OptionalPath
assetPackage android.OptionalPath
isLibrary bool
defaultManifestVersion string
useEmbeddedNativeLibs bool
useEmbeddedDex bool
usesNonSdkApis bool
hasNoCode bool
LoggingParent string
resourceFiles android.Paths
splitNames []string
splits []split
aaptProperties aaptProperties
resourcesNodesDepSet *android.DepSet[resourcesNode]
rroDirsDepSet *android.DepSet[rroDir]
manifestsDepSet *android.DepSet[android.Path]
}
type split struct {
@ -141,17 +142,16 @@ func propagateRROEnforcementMutator(ctx android.TopDownMutatorContext) {
func (a *aapt) ExportPackage() android.Path {
return a.exportPackage
}
func (a *aapt) ExportedRRODirs() []rroDir {
return a.rroDirs
func (a *aapt) ResourcesNodeDepSet() *android.DepSet[resourcesNode] {
return a.resourcesNodesDepSet
}
func (a *aapt) ExportedManifests() android.Paths {
return a.transitiveManifestPaths
func (a *aapt) RRODirsDepSet() *android.DepSet[rroDir] {
return a.rroDirsDepSet
}
func (a *aapt) ExportedAssets() android.OptionalPath {
return a.assetPackage
func (a *aapt) ManifestsDepSet() *android.DepSet[android.Path] {
return a.manifestsDepSet
}
func (a *aapt) SetRROEnforcedForDependent(enforce bool) {
@ -291,7 +291,7 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
classLoaderContexts dexpreopt.ClassLoaderContextMap, excludedLibs []string,
enforceDefaultTargetSdkVersion bool, extraLinkFlags ...string) {
transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, assetPackages, libDeps, libFlags :=
staticResourcesNodesDepSet, staticRRODirsDepSet, staticManifestsDepSet, sharedDeps, libFlags :=
aaptLibs(ctx, sdkContext, classLoaderContexts)
// Exclude any libraries from the supplied list.
@ -314,13 +314,20 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
EnforceDefaultTargetSdkVersion: enforceDefaultTargetSdkVersion,
})
staticDeps := transitiveAarDeps(staticResourcesNodesDepSet.ToList())
// Add additional manifest files to transitive manifests.
additionalManifests := android.PathsForModuleSrc(ctx, a.aaptProperties.Additional_manifests)
a.transitiveManifestPaths = append(android.Paths{manifestPath}, additionalManifests...)
a.transitiveManifestPaths = append(a.transitiveManifestPaths, transitiveStaticLibManifests...)
transitiveManifestPaths := append(android.Paths{manifestPath}, additionalManifests...)
// TODO(b/288358614): Soong has historically not merged manifests from dependencies of android_library_import
// modules. Merging manifests from dependencies could remove the need for pom2bp to generate the "-nodeps" copies
// of androidx libraries, but doing so triggers errors due to errors introduced by existing dependencies of
// android_library_import modules. If this is fixed, staticManifestsDepSet can be dropped completely in favor of
// staticResourcesNodesDepSet.manifests()
transitiveManifestPaths = append(transitiveManifestPaths, staticManifestsDepSet.ToList()...)
if len(a.transitiveManifestPaths) > 1 && !Bool(a.aaptProperties.Dont_merge_manifests) {
a.mergedManifestFile = manifestMerger(ctx, a.transitiveManifestPaths[0], a.transitiveManifestPaths[1:], a.isLibrary)
if len(transitiveManifestPaths) > 1 && !Bool(a.aaptProperties.Dont_merge_manifests) {
a.mergedManifestFile = manifestMerger(ctx, transitiveManifestPaths[0], transitiveManifestPaths[1:], a.isLibrary)
if !a.isLibrary {
// Only use the merged manifest for applications. For libraries, the transitive closure of manifests
// will be propagated to the final application and merged there. The merged manifest for libraries is
@ -333,9 +340,9 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
compileFlags, linkFlags, linkDeps, resDirs, overlayDirs, rroDirs, resZips := a.aapt2Flags(ctx, sdkContext, manifestPath)
rroDirs = append(rroDirs, staticRRODirs...)
linkFlags = append(linkFlags, libFlags...)
linkDeps = append(linkDeps, libDeps...)
linkDeps = append(linkDeps, sharedDeps...)
linkDeps = append(linkDeps, staticDeps.resPackages()...)
linkFlags = append(linkFlags, extraLinkFlags...)
if a.isLibrary {
linkFlags = append(linkFlags, "--static-lib")
@ -363,6 +370,10 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
var compiledRes, compiledOverlay android.Paths
// AAPT2 overlays are in lowest to highest priority order, reverse the topological order
// of transitiveStaticLibs.
transitiveStaticLibs := android.ReversePaths(staticDeps.resPackages())
compiledOverlay = append(compiledOverlay, transitiveStaticLibs...)
if len(transitiveStaticLibs) > 0 {
@ -404,12 +415,18 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
})
}
// No need to specify assets from dependencies to aapt2Link for libraries, all transitive assets will be
// provided to the final app aapt2Link step.
var transitiveAssets android.Paths
if !a.isLibrary {
transitiveAssets = android.ReverseSliceInPlace(staticDeps.assets())
}
aapt2Link(ctx, packageRes, srcJar, proguardOptionsFile, rTxt, extraPackages,
linkFlags, linkDeps, compiledRes, compiledOverlay, assetPackages, splitPackages)
linkFlags, linkDeps, compiledRes, compiledOverlay, transitiveAssets, splitPackages)
// Extract assets from the resource package output so that they can be used later in aapt2link
// for modules that depend on this one.
if android.PrefixInList(linkFlags, "-A ") || len(assetPackages) > 0 {
if android.PrefixInList(linkFlags, "-A ") {
assets := android.PathForModuleOut(ctx, "assets.zip")
ctx.Build(pctx, android.BuildParams{
Rule: extractAssetsRule,
@ -424,17 +441,62 @@ func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkCon
a.exportPackage = packageRes
a.manifestPath = manifestPath
a.proguardOptionsFile = proguardOptionsFile
a.rroDirs = rroDirs
a.extraAaptPackagesFile = extraPackages
a.rTxt = rTxt
a.splits = splits
a.resourcesNodesDepSet = android.NewDepSetBuilder[resourcesNode](android.TOPOLOGICAL).
Direct(resourcesNode{
resPackage: a.exportPackage,
manifest: a.manifestPath,
assets: a.assetPackage,
}).
Transitive(staticResourcesNodesDepSet).Build()
a.rroDirsDepSet = android.NewDepSetBuilder[rroDir](android.TOPOLOGICAL).
Direct(rroDirs...).
Transitive(staticRRODirsDepSet).Build()
a.manifestsDepSet = android.NewDepSetBuilder[android.Path](android.TOPOLOGICAL).
Direct(a.manifestPath).
Transitive(staticManifestsDepSet).Build()
}
type resourcesNode struct {
resPackage android.Path
manifest android.Path
assets android.OptionalPath
}
type transitiveAarDeps []resourcesNode
func (t transitiveAarDeps) resPackages() android.Paths {
var paths android.Paths
for _, dep := range t {
paths = append(paths, dep.resPackage)
}
return android.FirstUniquePaths(paths)
}
func (t transitiveAarDeps) manifests() android.Paths {
var paths android.Paths
for _, dep := range t {
paths = append(paths, dep.manifest)
}
return android.FirstUniquePaths(paths)
}
func (t transitiveAarDeps) assets() android.Paths {
var paths android.Paths
for _, dep := range t {
if dep.assets.Valid() {
paths = append(paths, dep.assets.Path())
}
}
return paths
}
// aaptLibs collects libraries from dependencies and sdk_version and converts them into paths
func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoaderContexts dexpreopt.ClassLoaderContextMap) (
transitiveStaticLibs, transitiveStaticLibManifests android.Paths, staticRRODirs []rroDir, assets, deps android.Paths, flags []string) {
var sharedLibs android.Paths
staticResourcesNodes *android.DepSet[resourcesNode], staticRRODirs *android.DepSet[rroDir],
staticManifests *android.DepSet[android.Path], sharedLibs android.Paths, flags []string) {
if classLoaderContexts == nil {
// Not all callers need to compute class loader context, those who don't just pass nil.
@ -447,6 +509,10 @@ func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoa
sharedLibs = append(sharedLibs, sdkDep.jars...)
}
var resourcesNodeDepSets []*android.DepSet[resourcesNode]
rroDirsDepSetBuilder := android.NewDepSetBuilder[rroDir](android.TOPOLOGICAL)
manifestsDepSetBuilder := android.NewDepSetBuilder[android.Path](android.TOPOLOGICAL)
ctx.VisitDirectDeps(func(module android.Module) {
depTag := ctx.OtherModuleDependencyTag(module)
@ -469,32 +535,28 @@ func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoa
}
case staticLibTag:
if exportPackage != nil {
transitiveStaticLibs = append(transitiveStaticLibs, aarDep.ExportedStaticPackages()...)
transitiveStaticLibs = append(transitiveStaticLibs, exportPackage)
transitiveStaticLibManifests = append(transitiveStaticLibManifests, aarDep.ExportedManifests()...)
if aarDep.ExportedAssets().Valid() {
assets = append(assets, aarDep.ExportedAssets().Path())
}
outer:
for _, d := range aarDep.ExportedRRODirs() {
for _, e := range staticRRODirs {
if d.path == e.path {
continue outer
}
}
staticRRODirs = append(staticRRODirs, d)
}
resourcesNodeDepSets = append(resourcesNodeDepSets, aarDep.ResourcesNodeDepSet())
rroDirsDepSetBuilder.Transitive(aarDep.RRODirsDepSet())
manifestsDepSetBuilder.Transitive(aarDep.ManifestsDepSet())
}
}
addCLCFromDep(ctx, module, classLoaderContexts)
})
deps = append(deps, sharedLibs...)
deps = append(deps, transitiveStaticLibs...)
// AAPT2 overlays are in lowest to highest priority order, the topological order will be reversed later.
// Reverse the dependency order now going into the depset so that it comes out in order after the second
// reverse later.
// NOTE: this is legacy and probably incorrect behavior, for most other cases (e.g. conflicting classes in
// dependencies) the highest priority dependency is listed first, but for resources the highest priority
// dependency has to be listed last.
staticResourcesNodes = android.NewDepSet(android.TOPOLOGICAL, nil,
android.ReverseSliceInPlace(resourcesNodeDepSets))
if len(transitiveStaticLibs) > 0 {
staticRRODirs = rroDirsDepSetBuilder.Build()
staticManifests = manifestsDepSetBuilder.Build()
if len(staticResourcesNodes.ToList()) > 0 {
flags = append(flags, "--auto-add-overlay")
}
@ -502,10 +564,7 @@ func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoa
flags = append(flags, "-I "+sharedLib.String())
}
transitiveStaticLibs = android.FirstUniquePaths(transitiveStaticLibs)
transitiveStaticLibManifests = android.FirstUniquePaths(transitiveStaticLibManifests)
return transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, assets, deps, flags
return staticResourcesNodes, staticRRODirs, staticManifests, sharedLibs, flags
}
type AndroidLibrary struct {
@ -516,8 +575,6 @@ type AndroidLibrary struct {
androidLibraryProperties androidLibraryProperties
aarFile android.WritablePath
exportedStaticPackages android.Paths
}
var _ android.OutputFileProducer = (*AndroidLibrary)(nil)
@ -532,10 +589,6 @@ func (a *AndroidLibrary) OutputFiles(tag string) (android.Paths, error) {
}
}
func (a *AndroidLibrary) ExportedStaticPackages() android.Paths {
return a.exportedStaticPackages
}
var _ AndroidLibraryDependency = (*AndroidLibrary)(nil)
func (a *AndroidLibrary) DepsMutator(ctx android.BottomUpMutatorContext) {
@ -579,19 +632,15 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext)
a.exportedProguardFlagFiles = append(a.exportedProguardFlagFiles,
android.PathsForModuleSrc(ctx, a.dexProperties.Optimize.Proguard_flags_files)...)
ctx.VisitDirectDeps(func(m android.Module) {
if ctx.OtherModuleDependencyTag(m) == staticLibTag {
if lib, ok := m.(LibraryDependency); ok {
a.exportedProguardFlagFiles = append(a.exportedProguardFlagFiles, lib.ExportedProguardFlagFiles()...)
}
if alib, ok := m.(AndroidLibraryDependency); ok {
a.exportedStaticPackages = append(a.exportedStaticPackages, alib.ExportPackage())
a.exportedStaticPackages = append(a.exportedStaticPackages, alib.ExportedStaticPackages()...)
}
}
})
a.exportedProguardFlagFiles = android.FirstUniquePaths(a.exportedProguardFlagFiles)
a.exportedStaticPackages = android.FirstUniquePaths(a.exportedStaticPackages)
prebuiltJniPackages := android.Paths{}
ctx.VisitDirectDeps(func(module android.Module) {
@ -681,7 +730,8 @@ type AARImport struct {
manifest android.WritablePath
assetsPackage android.WritablePath
exportedStaticPackages android.Paths
resourcesNodesDepSet *android.DepSet[resourcesNode]
manifestsDepSet *android.DepSet[android.Path]
hideApexVariantFromMake bool
@ -738,25 +788,20 @@ var _ AndroidLibraryDependency = (*AARImport)(nil)
func (a *AARImport) ExportPackage() android.Path {
return a.exportPackage
}
func (a *AARImport) ExportedProguardFlagFiles() android.Paths {
return android.Paths{a.proguardFlags}
}
func (a *AARImport) ExportedRRODirs() []rroDir {
return nil
func (a *AARImport) ResourcesNodeDepSet() *android.DepSet[resourcesNode] {
return a.resourcesNodesDepSet
}
func (a *AARImport) ExportedStaticPackages() android.Paths {
return a.exportedStaticPackages
func (a *AARImport) RRODirsDepSet() *android.DepSet[rroDir] {
return android.NewDepSet[rroDir](android.TOPOLOGICAL, nil, nil)
}
func (a *AARImport) ExportedManifests() android.Paths {
return android.Paths{a.manifest}
}
func (a *AARImport) ExportedAssets() android.OptionalPath {
return android.OptionalPathForPath(a.assetsPackage)
func (a *AARImport) ManifestsDepSet() *android.DepSet[android.Path] {
return a.manifestsDepSet
}
// RRO enforcement is not available on aar_import since its RRO dirs are not
@ -891,32 +936,44 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) {
linkFlags = append(linkFlags, "--manifest "+a.manifest.String())
linkDeps = append(linkDeps, a.manifest)
transitiveStaticLibs, staticLibManifests, staticRRODirs, transitiveAssets, libDeps, libFlags :=
staticResourcesNodesDepSet, staticRRODirsDepSet, staticManifestsDepSet, sharedLibs, libFlags :=
aaptLibs(ctx, android.SdkContext(a), nil)
_ = staticLibManifests
_ = staticRRODirs
_ = staticRRODirsDepSet
staticDeps := transitiveAarDeps(staticResourcesNodesDepSet.ToList())
linkDeps = append(linkDeps, libDeps...)
// AAPT2 overlays are in lowest to highest priority order, reverse the topological order
// of transitiveStaticLibs.
transitiveStaticLibs := android.ReversePaths(staticDeps.resPackages())
linkDeps = append(linkDeps, sharedLibs...)
linkDeps = append(linkDeps, transitiveStaticLibs...)
linkFlags = append(linkFlags, libFlags...)
overlayRes := append(android.Paths{flata}, transitiveStaticLibs...)
transitiveAssets := android.ReverseSliceInPlace(staticDeps.assets())
aapt2Link(ctx, a.exportPackage, srcJar, proguardOptionsFile, rTxt, a.extraAaptPackagesFile,
linkFlags, linkDeps, nil, overlayRes, transitiveAssets, nil)
// Merge this import's assets with its dependencies' assets (if there are any).
if len(transitiveAssets) > 0 {
mergedAssets := android.PathForModuleOut(ctx, "merged-assets.zip")
inputZips := append(android.Paths{a.assetsPackage}, transitiveAssets...)
ctx.Build(pctx, android.BuildParams{
Rule: mergeAssetsRule,
Inputs: inputZips,
Output: mergedAssets,
Description: "merge assets from dependencies and self",
})
a.assetsPackage = mergedAssets
}
resourcesNodesDepSetBuilder := android.NewDepSetBuilder[resourcesNode](android.TOPOLOGICAL)
resourcesNodesDepSetBuilder.Direct(resourcesNode{
resPackage: a.exportPackage,
manifest: a.manifest,
assets: android.OptionalPathForPath(a.assetsPackage),
})
resourcesNodesDepSetBuilder.Transitive(staticResourcesNodesDepSet)
a.resourcesNodesDepSet = resourcesNodesDepSetBuilder.Build()
manifestDepSetBuilder := android.NewDepSetBuilder[android.Path](android.TOPOLOGICAL).Direct(a.manifest)
// TODO(b/288358614): Soong has historically not merged manifests from dependencies of android_library_import
// modules. Merging manifests from dependencies could remove the need for pom2bp to generate the "-nodeps" copies
// of androidx libraries, but doing so triggers errors due to errors introduced by existing dependencies of
// android_library_import modules. If this is fixed, AndroidLibraryDependency.ManifestsDepSet can be dropped
// completely in favor of AndroidLibraryDependency.ResourceNodesDepSet.manifest
//manifestDepSetBuilder.Transitive(transitiveStaticDeps.manifests)
_ = staticManifestsDepSet
a.manifestsDepSet = manifestDepSetBuilder.Build()
a.collectTransitiveHeaderJars(ctx)
ctx.SetProvider(JavaInfoProvider, JavaInfo{

View file

@ -368,8 +368,13 @@ func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries {
filterRRO := func(filter overlayType) android.Paths {
var paths android.Paths
for _, d := range app.rroDirs {
seen := make(map[android.Path]bool)
for _, d := range app.rroDirsDepSet.ToList() {
if d.overlayType == filter {
if seen[d.path] {
continue
}
seen[d.path] = true
paths = append(paths, d.path)
}
}

View file

@ -204,8 +204,8 @@ func (a *AndroidApp) ExportedProguardFlagFiles() android.Paths {
return nil
}
func (a *AndroidApp) ExportedStaticPackages() android.Paths {
return nil
func (a *AndroidApp) ResourcesNodeDepSet() *android.DepSet[resourcesNode] {
return a.aapt.resourcesNodesDepSet
}
func (a *AndroidApp) OutputFile() android.Path {

View file

@ -599,7 +599,7 @@ func TestLibraryAssets(t *testing.T) {
android_library {
name: "lib3",
sdk_version: "current",
static_libs: ["lib4"],
static_libs: ["lib4", "import"],
}
android_library {
@ -607,6 +607,12 @@ func TestLibraryAssets(t *testing.T) {
sdk_version: "current",
asset_dirs: ["assets_b"],
}
android_library_import {
name: "import",
sdk_version: "current",
aars: ["import.aar"],
}
`
testCases := []struct {
@ -616,11 +622,12 @@ func TestLibraryAssets(t *testing.T) {
}{
{
name: "foo",
// lib1 has its own asset. lib3 doesn't have any, but provides lib4's transitively.
// lib1 has its own assets. lib3 doesn't have any, but lib4 and import have assets.
assetPackages: []string{
"out/soong/.intermediates/foo/android_common/aapt2/package-res.apk",
"out/soong/.intermediates/lib1/android_common/assets.zip",
"out/soong/.intermediates/lib3/android_common/assets.zip",
"out/soong/.intermediates/lib4/android_common/assets.zip",
"out/soong/.intermediates/import/android_common/assets.zip",
},
},
{
@ -632,10 +639,6 @@ func TestLibraryAssets(t *testing.T) {
},
{
name: "lib3",
assetPackages: []string{
"out/soong/.intermediates/lib3/android_common/aapt2/package-res.apk",
"out/soong/.intermediates/lib4/android_common/assets.zip",
},
},
{
name: "lib4",
@ -761,11 +764,14 @@ func TestAndroidResourceProcessor(t *testing.T) {
appResources: nil,
appOverlays: []string{
"out/soong/.intermediates/transitive/android_common/package-res.apk",
"out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk",
"out/soong/.intermediates/transitive_import/android_common/package-res.apk",
"out/soong/.intermediates/direct/android_common/package-res.apk",
"out/soong/.intermediates/direct_import_dep/android_common/package-res.apk",
"out/soong/.intermediates/direct_import/android_common/package-res.apk",
"out/soong/.intermediates/app/android_common/aapt2/app/res/values_strings.arsc.flat",
},
appImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"},
appSrcJars: []string{"out/soong/.intermediates/app/android_common/gen/android/R.srcjar"},
appClasspath: []string{
@ -782,9 +788,11 @@ func TestAndroidResourceProcessor(t *testing.T) {
directResources: nil,
directOverlays: []string{
"out/soong/.intermediates/transitive/android_common/package-res.apk",
"out/soong/.intermediates/transitive_import_dep/android_common/package-res.apk",
"out/soong/.intermediates/transitive_import/android_common/package-res.apk",
"out/soong/.intermediates/direct/android_common/aapt2/direct/res/values_strings.arsc.flat",
},
directImports: []string{"out/soong/.intermediates/default/java/framework-res/android_common/package-res.apk"},
directSrcJars: []string{"out/soong/.intermediates/direct/android_common/gen/android/R.srcjar"},
directClasspath: []string{
@ -1198,7 +1206,7 @@ func TestAndroidResourceOverlays(t *testing.T) {
overlayFiles = resourceListToFiles(module, android.PathsRelativeToTop(overlayList.Inputs))
}
for _, d := range module.Module().(AndroidLibraryDependency).ExportedRRODirs() {
for _, d := range module.Module().(AndroidLibraryDependency).RRODirsDepSet().ToList() {
var prefix string
if d.overlayType == device {
prefix = "device:"