platform_build_soong/java/hiddenapi_modular.go
Paul Duffin d2b1e0ca92 Add test to show issues with widest stub dex jars
The widest stub dex jars should include the widest stub dex jars
provided by each module. So, if module A has public, system and test
and module B has only public then the widest stub dex jars should
include module A's test and module B's public stub dex jars. Instead,
they just include module A's test.

That behaviour is needed so that when the "hiddenapi list" tool is run
against a module C that it is passed stub dex jars from both module A
and module B so that any references to the types provided by those APIs
can be resolved.

A follow up change will fix this issue.

Bug: 179354495
Test: m nothing
Change-Id: Ibd31964e8d2a33fa92fbd0b800c9fe054ee359c7
2021-06-27 22:46:14 +01:00

1185 lines
48 KiB
Go

// Copyright (C) 2021 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package java
import (
"fmt"
"strings"
"android/soong/android"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
)
// Contains support for processing hiddenAPI in a modular fashion.
// HiddenAPIScope encapsulates all the information that the hidden API processing needs about API
// scopes, i.e. what is called android.SdkKind and apiScope. It does not just use those as they do
// not provide the information needed by hidden API processing.
type HiddenAPIScope struct {
// The name of the scope, used for debug purposes.
name string
// The corresponding android.SdkKind, used for retrieving paths from java_sdk_library* modules.
sdkKind android.SdkKind
// The option needed to passed to "hiddenapi list".
hiddenAPIListOption string
// The name sof the source stub library modules that contain the API provided by the platform,
// i.e. by modules that are not in an APEX.
nonUpdatableSourceModule string
// The names of the prebuilt stub library modules that contain the API provided by the platform,
// i.e. by modules that are not in an APEX.
nonUpdatablePrebuiltModule string
}
// initHiddenAPIScope initializes the scope.
func initHiddenAPIScope(apiScope *HiddenAPIScope) *HiddenAPIScope {
sdkKind := apiScope.sdkKind
// The platform does not provide a core platform API.
if sdkKind != android.SdkCorePlatform {
kindAsString := sdkKind.String()
var insert string
if sdkKind == android.SdkPublic {
insert = ""
} else {
insert = "." + strings.ReplaceAll(kindAsString, "-", "_")
}
nonUpdatableModule := "android-non-updatable"
// Construct the name of the android-non-updatable source module for this scope.
apiScope.nonUpdatableSourceModule = fmt.Sprintf("%s.stubs%s", nonUpdatableModule, insert)
prebuiltModuleName := func(name string, kind string) string {
return fmt.Sprintf("sdk_%s_current_%s", kind, name)
}
// Construct the name of the android-non-updatable prebuilt module for this scope.
apiScope.nonUpdatablePrebuiltModule = prebuiltModuleName(nonUpdatableModule, kindAsString)
}
return apiScope
}
// android-non-updatable takes the name of a module and returns a possibly scope specific name of
// the module.
func (l *HiddenAPIScope) scopeSpecificStubModule(ctx android.BaseModuleContext, name string) string {
// The android-non-updatable is not a java_sdk_library but there are separate stub libraries for
// each scope.
// TODO(b/192067200): Remove special handling of android-non-updatable.
if name == "android-non-updatable" {
if ctx.Config().AlwaysUsePrebuiltSdks() {
return l.nonUpdatablePrebuiltModule
} else {
return l.nonUpdatableSourceModule
}
} else {
// Assume that the module is either a java_sdk_library (or equivalent) and so will provide
// separate stub jars for each scope or is a java_library (or equivalent) in which case it will
// have the same stub jar for each scope.
return name
}
}
func (l *HiddenAPIScope) String() string {
return fmt.Sprintf("HiddenAPIScope{%s}", l.name)
}
var (
PublicHiddenAPIScope = initHiddenAPIScope(&HiddenAPIScope{
name: "public",
sdkKind: android.SdkPublic,
hiddenAPIListOption: "--public-stub-classpath",
})
SystemHiddenAPIScope = initHiddenAPIScope(&HiddenAPIScope{
name: "system",
sdkKind: android.SdkSystem,
hiddenAPIListOption: "--system-stub-classpath",
})
TestHiddenAPIScope = initHiddenAPIScope(&HiddenAPIScope{
name: "test",
sdkKind: android.SdkTest,
hiddenAPIListOption: "--test-stub-classpath",
})
ModuleLibHiddenAPIScope = initHiddenAPIScope(&HiddenAPIScope{
name: "module-lib",
sdkKind: android.SdkModule,
})
CorePlatformHiddenAPIScope = initHiddenAPIScope(&HiddenAPIScope{
name: "core-platform",
sdkKind: android.SdkCorePlatform,
hiddenAPIListOption: "--core-platform-stub-classpath",
})
// hiddenAPIRelevantSdkKinds lists all the android.SdkKind instances that are needed by the hidden
// API processing.
//
// These are roughly in order from narrowest API surface to widest. Widest means the API stubs
// with the biggest API surface, e.g. test is wider than system is wider than public.
//
// Core platform is considered wider than system/module-lib because those modules that provide
// core platform APIs either do not have any system/module-lib APIs at all, or if they do it is
// because the core platform API is being converted to system/module-lib APIs. In either case the
// system/module-lib APIs are subsets of the core platform API.
//
// This is not strictly in order from narrowest to widest as the Test API is wider than system but
// is neither wider or narrower than the module-lib or core platform APIs. However, this works
// well enough at the moment.
// TODO(b/191644675): Correctly reflect the sub/superset relationships between APIs.
hiddenAPIScopes = []*HiddenAPIScope{
PublicHiddenAPIScope,
SystemHiddenAPIScope,
TestHiddenAPIScope,
ModuleLibHiddenAPIScope,
CorePlatformHiddenAPIScope,
}
// The HiddenAPIScope instances that are supported by a java_sdk_library.
//
// CorePlatformHiddenAPIScope is not used as the java_sdk_library does not have special support
// for core_platform API, instead it is implemented as a customized form of PublicHiddenAPIScope.
hiddenAPISdkLibrarySupportedScopes = []*HiddenAPIScope{
PublicHiddenAPIScope,
SystemHiddenAPIScope,
TestHiddenAPIScope,
ModuleLibHiddenAPIScope,
}
// The HiddenAPIScope instances that are supported by the `hiddenapi list`.
hiddenAPIFlagScopes = []*HiddenAPIScope{
PublicHiddenAPIScope,
SystemHiddenAPIScope,
TestHiddenAPIScope,
CorePlatformHiddenAPIScope,
}
)
type hiddenAPIStubsDependencyTag struct {
blueprint.BaseDependencyTag
// The api scope for which this dependency was added.
apiScope *HiddenAPIScope
// Indicates that the dependency is not for an API provided by the current bootclasspath fragment
// but is an additional API provided by a module that is not part of the current bootclasspath
// fragment.
fromAdditionalDependency bool
}
func (b hiddenAPIStubsDependencyTag) ExcludeFromApexContents() {
}
func (b hiddenAPIStubsDependencyTag) ReplaceSourceWithPrebuilt() bool {
return false
}
func (b hiddenAPIStubsDependencyTag) SdkMemberType(child android.Module) android.SdkMemberType {
// Do not add additional dependencies to the sdk.
if b.fromAdditionalDependency {
return nil
}
// If the module is a java_sdk_library then treat it as if it was specific in the java_sdk_libs
// property, otherwise treat if it was specified in the java_header_libs property.
if javaSdkLibrarySdkMemberType.IsInstance(child) {
return javaSdkLibrarySdkMemberType
}
return javaHeaderLibsSdkMemberType
}
func (b hiddenAPIStubsDependencyTag) ExportMember() bool {
// Export the module added via this dependency tag from the sdk.
return true
}
// Avoid having to make stubs content explicitly visible to dependent modules.
//
// This is a temporary workaround to make it easier to migrate to bootclasspath_fragment modules
// with proper dependencies.
// TODO(b/177892522): Remove this and add needed visibility.
func (b hiddenAPIStubsDependencyTag) ExcludeFromVisibilityEnforcement() {
}
var _ android.ExcludeFromVisibilityEnforcementTag = hiddenAPIStubsDependencyTag{}
var _ android.ReplaceSourceWithPrebuilt = hiddenAPIStubsDependencyTag{}
var _ android.ExcludeFromApexContentsTag = hiddenAPIStubsDependencyTag{}
var _ android.SdkMemberTypeDependencyTag = hiddenAPIStubsDependencyTag{}
// hiddenAPIComputeMonolithicStubLibModules computes the set of module names that provide stubs
// needed to produce the hidden API monolithic stub flags file.
func hiddenAPIComputeMonolithicStubLibModules(config android.Config) map[*HiddenAPIScope][]string {
var publicStubModules []string
var systemStubModules []string
var testStubModules []string
var corePlatformStubModules []string
if config.AlwaysUsePrebuiltSdks() {
// Build configuration mandates using prebuilt stub modules
publicStubModules = append(publicStubModules, "sdk_public_current_android")
systemStubModules = append(systemStubModules, "sdk_system_current_android")
testStubModules = append(testStubModules, "sdk_test_current_android")
} else {
// Use stub modules built from source
publicStubModules = append(publicStubModules, "android_stubs_current")
systemStubModules = append(systemStubModules, "android_system_stubs_current")
testStubModules = append(testStubModules, "android_test_stubs_current")
}
// We do not have prebuilts of the core platform api yet
corePlatformStubModules = append(corePlatformStubModules, "legacy.core.platform.api.stubs")
// Allow products to define their own stubs for custom product jars that apps can use.
publicStubModules = append(publicStubModules, config.ProductHiddenAPIStubs()...)
systemStubModules = append(systemStubModules, config.ProductHiddenAPIStubsSystem()...)
testStubModules = append(testStubModules, config.ProductHiddenAPIStubsTest()...)
if config.IsEnvTrue("EMMA_INSTRUMENT") {
// Add jacoco-stubs to public, system and test. It doesn't make any real difference as public
// allows everyone access but it is needed to ensure consistent flags between the
// bootclasspath fragment generated flags and the platform_bootclasspath generated flags.
publicStubModules = append(publicStubModules, "jacoco-stubs")
systemStubModules = append(systemStubModules, "jacoco-stubs")
testStubModules = append(testStubModules, "jacoco-stubs")
}
m := map[*HiddenAPIScope][]string{}
m[PublicHiddenAPIScope] = publicStubModules
m[SystemHiddenAPIScope] = systemStubModules
m[TestHiddenAPIScope] = testStubModules
m[CorePlatformHiddenAPIScope] = corePlatformStubModules
return m
}
// hiddenAPIAddStubLibDependencies adds dependencies onto the modules specified in
// apiScopeToStubLibModules. It adds them in a well known order and uses a HiddenAPIScope specific
// tag to identify the source of the dependency.
func hiddenAPIAddStubLibDependencies(ctx android.BottomUpMutatorContext, apiScopeToStubLibModules map[*HiddenAPIScope][]string) {
module := ctx.Module()
for _, apiScope := range hiddenAPIScopes {
modules := apiScopeToStubLibModules[apiScope]
ctx.AddDependency(module, hiddenAPIStubsDependencyTag{apiScope: apiScope}, modules...)
}
}
// hiddenAPIRetrieveDexJarBuildPath retrieves the DexJarBuildPath from the specified module, if
// available, or reports an error.
func hiddenAPIRetrieveDexJarBuildPath(ctx android.ModuleContext, module android.Module, kind android.SdkKind) android.Path {
var dexJar android.Path
if sdkLibrary, ok := module.(SdkLibraryDependency); ok {
dexJar = sdkLibrary.SdkApiStubDexJar(ctx, kind)
} else if j, ok := module.(UsesLibraryDependency); ok {
dexJar = j.DexJarBuildPath()
} else {
ctx.ModuleErrorf("dependency %s of module type %s does not support providing a dex jar", module, ctx.OtherModuleType(module))
return nil
}
if dexJar == nil {
ctx.ModuleErrorf("dependency %s does not provide a dex jar, consider setting compile_dex: true", module)
}
return dexJar
}
// buildRuleToGenerateHiddenAPIStubFlagsFile creates a rule to create a hidden API stub flags file.
//
// The rule is initialized but not built so that the caller can modify it and select an appropriate
// name.
func buildRuleToGenerateHiddenAPIStubFlagsFile(ctx android.BuilderContext, name, desc string, outputPath android.WritablePath, bootDexJars android.Paths, input HiddenAPIFlagInput, moduleStubFlagsPaths android.Paths) {
// Singleton rule which applies hiddenapi on all boot class path dex files.
rule := android.NewRuleBuilder(pctx, ctx)
tempPath := tempPathForRestat(ctx, outputPath)
// Find the widest API stubs provided by the fragments on which this depends, if any.
dependencyStubDexJars := input.DependencyStubDexJarsByScope.StubDexJarsForWidestAPIScope()
// Add widest API stubs from the additional dependencies of this, if any.
dependencyStubDexJars = append(dependencyStubDexJars, input.AdditionalStubDexJarsByScope.StubDexJarsForWidestAPIScope()...)
command := rule.Command().
Tool(ctx.Config().HostToolPath(ctx, "hiddenapi")).
Text("list").
FlagForEachInput("--dependency-stub-dex=", dependencyStubDexJars).
FlagForEachInput("--boot-dex=", bootDexJars)
// If no module stub flags paths are provided then this must be being called for a
// bootclasspath_fragment and not the whole platform_bootclasspath.
if moduleStubFlagsPaths == nil {
// This is being run on a fragment of the bootclasspath.
command.Flag("--fragment")
}
// Iterate over the api scopes in a fixed order.
for _, apiScope := range hiddenAPIFlagScopes {
// Merge in the stub dex jar paths for this api scope from the fragments on which it depends.
// They will be needed to resolve dependencies from this fragment's stubs to classes in the
// other fragment's APIs.
var paths android.Paths
paths = append(paths, input.DependencyStubDexJarsByScope[apiScope]...)
paths = append(paths, input.AdditionalStubDexJarsByScope[apiScope]...)
paths = append(paths, input.StubDexJarsByScope[apiScope]...)
if len(paths) > 0 {
option := apiScope.hiddenAPIListOption
command.FlagWithInputList(option+"=", paths, ":")
}
}
// Add the output path.
command.FlagWithOutput("--out-api-flags=", tempPath)
// If there are stub flag files that have been generated by fragments on which this depends then
// use them to validate the stub flag file generated by the rules created by this method.
if len(moduleStubFlagsPaths) > 0 {
validFile := buildRuleValidateOverlappingCsvFiles(ctx, name, desc, outputPath, moduleStubFlagsPaths)
// Add the file that indicates that the file generated by this is valid.
//
// This will cause the validation rule above to be run any time that the output of this rule
// changes but the validation will run in parallel with other rules that depend on this file.
command.Validation(validFile)
}
commitChangeForRestat(rule, tempPath, outputPath)
rule.Build(name, desc)
}
// HiddenAPIFlagFileProperties contains paths to the flag files that can be used to augment the
// information obtained from annotations within the source code in order to create the complete set
// of flags that should be applied to the dex implementation jars on the bootclasspath.
//
// Each property contains a list of paths. With the exception of the Unsupported_packages the paths
// of each property reference a plain text file that contains a java signature per line. The flags
// for each of those signatures will be updated in a property specific way.
//
// The Unsupported_packages property contains a list of paths, each of which is a plain text file
// with one Java package per line. All members of all classes within that package (but not nested
// packages) will be updated in a property specific way.
type HiddenAPIFlagFileProperties struct {
// Marks each signature in the referenced files as being unsupported.
Unsupported []string `android:"path"`
// Marks each signature in the referenced files as being unsupported because it has been removed.
// Any conflicts with other flags are ignored.
Removed []string `android:"path"`
// Marks each signature in the referenced files as being supported only for targetSdkVersion <= R
// and low priority.
Max_target_r_low_priority []string `android:"path"`
// Marks each signature in the referenced files as being supported only for targetSdkVersion <= Q.
Max_target_q []string `android:"path"`
// Marks each signature in the referenced files as being supported only for targetSdkVersion <= P.
Max_target_p []string `android:"path"`
// Marks each signature in the referenced files as being supported only for targetSdkVersion <= O
// and low priority. Any conflicts with other flags are ignored.
Max_target_o_low_priority []string `android:"path"`
// Marks each signature in the referenced files as being blocked.
Blocked []string `android:"path"`
// Marks each signature in every package in the referenced files as being unsupported.
Unsupported_packages []string `android:"path"`
}
type hiddenAPIFlagFileCategory struct {
// PropertyName is the name of the property for this category.
PropertyName string
// propertyValueReader retrieves the value of the property for this category from the set of
// properties.
propertyValueReader func(properties *HiddenAPIFlagFileProperties) []string
// commandMutator adds the appropriate command line options for this category to the supplied
// command
commandMutator func(command *android.RuleBuilderCommand, path android.Path)
}
// The flag file category for removed members of the API.
//
// This is extracted from HiddenAPIFlagFileCategories as it is needed to add the dex signatures
// list of removed API members that are generated automatically from the removed.txt files provided
// by API stubs.
var hiddenAPIRemovedFlagFileCategory = &hiddenAPIFlagFileCategory{
// See HiddenAPIFlagFileProperties.Removed
PropertyName: "removed",
propertyValueReader: func(properties *HiddenAPIFlagFileProperties) []string {
return properties.Removed
},
commandMutator: func(command *android.RuleBuilderCommand, path android.Path) {
command.FlagWithInput("--unsupported ", path).Flag("--ignore-conflicts ").FlagWithArg("--tag ", "removed")
},
}
var HiddenAPIFlagFileCategories = []*hiddenAPIFlagFileCategory{
// See HiddenAPIFlagFileProperties.Unsupported
{
PropertyName: "unsupported",
propertyValueReader: func(properties *HiddenAPIFlagFileProperties) []string {
return properties.Unsupported
},
commandMutator: func(command *android.RuleBuilderCommand, path android.Path) {
command.FlagWithInput("--unsupported ", path)
},
},
hiddenAPIRemovedFlagFileCategory,
// See HiddenAPIFlagFileProperties.Max_target_r_low_priority
{
PropertyName: "max_target_r_low_priority",
propertyValueReader: func(properties *HiddenAPIFlagFileProperties) []string {
return properties.Max_target_r_low_priority
},
commandMutator: func(command *android.RuleBuilderCommand, path android.Path) {
command.FlagWithInput("--max-target-r ", path).FlagWithArg("--tag ", "lo-prio")
},
},
// See HiddenAPIFlagFileProperties.Max_target_q
{
PropertyName: "max_target_q",
propertyValueReader: func(properties *HiddenAPIFlagFileProperties) []string {
return properties.Max_target_q
},
commandMutator: func(command *android.RuleBuilderCommand, path android.Path) {
command.FlagWithInput("--max-target-q ", path)
},
},
// See HiddenAPIFlagFileProperties.Max_target_p
{
PropertyName: "max_target_p",
propertyValueReader: func(properties *HiddenAPIFlagFileProperties) []string {
return properties.Max_target_p
},
commandMutator: func(command *android.RuleBuilderCommand, path android.Path) {
command.FlagWithInput("--max-target-p ", path)
},
},
// See HiddenAPIFlagFileProperties.Max_target_o_low_priority
{
PropertyName: "max_target_o_low_priority",
propertyValueReader: func(properties *HiddenAPIFlagFileProperties) []string {
return properties.Max_target_o_low_priority
},
commandMutator: func(command *android.RuleBuilderCommand, path android.Path) {
command.FlagWithInput("--max-target-o ", path).Flag("--ignore-conflicts ").FlagWithArg("--tag ", "lo-prio")
},
},
// See HiddenAPIFlagFileProperties.Blocked
{
PropertyName: "blocked",
propertyValueReader: func(properties *HiddenAPIFlagFileProperties) []string {
return properties.Blocked
},
commandMutator: func(command *android.RuleBuilderCommand, path android.Path) {
command.FlagWithInput("--blocked ", path)
},
},
// See HiddenAPIFlagFileProperties.Unsupported_packages
{
PropertyName: "unsupported_packages",
propertyValueReader: func(properties *HiddenAPIFlagFileProperties) []string {
return properties.Unsupported_packages
},
commandMutator: func(command *android.RuleBuilderCommand, path android.Path) {
command.FlagWithInput("--unsupported ", path).Flag("--packages ")
},
},
}
// FlagFilesByCategory maps a hiddenAPIFlagFileCategory to the paths to the files in that category.
type FlagFilesByCategory map[*hiddenAPIFlagFileCategory]android.Paths
// append appends the supplied flags files to the corresponding category in this map.
func (s FlagFilesByCategory) append(other FlagFilesByCategory) {
for _, category := range HiddenAPIFlagFileCategories {
s[category] = append(s[category], other[category]...)
}
}
// dedup removes duplicates in the flag files, while maintaining the order in which they were
// appended.
func (s FlagFilesByCategory) dedup() {
for category, paths := range s {
s[category] = android.FirstUniquePaths(paths)
}
}
// HiddenAPIInfo contains information provided by the hidden API processing.
//
// That includes paths resolved from HiddenAPIFlagFileProperties and also generated by hidden API
// processing.
type HiddenAPIInfo struct {
// FlagFilesByCategory maps from the flag file category to the paths containing information for
// that category.
FlagFilesByCategory FlagFilesByCategory
// The paths to the stub dex jars for each of the *HiddenAPIScope in hiddenAPIScopes.
TransitiveStubDexJarsByScope StubDexJarsByScope
// The output from the hidden API processing needs to be made available to other modules.
HiddenAPIFlagOutput
}
func newHiddenAPIInfo() *HiddenAPIInfo {
info := HiddenAPIInfo{
FlagFilesByCategory: FlagFilesByCategory{},
TransitiveStubDexJarsByScope: StubDexJarsByScope{},
}
return &info
}
func (i *HiddenAPIInfo) mergeFromFragmentDeps(ctx android.ModuleContext, fragments []android.Module) {
// Merge all the information from the fragments. The fragments form a DAG so it is possible that
// this will introduce duplicates so they will be resolved after processing all the fragments.
for _, fragment := range fragments {
if ctx.OtherModuleHasProvider(fragment, HiddenAPIInfoProvider) {
info := ctx.OtherModuleProvider(fragment, HiddenAPIInfoProvider).(HiddenAPIInfo)
i.TransitiveStubDexJarsByScope.append(info.TransitiveStubDexJarsByScope)
}
}
// Dedup and sort paths.
i.TransitiveStubDexJarsByScope.dedupAndSort()
}
var HiddenAPIInfoProvider = blueprint.NewProvider(HiddenAPIInfo{})
// StubDexJarsByScope maps a *HiddenAPIScope to the paths to stub dex jars appropriate for that
// scope. See hiddenAPIScopes for a list of the acceptable *HiddenAPIScope values.
type StubDexJarsByScope map[*HiddenAPIScope]android.Paths
// append appends the API scope specific stub dex jar args to the corresponding scope in this
// map.
func (s StubDexJarsByScope) append(other StubDexJarsByScope) {
for _, scope := range hiddenAPIScopes {
s[scope] = append(s[scope], other[scope]...)
}
}
// dedupAndSort removes duplicates in the stub dex jar paths and sorts them into a consistent and
// deterministic order.
func (s StubDexJarsByScope) dedupAndSort() {
for apiScope, paths := range s {
s[apiScope] = android.SortedUniquePaths(paths)
}
}
// StubDexJarsForWidestAPIScope returns the stub dex jars for the widest API scope provided by this
// map. The relative width of APIs is determined by their order in hiddenAPIScopes.
func (s StubDexJarsByScope) StubDexJarsForWidestAPIScope() android.Paths {
for i := len(hiddenAPIScopes) - 1; i >= 0; i-- {
apiScope := hiddenAPIScopes[i]
stubsForAPIScope := s[apiScope]
if len(stubsForAPIScope) != 0 {
return stubsForAPIScope
}
}
return nil
}
// HiddenAPIFlagInput encapsulates information obtained from a module and its dependencies that are
// needed for hidden API flag generation.
type HiddenAPIFlagInput struct {
// FlagFilesByCategory contains the flag files that override the initial flags that are derived
// from the stub dex files.
FlagFilesByCategory FlagFilesByCategory
// StubDexJarsByScope contains the stub dex jars for different *HiddenAPIScope and which determine
// the initial flags for each dex member.
StubDexJarsByScope StubDexJarsByScope
// DependencyStubDexJarsByScope contains the stub dex jars provided by the fragments on which this
// depends. It is the result of merging HiddenAPIInfo.TransitiveStubDexJarsByScope from each
// fragment on which this depends.
DependencyStubDexJarsByScope StubDexJarsByScope
// AdditionalStubDexJarsByScope contains stub dex jars provided by other modules in addition to
// the ones that are obtained from fragments on which this depends.
//
// These are kept separate from stub dex jars in HiddenAPIFlagInput.DependencyStubDexJarsByScope
// as there are not propagated transitively to other fragments that depend on this.
AdditionalStubDexJarsByScope StubDexJarsByScope
// RemovedTxtFiles is the list of removed.txt files provided by java_sdk_library modules that are
// specified in the bootclasspath_fragment's stub_libs and contents properties.
RemovedTxtFiles android.Paths
}
// newHiddenAPIFlagInput creates a new initialize HiddenAPIFlagInput struct.
func newHiddenAPIFlagInput() HiddenAPIFlagInput {
input := HiddenAPIFlagInput{
FlagFilesByCategory: FlagFilesByCategory{},
StubDexJarsByScope: StubDexJarsByScope{},
DependencyStubDexJarsByScope: StubDexJarsByScope{},
AdditionalStubDexJarsByScope: StubDexJarsByScope{},
}
return input
}
// canPerformHiddenAPIProcessing determines whether hidden API processing should be performed.
//
// A temporary workaround to avoid existing bootclasspath_fragments that do not provide the
// appropriate information needed for hidden API processing breaking the build.
// TODO(b/179354495): Remove this workaround.
func (i *HiddenAPIFlagInput) canPerformHiddenAPIProcessing(ctx android.ModuleContext, properties bootclasspathFragmentProperties) bool {
// Performing hidden API processing without stubs is not supported and it is unlikely to ever be
// required as the whole point of adding something to the bootclasspath fragment is to add it to
// the bootclasspath in order to be used by something else in the system. Without any stubs it
// cannot do that.
if len(i.StubDexJarsByScope) == 0 {
return false
}
// Hidden API processing is always enabled in tests.
if ctx.Config().TestProductVariables != nil {
return true
}
// A module that has fragments should have access to the information it needs in order to perform
// hidden API processing.
if len(properties.Fragments) != 0 {
return true
}
// The art bootclasspath fragment does not depend on any other fragments but already supports
// hidden API processing.
imageName := proptools.String(properties.Image_name)
if imageName == "art" {
return true
}
// Disable it for everything else.
return false
}
// gatherStubLibInfo gathers information from the stub libs needed by hidden API processing from the
// dependencies added in hiddenAPIAddStubLibDependencies.
//
// That includes paths to the stub dex jars as well as paths to the *removed.txt files.
func (i *HiddenAPIFlagInput) gatherStubLibInfo(ctx android.ModuleContext, contents []android.Module) {
addFromModule := func(ctx android.ModuleContext, module android.Module, apiScope *HiddenAPIScope) {
sdkKind := apiScope.sdkKind
dexJar := hiddenAPIRetrieveDexJarBuildPath(ctx, module, sdkKind)
if dexJar != nil {
i.StubDexJarsByScope[apiScope] = append(i.StubDexJarsByScope[apiScope], dexJar)
}
if sdkLibrary, ok := module.(SdkLibraryDependency); ok {
removedTxtFile := sdkLibrary.SdkRemovedTxtFile(ctx, sdkKind)
i.RemovedTxtFiles = append(i.RemovedTxtFiles, removedTxtFile.AsPaths()...)
}
}
// If the contents includes any java_sdk_library modules then add them to the stubs.
for _, module := range contents {
if _, ok := module.(SdkLibraryDependency); ok {
// Add information for every possible API scope needed by hidden API.
for _, apiScope := range hiddenAPISdkLibrarySupportedScopes {
addFromModule(ctx, module, apiScope)
}
}
}
ctx.VisitDirectDeps(func(module android.Module) {
tag := ctx.OtherModuleDependencyTag(module)
if hiddenAPIStubsTag, ok := tag.(hiddenAPIStubsDependencyTag); ok {
apiScope := hiddenAPIStubsTag.apiScope
if hiddenAPIStubsTag.fromAdditionalDependency {
dexJar := hiddenAPIRetrieveDexJarBuildPath(ctx, module, apiScope.sdkKind)
if dexJar != nil {
i.AdditionalStubDexJarsByScope[apiScope] = append(i.AdditionalStubDexJarsByScope[apiScope], dexJar)
}
} else {
addFromModule(ctx, module, apiScope)
}
}
})
// Normalize the paths, i.e. remove duplicates and sort.
i.StubDexJarsByScope.dedupAndSort()
i.DependencyStubDexJarsByScope.dedupAndSort()
i.RemovedTxtFiles = android.SortedUniquePaths(i.RemovedTxtFiles)
}
// extractFlagFilesFromProperties extracts the paths to flag files that are specified in the
// supplied properties and stores them in this struct.
func (i *HiddenAPIFlagInput) extractFlagFilesFromProperties(ctx android.ModuleContext, p *HiddenAPIFlagFileProperties) {
for _, category := range HiddenAPIFlagFileCategories {
paths := android.PathsForModuleSrc(ctx, category.propertyValueReader(p))
i.FlagFilesByCategory[category] = paths
}
}
func (i *HiddenAPIFlagInput) transitiveStubDexJarsByScope() StubDexJarsByScope {
transitive := i.DependencyStubDexJarsByScope
transitive.append(i.StubDexJarsByScope)
return transitive
}
// HiddenAPIFlagOutput contains paths to output files from the hidden API flag generation for a
// bootclasspath_fragment module.
type HiddenAPIFlagOutput struct {
// The path to the generated stub-flags.csv file.
StubFlagsPath android.Path
// The path to the generated annotation-flags.csv file.
AnnotationFlagsPath android.Path
// The path to the generated metadata.csv file.
MetadataPath android.Path
// The path to the generated index.csv file.
IndexPath android.Path
// The path to the generated all-flags.csv file.
AllFlagsPath android.Path
}
// bootDexJarByModule is a map from base module name (without prebuilt_ prefix) to the boot dex
// path.
type bootDexJarByModule map[string]android.Path
// addPath adds the path for a module to the map.
func (b bootDexJarByModule) addPath(module android.Module, path android.Path) {
b[android.RemoveOptionalPrebuiltPrefix(module.Name())] = path
}
// bootDexJars returns the boot dex jar paths sorted by their keys.
func (b bootDexJarByModule) bootDexJars() android.Paths {
paths := android.Paths{}
for _, k := range android.SortedStringKeys(b) {
paths = append(paths, b[k])
}
return paths
}
// bootDexJarsWithoutCoverage returns the boot dex jar paths sorted by their keys without coverage
// libraries if present.
func (b bootDexJarByModule) bootDexJarsWithoutCoverage() android.Paths {
paths := android.Paths{}
for _, k := range android.SortedStringKeys(b) {
if k == "jacocoagent" {
continue
}
paths = append(paths, b[k])
}
return paths
}
// HiddenAPIOutput encapsulates the output from the hidden API processing.
type HiddenAPIOutput struct {
HiddenAPIFlagOutput
// The map from base module name to the path to the encoded boot dex file.
EncodedBootDexFilesByModule bootDexJarByModule
}
// pathForValidation creates a path of the same type as the supplied type but with a name of
// <path>.valid.
//
// e.g. If path is an OutputPath for out/soong/hiddenapi/hiddenapi-flags.csv then this will return
// an OutputPath for out/soong/hiddenapi/hiddenapi-flags.csv.valid
func pathForValidation(ctx android.PathContext, path android.WritablePath) android.WritablePath {
extWithoutLeadingDot := strings.TrimPrefix(path.Ext(), ".")
return path.ReplaceExtension(ctx, extWithoutLeadingDot+".valid")
}
// buildRuleToGenerateHiddenApiFlags creates a rule to create the monolithic hidden API flags from
// the flags from all the modules, the stub flags, augmented with some additional configuration
// files.
//
// baseFlagsPath is the path to the flags file containing all the information from the stubs plus
// an entry for every single member in the dex implementation jars of the individual modules. Every
// signature in any of the other files MUST be included in this file.
//
// annotationFlags is the path to the annotation flags file generated from annotation information
// in each module.
//
// hiddenAPIInfo is a struct containing paths to files that augment the information provided by
// the annotationFlags.
func buildRuleToGenerateHiddenApiFlags(ctx android.BuilderContext, name, desc string,
outputPath android.WritablePath, baseFlagsPath android.Path, annotationFlagPaths android.Paths,
flagFilesByCategory FlagFilesByCategory, allFlagsPaths android.Paths, generatedRemovedDexSignatures android.OptionalPath) {
// Create the rule that will generate the flag files.
tempPath := tempPathForRestat(ctx, outputPath)
rule := android.NewRuleBuilder(pctx, ctx)
command := rule.Command().
BuiltTool("generate_hiddenapi_lists").
FlagWithInput("--csv ", baseFlagsPath).
Inputs(annotationFlagPaths).
FlagWithOutput("--output ", tempPath)
// Add the options for the different categories of flag files.
for _, category := range HiddenAPIFlagFileCategories {
paths := flagFilesByCategory[category]
for _, path := range paths {
category.commandMutator(command, path)
}
}
// If available then pass the automatically generated file containing dex signatures of removed
// API members to the rule so they can be marked as removed.
if generatedRemovedDexSignatures.Valid() {
hiddenAPIRemovedFlagFileCategory.commandMutator(command, generatedRemovedDexSignatures.Path())
}
commitChangeForRestat(rule, tempPath, outputPath)
// If there are flag files that have been generated by fragments on which this depends then use
// them to validate the flag file generated by the rules created by this method.
if len(allFlagsPaths) > 0 {
validFile := buildRuleValidateOverlappingCsvFiles(ctx, name, desc, outputPath, allFlagsPaths)
// Add the file that indicates that the file generated by this is valid.
//
// This will cause the validation rule above to be run any time that the output of this rule
// changes but the validation will run in parallel with other rules that depend on this file.
command.Validation(validFile)
}
rule.Build(name, desc)
}
// buildRuleValidateOverlappingCsvFiles checks that the modular CSV files, i.e. the files generated
// by the individual bootclasspath_fragment modules are subsets of the monolithic CSV file.
func buildRuleValidateOverlappingCsvFiles(ctx android.BuilderContext, name string, desc string, monolithicFilePath android.WritablePath, modularFilePaths android.Paths) android.WritablePath {
// The file which is used to record that the flags file is valid.
validFile := pathForValidation(ctx, monolithicFilePath)
// Create a rule to validate the output from the following rule.
rule := android.NewRuleBuilder(pctx, ctx)
rule.Command().
BuiltTool("verify_overlaps").
Input(monolithicFilePath).
Inputs(modularFilePaths).
// If validation passes then update the file that records that.
Text("&& touch").Output(validFile)
rule.Build(name+"Validation", desc+" validation")
return validFile
}
// hiddenAPIRulesForBootclasspathFragment will generate all the flags for a fragment of the
// bootclasspath and then encode the flags into the boot dex files.
//
// It takes:
// * Map from android.SdkKind to stub dex jar paths defining the API for that sdk kind.
// * The list of modules that are the contents of the fragment.
// * The additional manually curated flag files to use.
//
// It generates:
// * stub-flags.csv
// * annotation-flags.csv
// * metadata.csv
// * index.csv
// * all-flags.csv
// * encoded boot dex files
func hiddenAPIRulesForBootclasspathFragment(ctx android.ModuleContext, contents []android.Module, input HiddenAPIFlagInput) *HiddenAPIOutput {
hiddenApiSubDir := "modular-hiddenapi"
// Gather information about the boot dex files for the boot libraries provided by this fragment.
bootDexInfoByModule := extractBootDexInfoFromModules(ctx, contents)
// Generate the stub-flags.csv.
stubFlagsCSV := android.PathForModuleOut(ctx, hiddenApiSubDir, "stub-flags.csv")
buildRuleToGenerateHiddenAPIStubFlagsFile(ctx, "modularHiddenAPIStubFlagsFile", "modular hiddenapi stub flags", stubFlagsCSV, bootDexInfoByModule.bootDexJars(), input, nil)
// Extract the classes jars from the contents.
classesJars := extractClassesJarsFromModules(contents)
// Generate the set of flags from the annotations in the source code.
annotationFlagsCSV := android.PathForModuleOut(ctx, hiddenApiSubDir, "annotation-flags.csv")
buildRuleToGenerateAnnotationFlags(ctx, "modular hiddenapi annotation flags", classesJars, stubFlagsCSV, annotationFlagsCSV)
// Generate the metadata from the annotations in the source code.
metadataCSV := android.PathForModuleOut(ctx, hiddenApiSubDir, "metadata.csv")
buildRuleToGenerateMetadata(ctx, "modular hiddenapi metadata", classesJars, stubFlagsCSV, metadataCSV)
// Generate the index file from the CSV files in the classes jars.
indexCSV := android.PathForModuleOut(ctx, hiddenApiSubDir, "index.csv")
buildRuleToGenerateIndex(ctx, "modular hiddenapi index", classesJars, indexCSV)
// Removed APIs need to be marked and in order to do that the hiddenAPIInfo needs to specify files
// containing dex signatures of all the removed APIs. In the monolithic files that is done by
// manually combining all the removed.txt files for each API and then converting them to dex
// signatures, see the combined-removed-dex module. This does that automatically by using the
// *removed.txt files retrieved from the java_sdk_library modules that are specified in the
// stub_libs and contents properties of a bootclasspath_fragment.
removedDexSignatures := buildRuleToGenerateRemovedDexSignatures(ctx, input.RemovedTxtFiles)
// Generate the all-flags.csv which are the flags that will, in future, be encoded into the dex
// files.
allFlagsCSV := android.PathForModuleOut(ctx, hiddenApiSubDir, "all-flags.csv")
buildRuleToGenerateHiddenApiFlags(ctx, "modularHiddenApiAllFlags", "modular hiddenapi all flags", allFlagsCSV, stubFlagsCSV, android.Paths{annotationFlagsCSV}, input.FlagFilesByCategory, nil, removedDexSignatures)
// Encode the flags into the boot dex files.
encodedBootDexJarsByModule := map[string]android.Path{}
outputDir := android.PathForModuleOut(ctx, "hiddenapi-modular/encoded").OutputPath
for _, name := range android.SortedStringKeys(bootDexInfoByModule) {
bootDexInfo := bootDexInfoByModule[name]
unencodedDex := bootDexInfo.path
encodedDex := hiddenAPIEncodeDex(ctx, unencodedDex, allFlagsCSV, bootDexInfo.uncompressDex, outputDir)
encodedBootDexJarsByModule[name] = encodedDex
}
// Store the paths in the info for use by other modules and sdk snapshot generation.
output := HiddenAPIOutput{
HiddenAPIFlagOutput: HiddenAPIFlagOutput{
StubFlagsPath: stubFlagsCSV,
AnnotationFlagsPath: annotationFlagsCSV,
MetadataPath: metadataCSV,
IndexPath: indexCSV,
AllFlagsPath: allFlagsCSV,
},
EncodedBootDexFilesByModule: encodedBootDexJarsByModule,
}
return &output
}
func buildRuleToGenerateRemovedDexSignatures(ctx android.ModuleContext, removedTxtFiles android.Paths) android.OptionalPath {
if len(removedTxtFiles) == 0 {
return android.OptionalPath{}
}
output := android.PathForModuleOut(ctx, "modular-hiddenapi/removed-dex-signatures.txt")
rule := android.NewRuleBuilder(pctx, ctx)
rule.Command().
BuiltTool("metalava").
Flag("--no-banner").
Inputs(removedTxtFiles).
FlagWithOutput("--dex-api ", output)
rule.Build("modular-hiddenapi-removed-dex-signatures", "modular hiddenapi removed dex signatures")
return android.OptionalPathForPath(output)
}
// extractBootDexJarsFromModules extracts the boot dex jars from the supplied modules.
func extractBootDexJarsFromModules(ctx android.ModuleContext, contents []android.Module) bootDexJarByModule {
bootDexJars := bootDexJarByModule{}
for _, module := range contents {
hiddenAPIModule := hiddenAPIModuleFromModule(ctx, module)
if hiddenAPIModule == nil {
continue
}
bootDexJar := retrieveBootDexJarFromHiddenAPIModule(ctx, hiddenAPIModule)
bootDexJars.addPath(module, bootDexJar)
}
return bootDexJars
}
func hiddenAPIModuleFromModule(ctx android.BaseModuleContext, module android.Module) hiddenAPIModule {
if hiddenAPIModule, ok := module.(hiddenAPIModule); ok {
return hiddenAPIModule
} else if _, ok := module.(*DexImport); ok {
// Ignore this for the purposes of hidden API processing
} else {
ctx.ModuleErrorf("module %s does not implement hiddenAPIModule", module)
}
return nil
}
// bootDexInfo encapsulates both the path and uncompressDex status retrieved from a hiddenAPIModule.
type bootDexInfo struct {
// The path to the dex jar that has not had hidden API flags encoded into it.
path android.Path
// Indicates whether the dex jar needs uncompressing before encoding.
uncompressDex bool
}
// bootDexInfoByModule is a map from module name (as returned by module.Name()) to the boot dex
// path (as returned by hiddenAPIModule.bootDexJar()) and the uncompressDex flag.
type bootDexInfoByModule map[string]bootDexInfo
// bootDexJars returns the boot dex jar paths sorted by their keys.
func (b bootDexInfoByModule) bootDexJars() android.Paths {
paths := android.Paths{}
for _, m := range android.SortedStringKeys(b) {
paths = append(paths, b[m].path)
}
return paths
}
// extractBootDexInfoFromModules extracts the boot dex jar and uncompress dex state from
// each of the supplied modules which must implement hiddenAPIModule.
func extractBootDexInfoFromModules(ctx android.ModuleContext, contents []android.Module) bootDexInfoByModule {
bootDexJarsByModule := bootDexInfoByModule{}
for _, module := range contents {
hiddenAPIModule := module.(hiddenAPIModule)
bootDexJar := retrieveBootDexJarFromHiddenAPIModule(ctx, hiddenAPIModule)
bootDexJarsByModule[module.Name()] = bootDexInfo{
path: bootDexJar,
uncompressDex: *hiddenAPIModule.uncompressDex(),
}
}
return bootDexJarsByModule
}
// retrieveBootDexJarFromHiddenAPIModule retrieves the boot dex jar from the hiddenAPIModule.
//
// If the module does not provide a boot dex jar, i.e. the returned boot dex jar is nil, then that
// create a fake path and either report an error immediately or defer reporting of the error until
// the path is actually used.
func retrieveBootDexJarFromHiddenAPIModule(ctx android.ModuleContext, module hiddenAPIModule) android.Path {
bootDexJar := module.bootDexJar()
if bootDexJar == nil {
fake := android.PathForModuleOut(ctx, fmt.Sprintf("fake/boot-dex/%s.jar", module.Name()))
bootDexJar = fake
handleMissingDexBootFile(ctx, module, fake)
}
return bootDexJar
}
// extractClassesJarsFromModules extracts the class jars from the supplied modules.
func extractClassesJarsFromModules(contents []android.Module) android.Paths {
classesJars := android.Paths{}
for _, module := range contents {
classesJars = append(classesJars, retrieveClassesJarsFromModule(module)...)
}
return classesJars
}
// retrieveClassesJarsFromModule retrieves the classes jars from the supplied module.
func retrieveClassesJarsFromModule(module android.Module) android.Paths {
if hiddenAPIModule, ok := module.(hiddenAPIModule); ok {
return hiddenAPIModule.classesJars()
}
return nil
}
// deferReportingMissingBootDexJar returns true if a missing boot dex jar should not be reported by
// Soong but should instead only be reported in ninja if the file is actually built.
func deferReportingMissingBootDexJar(ctx android.ModuleContext, module android.Module) bool {
// TODO(b/179354495): Remove this workaround when it is unnecessary.
// Prebuilt modules like framework-wifi do not yet provide dex implementation jars. So,
// create a fake one that will cause a build error only if it is used.
if ctx.Config().AlwaysUsePrebuiltSdks() {
return true
}
// Any missing dependency should be allowed.
if ctx.Config().AllowMissingDependencies() {
return true
}
// This is called for both platform_bootclasspath and bootclasspath_fragment modules.
//
// A bootclasspath_fragment module should only use the APEX variant of source or prebuilt modules.
// Ideally, a bootclasspath_fragment module should never have a platform variant created for it
// but unfortunately, due to b/187910671 it does.
//
// That causes issues when obtaining a boot dex jar for a prebuilt module as a prebuilt module
// used by a bootclasspath_fragment can only provide a boot dex jar when it is part of APEX, i.e.
// has an APEX variant not a platform variant.
//
// There are some other situations when a prebuilt module used by a bootclasspath_fragment cannot
// provide a boot dex jar:
// 1. If the bootclasspath_fragment is not exported by the prebuilt_apex/apex_set module then it
// does not have an APEX variant and only has a platform variant and neither do its content
// modules.
// 2. Some build configurations, e.g. setting TARGET_BUILD_USE_PREBUILT_SDKS causes all
// java_sdk_library_import modules to be treated as preferred and as many of them are not part
// of an apex they cannot provide a boot dex jar.
//
// The first case causes problems when the affected prebuilt modules are preferred but that is an
// invalid configuration and it is ok for it to fail as the work to enable that is not yet
// complete. The second case is used for building targets that do not use boot dex jars and so
// deferring error reporting to ninja is fine as the affected ninja targets should never be built.
// That is handled above.
//
// A platform_bootclasspath module can use libraries from both platform and APEX variants. Unlike
// the bootclasspath_fragment it supports dex_import modules which provides the dex file. So, it
// can obtain a boot dex jar from a prebuilt that is not part of an APEX. However, it is assumed
// that if the library can be part of an APEX then it is the APEX variant that is used.
//
// This check handles the slightly different requirements of the bootclasspath_fragment and
// platform_bootclasspath modules by only deferring error reporting for the platform variant of
// a prebuilt modules that has other variants which are part of an APEX.
//
// TODO(b/187910671): Remove this once platform variants are no longer created unnecessarily.
if android.IsModulePrebuilt(module) {
if am, ok := module.(android.ApexModule); ok && am.InAnyApex() {
apexInfo := ctx.OtherModuleProvider(module, android.ApexInfoProvider).(android.ApexInfo)
if apexInfo.IsForPlatform() {
return true
}
}
}
// A bootclasspath module that is part of a versioned sdk never provides a boot dex jar as there
// is no equivalently versioned prebuilt APEX file from which it can be obtained. However,
// versioned bootclasspath modules are processed by Soong so in order to avoid them causing build
// failures missing boot dex jars need to be deferred.
if android.IsModuleInVersionedSdk(ctx.Module()) {
return true
}
return false
}
// handleMissingDexBootFile will either log a warning or create an error rule to create the fake
// file depending on the value returned from deferReportingMissingBootDexJar.
func handleMissingDexBootFile(ctx android.ModuleContext, module android.Module, fake android.WritablePath) {
if deferReportingMissingBootDexJar(ctx, module) {
// Create an error rule that pretends to create the output file but will actually fail if it
// is run.
ctx.Build(pctx, android.BuildParams{
Rule: android.ErrorRule,
Output: fake,
Args: map[string]string{
"error": fmt.Sprintf("missing dependencies: boot dex jar for %s", module),
},
})
} else {
ctx.ModuleErrorf("module %s does not provide a dex jar", module)
}
}
// retrieveEncodedBootDexJarFromModule returns a path to the boot dex jar from the supplied module's
// DexJarBuildPath() method.
//
// The returned path will usually be to a dex jar file that has been encoded with hidden API flags.
// However, under certain conditions, e.g. errors, or special build configurations it will return
// a path to a fake file.
func retrieveEncodedBootDexJarFromModule(ctx android.ModuleContext, module android.Module) android.Path {
bootDexJar := module.(interface{ DexJarBuildPath() android.Path }).DexJarBuildPath()
if bootDexJar == nil {
fake := android.PathForModuleOut(ctx, fmt.Sprintf("fake/encoded-dex/%s.jar", module.Name()))
bootDexJar = fake
handleMissingDexBootFile(ctx, module, fake)
}
return bootDexJar
}
// extractEncodedDexJarsFromModules extracts the encoded dex jars from the supplied modules.
func extractEncodedDexJarsFromModules(ctx android.ModuleContext, contents []android.Module) bootDexJarByModule {
encodedDexJarsByModuleName := bootDexJarByModule{}
for _, module := range contents {
path := retrieveEncodedBootDexJarFromModule(ctx, module)
encodedDexJarsByModuleName.addPath(module, path)
}
return encodedDexJarsByModuleName
}