tequilaOS/platform_build_soong
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use module type, size of srcs or deps to prioritize module

Browse source 
Instead of listing up module names, use another information from soong
such as module type, or size of inputs.

I focused on recall, not precision or accuracy, because false negative
module can slow down whole build

Bug: 273282046
Test: m --ninja_weight_source=ninja_log
Change-Id: I8fcc26f1312fda36197c787d240bc3d7b5346481
This commit is contained in:
Jeongik Cha 2023-04-11 14:28:43 +09:00
parent aea34a38ca
commit b745e2e205
2 changed files with 53 additions and 274 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

277
android/allowlists/allowlists.go
View file

@ -38,7 +38,13 @@ const (
// allows modules to opt-in.
Bp2BuildDefaultFalseRecursively
DEFAULT_NINJA_WEIGHT = 1000
// Modules with build time of more than half a minute should have high priority.
DEFAULT_PRIORITIZED_WEIGHT = 1000
// Modules with build time of more than a few minute should have higher priority.
HIGH_PRIORITIZED_WEIGHT = 10 * DEFAULT_PRIORITIZED_WEIGHT
// Modules with inputs greater than the threshold should have high priority.
// Adjust this threshold if there are lots of wrong predictions.
INPUT_SIZE_THRESHOLD = 50
)
var (
@ -1511,267 +1517,12 @@ var (
// "libstagefright_flacdec",
// "libutils",
// TODO(b/273282046): Make this list customizable to support various targets.
// The list of modules which are expected to spend lots of build time.
// With `--ninja_weight_source=soong`, ninja builds these modules and deps first.
HugeModulesMap = map[string]int{
"AccountManagementApp": DEFAULT_NINJA_WEIGHT,
"ActivityManagerPerfTestsStubApp1": DEFAULT_NINJA_WEIGHT,
"ActivityManagerPerfTestsStubApp2": DEFAULT_NINJA_WEIGHT,
"ActivityManagerPerfTestsStubApp3": DEFAULT_NINJA_WEIGHT,
"api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
"AppCompatibilityTest": DEFAULT_NINJA_WEIGHT,
"AppTransitionTests": DEFAULT_NINJA_WEIGHT,
"art_compiler_tests": DEFAULT_NINJA_WEIGHT,
"art.module.intra.core.api.stubs.source": DEFAULT_NINJA_WEIGHT,
"art.module.public.api.stubs.source": DEFAULT_NINJA_WEIGHT,
"AttestationVerificationTest": DEFAULT_NINJA_WEIGHT,
"BatteryUsageStatsProtoTests": DEFAULT_NINJA_WEIGHT,
"bluetooth_test_gd_unit": DEFAULT_NINJA_WEIGHT,
"Bluetooth": DEFAULT_NINJA_WEIGHT,
"BluetoothInstrumentationTests": DEFAULT_NINJA_WEIGHT,
"Calendar": DEFAULT_NINJA_WEIGHT,
"CalendarProvider": DEFAULT_NINJA_WEIGHT,
"Camera2": DEFAULT_NINJA_WEIGHT,
"CarRotaryControllerUnitTests": DEFAULT_NINJA_WEIGHT,
"CarSettingsForUnitTesting": DEFAULT_NINJA_WEIGHT,
"CarSettingsUnitTests": DEFAULT_NINJA_WEIGHT,
"CarSystemUI-tests": DEFAULT_NINJA_WEIGHT,
"CellBroadcastApp": DEFAULT_NINJA_WEIGHT,
"CellBroadcastLegacyApp": DEFAULT_NINJA_WEIGHT,
"CellBroadcastReceiverOemUnitTests": DEFAULT_NINJA_WEIGHT,
"CellBroadcastServiceModule": DEFAULT_NINJA_WEIGHT,
"CompanionDeviceManager": DEFAULT_NINJA_WEIGHT,
"ConnectivityChecker": DEFAULT_NINJA_WEIGHT,
"Contacts": DEFAULT_NINJA_WEIGHT,
"ContactsProvider": DEFAULT_NINJA_WEIGHT,
"ContentCapturePerfTests": DEFAULT_NINJA_WEIGHT,
"CorePerfTests": DEFAULT_NINJA_WEIGHT,
"crosvm": DEFAULT_NINJA_WEIGHT,
"CtsDomainVerificationDeviceMultiUserTestCases": DEFAULT_NINJA_WEIGHT,
"CtsLogdTestCases": DEFAULT_NINJA_WEIGHT,
"CtsMediaProviderTranscodeTests": DEFAULT_NINJA_WEIGHT,
"CtsRollbackManagerHostTestHelperApp": DEFAULT_NINJA_WEIGHT,
"CtsRollbackManagerHostTestHelperApp2": DEFAULT_NINJA_WEIGHT,
"CtsRootPackageInstallerTestCases": DEFAULT_NINJA_WEIGHT,
"CtsRootRollbackManagerHostTestHelperApp": DEFAULT_NINJA_WEIGHT,
"CtsTranscodeTestAppSupportsHevc": DEFAULT_NINJA_WEIGHT,
"CtsTranscodeTestAppSupportsSlowMotion": DEFAULT_NINJA_WEIGHT,
"CuttlefishDisplayHotplugHelperApp": DEFAULT_NINJA_WEIGHT,
"cvd-host_package": DEFAULT_NINJA_WEIGHT,
"DelegateTestApp": DEFAULT_NINJA_WEIGHT,
"DeskClock": DEFAULT_NINJA_WEIGHT,
"Development": DEFAULT_NINJA_WEIGHT,
"DeviceAdminTestApp": DEFAULT_NINJA_WEIGHT,
"DevicePolicyManagementRoleHolderTestApp": DEFAULT_NINJA_WEIGHT,
"dex2oatd": DEFAULT_NINJA_WEIGHT,
"DocumentsUI": DEFAULT_NINJA_WEIGHT,
"EasterEgg": DEFAULT_NINJA_WEIGHT,
"EffectProxyTest": DEFAULT_NINJA_WEIGHT,
"EmergencyInfo": DEFAULT_NINJA_WEIGHT,
"EmptyTestApp": DEFAULT_NINJA_WEIGHT,
"ExtServices": DEFAULT_NINJA_WEIGHT,
"FacebookAppsScenarioTests": DEFAULT_NINJA_WEIGHT,
"flickerlib-core": DEFAULT_NINJA_WEIGHT,
"flickerlib": DEFAULT_NINJA_WEIGHT,
"FlickerLibTest": DEFAULT_NINJA_WEIGHT,
"FlickerTests": DEFAULT_NINJA_WEIGHT,
"framework-minus-apex": DEFAULT_NINJA_WEIGHT,
"framework-res": DEFAULT_NINJA_WEIGHT,
"FrameworksCoreTests": DEFAULT_NINJA_WEIGHT,
"FrameworksMockingCoreTests": DEFAULT_NINJA_WEIGHT,
"FrameworksMockingServicesTests": DEFAULT_NINJA_WEIGHT,
"FrameworksNetSmokeTests": DEFAULT_NINJA_WEIGHT,
"FrameworksNetTests": DEFAULT_NINJA_WEIGHT,
"FrameworksServicesTests": DEFAULT_NINJA_WEIGHT,
"FrameworksTelephonyTests": DEFAULT_NINJA_WEIGHT,
"FrameworksUiServicesTests": DEFAULT_NINJA_WEIGHT,
"FrameworksVcnTests": DEFAULT_NINJA_WEIGHT,
"Gallery2": DEFAULT_NINJA_WEIGHT,
"GameCoreDevice": DEFAULT_NINJA_WEIGHT,
"GoogleBluetoothInstrumentationTests": DEFAULT_NINJA_WEIGHT,
"guice_munged_srcs": DEFAULT_NINJA_WEIGHT,
"HalfSheetUX": DEFAULT_NINJA_WEIGHT,
"ImePerfTests": DEFAULT_NINJA_WEIGHT,
"imgdiag": DEFAULT_NINJA_WEIGHT,
"ImsServiceEntitlement": DEFAULT_NINJA_WEIGHT,
"ImsServiceEntitlementUnitTests": DEFAULT_NINJA_WEIGHT,
"InputTests": DEFAULT_NINJA_WEIGHT,
"InstallTest": DEFAULT_NINJA_WEIGHT,
"IntentResolver": DEFAULT_NINJA_WEIGHT,
"JankBench": DEFAULT_NINJA_WEIGHT,
"jsilver": DEFAULT_NINJA_WEIGHT,
"KeyChain": DEFAULT_NINJA_WEIGHT,
"KeyChainTests": DEFAULT_NINJA_WEIGHT,
"keystore2": DEFAULT_NINJA_WEIGHT,
"LargeResourcesCompressed": DEFAULT_NINJA_WEIGHT,
"LatinIME": DEFAULT_NINJA_WEIGHT,
"Launcher3QuickStepLib": DEFAULT_NINJA_WEIGHT,
"libaom": DEFAULT_NINJA_WEIGHT,
"libart-broken": DEFAULT_NINJA_WEIGHT,
"libart-compiler": DEFAULT_NINJA_WEIGHT,
"libart-disassembler": DEFAULT_NINJA_WEIGHT,
"libart": DEFAULT_NINJA_WEIGHT,
"libartd": DEFAULT_NINJA_WEIGHT,
"libaudiohal@7.1": DEFAULT_NINJA_WEIGHT,
"libbluetooth_core_rs": DEFAULT_NINJA_WEIGHT,
"libbluetooth_gd_unit_tests": DEFAULT_NINJA_WEIGHT,
"libbluetooth_gd": DEFAULT_NINJA_WEIGHT,
"libbluetooth_rs": DEFAULT_NINJA_WEIGHT,
"libbluetooth-for-tests": DEFAULT_NINJA_WEIGHT,
"libbt_common": DEFAULT_NINJA_WEIGHT,
"libbt_packets_nonapex": DEFAULT_NINJA_WEIGHT,
"libbt_packets": DEFAULT_NINJA_WEIGHT,
"libbt_shim_ffi": DEFAULT_NINJA_WEIGHT,
"libbt_shim": DEFAULT_NINJA_WEIGHT,
"libbt-audio-hal-interface": DEFAULT_NINJA_WEIGHT,
"libbt-bta-core": DEFAULT_NINJA_WEIGHT,
"libbt-bta": DEFAULT_NINJA_WEIGHT,
"libbt-common": DEFAULT_NINJA_WEIGHT,
"libbt-hci": DEFAULT_NINJA_WEIGHT,
"libbt-platform-protos-lite": DEFAULT_NINJA_WEIGHT,
"libbt-protos-lite": DEFAULT_NINJA_WEIGHT,
"libbt-sbc-decoder": DEFAULT_NINJA_WEIGHT,
"libc": DEFAULT_NINJA_WEIGHT,
"libclap": DEFAULT_NINJA_WEIGHT,
"libcodec2_soft_av1dec_gav1": DEFAULT_NINJA_WEIGHT,
"libcompositionengine_test": DEFAULT_NINJA_WEIGHT,
"libdevices": DEFAULT_NINJA_WEIGHT,
"libfrontend_proto": DEFAULT_NINJA_WEIGHT,
"libhwtrust": DEFAULT_NINJA_WEIGHT,
"libjni": DEFAULT_NINJA_WEIGHT,
"libkeystore2": DEFAULT_NINJA_WEIGHT,
"libkmr_ta": DEFAULT_NINJA_WEIGHT,
"liblmp": DEFAULT_NINJA_WEIGHT,
"libopenjdkjvmtid": DEFAULT_NINJA_WEIGHT,
"libprotobuf_deprecated": DEFAULT_NINJA_WEIGHT,
"libprotobuf": DEFAULT_NINJA_WEIGHT,
"libregex": DEFAULT_NINJA_WEIGHT,
"LibStatsPullTests": DEFAULT_NINJA_WEIGHT,
"libstd": DEFAULT_NINJA_WEIGHT,
"libsurfaceflinger_unittest": DEFAULT_NINJA_WEIGHT,
"libsyn": DEFAULT_NINJA_WEIGHT,
"libtokio": DEFAULT_NINJA_WEIGHT,
"libuwb_core": DEFAULT_NINJA_WEIGHT,
"libuwb_uci_jni_rust": DEFAULT_NINJA_WEIGHT,
"libuwb_uci_packets": DEFAULT_NINJA_WEIGHT,
"libvpx": DEFAULT_NINJA_WEIGHT,
"libvulkan_enc": DEFAULT_NINJA_WEIGHT,
"libwebrtc": DEFAULT_NINJA_WEIGHT,
"LiveWallpapersPicker": DEFAULT_NINJA_WEIGHT,
"LockTaskApp": DEFAULT_NINJA_WEIGHT,
"LongevityPlatformLibTests": DEFAULT_NINJA_WEIGHT,
"ManagedProvisioning": DEFAULT_NINJA_WEIGHT,
"ManagedProvisioningTests": DEFAULT_NINJA_WEIGHT,
"MediaProvider": DEFAULT_NINJA_WEIGHT,
"MediaProviderClientTests": DEFAULT_NINJA_WEIGHT,
"MediaProviderLegacy": DEFAULT_NINJA_WEIGHT,
"messaging": DEFAULT_NINJA_WEIGHT,
"metalava": DEFAULT_NINJA_WEIGHT,
"MicrobenchmarkRunnerTests": DEFAULT_NINJA_WEIGHT,
"microdroid_manager": DEFAULT_NINJA_WEIGHT,
"minikin_tests": DEFAULT_NINJA_WEIGHT,
"MLCTestApp": DEFAULT_NINJA_WEIGHT,
"MmsService": DEFAULT_NINJA_WEIGHT,
"MmsServiceTests": DEFAULT_NINJA_WEIGHT,
"module-lib-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
"motion_tool_lib_tests": DEFAULT_NINJA_WEIGHT,
"MtpService": DEFAULT_NINJA_WEIGHT,
"MultiUserTests": DEFAULT_NINJA_WEIGHT,
"NearbyIntegrationUiTests": DEFAULT_NINJA_WEIGHT,
"net_test_bluetooth": DEFAULT_NINJA_WEIGHT,
"net_test_btif": DEFAULT_NINJA_WEIGHT,
"net_test_main_shim": DEFAULT_NINJA_WEIGHT,
"net_test_stack": DEFAULT_NINJA_WEIGHT,
"net-tests-utils": DEFAULT_NINJA_WEIGHT,
"NetworkStackCoverageTests": DEFAULT_NINJA_WEIGHT,
"NetworkStackIntegrationTests": DEFAULT_NINJA_WEIGHT,
"NetworkStackNext": DEFAULT_NINJA_WEIGHT,
"NfcNci": DEFAULT_NINJA_WEIGHT,
"NfcNciUnitTests": DEFAULT_NINJA_WEIGHT,
"NotEmptyTestApp": DEFAULT_NINJA_WEIGHT,
"NotificationFunctionalTests": DEFAULT_NINJA_WEIGHT,
"oatdumpd": DEFAULT_NINJA_WEIGHT,
"OsuLogin": DEFAULT_NINJA_WEIGHT,
"PackageInstaller": DEFAULT_NINJA_WEIGHT,
"PackageManagerComponentOverrideTests": DEFAULT_NINJA_WEIGHT,
"PackageManagerPerfTests": DEFAULT_NINJA_WEIGHT,
"PackageManagerServiceServerTests": DEFAULT_NINJA_WEIGHT,
"PackageManagerServiceUnitTests": DEFAULT_NINJA_WEIGHT,
"PackageWatchdogTest": DEFAULT_NINJA_WEIGHT,
"PandoraServerLib": DEFAULT_NINJA_WEIGHT,
"pdl": DEFAULT_NINJA_WEIGHT,
"perfetto_trace_java_protos": DEFAULT_NINJA_WEIGHT,
"perfetto_trace-full": DEFAULT_NINJA_WEIGHT,
"PermissionController": DEFAULT_NINJA_WEIGHT,
"PermissionControllerMockingTests": DEFAULT_NINJA_WEIGHT,
"PixelAppCompTests": DEFAULT_NINJA_WEIGHT,
"platform-bootclasspath": DEFAULT_NINJA_WEIGHT,
"PlatformCommonScenarioTests": DEFAULT_NINJA_WEIGHT,
"PlatformComposeCoreTests": DEFAULT_NINJA_WEIGHT,
"platformprotoslite": DEFAULT_NINJA_WEIGHT,
"PlatformRuleTests": DEFAULT_NINJA_WEIGHT,
"precompiled_sepolicy-without_apex": DEFAULT_NINJA_WEIGHT,
"PresencePolling": DEFAULT_NINJA_WEIGHT,
"PrintSpooler": DEFAULT_NINJA_WEIGHT,
"QuickSearchBox": DEFAULT_NINJA_WEIGHT,
"RemoteDPCTestApp": DEFAULT_NINJA_WEIGHT,
"RemoteProvisioningServiceTests": DEFAULT_NINJA_WEIGHT,
"RkpdAppUnitTests": DEFAULT_NINJA_WEIGHT,
"Robolectric_shadows_framework": DEFAULT_NINJA_WEIGHT,
"RoleHolderApp": DEFAULT_NINJA_WEIGHT,
"SdkSandbox": DEFAULT_NINJA_WEIGHT,
"service-appsearch": DEFAULT_NINJA_WEIGHT,
"service-connectivity": DEFAULT_NINJA_WEIGHT,
"service-uwb": DEFAULT_NINJA_WEIGHT,
"service-wifi": DEFAULT_NINJA_WEIGHT,
"services-non-updatable-stubs": DEFAULT_NINJA_WEIGHT,
"services": DEFAULT_NINJA_WEIGHT,
"Settings-core": DEFAULT_NINJA_WEIGHT,
"Settings": DEFAULT_NINJA_WEIGHT,
"SettingsIntelligence": DEFAULT_NINJA_WEIGHT,
"SettingsLibTests": DEFAULT_NINJA_WEIGHT,
"SettingsProvider": DEFAULT_NINJA_WEIGHT,
"Shell": DEFAULT_NINJA_WEIGHT,
"SimAppDialog": DEFAULT_NINJA_WEIGHT,
"sl4a": DEFAULT_NINJA_WEIGHT,
"SmsApp": DEFAULT_NINJA_WEIGHT,
"SoundPicker": DEFAULT_NINJA_WEIGHT,
"StagedInstallTest": DEFAULT_NINJA_WEIGHT,
"StatementService": DEFAULT_NINJA_WEIGHT,
"StatsdFrameworkTestApp": DEFAULT_NINJA_WEIGHT,
"StatsdFrameworkTestAppNoPermission": DEFAULT_NINJA_WEIGHT,
"statsdprotolite": DEFAULT_NINJA_WEIGHT,
"Stk": DEFAULT_NINJA_WEIGHT,
"StorageManager": DEFAULT_NINJA_WEIGHT,
"system-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
"SystemUI-core": DEFAULT_NINJA_WEIGHT,
"SystemUI-tests-base": DEFAULT_NINJA_WEIGHT,
"SystemUI-tests": DEFAULT_NINJA_WEIGHT,
"SystemUI": DEFAULT_NINJA_WEIGHT,
"SystemUIComposeFeatures": DEFAULT_NINJA_WEIGHT,
"SystemUIComposeFeaturesTests": DEFAULT_NINJA_WEIGHT,
"SystemUITests": DEFAULT_NINJA_WEIGHT,
"Tag": DEFAULT_NINJA_WEIGHT,
"Telecom": DEFAULT_NINJA_WEIGHT,
"TelecomUnitTests": DEFAULT_NINJA_WEIGHT,
"telephony-common": DEFAULT_NINJA_WEIGHT,
"TelephonyProvider": DEFAULT_NINJA_WEIGHT,
"TeleService": DEFAULT_NINJA_WEIGHT,
"test-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
"TetheringIntegrationTests": DEFAULT_NINJA_WEIGHT,
"TetheringNext": DEFAULT_NINJA_WEIGHT,
"ThemePickerTests": DEFAULT_NINJA_WEIGHT,
"Traceur": DEFAULT_NINJA_WEIGHT,
"UsbManagerTests": DEFAULT_NINJA_WEIGHT,
"UsbTests": DEFAULT_NINJA_WEIGHT,
"virtmgr": DEFAULT_NINJA_WEIGHT,
"WallpaperPicker2TestLib": DEFAULT_NINJA_WEIGHT,
"WallpaperPicker2Tests": DEFAULT_NINJA_WEIGHT,
"WifiDialog": DEFAULT_NINJA_WEIGHT,
"wm-proto-parsers": DEFAULT_NINJA_WEIGHT,
"WMShellFlickerTests": DEFAULT_NINJA_WEIGHT,
"WmTests": DEFAULT_NINJA_WEIGHT,
"wpa_supplicant": DEFAULT_NINJA_WEIGHT,
// The list of module types which are expected to spend lots of build time.
// With `--ninja_weight_source=soong`, ninja builds these module types and deps first.
HugeModuleTypePrefixMap = map[string]int{
"rust_": HIGH_PRIORITIZED_WEIGHT,
"droidstubs": DEFAULT_PRIORITIZED_WEIGHT,
"art_": DEFAULT_PRIORITIZED_WEIGHT,
"ndk_library": DEFAULT_PRIORITIZED_WEIGHT,
}
)

50
cmd/soong_build/main.go
View file

@ -30,6 +30,7 @@ import (
"android/soong/shared"
"android/soong/ui/metrics/bp2build_metrics_proto"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/deptools"
"github.com/google/blueprint/metrics"
@ -256,18 +257,45 @@ func apiBuildFileExcludes(ctx *android.Context) []string {
}
func writeNinjaHint(ctx *android.Context) error {
wantModules := make([]string, len(allowlists.HugeModulesMap))
i := 0
for k := range allowlists.HugeModulesMap {
wantModules[i] = k
i += 1
}
outputsMap := ctx.Context.GetOutputsFromModuleNames(wantModules)
var outputBuilder strings.Builder
for k, v := range allowlists.HugeModulesMap {
for _, output := range outputsMap[k] {
outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, v))
// The current predictor focuses on reducing false negatives.
// If there are too many false positives (e.g., most modules are marked as positive),
// real long-running jobs cannot run early.
// Therefore, the model should be adjusted in this case.
// The model should also be adjusted if there are critical false negatives.
predicate := func(j *blueprint.JsonModule) (prioritized bool, weight int) {
prioritized = false
weight = 0
for prefix, w := range allowlists.HugeModuleTypePrefixMap {
if strings.HasPrefix(j.Type, prefix) {
prioritized = true
weight = w
return
}
}
dep_count := len(j.Deps)
src_count := 0
for _, a := range j.Module["Actions"].([]blueprint.JSONAction) {
src_count += len(a.Inputs)
}
input_size := dep_count + src_count
// Current threshold is an arbitrary value which only consider recall rather than accuracy.
if input_size > allowlists.INPUT_SIZE_THRESHOLD {
prioritized = true
weight += ((input_size) / allowlists.INPUT_SIZE_THRESHOLD) * allowlists.DEFAULT_PRIORITIZED_WEIGHT
// To prevent some modules from having too large a priority value.
if weight > allowlists.HIGH_PRIORITIZED_WEIGHT {
weight = allowlists.HIGH_PRIORITIZED_WEIGHT
}
}
return
}
outputsMap := ctx.Context.GetWeightedOutputsFromPredicate(predicate)
var outputBuilder strings.Builder
for output, weight := range outputsMap {
outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, weight))
}
weightListFile := filepath.Join(topDir, ctx.Config().OutDir(), ".ninja_weight_list")