StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Use module type, size of srcs or deps to prioritize module"

Browse Source
This commit is contained in:
Jeongik Cha
2023-05-04 06:10:32 +00:00
committed by Gerrit Code Review
parent 0428f71b30 b745e2e205
commit 291cc5f9d9
2 changed files with 53 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
android/allowlists/allowlists.go
View File

@@ -38,7 +38,13 @@ const (
// allows modules to opt-in. // allows modules to opt-in.
Bp2BuildDefaultFalseRecursively Bp2BuildDefaultFalseRecursively
DEFAULT_NINJA_WEIGHT = 1000 // Modules with build time of more than half a minute should have high priority.
DEFAULT_PRIORITIZED_WEIGHT = 1000
// Modules with build time of more than a few minute should have higher priority.
HIGH_PRIORITIZED_WEIGHT = 10 * DEFAULT_PRIORITIZED_WEIGHT
// Modules with inputs greater than the threshold should have high priority.
// Adjust this threshold if there are lots of wrong predictions.
INPUT_SIZE_THRESHOLD = 50
) )
var ( var (
@@ -1562,267 +1568,12 @@ var (
// "libstagefright_flacdec", // "libstagefright_flacdec",
// "libutils", // "libutils",
// TODO(b/273282046): Make this list customizable to support various targets. // The list of module types which are expected to spend lots of build time.
// The list of modules which are expected to spend lots of build time. // With `--ninja_weight_source=soong`, ninja builds these module types and deps first.
// With `--ninja_weight_source=soong`, ninja builds these modules and deps first. HugeModuleTypePrefixMap = map[string]int{
HugeModulesMap = map[string]int{ "rust_": HIGH_PRIORITIZED_WEIGHT,
"AccountManagementApp": DEFAULT_NINJA_WEIGHT, "droidstubs": DEFAULT_PRIORITIZED_WEIGHT,
"ActivityManagerPerfTestsStubApp1": DEFAULT_NINJA_WEIGHT, "art_": DEFAULT_PRIORITIZED_WEIGHT,
"ActivityManagerPerfTestsStubApp2": DEFAULT_NINJA_WEIGHT, "ndk_library": DEFAULT_PRIORITIZED_WEIGHT,
"ActivityManagerPerfTestsStubApp3": DEFAULT_NINJA_WEIGHT,
"api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
"AppCompatibilityTest": DEFAULT_NINJA_WEIGHT,
"AppTransitionTests": DEFAULT_NINJA_WEIGHT,
"art_compiler_tests": DEFAULT_NINJA_WEIGHT,
"art.module.intra.core.api.stubs.source": DEFAULT_NINJA_WEIGHT,
"art.module.public.api.stubs.source": DEFAULT_NINJA_WEIGHT,
"AttestationVerificationTest": DEFAULT_NINJA_WEIGHT,
"BatteryUsageStatsProtoTests": DEFAULT_NINJA_WEIGHT,
"bluetooth_test_gd_unit": DEFAULT_NINJA_WEIGHT,
"Bluetooth": DEFAULT_NINJA_WEIGHT,
"BluetoothInstrumentationTests": DEFAULT_NINJA_WEIGHT,
"Calendar": DEFAULT_NINJA_WEIGHT,
"CalendarProvider": DEFAULT_NINJA_WEIGHT,
"Camera2": DEFAULT_NINJA_WEIGHT,
"CarRotaryControllerUnitTests": DEFAULT_NINJA_WEIGHT,
"CarSettingsForUnitTesting": DEFAULT_NINJA_WEIGHT,
"CarSettingsUnitTests": DEFAULT_NINJA_WEIGHT,
"CarSystemUI-tests": DEFAULT_NINJA_WEIGHT,
"CellBroadcastApp": DEFAULT_NINJA_WEIGHT,
"CellBroadcastLegacyApp": DEFAULT_NINJA_WEIGHT,
"CellBroadcastReceiverOemUnitTests": DEFAULT_NINJA_WEIGHT,
"CellBroadcastServiceModule": DEFAULT_NINJA_WEIGHT,
"CompanionDeviceManager": DEFAULT_NINJA_WEIGHT,
"ConnectivityChecker": DEFAULT_NINJA_WEIGHT,
"Contacts": DEFAULT_NINJA_WEIGHT,
"ContactsProvider": DEFAULT_NINJA_WEIGHT,
"ContentCapturePerfTests": DEFAULT_NINJA_WEIGHT,
"CorePerfTests": DEFAULT_NINJA_WEIGHT,
"crosvm": DEFAULT_NINJA_WEIGHT,
"CtsDomainVerificationDeviceMultiUserTestCases": DEFAULT_NINJA_WEIGHT,
"CtsLogdTestCases": DEFAULT_NINJA_WEIGHT,
"CtsMediaProviderTranscodeTests": DEFAULT_NINJA_WEIGHT,
"CtsRollbackManagerHostTestHelperApp": DEFAULT_NINJA_WEIGHT,
"CtsRollbackManagerHostTestHelperApp2": DEFAULT_NINJA_WEIGHT,
"CtsRootPackageInstallerTestCases": DEFAULT_NINJA_WEIGHT,
"CtsRootRollbackManagerHostTestHelperApp": DEFAULT_NINJA_WEIGHT,
"CtsTranscodeTestAppSupportsHevc": DEFAULT_NINJA_WEIGHT,
"CtsTranscodeTestAppSupportsSlowMotion": DEFAULT_NINJA_WEIGHT,
"CuttlefishDisplayHotplugHelperApp": DEFAULT_NINJA_WEIGHT,
"cvd-host_package": DEFAULT_NINJA_WEIGHT,
"DelegateTestApp": DEFAULT_NINJA_WEIGHT,
"DeskClock": DEFAULT_NINJA_WEIGHT,
"Development": DEFAULT_NINJA_WEIGHT,
"DeviceAdminTestApp": DEFAULT_NINJA_WEIGHT,
"DevicePolicyManagementRoleHolderTestApp": DEFAULT_NINJA_WEIGHT,
"dex2oatd": DEFAULT_NINJA_WEIGHT,
"DocumentsUI": DEFAULT_NINJA_WEIGHT,
"EasterEgg": DEFAULT_NINJA_WEIGHT,
"EffectProxyTest": DEFAULT_NINJA_WEIGHT,
"EmergencyInfo": DEFAULT_NINJA_WEIGHT,
"EmptyTestApp": DEFAULT_NINJA_WEIGHT,
"ExtServices": DEFAULT_NINJA_WEIGHT,
"FacebookAppsScenarioTests": DEFAULT_NINJA_WEIGHT,
"flickerlib-core": DEFAULT_NINJA_WEIGHT,
"flickerlib": DEFAULT_NINJA_WEIGHT,
"FlickerLibTest": DEFAULT_NINJA_WEIGHT,
"FlickerTests": DEFAULT_NINJA_WEIGHT,
"framework-minus-apex": DEFAULT_NINJA_WEIGHT,
"framework-res": DEFAULT_NINJA_WEIGHT,
"FrameworksCoreTests": DEFAULT_NINJA_WEIGHT,
"FrameworksMockingCoreTests": DEFAULT_NINJA_WEIGHT,
"FrameworksMockingServicesTests": DEFAULT_NINJA_WEIGHT,
"FrameworksNetSmokeTests": DEFAULT_NINJA_WEIGHT,
"FrameworksNetTests": DEFAULT_NINJA_WEIGHT,
"FrameworksServicesTests": DEFAULT_NINJA_WEIGHT,
"FrameworksTelephonyTests": DEFAULT_NINJA_WEIGHT,
"FrameworksUiServicesTests": DEFAULT_NINJA_WEIGHT,
"FrameworksVcnTests": DEFAULT_NINJA_WEIGHT,
"Gallery2": DEFAULT_NINJA_WEIGHT,
"GameCoreDevice": DEFAULT_NINJA_WEIGHT,
"GoogleBluetoothInstrumentationTests": DEFAULT_NINJA_WEIGHT,
"guice_munged_srcs": DEFAULT_NINJA_WEIGHT,
"HalfSheetUX": DEFAULT_NINJA_WEIGHT,
"ImePerfTests": DEFAULT_NINJA_WEIGHT,
"imgdiag": DEFAULT_NINJA_WEIGHT,
"ImsServiceEntitlement": DEFAULT_NINJA_WEIGHT,
"ImsServiceEntitlementUnitTests": DEFAULT_NINJA_WEIGHT,
"InputTests": DEFAULT_NINJA_WEIGHT,
"InstallTest": DEFAULT_NINJA_WEIGHT,
"IntentResolver": DEFAULT_NINJA_WEIGHT,
"JankBench": DEFAULT_NINJA_WEIGHT,
"jsilver": DEFAULT_NINJA_WEIGHT,
"KeyChain": DEFAULT_NINJA_WEIGHT,
"KeyChainTests": DEFAULT_NINJA_WEIGHT,
"keystore2": DEFAULT_NINJA_WEIGHT,
"LargeResourcesCompressed": DEFAULT_NINJA_WEIGHT,
"LatinIME": DEFAULT_NINJA_WEIGHT,
"Launcher3QuickStepLib": DEFAULT_NINJA_WEIGHT,
"libaom": DEFAULT_NINJA_WEIGHT,
"libart-broken": DEFAULT_NINJA_WEIGHT,
"libart-compiler": DEFAULT_NINJA_WEIGHT,
"libart-disassembler": DEFAULT_NINJA_WEIGHT,
"libart": DEFAULT_NINJA_WEIGHT,
"libartd": DEFAULT_NINJA_WEIGHT,
"libaudiohal@7.1": DEFAULT_NINJA_WEIGHT,
"libbluetooth_core_rs": DEFAULT_NINJA_WEIGHT,
"libbluetooth_gd_unit_tests": DEFAULT_NINJA_WEIGHT,
"libbluetooth_gd": DEFAULT_NINJA_WEIGHT,
"libbluetooth_rs": DEFAULT_NINJA_WEIGHT,
"libbluetooth-for-tests": DEFAULT_NINJA_WEIGHT,
"libbt_common": DEFAULT_NINJA_WEIGHT,
"libbt_packets_nonapex": DEFAULT_NINJA_WEIGHT,
"libbt_packets": DEFAULT_NINJA_WEIGHT,
"libbt_shim_ffi": DEFAULT_NINJA_WEIGHT,
"libbt_shim": DEFAULT_NINJA_WEIGHT,
"libbt-audio-hal-interface": DEFAULT_NINJA_WEIGHT,
"libbt-bta-core": DEFAULT_NINJA_WEIGHT,
"libbt-bta": DEFAULT_NINJA_WEIGHT,
"libbt-common": DEFAULT_NINJA_WEIGHT,
"libbt-hci": DEFAULT_NINJA_WEIGHT,
"libbt-platform-protos-lite": DEFAULT_NINJA_WEIGHT,
"libbt-protos-lite": DEFAULT_NINJA_WEIGHT,
"libbt-sbc-decoder": DEFAULT_NINJA_WEIGHT,
"libc": DEFAULT_NINJA_WEIGHT,
"libclap": DEFAULT_NINJA_WEIGHT,
"libcodec2_soft_av1dec_gav1": DEFAULT_NINJA_WEIGHT,
"libcompositionengine_test": DEFAULT_NINJA_WEIGHT,
"libdevices": DEFAULT_NINJA_WEIGHT,
"libfrontend_proto": DEFAULT_NINJA_WEIGHT,
"libhwtrust": DEFAULT_NINJA_WEIGHT,
"libjni": DEFAULT_NINJA_WEIGHT,
"libkeystore2": DEFAULT_NINJA_WEIGHT,
"libkmr_ta": DEFAULT_NINJA_WEIGHT,
"liblmp": DEFAULT_NINJA_WEIGHT,
"libopenjdkjvmtid": DEFAULT_NINJA_WEIGHT,
"libprotobuf_deprecated": DEFAULT_NINJA_WEIGHT,
"libprotobuf": DEFAULT_NINJA_WEIGHT,
"libregex": DEFAULT_NINJA_WEIGHT,
"LibStatsPullTests": DEFAULT_NINJA_WEIGHT,
"libstd": DEFAULT_NINJA_WEIGHT,
"libsurfaceflinger_unittest": DEFAULT_NINJA_WEIGHT,
"libsyn": DEFAULT_NINJA_WEIGHT,
"libtokio": DEFAULT_NINJA_WEIGHT,
"libuwb_core": DEFAULT_NINJA_WEIGHT,
"libuwb_uci_jni_rust": DEFAULT_NINJA_WEIGHT,
"libuwb_uci_packets": DEFAULT_NINJA_WEIGHT,
"libvpx": DEFAULT_NINJA_WEIGHT,
"libvulkan_enc": DEFAULT_NINJA_WEIGHT,
"libwebrtc": DEFAULT_NINJA_WEIGHT,
"LiveWallpapersPicker": DEFAULT_NINJA_WEIGHT,
"LockTaskApp": DEFAULT_NINJA_WEIGHT,
"LongevityPlatformLibTests": DEFAULT_NINJA_WEIGHT,
"ManagedProvisioning": DEFAULT_NINJA_WEIGHT,
"ManagedProvisioningTests": DEFAULT_NINJA_WEIGHT,
"MediaProvider": DEFAULT_NINJA_WEIGHT,
"MediaProviderClientTests": DEFAULT_NINJA_WEIGHT,
"MediaProviderLegacy": DEFAULT_NINJA_WEIGHT,
"messaging": DEFAULT_NINJA_WEIGHT,
"metalava": DEFAULT_NINJA_WEIGHT,
"MicrobenchmarkRunnerTests": DEFAULT_NINJA_WEIGHT,
"microdroid_manager": DEFAULT_NINJA_WEIGHT,
"minikin_tests": DEFAULT_NINJA_WEIGHT,
"MLCTestApp": DEFAULT_NINJA_WEIGHT,
"MmsService": DEFAULT_NINJA_WEIGHT,
"MmsServiceTests": DEFAULT_NINJA_WEIGHT,
"module-lib-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
"motion_tool_lib_tests": DEFAULT_NINJA_WEIGHT,
"MtpService": DEFAULT_NINJA_WEIGHT,
"MultiUserTests": DEFAULT_NINJA_WEIGHT,
"NearbyIntegrationUiTests": DEFAULT_NINJA_WEIGHT,
"net_test_bluetooth": DEFAULT_NINJA_WEIGHT,
"net_test_btif": DEFAULT_NINJA_WEIGHT,
"net_test_main_shim": DEFAULT_NINJA_WEIGHT,
"net_test_stack": DEFAULT_NINJA_WEIGHT,
"net-tests-utils": DEFAULT_NINJA_WEIGHT,
"NetworkStackCoverageTests": DEFAULT_NINJA_WEIGHT,
"NetworkStackIntegrationTests": DEFAULT_NINJA_WEIGHT,
"NetworkStackNext": DEFAULT_NINJA_WEIGHT,
"NfcNci": DEFAULT_NINJA_WEIGHT,
"NfcNciUnitTests": DEFAULT_NINJA_WEIGHT,
"NotEmptyTestApp": DEFAULT_NINJA_WEIGHT,
"NotificationFunctionalTests": DEFAULT_NINJA_WEIGHT,
"oatdumpd": DEFAULT_NINJA_WEIGHT,
"OsuLogin": DEFAULT_NINJA_WEIGHT,
"PackageInstaller": DEFAULT_NINJA_WEIGHT,
"PackageManagerComponentOverrideTests": DEFAULT_NINJA_WEIGHT,
"PackageManagerPerfTests": DEFAULT_NINJA_WEIGHT,
"PackageManagerServiceServerTests": DEFAULT_NINJA_WEIGHT,
"PackageManagerServiceUnitTests": DEFAULT_NINJA_WEIGHT,
"PackageWatchdogTest": DEFAULT_NINJA_WEIGHT,
"PandoraServerLib": DEFAULT_NINJA_WEIGHT,
"pdl": DEFAULT_NINJA_WEIGHT,
"perfetto_trace_java_protos": DEFAULT_NINJA_WEIGHT,
"perfetto_trace-full": DEFAULT_NINJA_WEIGHT,
"PermissionController": DEFAULT_NINJA_WEIGHT,
"PermissionControllerMockingTests": DEFAULT_NINJA_WEIGHT,
"PixelAppCompTests": DEFAULT_NINJA_WEIGHT,
"platform-bootclasspath": DEFAULT_NINJA_WEIGHT,
"PlatformCommonScenarioTests": DEFAULT_NINJA_WEIGHT,
"PlatformComposeCoreTests": DEFAULT_NINJA_WEIGHT,
"platformprotoslite": DEFAULT_NINJA_WEIGHT,
"PlatformRuleTests": DEFAULT_NINJA_WEIGHT,
"precompiled_sepolicy-without_apex": DEFAULT_NINJA_WEIGHT,
"PresencePolling": DEFAULT_NINJA_WEIGHT,
"PrintSpooler": DEFAULT_NINJA_WEIGHT,
"QuickSearchBox": DEFAULT_NINJA_WEIGHT,
"RemoteDPCTestApp": DEFAULT_NINJA_WEIGHT,
"RemoteProvisioningServiceTests": DEFAULT_NINJA_WEIGHT,
"RkpdAppUnitTests": DEFAULT_NINJA_WEIGHT,
"Robolectric_shadows_framework": DEFAULT_NINJA_WEIGHT,
"RoleHolderApp": DEFAULT_NINJA_WEIGHT,
"SdkSandbox": DEFAULT_NINJA_WEIGHT,
"service-appsearch": DEFAULT_NINJA_WEIGHT,
"service-connectivity": DEFAULT_NINJA_WEIGHT,
"service-uwb": DEFAULT_NINJA_WEIGHT,
"service-wifi": DEFAULT_NINJA_WEIGHT,
"services-non-updatable-stubs": DEFAULT_NINJA_WEIGHT,
"services": DEFAULT_NINJA_WEIGHT,
"Settings-core": DEFAULT_NINJA_WEIGHT,
"Settings": DEFAULT_NINJA_WEIGHT,
"SettingsIntelligence": DEFAULT_NINJA_WEIGHT,
"SettingsLibTests": DEFAULT_NINJA_WEIGHT,
"SettingsProvider": DEFAULT_NINJA_WEIGHT,
"Shell": DEFAULT_NINJA_WEIGHT,
"SimAppDialog": DEFAULT_NINJA_WEIGHT,
"sl4a": DEFAULT_NINJA_WEIGHT,
"SmsApp": DEFAULT_NINJA_WEIGHT,
"SoundPicker": DEFAULT_NINJA_WEIGHT,
"StagedInstallTest": DEFAULT_NINJA_WEIGHT,
"StatementService": DEFAULT_NINJA_WEIGHT,
"StatsdFrameworkTestApp": DEFAULT_NINJA_WEIGHT,
"StatsdFrameworkTestAppNoPermission": DEFAULT_NINJA_WEIGHT,
"statsdprotolite": DEFAULT_NINJA_WEIGHT,
"Stk": DEFAULT_NINJA_WEIGHT,
"StorageManager": DEFAULT_NINJA_WEIGHT,
"system-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
"SystemUI-core": DEFAULT_NINJA_WEIGHT,
"SystemUI-tests-base": DEFAULT_NINJA_WEIGHT,
"SystemUI-tests": DEFAULT_NINJA_WEIGHT,
"SystemUI": DEFAULT_NINJA_WEIGHT,
"SystemUIComposeFeatures": DEFAULT_NINJA_WEIGHT,
"SystemUIComposeFeaturesTests": DEFAULT_NINJA_WEIGHT,
"SystemUITests": DEFAULT_NINJA_WEIGHT,
"Tag": DEFAULT_NINJA_WEIGHT,
"Telecom": DEFAULT_NINJA_WEIGHT,
"TelecomUnitTests": DEFAULT_NINJA_WEIGHT,
"telephony-common": DEFAULT_NINJA_WEIGHT,
"TelephonyProvider": DEFAULT_NINJA_WEIGHT,
"TeleService": DEFAULT_NINJA_WEIGHT,
"test-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT,
"TetheringIntegrationTests": DEFAULT_NINJA_WEIGHT,
"TetheringNext": DEFAULT_NINJA_WEIGHT,
"ThemePickerTests": DEFAULT_NINJA_WEIGHT,
"Traceur": DEFAULT_NINJA_WEIGHT,
"UsbManagerTests": DEFAULT_NINJA_WEIGHT,
"UsbTests": DEFAULT_NINJA_WEIGHT,
"virtmgr": DEFAULT_NINJA_WEIGHT,
"WallpaperPicker2TestLib": DEFAULT_NINJA_WEIGHT,
"WallpaperPicker2Tests": DEFAULT_NINJA_WEIGHT,
"WifiDialog": DEFAULT_NINJA_WEIGHT,
"wm-proto-parsers": DEFAULT_NINJA_WEIGHT,
"WMShellFlickerTests": DEFAULT_NINJA_WEIGHT,
"WmTests": DEFAULT_NINJA_WEIGHT,
"wpa_supplicant": DEFAULT_NINJA_WEIGHT,
} }
) )

50
cmd/soong_build/main.go
View File

@@ -30,6 +30,7 @@ import (
"android/soong/shared" "android/soong/shared"
"android/soong/ui/metrics/bp2build_metrics_proto" "android/soong/ui/metrics/bp2build_metrics_proto"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap" "github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/deptools" "github.com/google/blueprint/deptools"
"github.com/google/blueprint/metrics" "github.com/google/blueprint/metrics"
@@ -256,18 +257,45 @@ func apiBuildFileExcludes(ctx *android.Context) []string {
} }
func writeNinjaHint(ctx *android.Context) error { func writeNinjaHint(ctx *android.Context) error {
wantModules := make([]string, len(allowlists.HugeModulesMap)) // The current predictor focuses on reducing false negatives.
i := 0 // If there are too many false positives (e.g., most modules are marked as positive),
for k := range allowlists.HugeModulesMap { // real long-running jobs cannot run early.
wantModules[i] = k // Therefore, the model should be adjusted in this case.
i += 1 // The model should also be adjusted if there are critical false negatives.
} predicate := func(j *blueprint.JsonModule) (prioritized bool, weight int) {
outputsMap := ctx.Context.GetOutputsFromModuleNames(wantModules) prioritized = false
var outputBuilder strings.Builder weight = 0
for k, v := range allowlists.HugeModulesMap { for prefix, w := range allowlists.HugeModuleTypePrefixMap {
for _, output := range outputsMap[k] { if strings.HasPrefix(j.Type, prefix) {
outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, v)) prioritized = true
weight = w
return
}
} }
dep_count := len(j.Deps)
src_count := 0
for _, a := range j.Module["Actions"].([]blueprint.JSONAction) {
src_count += len(a.Inputs)
}
input_size := dep_count + src_count
// Current threshold is an arbitrary value which only consider recall rather than accuracy.
if input_size > allowlists.INPUT_SIZE_THRESHOLD {
prioritized = true
weight += ((input_size) / allowlists.INPUT_SIZE_THRESHOLD) * allowlists.DEFAULT_PRIORITIZED_WEIGHT
// To prevent some modules from having too large a priority value.
if weight > allowlists.HIGH_PRIORITIZED_WEIGHT {
weight = allowlists.HIGH_PRIORITIZED_WEIGHT
}
}
return
}
outputsMap := ctx.Context.GetWeightedOutputsFromPredicate(predicate)
var outputBuilder strings.Builder
for output, weight := range outputsMap {
outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, weight))
} }
weightListFile := filepath.Join(topDir, ctx.Config().OutDir(), ".ninja_weight_list") weightListFile := filepath.Join(topDir, ctx.Config().OutDir(), ".ninja_weight_list")