From 9ba460fd8f4f2de0e0960ec0e5ffcb2172cebed0 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Tue, 4 Aug 2020 09:55:13 -0700 Subject: [PATCH 01/53] Add property api_levels_jar_filename to droidstubs The default is android.jar to maintain current behavior but allows users to specify a different filepath to support generating api-versions for docs other than the platform (e.g. auto). Bug: 162552181 Test: m doc Test: go test java_test Merged-In: I9e51abaf7d5451d68ca782157d9b739f76c5da82 Change-Id: I9e51abaf7d5451d68ca782157d9b739f76c5da82 --- java/droiddoc.go | 60 ++++++++++++++++++++++++++--------------------- java/java_test.go | 56 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 27 deletions(-) diff --git a/java/droiddoc.go b/java/droiddoc.go index 230b1f026..b564fea01 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -287,6 +287,9 @@ type DroidstubsProperties struct { // the dirs which Metalava extracts API levels annotations from. Api_levels_annotations_dirs []string + // the filename which Metalava extracts API levels annotations from. Defaults to android.jar. + Api_levels_jar_filename *string + // if set to true, collect the values used by the Dev tools and // write them in files packaged with the SDK. Defaults to false. Write_sdk_values *bool @@ -1387,34 +1390,37 @@ func (d *Droidstubs) inclusionAnnotationsFlags(ctx android.ModuleContext, cmd *a } func (d *Droidstubs) apiLevelsAnnotationsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { - if Bool(d.properties.Api_levels_annotations_enabled) { - d.apiVersionsXml = android.PathForModuleOut(ctx, "api-versions.xml") - - if len(d.properties.Api_levels_annotations_dirs) == 0 { - ctx.PropertyErrorf("api_levels_annotations_dirs", - "has to be non-empty if api levels annotations was enabled!") - } - - cmd.FlagWithOutput("--generate-api-levels ", d.apiVersionsXml) - cmd.FlagWithInput("--apply-api-levels ", d.apiVersionsXml) - cmd.FlagWithArg("--current-version ", ctx.Config().PlatformSdkVersion()) - cmd.FlagWithArg("--current-codename ", ctx.Config().PlatformSdkCodename()) - - ctx.VisitDirectDepsWithTag(metalavaAPILevelsAnnotationsDirTag, func(m android.Module) { - if t, ok := m.(*ExportedDroiddocDir); ok { - for _, dep := range t.deps { - if strings.HasSuffix(dep.String(), "android.jar") { - cmd.Implicit(dep) - } - } - cmd.FlagWithArg("--android-jar-pattern ", t.dir.String()+"/%/public/android.jar") - } else { - ctx.PropertyErrorf("api_levels_annotations_dirs", - "module %q is not a metalava api-levels-annotations dir", ctx.OtherModuleName(m)) - } - }) - + if !Bool(d.properties.Api_levels_annotations_enabled) { + return } + + d.apiVersionsXml = android.PathForModuleOut(ctx, "api-versions.xml") + + if len(d.properties.Api_levels_annotations_dirs) == 0 { + ctx.PropertyErrorf("api_levels_annotations_dirs", + "has to be non-empty if api levels annotations was enabled!") + } + + cmd.FlagWithOutput("--generate-api-levels ", d.apiVersionsXml) + cmd.FlagWithInput("--apply-api-levels ", d.apiVersionsXml) + cmd.FlagWithArg("--current-version ", ctx.Config().PlatformSdkVersion()) + cmd.FlagWithArg("--current-codename ", ctx.Config().PlatformSdkCodename()) + + filename := proptools.StringDefault(d.properties.Api_levels_jar_filename, "android.jar") + + ctx.VisitDirectDepsWithTag(metalavaAPILevelsAnnotationsDirTag, func(m android.Module) { + if t, ok := m.(*ExportedDroiddocDir); ok { + for _, dep := range t.deps { + if strings.HasSuffix(dep.String(), filename) { + cmd.Implicit(dep) + } + } + cmd.FlagWithArg("--android-jar-pattern ", t.dir.String()+"/%/public/"+filename) + } else { + ctx.PropertyErrorf("api_levels_annotations_dirs", + "module %q is not a metalava api-levels-annotations dir", ctx.OtherModuleName(m)) + } + }) } func (d *Droidstubs) apiToXmlFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { diff --git a/java/java_test.go b/java/java_test.go index f0de52fb9..8797119b5 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -1006,6 +1006,62 @@ func TestDroiddoc(t *testing.T) { } } +func TestDroidstubs(t *testing.T) { + ctx, _ := testJavaWithFS(t, ` + droiddoc_exported_dir { + name: "droiddoc-templates-sdk", + path: ".", + } + + droidstubs { + name: "bar-stubs", + srcs: [ + "bar-doc/a.java", + ], + api_levels_annotations_dirs: [ + "droiddoc-templates-sdk", + ], + api_levels_annotations_enabled: true, + } + + droidstubs { + name: "bar-stubs-other", + srcs: [ + "bar-doc/a.java", + ], + api_levels_annotations_dirs: [ + "droiddoc-templates-sdk", + ], + api_levels_annotations_enabled: true, + api_levels_jar_filename: "android.other.jar", + } + `, + map[string][]byte{ + "bar-doc/a.java": nil, + }) + testcases := []struct { + moduleName string + expectedJarFilename string + }{ + { + moduleName: "bar-stubs", + expectedJarFilename: "android.jar", + }, + { + moduleName: "bar-stubs-other", + expectedJarFilename: "android.other.jar", + }, + } + for _, c := range testcases { + m := ctx.ModuleForTests(c.moduleName, "android_common") + metalava := m.Rule("metalava") + expected := "--android-jar-pattern ./%/public/" + c.expectedJarFilename + if actual := metalava.RuleParams.Command; !strings.Contains(actual, expected) { + t.Errorf("For %q, expected metalava argument %q, but was not found %q", c.moduleName, expected, actual) + } + } +} + func TestDroidstubsWithSystemModules(t *testing.T) { ctx, _ := testJava(t, ` droidstubs { From ba3876a91ab638157b1cf04d57b4aa8dedf6fd5e Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Thu, 28 May 2020 15:28:00 -0700 Subject: [PATCH 02/53] Remove obsolete PDK build functionality This hasn't worked for a couple years, and continues to bitrot. Just remove it. Adds a bpfix rule so that we can eventually remove the product_variables.pdk definition, which is now always a no-op. Test: treehugger Change-Id: I830b54d419b59f6db1d4617b45e61a78234f57a7 --- android/config.go | 4 --- android/variable.go | 1 - bpfix/bpfix/bpfix.go | 23 +++++++++++++++ bpfix/bpfix/bpfix_test.go | 58 ++++++++++++++++++++++++++++++++++++++ cc/rs.go | 2 +- java/config/config.go | 8 +++--- java/dexpreopt_bootjars.go | 6 ++-- java/droiddoc.go | 20 +++++-------- java/sdk.go | 30 ++------------------ java/sdk_library.go | 28 ++++++++---------- java/sdk_test.go | 42 --------------------------- ui/build/config.go | 10 ------- ui/build/dumpvars.go | 3 -- ui/build/kati.go | 6 +--- 14 files changed, 110 insertions(+), 131 deletions(-) diff --git a/android/config.go b/android/config.go index a6284f0d3..a1e97c981 100644 --- a/android/config.go +++ b/android/config.go @@ -742,10 +742,6 @@ func (c *config) Fuchsia() bool { return Bool(c.productVariables.Fuchsia) } -func (c *config) IsPdkBuild() bool { - return Bool(c.productVariables.Pdk) -} - func (c *config) MinimizeJavaDebugInfo() bool { return Bool(c.productVariables.MinimizeJavaDebugInfo) && !Bool(c.productVariables.Eng) } diff --git a/android/variable.go b/android/variable.go index 9fb857744..1f21f3498 100644 --- a/android/variable.go +++ b/android/variable.go @@ -233,7 +233,6 @@ type productVariables struct { Eng *bool `json:",omitempty"` Treble_linker_namespaces *bool `json:",omitempty"` Enforce_vintf_manifest *bool `json:",omitempty"` - Pdk *bool `json:",omitempty"` Uml *bool `json:",omitempty"` Use_lmkd_stats_log *bool `json:",omitempty"` Arc *bool `json:",omitempty"` diff --git a/bpfix/bpfix/bpfix.go b/bpfix/bpfix/bpfix.go index 689cbd15b..faec473a4 100644 --- a/bpfix/bpfix/bpfix.go +++ b/bpfix/bpfix/bpfix.go @@ -128,6 +128,10 @@ var fixSteps = []FixStep{ Name: "removeSoongConfigBoolVariable", Fix: removeSoongConfigBoolVariable, }, + { + Name: "removePdkProperty", + Fix: runPatchListMod(removePdkProperty), + }, } func NewFixRequest() FixRequest { @@ -993,6 +997,25 @@ func removeTags(mod *parser.Module, buf []byte, patchlist *parser.PatchList) err return patchlist.Add(prop.Pos().Offset, prop.End().Offset+2, replaceStr) } +func removePdkProperty(mod *parser.Module, buf []byte, patchlist *parser.PatchList) error { + prop, ok := mod.GetProperty("product_variables") + if !ok { + return nil + } + propMap, ok := prop.Value.(*parser.Map) + if !ok { + return nil + } + pdkProp, ok := propMap.GetProperty("pdk") + if !ok { + return nil + } + if len(propMap.Properties) > 1 { + return patchlist.Add(pdkProp.Pos().Offset, pdkProp.End().Offset+2, "") + } + return patchlist.Add(prop.Pos().Offset, prop.End().Offset+2, "") +} + func mergeMatchingModuleProperties(mod *parser.Module, buf []byte, patchlist *parser.PatchList) error { return mergeMatchingProperties(&mod.Properties, buf, patchlist) } diff --git a/bpfix/bpfix/bpfix_test.go b/bpfix/bpfix/bpfix_test.go index 8988177b1..ef9814fb8 100644 --- a/bpfix/bpfix/bpfix_test.go +++ b/bpfix/bpfix/bpfix_test.go @@ -998,3 +998,61 @@ func TestRemoveSoongConfigBoolVariable(t *testing.T) { }) } } + +func TestRemovePdkProperty(t *testing.T) { + tests := []struct { + name string + in string + out string + }{ + { + name: "remove property", + in: ` + cc_library_shared { + name: "foo", + product_variables: { + other: { + bar: true, + }, + pdk: { + enabled: false, + }, + }, + } + `, + out: ` + cc_library_shared { + name: "foo", + product_variables: { + other: { + bar: true, + }, + }, + } + `, + }, + { + name: "remove property and empty product_variables", + in: ` + cc_library_shared { + name: "foo", + product_variables: { + pdk: { + enabled: false, + }, + }, + } + `, + out: ` + cc_library_shared { + name: "foo", + } + `, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runPass(t, test.in, test.out, runPatchListMod(removePdkProperty)) + }) + } +} diff --git a/cc/rs.go b/cc/rs.go index de3e64b3d..ba69f2339 100644 --- a/cc/rs.go +++ b/cc/rs.go @@ -26,7 +26,7 @@ import ( func init() { pctx.VariableFunc("rsCmd", func(ctx android.PackageVarContext) string { if ctx.Config().AlwaysUsePrebuiltSdks() { - // Use RenderScript prebuilts for unbundled builds but not PDK builds + // Use RenderScript prebuilts for unbundled builds return filepath.Join("prebuilts/sdk/tools", runtime.GOOS, "bin/llvm-rs-cc") } else { return ctx.Config().HostToolPath(ctx, "llvm-rs-cc").String() diff --git a/java/config/config.go b/java/config/config.go index a093e98c9..0fd09e10e 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -180,7 +180,7 @@ func init() { func hostBinToolVariableWithSdkToolsPrebuilt(name, tool string) { pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { - if ctx.Config().AlwaysUsePrebuiltSdks() || ctx.Config().IsPdkBuild() { + if ctx.Config().AlwaysUsePrebuiltSdks() { return filepath.Join("prebuilts/sdk/tools", runtime.GOOS, "bin", tool) } else { return ctx.Config().HostToolPath(ctx, tool).String() @@ -190,7 +190,7 @@ func hostBinToolVariableWithSdkToolsPrebuilt(name, tool string) { func hostJavaToolVariableWithSdkToolsPrebuilt(name, tool string) { pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { - if ctx.Config().AlwaysUsePrebuiltSdks() || ctx.Config().IsPdkBuild() { + if ctx.Config().AlwaysUsePrebuiltSdks() { return filepath.Join("prebuilts/sdk/tools/lib", tool+".jar") } else { return ctx.Config().HostJavaToolPath(ctx, tool+".jar").String() @@ -200,7 +200,7 @@ func hostJavaToolVariableWithSdkToolsPrebuilt(name, tool string) { func hostJNIToolVariableWithSdkToolsPrebuilt(name, tool string) { pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { - if ctx.Config().AlwaysUsePrebuiltSdks() || ctx.Config().IsPdkBuild() { + if ctx.Config().AlwaysUsePrebuiltSdks() { ext := ".so" if runtime.GOOS == "darwin" { ext = ".dylib" @@ -214,7 +214,7 @@ func hostJNIToolVariableWithSdkToolsPrebuilt(name, tool string) { func hostBinToolVariableWithBuildToolsPrebuilt(name, tool string) { pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { - if ctx.Config().AlwaysUsePrebuiltSdks() || ctx.Config().IsPdkBuild() { + if ctx.Config().AlwaysUsePrebuiltSdks() { return filepath.Join("prebuilts/build-tools", ctx.Config().PrebuiltOS(), "bin", tool) } else { return ctx.Config().HostToolPath(ctx, tool).String() diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 3abe78271..b44545647 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -505,7 +505,7 @@ func bootImageProfileRule(ctx android.SingletonContext, image *bootImageConfig, globalSoong := dexpreopt.GetCachedGlobalSoongConfig(ctx) global := dexpreopt.GetGlobalConfig(ctx) - if global.DisableGenerateProfile || ctx.Config().IsPdkBuild() || ctx.Config().UnbundledBuild() { + if global.DisableGenerateProfile || ctx.Config().UnbundledBuild() { return nil } profile := ctx.Config().Once(bootImageProfileRuleKey, func() interface{} { @@ -560,7 +560,7 @@ func bootFrameworkProfileRule(ctx android.SingletonContext, image *bootImageConf globalSoong := dexpreopt.GetCachedGlobalSoongConfig(ctx) global := dexpreopt.GetGlobalConfig(ctx) - if global.DisableGenerateProfile || ctx.Config().IsPdkBuild() || ctx.Config().UnbundledBuild() { + if global.DisableGenerateProfile || ctx.Config().UnbundledBuild() { return nil } return ctx.Config().Once(bootFrameworkProfileRuleKey, func() interface{} { @@ -602,7 +602,7 @@ func bootFrameworkProfileRule(ctx android.SingletonContext, image *bootImageConf var bootFrameworkProfileRuleKey = android.NewOnceKey("bootFrameworkProfileRule") func updatableBcpPackagesRule(ctx android.SingletonContext, image *bootImageConfig, missingDeps []string) android.WritablePath { - if ctx.Config().IsPdkBuild() || ctx.Config().UnbundledBuild() { + if ctx.Config().UnbundledBuild() { return nil } diff --git a/java/droiddoc.go b/java/droiddoc.go index 6b8ff7997..3b192ba55 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -1081,9 +1081,7 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { rule.Build(pctx, ctx, "javadoc", desc) - if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") && - !ctx.Config().IsPdkBuild() { - + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") { apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Api_file)) removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Removed_api_file)) @@ -1150,9 +1148,7 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { rule.Build(pctx, ctx, "doclavaCurrentApiUpdate", "update current API") } - if apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") && - !ctx.Config().IsPdkBuild() { - + if apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") { apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Api_file)) removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Removed_api_file)) @@ -1444,7 +1440,7 @@ func (d *Droidstubs) apiLevelsAnnotationsFlags(ctx android.ModuleContext, cmd *a } func (d *Droidstubs) apiToXmlFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { - if Bool(d.properties.Jdiff_enabled) && !ctx.Config().IsPdkBuild() && d.apiFile != nil { + if Bool(d.properties.Jdiff_enabled) && d.apiFile != nil { if d.apiFile.String() == "" { ctx.ModuleErrorf("API signature file has to be specified in Metalava when jdiff is enabled.") } @@ -1592,7 +1588,7 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Add API lint options. - if BoolDefault(d.properties.Check_api.Api_lint.Enabled, false) && !ctx.Config().IsPdkBuild() { + if BoolDefault(d.properties.Check_api.Api_lint.Enabled, false) { doApiLint = true newSince := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Api_lint.New_since) @@ -1659,8 +1655,7 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Add "check released" options. (Detect incompatible API changes from the last public release) - if apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") && - !ctx.Config().IsPdkBuild() { + if apiCheckEnabled(ctx, d.properties.Check_api.Last_released, "last_released") { doCheckReleased = true if len(d.Javadoc.properties.Out) > 0 { @@ -1737,8 +1732,7 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { rule.Build(pctx, ctx, "metalava", "metalava merged") - if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") && - !ctx.Config().IsPdkBuild() { + if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") { if len(d.Javadoc.properties.Out) > 0 { ctx.PropertyErrorf("out", "out property may not be combined with check_api") @@ -1852,7 +1846,7 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { rule.Build(pctx, ctx, "nullabilityWarningsCheck", "nullability warnings check") } - if Bool(d.properties.Jdiff_enabled) && !ctx.Config().IsPdkBuild() { + if Bool(d.properties.Jdiff_enabled) { if len(d.Javadoc.properties.Out) > 0 { ctx.PropertyErrorf("out", "out property may not be combined with jdiff") } diff --git a/java/sdk.go b/java/sdk.go index fa4c05364..b44cd8e1a 100644 --- a/java/sdk.go +++ b/java/sdk.go @@ -191,23 +191,6 @@ func (s sdkSpec) prebuiltSdkAvailableForUnbundledBuild() bool { return s.kind != sdkPrivate && s.kind != sdkNone && s.kind != sdkCorePlatform } -// forPdkBuild converts this sdkSpec into another sdkSpec that is for the PDK builds. -func (s sdkSpec) forPdkBuild(ctx android.EarlyModuleContext) sdkSpec { - // For PDK builds, use the latest SDK version instead of "current" or "" - if s.kind == sdkPrivate || s.kind == sdkPublic { - kind := s.kind - if kind == sdkPrivate { - // We don't have prebuilt SDK for private APIs, so use the public SDK - // instead. This looks odd, but that's how it has been done. - // TODO(b/148271073): investigate the need for this. - kind = sdkPublic - } - version := sdkVersion(LatestSdkVersionInt(ctx)) - return sdkSpec{kind, version, s.raw} - } - return s -} - // usePrebuilt determines whether prebuilt SDK should be used for this sdkSpec with the given context. func (s sdkSpec) usePrebuilt(ctx android.EarlyModuleContext) bool { if s.version.isCurrent() { @@ -233,9 +216,6 @@ func (s sdkSpec) effectiveVersion(ctx android.EarlyModuleContext) (sdkVersion, e if !s.valid() { return s.version, fmt.Errorf("invalid sdk version %q", s.raw) } - if ctx.Config().IsPdkBuild() { - s = s.forPdkBuild(ctx) - } if s.version.isNumbered() { return s.version, nil } @@ -350,9 +330,6 @@ func decodeSdkDep(ctx android.EarlyModuleContext, sdkContext sdkContext) sdkDep return sdkDep{} } - if ctx.Config().IsPdkBuild() { - sdkVersion = sdkVersion.forPdkBuild(ctx) - } if !sdkVersion.validateSystemSdk(ctx) { return sdkDep{} } @@ -511,7 +488,7 @@ func sdkSingletonFactory() android.Singleton { type sdkSingleton struct{} func (sdkSingleton) GenerateBuildActions(ctx android.SingletonContext) { - if ctx.Config().AlwaysUsePrebuiltSdks() || ctx.Config().IsPdkBuild() { + if ctx.Config().AlwaysUsePrebuiltSdks() { return } @@ -631,9 +608,6 @@ func createAPIFingerprint(ctx android.SingletonContext) { if ctx.Config().PlatformSdkCodename() == "REL" { cmd.Text("echo REL >").Output(out) - } else if ctx.Config().IsPdkBuild() { - // TODO: get this from the PDK artifacts? - cmd.Text("echo PDK >").Output(out) } else if !ctx.Config().AlwaysUsePrebuiltSdks() { in, err := ctx.GlobWithDeps("frameworks/base/api/*current.txt", nil) if err != nil { @@ -663,7 +637,7 @@ func ApiFingerprintPath(ctx android.PathContext) android.OutputPath { } func sdkMakeVars(ctx android.MakeVarsContext) { - if ctx.Config().AlwaysUsePrebuiltSdks() || ctx.Config().IsPdkBuild() { + if ctx.Config().AlwaysUsePrebuiltSdks() { return } diff --git a/java/sdk_library.go b/java/sdk_library.go index 936d147a8..8b1284d52 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -1127,22 +1127,17 @@ func (module *SdkLibrary) createImplLibrary(mctx android.DefaultableHookContext) // Creates a static java library that has API stubs func (module *SdkLibrary) createStubsLibrary(mctx android.DefaultableHookContext, apiScope *apiScope) { props := struct { - Name *string - Visibility []string - Srcs []string - Installable *bool - Sdk_version *string - System_modules *string - Patch_module *string - Libs []string - Compile_dex *bool - Java_version *string - Product_variables struct { - Pdk struct { - Enabled *bool - } - } - Openjdk9 struct { + Name *string + Visibility []string + Srcs []string + Installable *bool + Sdk_version *string + System_modules *string + Patch_module *string + Libs []string + Compile_dex *bool + Java_version *string + Openjdk9 struct { Srcs []string Javacflags []string } @@ -1174,7 +1169,6 @@ func (module *SdkLibrary) createStubsLibrary(mctx android.DefaultableHookContext if proptools.Bool(module.sdkLibraryProperties.Annotations_enabled) { props.Libs = append(props.Libs, "stub-annotations") } - props.Product_variables.Pdk.Enabled = proptools.BoolPtr(false) props.Openjdk9.Srcs = module.properties.Openjdk9.Srcs props.Openjdk9.Javacflags = module.properties.Openjdk9.Javacflags // We compile the stubs for 1.8 in line with the main android.jar stubs, and potential diff --git a/java/sdk_test.go b/java/sdk_test.go index 13fc915ee..776069dc9 100644 --- a/java/sdk_test.go +++ b/java/sdk_test.go @@ -30,7 +30,6 @@ func TestClasspath(t *testing.T) { var classpathTestcases = []struct { name string unbundled bool - pdk bool moduleType string host android.OsClass properties string @@ -216,35 +215,6 @@ func TestClasspath(t *testing.T) { aidl: "-pprebuilts/sdk/current/public/framework.aidl", }, - { - name: "pdk default", - pdk: true, - bootclasspath: []string{`""`}, - system: "sdk_public_30_system_modules", - java8classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - java9classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/30/public/framework.aidl", - }, - { - name: "pdk current", - pdk: true, - properties: `sdk_version: "current",`, - bootclasspath: []string{`""`}, - system: "sdk_public_30_system_modules", - java8classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - java9classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/30/public/framework.aidl", - }, - { - name: "pdk 29", - pdk: true, - properties: `sdk_version: "29",`, - bootclasspath: []string{`""`}, - system: "sdk_public_30_system_modules", - java8classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - java9classpath: []string{"prebuilts/sdk/30/public/android.jar", "prebuilts/sdk/tools/core-lambda-stubs.jar"}, - aidl: "-pprebuilts/sdk/30/public/framework.aidl", - }, { name: "module_current", properties: `sdk_version: "module_current",`, @@ -386,9 +356,6 @@ func TestClasspath(t *testing.T) { config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true) config.TestProductVariables.Always_use_prebuilt_sdks = proptools.BoolPtr(true) } - if testcase.pdk { - config.TestProductVariables.Pdk = proptools.BoolPtr(true) - } ctx := testContext() run(t, ctx, config) @@ -410,9 +377,6 @@ func TestClasspath(t *testing.T) { config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true) config.TestProductVariables.Always_use_prebuilt_sdks = proptools.BoolPtr(true) } - if testcase.pdk { - config.TestProductVariables.Pdk = proptools.BoolPtr(true) - } ctx := testContext() run(t, ctx, config) @@ -437,9 +401,6 @@ func TestClasspath(t *testing.T) { config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true) config.TestProductVariables.Always_use_prebuilt_sdks = proptools.BoolPtr(true) } - if testcase.pdk { - config.TestProductVariables.Pdk = proptools.BoolPtr(true) - } ctx := testContext() run(t, ctx, config) @@ -456,9 +417,6 @@ func TestClasspath(t *testing.T) { config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true) config.TestProductVariables.Always_use_prebuilt_sdks = proptools.BoolPtr(true) } - if testcase.pdk { - config.TestProductVariables.Pdk = proptools.BoolPtr(true) - } ctx := testContext() run(t, ctx, config) diff --git a/ui/build/config.go b/ui/build/config.go index 3fa047900..b263a98a5 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -53,8 +53,6 @@ type configImpl struct { // Autodetected totalRAM uint64 - pdkBuild bool - brokenDupRules bool brokenUsesNetwork bool brokenNinjaEnvVars []string @@ -968,14 +966,6 @@ func (c *configImpl) TargetDeviceDir() string { return c.targetDeviceDir } -func (c *configImpl) SetPdkBuild(pdk bool) { - c.pdkBuild = pdk -} - -func (c *configImpl) IsPdkBuild() bool { - return c.pdkBuild -} - func (c *configImpl) BuildDateTime() string { return c.buildDateTime } diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index e22985675..999af0796 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -161,8 +161,6 @@ var BannerVars = []string{ "BUILD_ID", "OUT_DIR", "AUX_OS_VARIANT_LIST", - "TARGET_BUILD_PDK", - "PDK_FUSION_PLATFORM_ZIP", "PRODUCT_SOONG_NAMESPACES", } @@ -281,7 +279,6 @@ func runMakeProductConfig(ctx Context, config Config) { config.SetTargetDevice(make_vars["TARGET_DEVICE"]) config.SetTargetDeviceDir(make_vars["TARGET_DEVICE_DIR"]) - config.SetPdkBuild(make_vars["TARGET_BUILD_PDK"] == "true") config.SetBuildBrokenDupRules(make_vars["BUILD_BROKEN_DUP_RULES"] == "true") config.SetBuildBrokenUsesNetwork(make_vars["BUILD_BROKEN_USES_NETWORK"] == "true") config.SetBuildBrokenNinjaUsesEnvVars(strings.Fields(make_vars["BUILD_BROKEN_NINJA_USES_ENV_VARS"])) diff --git a/ui/build/kati.go b/ui/build/kati.go index 1cd5fea32..f6d3a5741 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -134,14 +134,10 @@ func runKatiBuild(ctx Context, config Config) { args := []string{ "--writable", config.OutDir() + "/", + "--werror_implicit_rules", "-f", "build/make/core/main.mk", } - // PDK builds still uses a few implicit rules - if !config.IsPdkBuild() { - args = append(args, "--werror_implicit_rules") - } - if !config.BuildBrokenDupRules() { args = append(args, "--werror_overriding_commands") } From 9cd4216c56b7d62c1584ef741a77d18c647041e1 Mon Sep 17 00:00:00 2001 From: Jaewoong Jung Date: Mon, 29 Jun 2020 19:18:44 -0700 Subject: [PATCH 03/53] Output apkcerts file for android_app_set. Soong and Make have no ways to figure out what splits will be outputted from a given android_app_set, so it's impossible for them to provide full PACKAGES.$(LOCAL_MODULE).CERTIFICATE entries, which are required to build a final apkcerts.txt. This change makes extract_apks produce apkcerts.txt files for each input modules instead. The Make-side counterpart of this change merges all local apkcerts.txt into a final one. Bug: 160119159 Bug: 162464887 Test: main_test.go Test: m apkcerts-list Merged-In: I321e80fd636a955213761f56a3ac64bfe7f7f7c0 Change-Id: I321e80fd636a955213761f56a3ac64bfe7f7f7c0 --- android/module.go | 27 ++++++++++++++++++++++++ cmd/extract_apks/main.go | 39 ++++++++++++++++++++++++++++------- cmd/extract_apks/main_test.go | 38 ++++++++++++++++++++++++---------- java/androidmk.go | 1 + java/app.go | 13 ++++++++---- java/app_test.go | 3 +++ java/builder.go | 3 ++- 7 files changed, 101 insertions(+), 23 deletions(-) diff --git a/android/module.go b/android/module.go index 6a796227d..8bdc82e78 100644 --- a/android/module.go +++ b/android/module.go @@ -614,6 +614,33 @@ func (a *ModuleBase) ProductServicesSpecific() bool { return Bool(a.commonProperties.Product_services_specific) } +func (m *ModuleBase) PartitionTag(config DeviceConfig) string { + partition := "system" + if m.SocSpecific() { + // A SoC-specific module could be on the vendor partition at + // "vendor" or the system partition at "system/vendor". + if config.VendorPath() == "vendor" { + partition = "vendor" + } + } else if m.DeviceSpecific() { + // A device-specific module could be on the odm partition at + // "odm", the vendor partition at "vendor/odm", or the system + // partition at "system/vendor/odm". + if config.OdmPath() == "odm" { + partition = "odm" + } else if strings.HasPrefix(config.OdmPath(), "vendor/") { + partition = "vendor" + } + } else if m.ProductSpecific() { + // A product-specific module could be on the product partition + // at "product" or the system partition at "system/product". + if config.ProductPath() == "product" { + partition = "product" + } + } + return partition +} + func (a *ModuleBase) Enabled() bool { if a.commonProperties.Enabled == nil { return !a.Os().DefaultDisabled diff --git a/cmd/extract_apks/main.go b/cmd/extract_apks/main.go index e9a850ee8..85dd6d122 100644 --- a/cmd/extract_apks/main.go +++ b/cmd/extract_apks/main.go @@ -24,6 +24,7 @@ import ( "math" "os" "regexp" + "sort" "strings" "github.com/golang/protobuf/proto" @@ -355,7 +356,7 @@ type Zip2ZipWriter interface { // Writes out selected entries, renaming them as needed func (apkSet *ApkSet) writeApks(selected SelectionResult, config TargetConfig, - writer Zip2ZipWriter) error { + writer Zip2ZipWriter, partition string) ([]string, error) { // Renaming rules: // splits/MODULE-master.apk to STEM.apk // else @@ -389,10 +390,11 @@ func (apkSet *ApkSet) writeApks(selected SelectionResult, config TargetConfig, } entryOrigin := make(map[string]string) // output entry to input entry + var apkcerts []string for _, apk := range selected.entries { apkFile, ok := apkSet.entries[apk] if !ok { - return fmt.Errorf("TOC refers to an entry %s which does not exist", apk) + return nil, fmt.Errorf("TOC refers to an entry %s which does not exist", apk) } inName := apkFile.Name outName, ok := renamer(inName) @@ -405,10 +407,15 @@ func (apkSet *ApkSet) writeApks(selected SelectionResult, config TargetConfig, } entryOrigin[outName] = inName if err := writer.CopyFrom(apkFile, outName); err != nil { - return err + return nil, err + } + if partition != "" { + apkcerts = append(apkcerts, fmt.Sprintf( + `name="%s" certificate="PRESIGNED" private_key=""`, outName)) } } - return nil + sort.Strings(apkcerts) + return apkcerts, nil } func (apkSet *ApkSet) extractAndCopySingle(selected SelectionResult, outFile *os.File) error { @@ -433,6 +440,9 @@ var ( } extractSingle = flag.Bool("extract-single", false, "extract a single target and output it uncompressed. only available for standalone apks and apexes.") + apkcertsOutput = flag.String("apkcerts", "", + "optional apkcerts.txt output file containing signing info of all outputted apks") + partition = flag.String("partition", "", "partition string. required when -apkcerts is used.") ) // Parse abi values @@ -485,7 +495,8 @@ func (s screenDensityFlagValue) Set(densityList string) error { func processArgs() { flag.Usage = func() { fmt.Fprintln(os.Stderr, `usage: extract_apks -o -sdk-version value -abis value `+ - `-screen-densities value {-stem value | -extract-single} [-allow-prereleased] `) + `-screen-densities value {-stem value | -extract-single} [-allow-prereleased] `+ + `[-apkcerts -partition ] `) flag.PrintDefaults() os.Exit(2) } @@ -498,7 +509,8 @@ func processArgs() { "allow prereleased") flag.StringVar(&targetConfig.stem, "stem", "", "output entries base name in the output zip file") flag.Parse() - if (*outputFile == "") || len(flag.Args()) != 1 || *version == 0 || (targetConfig.stem == "" && !*extractSingle) { + if (*outputFile == "") || len(flag.Args()) != 1 || *version == 0 || + (targetConfig.stem == "" && !*extractSingle) || (*apkcertsOutput != "" && *partition == "") { flag.Usage() } targetConfig.sdkVersion = int32(*version) @@ -536,7 +548,20 @@ func main() { log.Fatal(err) } }() - err = apkSet.writeApks(sel, targetConfig, writer) + apkcerts, err := apkSet.writeApks(sel, targetConfig, writer, *partition) + if err == nil && *apkcertsOutput != "" { + apkcertsFile, err := os.Create(*apkcertsOutput) + if err != nil { + log.Fatal(err) + } + defer apkcertsFile.Close() + for _, a := range apkcerts { + _, err = apkcertsFile.WriteString(a + "\n") + if err != nil { + log.Fatal(err) + } + } + } } if err != nil { log.Fatal(err) diff --git a/cmd/extract_apks/main_test.go b/cmd/extract_apks/main_test.go index bdd4becce..356faab82 100644 --- a/cmd/extract_apks/main_test.go +++ b/cmd/extract_apks/main_test.go @@ -16,10 +16,11 @@ package main import ( "fmt" - "github.com/golang/protobuf/proto" "reflect" "testing" + "github.com/golang/protobuf/proto" + bp "android/soong/cmd/extract_apks/bundle_proto" "android/soong/third_party/zip" ) @@ -430,48 +431,63 @@ func (w testZip2ZipWriter) CopyFrom(file *zip.File, out string) error { return nil } -type testCaseWriteZip struct { +type testCaseWriteApks struct { name string moduleName string stem string + partition string // what we write from what - expected map[string]string + expectedZipEntries map[string]string + expectedApkcerts []string } -func TestWriteZip(t *testing.T) { - testCases := []testCaseWriteZip{ +func TestWriteApks(t *testing.T) { + testCases := []testCaseWriteApks{ { name: "splits", moduleName: "mybase", stem: "Foo", - expected: map[string]string{ + partition: "system", + expectedZipEntries: map[string]string{ "Foo.apk": "splits/mybase-master.apk", "Foo-xhdpi.apk": "splits/mybase-xhdpi.apk", }, + expectedApkcerts: []string{ + `name="Foo-xhdpi.apk" certificate="PRESIGNED" private_key=""`, + `name="Foo.apk" certificate="PRESIGNED" private_key=""`, + }, }, { name: "universal", moduleName: "base", stem: "Bar", - expected: map[string]string{ + partition: "product", + expectedZipEntries: map[string]string{ "Bar.apk": "universal.apk", }, + expectedApkcerts: []string{ + `name="Bar.apk" certificate="PRESIGNED" private_key=""`, + }, }, } for _, testCase := range testCases { apkSet := ApkSet{entries: make(map[string]*zip.File)} sel := SelectionResult{moduleName: testCase.moduleName} - for _, in := range testCase.expected { + for _, in := range testCase.expectedZipEntries { apkSet.entries[in] = &zip.File{FileHeader: zip.FileHeader{Name: in}} sel.entries = append(sel.entries, in) } writer := testZip2ZipWriter{make(map[string]string)} config := TargetConfig{stem: testCase.stem} - if err := apkSet.writeApks(sel, config, writer); err != nil { + apkcerts, err := apkSet.writeApks(sel, config, writer, testCase.partition) + if err != nil { t.Error(err) } - if !reflect.DeepEqual(testCase.expected, writer.entries) { - t.Errorf("expected %v, got %v", testCase.expected, writer.entries) + if !reflect.DeepEqual(testCase.expectedZipEntries, writer.entries) { + t.Errorf("expected zip entries %v, got %v", testCase.expectedZipEntries, writer.entries) + } + if !reflect.DeepEqual(testCase.expectedApkcerts, apkcerts) { + t.Errorf("expected apkcerts %v, got %v", testCase.expectedApkcerts, apkcerts) } } } diff --git a/java/androidmk.go b/java/androidmk.go index ed8295549..45fd1c1c0 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -607,6 +607,7 @@ func (apkSet *AndroidAppSet) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, "LOCAL_PRIVILEGED_MODULE := true") } fmt.Fprintln(w, "LOCAL_APK_SET_MASTER_FILE := ", apkSet.masterFile) + fmt.Fprintln(w, "LOCAL_APKCERTS_FILE := ", apkSet.apkcertsFile) fmt.Fprintln(w, "LOCAL_OVERRIDES_PACKAGES :=", strings.Join(apkSet.properties.Overrides, " ")) }, }, diff --git a/java/app.go b/java/app.go index f755d8b8f..586b66d85 100644 --- a/java/app.go +++ b/java/app.go @@ -64,6 +64,7 @@ type AndroidAppSet struct { properties AndroidAppSetProperties packedOutput android.WritablePath masterFile string + apkcertsFile android.ModuleOutPath } func (as *AndroidAppSet) Name() string { @@ -107,6 +108,7 @@ func SupportedAbis(ctx android.ModuleContext) []string { func (as *AndroidAppSet) GenerateAndroidBuildActions(ctx android.ModuleContext) { as.packedOutput = android.PathForModuleOut(ctx, "extracted.zip") + as.apkcertsFile = android.PathForModuleOut(ctx, "apkcerts.txt") // We are assuming here that the master file in the APK // set has `.apk` suffix. If it doesn't the build will fail. // APK sets containing APEX files are handled elsewhere. @@ -119,16 +121,19 @@ func (as *AndroidAppSet) GenerateAndroidBuildActions(ctx android.ModuleContext) // TODO(asmundak): do we support device features ctx.Build(pctx, android.BuildParams{ - Rule: extractMatchingApks, - Description: "Extract APKs from APK set", - Output: as.packedOutput, - Inputs: android.Paths{as.prebuilt.SingleSourcePath(ctx)}, + Rule: extractMatchingApks, + Description: "Extract APKs from APK set", + Output: as.packedOutput, + ImplicitOutput: as.apkcertsFile, + Inputs: android.Paths{as.prebuilt.SingleSourcePath(ctx)}, Args: map[string]string{ "abis": strings.Join(SupportedAbis(ctx), ","), "allow-prereleased": strconv.FormatBool(proptools.Bool(as.properties.Prerelease)), "screen-densities": screenDensities, "sdk-version": ctx.Config().PlatformSdkVersion(), "stem": ctx.ModuleName(), + "apkcerts": as.apkcertsFile.String(), + "partition": as.PartitionTag(ctx.DeviceConfig()), }, }) // TODO(asmundak): add this (it's wrong now, will cause copying extracted.zip) diff --git a/java/app_test.go b/java/app_test.go index 4f174d8a3..1d497703a 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -153,6 +153,9 @@ func TestAndroidAppSet(t *testing.T) { if s := params.Args["allow-prereleased"]; s != "true" { t.Errorf("wrong allow-prereleased value: '%s', expected 'true'", s) } + if s := params.Args["partition"]; s != "system" { + t.Errorf("wrong partition value: '%s', expected 'system'", s) + } } func TestAndroidAppSet_Variants(t *testing.T) { diff --git a/java/builder.go b/java/builder.go index b552b2c8e..2fe4d8ab3 100644 --- a/java/builder.go +++ b/java/builder.go @@ -68,10 +68,11 @@ var ( `${config.ExtractApksCmd} -o "${out}" -allow-prereleased=${allow-prereleased} ` + `-sdk-version=${sdk-version} -abis=${abis} ` + `--screen-densities=${screen-densities} --stem=${stem} ` + + `-apkcerts=${apkcerts} -partition=${partition} ` + `${in}`, CommandDeps: []string{"${config.ExtractApksCmd}"}, }, - "abis", "allow-prereleased", "screen-densities", "sdk-version", "stem") + "abis", "allow-prereleased", "screen-densities", "sdk-version", "stem", "apkcerts", "partition") turbine = pctx.AndroidStaticRule("turbine", blueprint.RuleParams{ From 4a9f512b4058cb4727b94e5ad61c8c9249dd2faf Mon Sep 17 00:00:00 2001 From: Jiyong Park Date: Fri, 12 Jun 2020 17:26:31 +0900 Subject: [PATCH 04/53] apex_sets is added to apexkeys.txt apex_sets is a new module type that can be used to deliver pre-signed APEXes, which previously could be done only via prebuilt_apex. Soon gnow understands apex_sets module types and emits the signing info of the modules to apexkeys.txt Bug: 162464887 Bug: 158729168 Test: m Merged-In: I9507375342ec053309660d94c931a79bf4f21218 Change-Id: I9507375342ec053309660d94c931a79bf4f21218 --- apex/apex.go | 4 +++ apex/apex_test.go | 34 +++++++++++++++++++++ apex/key.go | 75 ++++++++++++++++++++++++++++++++++------------- 3 files changed, 93 insertions(+), 20 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index cac519b54..4dca96153 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1528,6 +1528,10 @@ func (a *ApexSet) Name() string { return a.prebuilt.Name(a.ModuleBase.Name()) } +func (a *ApexSet) Overrides() []string { + return a.properties.Overrides +} + // prebuilt_apex imports an `.apex` file into the build graph as if it was built with apex. func apexSetFactory() android.Module { module := &ApexSet{} diff --git a/apex/apex_test.go b/apex/apex_test.go index a2610f667..08011fd5e 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -77,6 +77,7 @@ func testApex(t *testing.T, bp string, handlers ...testCustomizer) *android.Test ctx.BottomUp("version", cc.VersionMutator).Parallel() ctx.BottomUp("begin", cc.BeginMutator).Parallel() }) + ctx.RegisterSingletonType("apex_keys_text", android.SingletonFactoryAdaptor(apexKeysTextFactory)) ctx.Register() @@ -1331,3 +1332,36 @@ func TestApexSet(t *testing.T) { t.Errorf("Unexpected abis parameter - expected %q vs actual %q", expected, actual) } } + +func TestApexKeysTxt(t *testing.T) { + ctx := testApex(t, ` + prebuilt_apex { + name: "myapex", + prefer: true, + arch: { + arm64: { + src: "myapex-arm64.apex", + }, + arm: { + src: "myapex-arm.apex", + }, + }, + } + + apex_set { + name: "myapex_set", + set: "myapex.apks", + filename: "myapex_set.apex", + overrides: ["myapex"], + } + `, func(fs map[string][]byte, config android.Config) { + config.TestProductVariables.Platform_sdk_version = intPtr(30) + config.TestProductVariables.DeviceArch = proptools.StringPtr("arm") + config.TestProductVariables.DeviceSecondaryArch = proptools.StringPtr("arm64") + }) + + apexKeysText := ctx.SingletonForTests("apex_keys_text") + content := apexKeysText.MaybeDescription("apexkeys.txt").BuildParams.Args["content"] + ensureContains(t, content, `name="myapex_set.apex" public_key="PRESIGNED" private_key="PRESIGNED" container_certificate="PRESIGNED" container_private_key="PRESIGNED"`) + ensureNotContains(t, content, "myapex.apex") +} diff --git a/apex/key.go b/apex/key.go index 6ee3dca3f..ec28134cc 100644 --- a/apex/key.go +++ b/apex/key.go @@ -107,10 +107,36 @@ type apexKeysText struct { func (s *apexKeysText) GenerateBuildActions(ctx android.SingletonContext) { s.output = android.PathForOutput(ctx, "apexkeys.txt") - apexModulesMap := make(map[string]android.Module) + type apexKeyEntry struct { + name string + presigned bool + public_key string + private_key string + container_certificate string + container_private_key string + partition string + } + toString := func(e apexKeyEntry) string { + format := "name=%q public_key=%q private_key=%q container_certificate=%q container_private_key=%q\\n" + if e.presigned { + return fmt.Sprintf(format, e.name, "PRESIGNED", "PRESIGNED", "PRESIGNED", "PRESIGNED") + } else { + return fmt.Sprintf(format, e.name, e.public_key, e.private_key, e.container_certificate, e.container_private_key) + } + } + + apexKeyMap := make(map[string]apexKeyEntry) ctx.VisitAllModules(func(module android.Module) { if m, ok := module.(*apexBundle); ok && m.Enabled() && m.installable() { - apexModulesMap[m.Name()] = m + apexKeyMap[m.Name()] = apexKeyEntry{ + name: m.Name() + ".apex", + presigned: false, + public_key: m.public_key_file.String(), + private_key: m.private_key_file.String(), + container_certificate: m.container_certificate_file.String(), + container_private_key: m.container_private_key_file.String(), + partition: m.PartitionTag(ctx.DeviceConfig()), + } } }) @@ -118,34 +144,43 @@ func (s *apexKeysText) GenerateBuildActions(ctx android.SingletonContext) { ctx.VisitAllModules(func(module android.Module) { if m, ok := module.(*Prebuilt); ok && m.Enabled() && m.installable() && m.Prebuilt().UsePrebuilt() { - apexModulesMap[m.BaseModuleName()] = m + apexKeyMap[m.BaseModuleName()] = apexKeyEntry{ + name: m.InstallFilename(), + presigned: true, + partition: m.PartitionTag(ctx.DeviceConfig()), + } + } + }) + + // Find apex_set and let them override apexBundle or prebuilts. This is done in a separate pass + // so that apex_set are not overridden by prebuilts. + ctx.VisitAllModules(func(module android.Module) { + if m, ok := module.(*ApexSet); ok && m.Enabled() { + entry := apexKeyEntry{ + name: m.InstallFilename(), + presigned: true, + partition: m.PartitionTag(ctx.DeviceConfig()), + } + + for _, om := range m.Overrides() { + if _, ok := apexKeyMap[om]; ok { + delete(apexKeyMap, om) + } + } + apexKeyMap[m.BaseModuleName()] = entry } }) // iterating over map does not give consistent ordering in golang var moduleNames []string - for key, _ := range apexModulesMap { + for key, _ := range apexKeyMap { moduleNames = append(moduleNames, key) } sort.Strings(moduleNames) var filecontent strings.Builder - for _, key := range moduleNames { - module := apexModulesMap[key] - if m, ok := module.(*apexBundle); ok { - fmt.Fprintf(&filecontent, - "name=%q public_key=%q private_key=%q container_certificate=%q container_private_key=%q\\n", - m.Name()+".apex", - m.public_key_file.String(), - m.private_key_file.String(), - m.container_certificate_file.String(), - m.container_private_key_file.String()) - } else if m, ok := module.(*Prebuilt); ok { - fmt.Fprintf(&filecontent, - "name=%q public_key=%q private_key=%q container_certificate=%q container_private_key=%q\\n", - m.InstallFilename(), - "PRESIGNED", "PRESIGNED", "PRESIGNED", "PRESIGNED") - } + for _, name := range moduleNames { + fmt.Fprintf(&filecontent, "%s", toString(apexKeyMap[name])) } ctx.Build(pctx, android.BuildParams{ From 81aa8a42a1f14554f3f4bc117a0ba591fe134fa9 Mon Sep 17 00:00:00 2001 From: Jiyong Park Date: Thu, 18 Jun 2020 19:34:42 +0900 Subject: [PATCH 05/53] Don't remove entries for overriddable modules Previously, when there is apex_set that is overriding another module, the entry for the overridden module is removed from apexkeys.txt. However, this is wrong because the existence of the apex_set module doesn't necessary mean that the module is insatalled instead of the overridden module. That is determined by PRODUCT_PACKAGES which Soong has no knowledge of. Therefore, we don't delete the entry for the overridden (actually possibly overridable) modules in the file. Bug: 158729168 Bug: 162464887 Test: m Merged-In: I85d0c756f862323bae556bf657d66ec50038985f (cherry picked from commit ac5e79f9009400d1020bf45f30c3864bde163b76) Change-Id: I85d0c756f862323bae556bf657d66ec50038985f --- apex/apex_test.go | 2 +- apex/key.go | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index 08011fd5e..406ade889 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -1363,5 +1363,5 @@ func TestApexKeysTxt(t *testing.T) { apexKeysText := ctx.SingletonForTests("apex_keys_text") content := apexKeysText.MaybeDescription("apexkeys.txt").BuildParams.Args["content"] ensureContains(t, content, `name="myapex_set.apex" public_key="PRESIGNED" private_key="PRESIGNED" container_certificate="PRESIGNED" container_private_key="PRESIGNED"`) - ensureNotContains(t, content, "myapex.apex") + ensureContains(t, content, `name="myapex.apex" public_key="PRESIGNED" private_key="PRESIGNED" container_certificate="PRESIGNED" container_private_key="PRESIGNED"`) } diff --git a/apex/key.go b/apex/key.go index ec28134cc..ec33763dd 100644 --- a/apex/key.go +++ b/apex/key.go @@ -161,12 +161,6 @@ func (s *apexKeysText) GenerateBuildActions(ctx android.SingletonContext) { presigned: true, partition: m.PartitionTag(ctx.DeviceConfig()), } - - for _, om := range m.Overrides() { - if _, ok := apexKeyMap[om]; ok { - delete(apexKeyMap, om) - } - } apexKeyMap[m.BaseModuleName()] = entry } }) From cb70bdb346fc2d95c1e44268ca909cc8edb0a289 Mon Sep 17 00:00:00 2001 From: Theotime Combes Date: Fri, 12 Jun 2020 12:46:59 +0000 Subject: [PATCH 06/53] Add F2FS support for APEX build system Add the new property payload_fs_type in apex module type. Either 'f2fs' or 'ext4'. Default 'ext4'. Exempt-From-Owner-Approval: Jiyong reviewed this change Test: m Bug: 158453869 Merged-In: I36f373251bd597e11acb62af75437a87e2a531ec Change-Id: I36f373251bd597e11acb62af75437a87e2a531ec (cherry picked from commit b20d6629aefd2100ec95583ed946de852f1d6bf4) (cherry picked from commit 1c751e14535a32bb03ce50e84331a89337dfd105) --- apex/apex.go | 36 ++++++++++++++++++++++++++++++++++++ apex/builder.go | 8 ++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index d1f5714ca..1b995c3f2 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -41,6 +41,9 @@ const ( imageApexType = "image" zipApexType = "zip" flattenedApexType = "flattened" + + ext4FsType = "ext4" + f2fsFsType = "f2fs" ) type dependencyTag struct { @@ -1142,6 +1145,10 @@ type apexBundleProperties struct { // Should be only used in non-system apexes (e.g. vendor: true). // Default is false. Use_vndk_as_stable *bool + + // The type of filesystem to use for an image apex. Either 'ext4' or 'f2fs'. + // Default 'ext4'. + Payload_fs_type *string } type apexTargetBundleProperties struct { @@ -1349,6 +1356,24 @@ func (af *apexFile) AvailableToPlatform() bool { return false } +type fsType int + +const ( + ext4 fsType = iota + f2fs +) + +func (f fsType) string() string { + switch f { + case ext4: + return ext4FsType + case f2fs: + return f2fsFsType + default: + panic(fmt.Errorf("unknown APEX payload type %d", f)) + } +} + type apexBundle struct { android.ModuleBase android.DefaultableModuleBase @@ -1414,6 +1439,8 @@ type apexBundle struct { // Optional list of lint report zip files for apexes that contain java or app modules lintReports android.Paths + + payloadFsType fsType } func addDependenciesForNativeModules(ctx android.BottomUpMutatorContext, @@ -2379,6 +2406,15 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.installDir = android.PathForModuleInstall(ctx, "apex") a.filesInfo = filesInfo + switch proptools.StringDefault(a.properties.Payload_fs_type, ext4FsType) { + case ext4FsType: + a.payloadFsType = ext4 + case f2fsFsType: + a.payloadFsType = f2fs + default: + ctx.PropertyErrorf("payload_fs_type", "%q is not a valid filesystem for apex [ext4, f2fs]", *a.properties.Payload_fs_type) + } + // Optimization. If we are building bundled APEX, for the files that are gathered due to the // transitive dependencies, don't place them inside the APEX, but place a symlink pointing // the same library in the system partition, thus effectively sharing the same libraries diff --git a/apex/builder.go b/apex/builder.go index 1cb2df2a1..307142097 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -63,6 +63,8 @@ func init() { pctx.HostBinToolVariable("jsonmodify", "jsonmodify") pctx.HostBinToolVariable("conv_apex_manifest", "conv_apex_manifest") pctx.HostBinToolVariable("extract_apks", "extract_apks") + pctx.HostBinToolVariable("make_f2fs", "make_f2fs") + pctx.HostBinToolVariable("sload_f2fs", "sload_f2fs") } var ( @@ -116,12 +118,12 @@ var ( `--payload_type image ` + `--key ${key} ${opt_flags} ${image_dir} ${out} `, CommandDeps: []string{"${apexer}", "${avbtool}", "${e2fsdroid}", "${merge_zips}", - "${mke2fs}", "${resize2fs}", "${sefcontext_compile}", + "${mke2fs}", "${resize2fs}", "${sefcontext_compile}", "${make_f2fs}", "${sload_f2fs}", "${soong_zip}", "${zipalign}", "${aapt2}", "prebuilts/sdk/current/public/android.jar"}, Rspfile: "${out}.copy_commands", RspfileContent: "${copy_commands}", Description: "APEX ${image_dir} => ${out}", - }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest") + }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest", "payload_fs_type") zipApexRule = pctx.StaticRule("zipApexRule", blueprint.RuleParams{ Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + @@ -582,6 +584,8 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { optFlags = append(optFlags, "--manifest_json "+a.manifestJsonOut.String()) } + optFlags = append(optFlags, "--payload_fs_type "+a.payloadFsType.string()) + ctx.Build(pctx, android.BuildParams{ Rule: apexRule, Implicits: implicitInputs, From adf591a50580e9724d9a38a2c1b25dde946db4da Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Wed, 17 Jul 2019 12:30:04 +0000 Subject: [PATCH 07/53] [DO NOT MERGE] Add USE_RBE support to soong. Test: Built aosp_arm-user with and without USE_RBE. USE_RBE uses a proxy script in place of rewrapper. Bug: b/166182389 Change-Id: I5bf008a940513872d70b5b215bd6209f759826ae Merged-In: I5bf008a940513872d70b5b215bd6209f759826ae --- android/config.go | 4 ++++ android/variable.go | 1 + ui/build/Android.bp | 1 + ui/build/build.go | 5 +++++ ui/build/config.go | 24 +++++++++++++++++++++ ui/build/ninja.go | 2 +- ui/build/rbe.go | 52 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 ui/build/rbe.go diff --git a/android/config.go b/android/config.go index 3ea361e8a..789911935 100644 --- a/android/config.go +++ b/android/config.go @@ -714,6 +714,10 @@ func (c *config) UseGoma() bool { return Bool(c.productVariables.UseGoma) } +func (c *config) UseRBE() bool { + return Bool(c.productVariables.UseRBE) +} + func (c *config) RunErrorProne() bool { return c.IsEnvTrue("RUN_ERROR_PRONE") } diff --git a/android/variable.go b/android/variable.go index e643c0ebd..28a8d0c0f 100644 --- a/android/variable.go +++ b/android/variable.go @@ -192,6 +192,7 @@ type productVariables struct { HostStaticBinaries *bool `json:",omitempty"` Binder32bit *bool `json:",omitempty"` UseGoma *bool `json:",omitempty"` + UseRBE *bool `json:",omitempty"` Debuggable *bool `json:",omitempty"` Eng *bool `json:",omitempty"` Treble_linker_namespaces *bool `json:",omitempty"` diff --git a/ui/build/Android.bp b/ui/build/Android.bp index 1ddaf68a6..6f81c1730 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -52,6 +52,7 @@ bootstrap_go_package { "ninja.go", "path.go", "proc_sync.go", + "rbe.go", "signal.go", "soong.go", "test_build.go", diff --git a/ui/build/build.go b/ui/build/build.go index 0ae06d61e..58b6da94e 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -161,6 +161,11 @@ func Build(ctx Context, config Config, what int) { startGoma(ctx, config) } + if config.StartRBE() { + // Ensure RBE proxy is started + startRBE(ctx, config) + } + if what&BuildProductConfig != 0 { // Run make for product config runMakeProductConfig(ctx, config) diff --git a/ui/build/config.go b/ui/build/config.go index 7eb3a7252..b105dc0de 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -493,6 +493,30 @@ func (c *configImpl) StartGoma() bool { return true } +func (c *configImpl) UseRBE() bool { + if v, ok := c.environ.Get("USE_RBE"); ok { + v = strings.TrimSpace(v) + if v != "" && v != "false" { + return true + } + } + return false +} + +func (c *configImpl) StartRBE() bool { + if !c.UseRBE() { + return false + } + + if v, ok := c.environ.Get("NOSTART_RBE"); ok { + v = strings.TrimSpace(v) + if v != "" && v != "false" { + return false + } + } + return true +} + // RemoteParallel controls how many remote jobs (i.e., commands which contain // gomacc) are run in parallel. Note the parallelism of all other jobs is // still limited by Parallel() diff --git a/ui/build/ninja.go b/ui/build/ninja.go index 7994f3a2d..7497c94bd 100644 --- a/ui/build/ninja.go +++ b/ui/build/ninja.go @@ -43,7 +43,7 @@ func runNinja(ctx Context, config Config) { args = append(args, config.NinjaArgs()...) var parallel int - if config.UseGoma() { + if config.UseGoma() || config.UseRBE() { parallel = config.RemoteParallel() } else { parallel = config.Parallel() diff --git a/ui/build/rbe.go b/ui/build/rbe.go new file mode 100644 index 000000000..c80b8ea7d --- /dev/null +++ b/ui/build/rbe.go @@ -0,0 +1,52 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "path/filepath" + + "android/soong/ui/metrics" +) + +const bootstrapCmd = "bootstrap" +const rbeLeastNProcs = 2500 +const rbeLeastNFiles = 16000 + +func startRBE(ctx Context, config Config) { + ctx.BeginTrace(metrics.RunSetupTool, "rbe_bootstrap") + defer ctx.EndTrace() + + if u := ulimitOrFatal(ctx, config, "-u"); u < rbeLeastNProcs { + ctx.Fatalf("max user processes is insufficient: %d; want >= %d.\n", u, rbeLeastNProcs) + } + if n := ulimitOrFatal(ctx, config, "-n"); n < rbeLeastNFiles { + ctx.Fatalf("max open files is insufficient: %d; want >= %d.\n", n, rbeLeastNFiles) + } + + var rbeBootstrap string + if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok { + rbeBootstrap = filepath.Join(rbeDir, bootstrapCmd) + } else if home, ok := config.Environment().Get("HOME"); ok { + rbeBootstrap = filepath.Join(home, "rbe", bootstrapCmd) + } else { + ctx.Fatalln("rbe bootstrap not found") + } + + cmd := Command(ctx, config, "bootstrap", rbeBootstrap) + + if output, err := cmd.CombinedOutput(); err != nil { + ctx.Fatalf("rbe bootstrap failed with: %v\n%s\n", err, output) + } +} From 80059e69c7130c4bc324ce0bab921b6e9acbec0b Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 25 Sep 2019 13:31:46 -0700 Subject: [PATCH 08/53] [DO NOT MERGE] Use localPool consistently for UseGoma() == true Remove the distinction between pctx.StaticRule and pctx.AndroidStaticRule so that all of the local rules correctly get assigned to the localPool. Also put Module and Singleton rules into the localPool. Bug: b/166182389 Test: compare out/soong/build.ninja Change-Id: Id2bb38eff3c7209340fe55bc9006f00bd3661d81 Merged-In: Id2bb38eff3c7209340fe55bc9006f00bd3661d81 --- android/module.go | 6 ++++++ android/package_ctx.go | 34 +++++++++++++++++----------------- android/singleton.go | 5 +++++ cc/builder.go | 2 +- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/android/module.go b/android/module.go index 8bdc82e78..2342eae2f 100644 --- a/android/module.go +++ b/android/module.go @@ -1001,6 +1001,12 @@ func (a *androidModuleContext) Variable(pctx PackageContext, name, value string) func (a *androidModuleContext) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule { + if a.config.UseGoma() && params.Pool == nil { + // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the + // local parallelism value + params.Pool = localPool + } + rule := a.ModuleContext.Rule(pctx.PackageContext, name, params, argNames...) if a.config.captureBuild { diff --git a/android/package_ctx.go b/android/package_ctx.go index 00b99ff62..548450e44 100644 --- a/android/package_ctx.go +++ b/android/package_ctx.go @@ -104,7 +104,8 @@ func (p PackageContext) PoolFunc(name string, } // RuleFunc wraps blueprint.PackageContext.RuleFunc, converting the interface{} config -// argument to a Context that supports Config(). +// argument to a Context that supports Config(), and provides a default Pool if none is +// specified. func (p PackageContext) RuleFunc(name string, f func(PackageRuleContext) blueprint.RuleParams, argNames ...string) blueprint.Rule { @@ -114,6 +115,11 @@ func (p PackageContext) RuleFunc(name string, if len(ctx.errors) > 0 { return params, ctx.errors[0] } + if ctx.Config().UseGoma() && params.Pool == nil { + // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the + // local parallelism value + params.Pool = localPool + } return params, nil }, argNames...) } @@ -234,10 +240,16 @@ func (p PackageContext) PrefixedExistentPathsForSourcesVariable( }) } -// AndroidStaticRule wraps blueprint.StaticRule and provides a default Pool if none is specified +// AndroidStaticRule is an alias for StaticRule. func (p PackageContext) AndroidStaticRule(name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule { - return p.AndroidRuleFunc(name, func(PackageRuleContext) blueprint.RuleParams { + return p.StaticRule(name, params, argNames...) +} + +// StaticRule wraps blueprint.StaticRule and provides a default Pool if none is specified. +func (p PackageContext) StaticRule(name string, params blueprint.RuleParams, + argNames ...string) blueprint.Rule { + return p.RuleFunc(name, func(PackageRuleContext) blueprint.RuleParams { return params }, argNames...) } @@ -245,18 +257,6 @@ func (p PackageContext) AndroidStaticRule(name string, params blueprint.RulePara // AndroidGomaStaticRule wraps blueprint.StaticRule but uses goma's parallelism if goma is enabled func (p PackageContext) AndroidGomaStaticRule(name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule { - return p.StaticRule(name, params, argNames...) -} - -func (p PackageContext) AndroidRuleFunc(name string, - f func(PackageRuleContext) blueprint.RuleParams, argNames ...string) blueprint.Rule { - return p.RuleFunc(name, func(ctx PackageRuleContext) blueprint.RuleParams { - params := f(ctx) - if ctx.Config().UseGoma() && params.Pool == nil { - // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the - // local parallelism value - params.Pool = localPool - } - return params - }, argNames...) + // bypass android.PackageContext.StaticRule so that Pool does not get set to local_pool. + return p.PackageContext.StaticRule(name, params, argNames...) } diff --git a/android/singleton.go b/android/singleton.go index a59d54aa2..7f9c21606 100644 --- a/android/singleton.go +++ b/android/singleton.go @@ -127,6 +127,11 @@ func (s *singletonContextAdaptor) Variable(pctx PackageContext, name, value stri } func (s *singletonContextAdaptor) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule { + if s.Config().UseGoma() && params.Pool == nil { + // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the + // local parallelism value + params.Pool = localPool + } rule := s.SingletonContext.Rule(pctx.PackageContext, name, params, argNames...) if s.Config().captureBuild { s.ruleParams[rule] = params diff --git a/cc/builder.go b/cc/builder.go index a9ee4e92c..e448cf3eb 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -209,7 +209,7 @@ var ( _ = pctx.SourcePathVariable("sAbiDiffer", "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/header-abi-diff") - sAbiDiff = pctx.AndroidRuleFunc("sAbiDiff", + sAbiDiff = pctx.RuleFunc("sAbiDiff", func(ctx android.PackageRuleContext) blueprint.RuleParams { // TODO(b/78139997): Add -check-all-apis back commandStr := "($sAbiDiffer ${allowFlags} -lib ${libName} -arch ${arch} -o ${out} -new ${in} -old ${referenceDump})" From c0d95df5d00762a52f42368a2abeb3aa7bf69be4 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Mon, 4 Nov 2019 18:16:11 -0500 Subject: [PATCH 09/53] [DO NOT MERGE] Run non-RBE supported actions in the local pool when USE_RBE is set. Bug: 143938974 Test: ran CTS build at -j500 successfully. Bug: b/166182389 Change-Id: I55074bd67308cd716972e24fb56a20bc393d5d9d Merged-In: I55074bd67308cd716972e24fb56a20bc393d5d9d --- android/module.go | 6 +++--- android/package_ctx.go | 40 +++++++++++++++++++++++++++++++++------- android/singleton.go | 6 +++--- bpf/bpf.go | 2 +- cc/builder.go | 4 ++-- java/builder.go | 2 +- java/kotlin.go | 4 ++-- 7 files changed, 45 insertions(+), 19 deletions(-) diff --git a/android/module.go b/android/module.go index 2342eae2f..d34916fc4 100644 --- a/android/module.go +++ b/android/module.go @@ -1001,9 +1001,9 @@ func (a *androidModuleContext) Variable(pctx PackageContext, name, value string) func (a *androidModuleContext) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule { - if a.config.UseGoma() && params.Pool == nil { - // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the - // local parallelism value + if (a.config.UseGoma() || a.config.UseRBE()) && params.Pool == nil { + // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by goma/RBE, restrict + // jobs to the local parallelism value params.Pool = localPool } diff --git a/android/package_ctx.go b/android/package_ctx.go index 548450e44..cf8facef4 100644 --- a/android/package_ctx.go +++ b/android/package_ctx.go @@ -115,9 +115,9 @@ func (p PackageContext) RuleFunc(name string, if len(ctx.errors) > 0 { return params, ctx.errors[0] } - if ctx.Config().UseGoma() && params.Pool == nil { - // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the - // local parallelism value + if (ctx.Config().UseGoma() || ctx.Config().UseRBE()) && params.Pool == nil { + // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by + // goma/RBE, restrict jobs to the local parallelism value params.Pool = localPool } return params, nil @@ -254,9 +254,35 @@ func (p PackageContext) StaticRule(name string, params blueprint.RuleParams, }, argNames...) } -// AndroidGomaStaticRule wraps blueprint.StaticRule but uses goma's parallelism if goma is enabled -func (p PackageContext) AndroidGomaStaticRule(name string, params blueprint.RuleParams, +// RemoteRuleSupports selects if a AndroidRemoteStaticRule supports goma, RBE, or both. +type RemoteRuleSupports int + +const ( + SUPPORTS_NONE = 0 + SUPPORTS_GOMA = 1 << iota + SUPPORTS_RBE = 1 << iota + SUPPORTS_BOTH = SUPPORTS_GOMA | SUPPORTS_RBE +) + +// AndroidRemoteStaticRule wraps blueprint.StaticRule but uses goma or RBE's parallelism if goma or RBE are enabled +// and the appropriate SUPPORTS_* flag is set. +func (p PackageContext) AndroidRemoteStaticRule(name string, supports RemoteRuleSupports, params blueprint.RuleParams, argNames ...string) blueprint.Rule { - // bypass android.PackageContext.StaticRule so that Pool does not get set to local_pool. - return p.PackageContext.StaticRule(name, params, argNames...) + + return p.PackageContext.RuleFunc(name, func(config interface{}) (blueprint.RuleParams, error) { + ctx := &configErrorWrapper{p, config.(Config), nil} + if ctx.Config().UseGoma() && supports&SUPPORTS_GOMA == 0 { + // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the + // local parallelism value + params.Pool = localPool + } + + if ctx.Config().UseRBE() && supports&SUPPORTS_RBE == 0 { + // When USE_RBE=true is set and the rule is not supported by RBE, restrict jobs to the + // local parallelism value + params.Pool = localPool + } + + return params, nil + }, argNames...) } diff --git a/android/singleton.go b/android/singleton.go index 7f9c21606..0266d7798 100644 --- a/android/singleton.go +++ b/android/singleton.go @@ -127,9 +127,9 @@ func (s *singletonContextAdaptor) Variable(pctx PackageContext, name, value stri } func (s *singletonContextAdaptor) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule { - if s.Config().UseGoma() && params.Pool == nil { - // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the - // local parallelism value + if (s.Config().UseGoma() || s.Config().UseRBE()) && params.Pool == nil { + // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by goma/RBE, restrict + // jobs to the local parallelism value params.Pool = localPool } rule := s.SingletonContext.Rule(pctx.PackageContext, name, params, argNames...) diff --git a/bpf/bpf.go b/bpf/bpf.go index 13468c73f..85eb7daa1 100644 --- a/bpf/bpf.go +++ b/bpf/bpf.go @@ -33,7 +33,7 @@ func init() { var ( pctx = android.NewPackageContext("android/soong/bpf") - cc = pctx.AndroidGomaStaticRule("cc", + cc = pctx.AndroidRemoteStaticRule("cc", android.SUPPORTS_GOMA, blueprint.RuleParams{ Depfile: "${out}.d", Deps: blueprint.DepsGCC, diff --git a/cc/builder.go b/cc/builder.go index e448cf3eb..773e7e816 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -46,7 +46,7 @@ var ( var ( pctx = android.NewPackageContext("android/soong/cc") - cc = pctx.AndroidGomaStaticRule("cc", + cc = pctx.AndroidRemoteStaticRule("cc", android.SUPPORTS_BOTH, blueprint.RuleParams{ Depfile: "${out}.d", Deps: blueprint.DepsGCC, @@ -55,7 +55,7 @@ var ( }, "ccCmd", "cFlags") - ccNoDeps = pctx.AndroidGomaStaticRule("ccNoDeps", + ccNoDeps = pctx.AndroidRemoteStaticRule("ccNoDeps", android.SUPPORTS_GOMA, blueprint.RuleParams{ Command: "$relPwd ${config.CcWrapper}$ccCmd -c $cFlags -o $out $in", CommandDeps: []string{"$ccCmd"}, diff --git a/java/builder.go b/java/builder.go index 2fe4d8ab3..a8e54cc6f 100644 --- a/java/builder.go +++ b/java/builder.go @@ -38,7 +38,7 @@ var ( // this, all java rules write into separate directories and then are combined into a .jar file // (if the rule produces .class files) or a .srcjar file (if the rule produces .java files). // .srcjar files are unzipped into a temporary directory when compiled with javac. - javac = pctx.AndroidGomaStaticRule("javac", + javac = pctx.AndroidRemoteStaticRule("javac", android.SUPPORTS_GOMA, blueprint.RuleParams{ Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + diff --git a/java/kotlin.go b/java/kotlin.go index 81b89f96f..72560301e 100644 --- a/java/kotlin.go +++ b/java/kotlin.go @@ -25,7 +25,7 @@ import ( "github.com/google/blueprint" ) -var kotlinc = pctx.AndroidGomaStaticRule("kotlinc", +var kotlinc = pctx.AndroidRemoteStaticRule("kotlinc", android.SUPPORTS_GOMA, blueprint.RuleParams{ Command: `rm -rf "$classesDir" "$srcJarDir" "$kotlinBuildFile" && mkdir -p "$classesDir" "$srcJarDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + @@ -74,7 +74,7 @@ func kotlinCompile(ctx android.ModuleContext, outputFile android.WritablePath, }) } -var kapt = pctx.AndroidGomaStaticRule("kapt", +var kapt = pctx.AndroidRemoteStaticRule("kapt", android.SUPPORTS_GOMA, blueprint.RuleParams{ Command: `rm -rf "$srcJarDir" "$kotlinBuildFile" "$kaptDir" && mkdir -p "$srcJarDir" "$kaptDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + From 4807a1b5ec60689e553b225c0cf16ace5873d67d Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Mon, 27 Jan 2020 14:19:44 -0500 Subject: [PATCH 10/53] [DO NOT MERGE] Add support for experimentally enabling RBE support on specific rules. This CL adds RBE support to javac, r8, and d8 rules which is only enabled if respective environment variables are set. Test: an aosp_crosshatch build with and without the new variables. Bug: b/166182389 Change-Id: Ic82f3627944f6a5ee7b9f3228170c2709b1bfcb8 Merged-In: Ic82f3627944f6a5ee7b9f3228170c2709b1bfcb8 --- android/config.go | 16 +++++++++++++++ android/package_ctx.go | 44 ++++++++++++++++++++++++++++++++++-------- android/variable.go | 3 +++ bpf/bpf.go | 2 +- cc/builder.go | 4 ++-- java/builder.go | 2 +- java/config/config.go | 14 ++++++++++++++ java/dex.go | 8 ++++---- java/kotlin.go | 4 ++-- ui/build/config.go | 42 ++++++++++++++++++++++++++++++++++++++++ ui/build/dumpvars.go | 2 ++ 11 files changed, 123 insertions(+), 18 deletions(-) diff --git a/android/config.go b/android/config.go index 789911935..2e0c247bb 100644 --- a/android/config.go +++ b/android/config.go @@ -718,6 +718,22 @@ func (c *config) UseRBE() bool { return Bool(c.productVariables.UseRBE) } +func (c *config) UseRBEJAVAC() bool { + return Bool(c.productVariables.UseRBEJAVAC) +} + +func (c *config) UseRBER8() bool { + return Bool(c.productVariables.UseRBER8) +} + +func (c *config) UseRBED8() bool { + return Bool(c.productVariables.UseRBED8) +} + +func (c *config) UseRemoteBuild() bool { + return c.UseGoma() || c.UseRBE() +} + func (c *config) RunErrorProne() bool { return c.IsEnvTrue("RUN_ERROR_PRONE") } diff --git a/android/package_ctx.go b/android/package_ctx.go index cf8facef4..ff10fe78e 100644 --- a/android/package_ctx.go +++ b/android/package_ctx.go @@ -254,16 +254,32 @@ func (p PackageContext) StaticRule(name string, params blueprint.RuleParams, }, argNames...) } -// RemoteRuleSupports selects if a AndroidRemoteStaticRule supports goma, RBE, or both. -type RemoteRuleSupports int +// RBEExperimentalFlag indicates which flag should be set for the AndroidRemoteStaticRule +// to use RBE. +type RBEExperimentalFlag int const ( - SUPPORTS_NONE = 0 - SUPPORTS_GOMA = 1 << iota - SUPPORTS_RBE = 1 << iota - SUPPORTS_BOTH = SUPPORTS_GOMA | SUPPORTS_RBE + // RBE_NOT_EXPERIMENTAL indicates the rule should use RBE in every build that has + // UseRBE set. + RBE_NOT_EXPERIMENTAL RBEExperimentalFlag = iota + // RBE_JAVAC indicates the rule should use RBE only if the RBE_JAVAC variable is + // set in an RBE enabled build. + RBE_JAVAC + // RBE_R8 indicates the rule should use RBE only if the RBE_R8 variable is set in + // an RBE enabled build. + RBE_R8 + // RBE_D8 indicates the rule should use RBE only if the RBE_D8 variable is set in + // an RBE enabled build. + RBE_D8 ) +// RemoteRuleSupports configures rules with whether they have Goma and/or RBE support. +type RemoteRuleSupports struct { + Goma bool + RBE bool + RBEFlag RBEExperimentalFlag +} + // AndroidRemoteStaticRule wraps blueprint.StaticRule but uses goma or RBE's parallelism if goma or RBE are enabled // and the appropriate SUPPORTS_* flag is set. func (p PackageContext) AndroidRemoteStaticRule(name string, supports RemoteRuleSupports, params blueprint.RuleParams, @@ -271,18 +287,30 @@ func (p PackageContext) AndroidRemoteStaticRule(name string, supports RemoteRule return p.PackageContext.RuleFunc(name, func(config interface{}) (blueprint.RuleParams, error) { ctx := &configErrorWrapper{p, config.(Config), nil} - if ctx.Config().UseGoma() && supports&SUPPORTS_GOMA == 0 { + if ctx.Config().UseGoma() && !supports.Goma { // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the // local parallelism value params.Pool = localPool } - if ctx.Config().UseRBE() && supports&SUPPORTS_RBE == 0 { + if ctx.Config().UseRBE() && !supports.RBE { // When USE_RBE=true is set and the rule is not supported by RBE, restrict jobs to the // local parallelism value params.Pool = localPool } + if ctx.Config().UseRBE() && supports.RBE { + if supports.RBEFlag == RBE_JAVAC && !ctx.Config().UseRBEJAVAC() { + params.Pool = localPool + } + if supports.RBEFlag == RBE_R8 && !ctx.Config().UseRBER8() { + params.Pool = localPool + } + if supports.RBEFlag == RBE_D8 && !ctx.Config().UseRBED8() { + params.Pool = localPool + } + } + return params, nil }, argNames...) } diff --git a/android/variable.go b/android/variable.go index 28a8d0c0f..a88fc9d51 100644 --- a/android/variable.go +++ b/android/variable.go @@ -193,6 +193,9 @@ type productVariables struct { Binder32bit *bool `json:",omitempty"` UseGoma *bool `json:",omitempty"` UseRBE *bool `json:",omitempty"` + UseRBEJAVAC *bool `json:",omitempty"` + UseRBER8 *bool `json:",omitempty"` + UseRBED8 *bool `json:",omitempty"` Debuggable *bool `json:",omitempty"` Eng *bool `json:",omitempty"` Treble_linker_namespaces *bool `json:",omitempty"` diff --git a/bpf/bpf.go b/bpf/bpf.go index 85eb7daa1..0047636f0 100644 --- a/bpf/bpf.go +++ b/bpf/bpf.go @@ -33,7 +33,7 @@ func init() { var ( pctx = android.NewPackageContext("android/soong/bpf") - cc = pctx.AndroidRemoteStaticRule("cc", android.SUPPORTS_GOMA, + cc = pctx.AndroidRemoteStaticRule("cc", android.RemoteRuleSupports{Goma: true}, blueprint.RuleParams{ Depfile: "${out}.d", Deps: blueprint.DepsGCC, diff --git a/cc/builder.go b/cc/builder.go index 773e7e816..972f967ae 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -46,7 +46,7 @@ var ( var ( pctx = android.NewPackageContext("android/soong/cc") - cc = pctx.AndroidRemoteStaticRule("cc", android.SUPPORTS_BOTH, + cc = pctx.AndroidRemoteStaticRule("cc", android.RemoteRuleSupports{Goma: true, RBE: true}, blueprint.RuleParams{ Depfile: "${out}.d", Deps: blueprint.DepsGCC, @@ -55,7 +55,7 @@ var ( }, "ccCmd", "cFlags") - ccNoDeps = pctx.AndroidRemoteStaticRule("ccNoDeps", android.SUPPORTS_GOMA, + ccNoDeps = pctx.AndroidRemoteStaticRule("ccNoDeps", android.RemoteRuleSupports{Goma: true}, blueprint.RuleParams{ Command: "$relPwd ${config.CcWrapper}$ccCmd -c $cFlags -o $out $in", CommandDeps: []string{"$ccCmd"}, diff --git a/java/builder.go b/java/builder.go index a8e54cc6f..e5ac7b00a 100644 --- a/java/builder.go +++ b/java/builder.go @@ -38,7 +38,7 @@ var ( // this, all java rules write into separate directories and then are combined into a .jar file // (if the rule produces .class files) or a .srcjar file (if the rule produces .java files). // .srcjar files are unzipped into a temporary directory when compiled with javac. - javac = pctx.AndroidRemoteStaticRule("javac", android.SUPPORTS_GOMA, + javac = pctx.AndroidRemoteStaticRule("javac", android.RemoteRuleSupports{Goma: true, RBE: true, RBEFlag: android.RBE_JAVAC}, blueprint.RuleParams{ Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + diff --git a/java/config/config.go b/java/config/config.go index f6688a281..f76a39315 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -129,6 +129,20 @@ func init() { return "" }) + pctx.VariableFunc("R8Wrapper", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("R8_WRAPPER"); override != "" { + return override + " " + } + return "" + }) + + pctx.VariableFunc("D8Wrapper", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("D8_WRAPPER"); override != "" { + return override + " " + } + return "" + }) + pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar") hostBinToolVariableWithPrebuilt := func(name, prebuiltDir, tool string) { diff --git a/java/dex.go b/java/dex.go index c8a4fa8ed..86a28fc18 100644 --- a/java/dex.go +++ b/java/dex.go @@ -22,10 +22,10 @@ import ( "android/soong/android" ) -var d8 = pctx.AndroidStaticRule("d8", +var d8 = pctx.AndroidRemoteStaticRule("d8", android.RemoteRuleSupports{RBE: true, RBEFlag: android.RBE_D8}, blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + - `${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` + + `${config.D8Wrapper}${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` + `${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + `${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`, CommandDeps: []string{ @@ -36,11 +36,11 @@ var d8 = pctx.AndroidStaticRule("d8", }, "outDir", "d8Flags", "zipFlags") -var r8 = pctx.AndroidStaticRule("r8", +var r8 = pctx.AndroidRemoteStaticRule("r8", android.RemoteRuleSupports{RBE: true, RBEFlag: android.RBE_R8}, blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + `rm -f "$outDict" && ` + - `${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` + + `${config.R8Wrapper}${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` + `--force-proguard-compatibility ` + `--no-data-resources ` + `-printmapping $outDict ` + diff --git a/java/kotlin.go b/java/kotlin.go index 72560301e..58dc64cd1 100644 --- a/java/kotlin.go +++ b/java/kotlin.go @@ -25,7 +25,7 @@ import ( "github.com/google/blueprint" ) -var kotlinc = pctx.AndroidRemoteStaticRule("kotlinc", android.SUPPORTS_GOMA, +var kotlinc = pctx.AndroidRemoteStaticRule("kotlinc", android.RemoteRuleSupports{Goma: true}, blueprint.RuleParams{ Command: `rm -rf "$classesDir" "$srcJarDir" "$kotlinBuildFile" && mkdir -p "$classesDir" "$srcJarDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + @@ -74,7 +74,7 @@ func kotlinCompile(ctx android.ModuleContext, outputFile android.WritablePath, }) } -var kapt = pctx.AndroidRemoteStaticRule("kapt", android.SUPPORTS_GOMA, +var kapt = pctx.AndroidRemoteStaticRule("kapt", android.RemoteRuleSupports{Goma: true}, blueprint.RuleParams{ Command: `rm -rf "$srcJarDir" "$kotlinBuildFile" "$kaptDir" && mkdir -p "$srcJarDir" "$kaptDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + diff --git a/ui/build/config.go b/ui/build/config.go index b105dc0de..daefb2f73 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -503,6 +503,48 @@ func (c *configImpl) UseRBE() bool { return false } +func (c *configImpl) UseRBEJAVAC() bool { + if !c.UseRBE() { + return false + } + + if v, ok := c.environ.Get("RBE_JAVAC"); ok { + v = strings.TrimSpace(v) + if v != "" && v != "false" { + return true + } + } + return false +} + +func (c *configImpl) UseRBER8() bool { + if !c.UseRBE() { + return false + } + + if v, ok := c.environ.Get("RBE_R8"); ok { + v = strings.TrimSpace(v) + if v != "" && v != "false" { + return true + } + } + return false +} + +func (c *configImpl) UseRBED8() bool { + if !c.UseRBE() { + return false + } + + if v, ok := c.environ.Get("RBE_D8"); ok { + v = strings.TrimSpace(v) + if v != "" && v != "false" { + return true + } + } + return false +} + func (c *configImpl) StartRBE() bool { if !c.UseRBE() { return false diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index 3e387c1ee..2a6a9ca7e 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -178,6 +178,8 @@ func runMakeProductConfig(ctx Context, config Config) { "CC_WRAPPER", "CXX_WRAPPER", "JAVAC_WRAPPER", + "R8_WRAPPER", + "D8_WRAPPER", // ccache settings "CCACHE_COMPILERCHECK", From 65e10fba52d3eccfc44c9d66820909adcb87a2f8 Mon Sep 17 00:00:00 2001 From: Kousik Kumar Date: Mon, 17 Feb 2020 00:26:55 -0800 Subject: [PATCH 11/53] [DO NOT MERGE] Do not add ccWrapper to ccNoDeps rule The "ccNoDeps" rule was introduced to separate out ".s" compilations from other ".S" and ".c*" compilations. The ".s" compilation does not produce a dependency file and does not support running a preprocessor through it. However, it does have ".include" directives, that do NOT take a macro, but still does the equivalent of a "#include". The compilation of ".s" assembly files also do NOT produce a dependency file. Because they don't produce a dependency file AND because RBE's input processor does not yet support finding dependencies for these files, I'm making this change to not prepend rewrapper for these commands. Test: Tested by running a build with this change and ensuring the ".s" compilations aren't being sent to rewrapper. Bug: b/166182389 Change-Id: I60bb14ff92596e4992e9f675bdc199f1440d4327 Merged-In: I60bb14ff92596e4992e9f675bdc199f1440d4327 --- cc/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cc/builder.go b/cc/builder.go index 972f967ae..5641b7d15 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -55,9 +55,9 @@ var ( }, "ccCmd", "cFlags") - ccNoDeps = pctx.AndroidRemoteStaticRule("ccNoDeps", android.RemoteRuleSupports{Goma: true}, + ccNoDeps = pctx.AndroidStaticRule("ccNoDeps", blueprint.RuleParams{ - Command: "$relPwd ${config.CcWrapper}$ccCmd -c $cFlags -o $out $in", + Command: "$relPwd $ccCmd -c $cFlags -o $out $in", CommandDeps: []string{"$ccCmd"}, }, "ccCmd", "cFlags") From b2da01d455cfab87fda2c42895673a848329b91a Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Fri, 3 Apr 2020 08:40:19 -0700 Subject: [PATCH 12/53] Add a new protobuf message named Upload to upload build metrics. A new protobuf message named Upload is defined to send build metrics to the Android Build Server API after a build is completed. These metrics are collected from googlers only, as described in go/local-developers-metrics-wfh. Bug: b/140638454 Test: m nothing Change-Id: I5e49b5ace053913be2b75673cb1375f4cef37825 Merged-In: I5e49b5ace053913be2b75673cb1375f4cef37825 --- ui/metrics/Android.bp | 9 ++ ui/metrics/upload_proto/regen.sh | 17 ++++ ui/metrics/upload_proto/upload.pb.go | 122 +++++++++++++++++++++++++++ ui/metrics/upload_proto/upload.proto | 35 ++++++++ 4 files changed, 183 insertions(+) create mode 100755 ui/metrics/upload_proto/regen.sh create mode 100644 ui/metrics/upload_proto/upload.pb.go create mode 100644 ui/metrics/upload_proto/upload.proto diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp index 529639de5..3596e1019 100644 --- a/ui/metrics/Android.bp +++ b/ui/metrics/Android.bp @@ -17,6 +17,7 @@ bootstrap_go_package { pkgPath: "android/soong/ui/metrics", deps: [ "golang-protobuf-proto", + "soong-ui-metrics_upload_proto", "soong-ui-metrics_proto", "soong-ui-tracer", ], @@ -35,3 +36,11 @@ bootstrap_go_package { ], } +bootstrap_go_package { + name: "soong-ui-metrics_upload_proto", + pkgPath: "android/soong/ui/metrics/upload_proto", + deps: ["golang-protobuf-proto"], + srcs: [ + "upload_proto/upload.pb.go", + ], +} diff --git a/ui/metrics/upload_proto/regen.sh b/ui/metrics/upload_proto/regen.sh new file mode 100755 index 000000000..4521df7bd --- /dev/null +++ b/ui/metrics/upload_proto/regen.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Generates the golang source file of upload.proto file. + +set -e + +function die() { echo "ERROR: $1" >&2; exit 1; } + +readonly error_msg="Maybe you need to run 'lunch aosp_arm-eng && m aprotoc blueprint_tools'?" + +if ! hash aprotoc &>/dev/null; then + die "could not find aprotoc. ${error_msg}" +fi + +if ! aprotoc --go_out=paths=source_relative:. upload.proto; then + die "build failed. ${error_msg}" +fi diff --git a/ui/metrics/upload_proto/upload.pb.go b/ui/metrics/upload_proto/upload.pb.go new file mode 100644 index 000000000..1b1e5e843 --- /dev/null +++ b/ui/metrics/upload_proto/upload.pb.go @@ -0,0 +1,122 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: upload.proto + +package soong_metrics_upload_proto + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type Upload struct { + // The timestamp in milliseconds that the build was created. + CreationTimestampMs *uint64 `protobuf:"varint,1,opt,name=creation_timestamp_ms,json=creationTimestampMs" json:"creation_timestamp_ms,omitempty"` + // The timestamp in milliseconds when the build was completed. + CompletionTimestampMs *uint64 `protobuf:"varint,2,opt,name=completion_timestamp_ms,json=completionTimestampMs" json:"completion_timestamp_ms,omitempty"` + // The branch name. + BranchName *string `protobuf:"bytes,3,opt,name=branch_name,json=branchName" json:"branch_name,omitempty"` + // The target name. + TargetName *string `protobuf:"bytes,4,opt,name=target_name,json=targetName" json:"target_name,omitempty"` + // A list of metrics filepaths to upload. + MetricsFiles []string `protobuf:"bytes,5,rep,name=metrics_files,json=metricsFiles" json:"metrics_files,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Upload) Reset() { *m = Upload{} } +func (m *Upload) String() string { return proto.CompactTextString(m) } +func (*Upload) ProtoMessage() {} +func (*Upload) Descriptor() ([]byte, []int) { + return fileDescriptor_91b94b655bd2a7e5, []int{0} +} + +func (m *Upload) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Upload.Unmarshal(m, b) +} +func (m *Upload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Upload.Marshal(b, m, deterministic) +} +func (m *Upload) XXX_Merge(src proto.Message) { + xxx_messageInfo_Upload.Merge(m, src) +} +func (m *Upload) XXX_Size() int { + return xxx_messageInfo_Upload.Size(m) +} +func (m *Upload) XXX_DiscardUnknown() { + xxx_messageInfo_Upload.DiscardUnknown(m) +} + +var xxx_messageInfo_Upload proto.InternalMessageInfo + +func (m *Upload) GetCreationTimestampMs() uint64 { + if m != nil && m.CreationTimestampMs != nil { + return *m.CreationTimestampMs + } + return 0 +} + +func (m *Upload) GetCompletionTimestampMs() uint64 { + if m != nil && m.CompletionTimestampMs != nil { + return *m.CompletionTimestampMs + } + return 0 +} + +func (m *Upload) GetBranchName() string { + if m != nil && m.BranchName != nil { + return *m.BranchName + } + return "" +} + +func (m *Upload) GetTargetName() string { + if m != nil && m.TargetName != nil { + return *m.TargetName + } + return "" +} + +func (m *Upload) GetMetricsFiles() []string { + if m != nil { + return m.MetricsFiles + } + return nil +} + +func init() { + proto.RegisterType((*Upload)(nil), "soong_metrics_upload.Upload") +} + +func init() { + proto.RegisterFile("upload.proto", fileDescriptor_91b94b655bd2a7e5) +} + +var fileDescriptor_91b94b655bd2a7e5 = []byte{ + // 201 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x2d, 0xc8, 0xc9, + 0x4f, 0x4c, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x29, 0xce, 0xcf, 0xcf, 0x4b, 0x8f, + 0xcf, 0x4d, 0x2d, 0x29, 0xca, 0x4c, 0x2e, 0x8e, 0x87, 0xc8, 0x29, 0xdd, 0x66, 0xe4, 0x62, 0x0b, + 0x05, 0x33, 0x85, 0x8c, 0xb8, 0x44, 0x93, 0x8b, 0x52, 0x13, 0x4b, 0x32, 0xf3, 0xf3, 0xe2, 0x4b, + 0x32, 0x73, 0x53, 0x8b, 0x4b, 0x12, 0x73, 0x0b, 0xe2, 0x73, 0x8b, 0x25, 0x18, 0x15, 0x18, 0x35, + 0x58, 0x82, 0x84, 0x61, 0x92, 0x21, 0x30, 0x39, 0xdf, 0x62, 0x21, 0x33, 0x2e, 0xf1, 0xe4, 0xfc, + 0xdc, 0x82, 0x9c, 0x54, 0x4c, 0x5d, 0x4c, 0x60, 0x5d, 0xa2, 0x08, 0x69, 0x64, 0x7d, 0xf2, 0x5c, + 0xdc, 0x49, 0x45, 0x89, 0x79, 0xc9, 0x19, 0xf1, 0x79, 0x89, 0xb9, 0xa9, 0x12, 0xcc, 0x0a, 0x8c, + 0x1a, 0x9c, 0x41, 0x5c, 0x10, 0x21, 0xbf, 0xc4, 0xdc, 0x54, 0x90, 0x82, 0x92, 0xc4, 0xa2, 0xf4, + 0xd4, 0x12, 0x88, 0x02, 0x16, 0x88, 0x02, 0x88, 0x10, 0x58, 0x81, 0x32, 0x17, 0x2f, 0xcc, 0x2b, + 0x69, 0x99, 0x39, 0xa9, 0xc5, 0x12, 0xac, 0x0a, 0xcc, 0x1a, 0x9c, 0x41, 0x3c, 0x50, 0x41, 0x37, + 0x90, 0x98, 0x93, 0x4c, 0x94, 0x14, 0x36, 0x5f, 0xc7, 0x83, 0x43, 0x04, 0x10, 0x00, 0x00, 0xff, + 0xff, 0xe2, 0x01, 0x74, 0x65, 0x20, 0x01, 0x00, 0x00, +} diff --git a/ui/metrics/upload_proto/upload.proto b/ui/metrics/upload_proto/upload.proto new file mode 100644 index 000000000..7a9f080e0 --- /dev/null +++ b/ui/metrics/upload_proto/upload.proto @@ -0,0 +1,35 @@ +// Copyright 2020 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package soong_metrics_upload; +option go_package = "soong_metrics_upload_proto"; + +message Upload { + // The timestamp in milliseconds that the build was created. + optional uint64 creation_timestamp_ms = 1; + + // The timestamp in milliseconds when the build was completed. + optional uint64 completion_timestamp_ms = 2; + + // The branch name. + optional string branch_name = 3; + + // The target name. + optional string target_name = 4; + + // A list of metrics filepaths to upload. + repeated string metrics_files = 5; +} From 2ce077605ab2e4b0a16f871f9e04b4af7b16ad10 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Thu, 4 Jun 2020 19:34:41 +0000 Subject: [PATCH 13/53] Pass the TMPDIR directly to the command when dumping makefile variables. External applications such as metrics uploader depends on the TMPDIR. DumpMakeVars is changing the TMPDIR environment variable value. Once the work is done, TMPDIR directory is deleted. Additional flow of the tool or external applications can panic since TMPDIR directory does not exist. Since this is an isolated case, pass the TMPDIR directory to the command of dumping the makefile variables. Bug: b/140638454 Test: * Ran lunch command with ANDROID_ENABLE_METRICS_UPLOAD set and metrics_uploader did not crash. * m nothing Change-Id: Ib78a3f8cfb5da48f65bd9bce1511e9abdf3d9cca Merged-In: Ib78a3f8cfb5da48f65bd9bce1511e9abdf3d9cca --- ui/build/dumpvars.go | 15 ++++++++------- ui/build/path.go | 12 ++++++++---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index a3234e339..bd073e53f 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -54,18 +54,16 @@ func DumpMakeVars(ctx Context, config Config, goals, vars []string) (map[string] var ret map[string]string if len(makeVars) > 0 { + // It's not safe to use the same TMPDIR as the build, as that can be removed. tmpDir, err := ioutil.TempDir("", "dumpvars") if err != nil { return nil, err } defer os.RemoveAll(tmpDir) - // It's not safe to use the same TMPDIR as the build, as that can be removed. - config.Environment().Set("TMPDIR", tmpDir) + SetupLitePath(ctx, config, tmpDir) - SetupLitePath(ctx, config) - - ret, err = dumpMakeVars(ctx, config, goals, makeVars, false) + ret, err = dumpMakeVars(ctx, config, goals, makeVars, false, tmpDir) if err != nil { return ret, err } @@ -82,7 +80,7 @@ func DumpMakeVars(ctx Context, config Config, goals, vars []string) (map[string] return ret, nil } -func dumpMakeVars(ctx Context, config Config, goals, vars []string, write_soong_vars bool) (map[string]string, error) { +func dumpMakeVars(ctx Context, config Config, goals, vars []string, write_soong_vars bool, tmpDir string) (map[string]string, error) { ctx.BeginTrace(metrics.RunKati, "dumpvars") defer ctx.EndTrace() @@ -98,6 +96,9 @@ func dumpMakeVars(ctx Context, config Config, goals, vars []string, write_soong_ cmd.Environment.Set("WRITE_SOONG_VARIABLES", "true") } cmd.Environment.Set("DUMP_MANY_VARS", strings.Join(vars, " ")) + if tmpDir != "" { + cmd.Environment.Set("TMPDIR", tmpDir) + } cmd.Sandbox = dumpvarsSandbox output := bytes.Buffer{} cmd.Stdout = &output @@ -262,7 +263,7 @@ func runMakeProductConfig(ctx Context, config Config) { "BUILD_BROKEN_USES_BUILD_TARGET_TEST_CONFIG", }, exportEnvVars...), BannerVars...) - make_vars, err := dumpMakeVars(ctx, config, config.Arguments(), allVars, true) + make_vars, err := dumpMakeVars(ctx, config, config.Arguments(), allVars, true, "") if err != nil { ctx.Fatalln("Error dumping make vars:", err) } diff --git a/ui/build/path.go b/ui/build/path.go index c34ba1b52..f515775e3 100644 --- a/ui/build/path.go +++ b/ui/build/path.go @@ -55,8 +55,9 @@ func parsePathDir(dir string) []string { } // A "lite" version of SetupPath used for dumpvars, or other places that need -// minimal overhead (but at the expense of logging). -func SetupLitePath(ctx Context, config Config) { +// minimal overhead (but at the expense of logging). If tmpDir is empty, the +// default TMPDIR is used from config. +func SetupLitePath(ctx Context, config Config, tmpDir string) { if config.pathReplaced { return } @@ -65,8 +66,11 @@ func SetupLitePath(ctx Context, config Config) { defer ctx.EndTrace() origPath, _ := config.Environment().Get("PATH") - myPath, _ := config.Environment().Get("TMPDIR") - myPath = filepath.Join(myPath, "path") + + if tmpDir == "" { + tmpDir, _ = config.Environment().Get("TMPDIR") + } + myPath := filepath.Join(tmpDir, "path") ensureEmptyDirectoriesExist(ctx, myPath) os.Setenv("PATH", origPath) From 4f7f60b54bf9b2df3c6d6fdce484cfe86886a83e Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Mon, 1 Jun 2020 17:29:30 +0000 Subject: [PATCH 14/53] Upload build metrics after a build is completed. Soong now supports the ability to upload metrics to another location by setting the ANDROID_ENABLE_METRICS_UPLOAD to an uploader that accepts the upload.proto proto buffer message. When the environment variable is set, a set of build metrics files (soong_metrics, rbe_metrics.pb and build_error) is uploaded. Bug: 140638454 Test: * Wrote unit test cases * Setup the uploader, built a succcessful and failed aosp_arm-eng target and monitor the uploading of the metrics. Change-Id: I76a65739c557dc90345e098ca03119a950ece2d2 Merged-In: I76a65739c557dc90345e098ca03119a950ece2d2 --- cmd/soong_ui/main.go | 11 +++- ui/build/Android.bp | 2 + ui/build/config.go | 11 ++++ ui/build/config_test.go | 2 + ui/build/upload.go | 80 +++++++++++++++++++++++++++ ui/build/upload_test.go | 120 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 ui/build/upload.go create mode 100644 ui/build/upload_test.go diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index db61fba5c..7ae7fd681 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -117,6 +117,8 @@ func inList(s string, list []string) bool { // Command is the type of soong_ui execution. Only one type of // execution is specified. The args are specific to the command. func main() { + buildStartedMilli := time.Now().UnixNano() / int64(time.Millisecond) + c, args := getCommand(os.Args) if c == nil { fmt.Fprintf(os.Stderr, "The `soong` native UI is not yet available.\n") @@ -166,19 +168,24 @@ func main() { logsDir = filepath.Join(config.DistDir(), "logs") } + buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error") + rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb") + soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics") + defer build.UploadMetrics(buildCtx, config, buildStartedMilli, buildErrorFile, rbeMetricsFile, soongMetricsFile) + os.MkdirAll(logsDir, 0777) log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log")) trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace")) stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log"))) stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log"))) - stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"build_error"))) + stat.AddOutput(status.NewProtoErrorLog(log, buildErrorFile)) stat.AddOutput(status.NewCriticalPath(log)) buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(1024*1024*1024)) buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v", config.Parallel(), config.RemoteParallel(), config.HighmemParallel()) - defer met.Dump(filepath.Join(logsDir, c.logsPrefix+"soong_metrics")) + defer met.Dump(soongMetricsFile) if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { if !strings.HasSuffix(start, "N") { diff --git a/ui/build/Android.bp b/ui/build/Android.bp index 2a5a51adb..0a0bb1614 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -56,12 +56,14 @@ bootstrap_go_package { "signal.go", "soong.go", "test_build.go", + "upload.go", "util.go", ], testSrcs: [ "cleanbuild_test.go", "config_test.go", "environment_test.go", + "upload_test.go", "util_test.go", "proc_sync_test.go", ], diff --git a/ui/build/config.go b/ui/build/config.go index 3a1188bdf..a85b38755 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -962,3 +962,14 @@ func (c *configImpl) SetPdkBuild(pdk bool) { func (c *configImpl) IsPdkBuild() bool { return c.pdkBuild } + +func (c *configImpl) BuildDateTime() string { + return c.buildDateTime +} + +func (c *configImpl) MetricsUploaderApp() string { + if p, ok := c.environ.Get("ANDROID_ENABLE_METRICS_UPLOAD"); ok { + return p + } + return "" +} diff --git a/ui/build/config_test.go b/ui/build/config_test.go index df618c4ec..7b14c4703 100644 --- a/ui/build/config_test.go +++ b/ui/build/config_test.go @@ -26,6 +26,7 @@ import ( "testing" "android/soong/ui/logger" + "android/soong/ui/status" ) func testContext() Context { @@ -33,6 +34,7 @@ func testContext() Context { Context: context.Background(), Logger: logger.New(&bytes.Buffer{}), Writer: &bytes.Buffer{}, + Status: &status.Status{}, }} } diff --git a/ui/build/upload.go b/ui/build/upload.go new file mode 100644 index 000000000..75a5e2fc8 --- /dev/null +++ b/ui/build/upload.go @@ -0,0 +1,80 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +// This file contains the functionality to upload data from one location to +// another. + +import ( + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/golang/protobuf/proto" + + upload_proto "android/soong/ui/metrics/upload_proto" +) + +const ( + uploadPbFilename = ".uploader.pb" +) + +// UploadMetrics uploads a set of metrics files to a server for analysis. An +// uploader full path is required to be specified in order to upload the set +// of metrics files. This is accomplished by defining the ANDROID_ENABLE_METRICS_UPLOAD +// environment variable. +func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files ...string) { + uploader := config.MetricsUploaderApp() + // No metrics to upload if the path to the uploader was not specified. + if uploader == "" { + return + } + + // Some files may not exist. For example, build errors protobuf file + // may not exist since the build was successful. + var metricsFiles []string + for _, f := range files { + if _, err := os.Stat(f); err == nil { + metricsFiles = append(metricsFiles, f) + } + } + + if len(metricsFiles) == 0 { + return + } + + // For platform builds, the branch and target name is hardcoded to specific + // values for later extraction of the metrics in the data metrics pipeline. + data, err := proto.Marshal(&upload_proto.Upload{ + CreationTimestampMs: proto.Uint64(uint64(buildStartedMilli)), + CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))), + BranchName: proto.String("developer-metrics"), + TargetName: proto.String("platform-build-systems-metrics"), + MetricsFiles: metricsFiles, + }) + if err != nil { + ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err) + } + + pbFile := filepath.Join(config.OutDir(), uploadPbFilename) + if err := ioutil.WriteFile(pbFile, data, 0644); err != nil { + ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err) + } + // Remove the upload file as it's not longer needed after it has been processed by the uploader. + defer os.Remove(pbFile) + + Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile).RunAndStreamOrFatal() +} diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go new file mode 100644 index 000000000..adaa08d61 --- /dev/null +++ b/ui/build/upload_test.go @@ -0,0 +1,120 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "android/soong/ui/logger" +) + +func TestUploadMetrics(t *testing.T) { + ctx := testContext() + tests := []struct { + description string + uploader string + createFiles bool + files []string + }{{ + description: "ANDROID_ENABLE_METRICS_UPLOAD not set", + }, { + description: "no metrics files to upload", + uploader: "fake", + }, { + description: "non-existent metrics files no upload", + uploader: "fake", + files: []string{"metrics_file_1", "metrics_file_2", "metrics_file_3"}, + }, { + description: "trigger upload", + uploader: "echo", + createFiles: true, + files: []string{"metrics_file_1", "metrics_file_2"}, + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + t.Fatalf("got unexpected error: %v", err) + }) + + outDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create out directory: %v", outDir) + } + defer os.RemoveAll(outDir) + + var metricsFiles []string + if tt.createFiles { + for _, f := range tt.files { + filename := filepath.Join(outDir, f) + metricsFiles = append(metricsFiles, filename) + if err := ioutil.WriteFile(filename, []byte("test file"), 0644); err != nil { + t.Fatalf("failed to create a fake metrics file %q for uploading: %v", filename, err) + } + } + } + + config := Config{&configImpl{ + environ: &Environment{ + "OUT_DIR=" + outDir, + "ANDROID_ENABLE_METRICS_UPLOAD=" + tt.uploader, + }, + buildDateTime: strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10), + }} + + UploadMetrics(ctx, config, 1591031903, metricsFiles...) + + if _, err := os.Stat(filepath.Join(outDir, uploadPbFilename)); err == nil { + t.Error("got true, want false for upload protobuf file to exist") + } + }) + } +} + +func TestUploadMetricsErrors(t *testing.T) { + expectedErr := "failed to write the marshaled" + defer logger.Recover(func(err error) { + got := err.Error() + if !strings.Contains(got, expectedErr) { + t.Errorf("got %q, want %q to be contained in error", got, expectedErr) + } + }) + + outDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create out directory: %v", outDir) + } + defer os.RemoveAll(outDir) + + metricsFile := filepath.Join(outDir, "metrics_file_1") + if err := ioutil.WriteFile(metricsFile, []byte("test file"), 0644); err != nil { + t.Fatalf("failed to create a fake metrics file %q for uploading: %v", metricsFile, err) + } + + config := Config{&configImpl{ + environ: &Environment{ + "ANDROID_ENABLE_METRICS_UPLOAD=fake", + "OUT_DIR=/bad", + }}} + + UploadMetrics(testContext(), config, 1591031903, metricsFile) + t.Errorf("got nil, expecting %q as a failure", expectedErr) +} From de44afac93ac5bba06f29f40caec91ce56a31a98 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Wed, 10 Jun 2020 18:48:01 +0000 Subject: [PATCH 15/53] Run the metrics uploader in the background. The metrics uploader was currently running on foreground where it would copy the metrics files in a separate directory and then forked into the background for the upload process. As a result, running the lunch command would take a second longer to run since each metrics uploader run had an average of half a second. Bug: 140638454 Test: * Wrote and updated unit test cases. * Set ANDROID_ENABLE_METRICS_UPLOAD to point to the latest metrics_uploader bash script. Executed the "lunch 1" command and measured the running time. Executed "m nothing" command and checked that the metrics were uploaded. * Ran "lunch 1" and "m nothing" with ANDROID_ENABLE_METRICS_UPLOAD="" * Removed oauth from metrics_uploader and ran "m nothing" and "lunch 1". The oauth Message appeared only to "m nothing" Change-Id: I13c61e666c8f44613dee291a704cef6a27335188 Merged-In: I13c61e666c8f44613dee291a704cef6a27335188 --- cmd/soong_ui/main.go | 2 +- ui/build/upload.go | 45 +++++++++++-- ui/build/upload_test.go | 98 +++++++++++++++++++--------- ui/build/util.go | 18 +++++ ui/build/util_test.go | 70 ++++++++++++++++++++ ui/metrics/upload_proto/upload.pb.go | 42 +++++++----- ui/metrics/upload_proto/upload.proto | 4 ++ 7 files changed, 227 insertions(+), 52 deletions(-) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 7ae7fd681..0471a6ef2 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -171,7 +171,7 @@ func main() { buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error") rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb") soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics") - defer build.UploadMetrics(buildCtx, config, buildStartedMilli, buildErrorFile, rbeMetricsFile, soongMetricsFile) + defer build.UploadMetrics(buildCtx, config, c.forceDumbOutput, buildStartedMilli, buildErrorFile, rbeMetricsFile, soongMetricsFile) os.MkdirAll(logsDir, 0777) log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log")) diff --git a/ui/build/upload.go b/ui/build/upload.go index 75a5e2fc8..3a23a8013 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -23,6 +23,7 @@ import ( "path/filepath" "time" + "android/soong/ui/metrics" "github.com/golang/protobuf/proto" upload_proto "android/soong/ui/metrics/upload_proto" @@ -32,11 +33,21 @@ const ( uploadPbFilename = ".uploader.pb" ) +var ( + // For testing purpose + getTmpDir = ioutil.TempDir +) + // UploadMetrics uploads a set of metrics files to a server for analysis. An // uploader full path is required to be specified in order to upload the set // of metrics files. This is accomplished by defining the ANDROID_ENABLE_METRICS_UPLOAD -// environment variable. -func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files ...string) { +// environment variable. The metrics files are copied to a temporary directory +// and the uploader is then executed in the background to allow the user to continue +// working. +func UploadMetrics(ctx Context, config Config, forceDumbOutput bool, buildStartedMilli int64, files ...string) { + ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics") + defer ctx.EndTrace() + uploader := config.MetricsUploaderApp() // No metrics to upload if the path to the uploader was not specified. if uploader == "" { @@ -56,6 +67,22 @@ func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files .. return } + // The temporary directory cannot be deleted as the metrics uploader is started + // in the background and requires to exist until the operation is done. The + // uploader can delete the directory as it is specified in the upload proto. + tmpDir, err := getTmpDir("", "upload_metrics") + if err != nil { + ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err) + } + + for i, src := range metricsFiles { + dst := filepath.Join(tmpDir, filepath.Base(src)) + if _, err := copyFile(src, dst); err != nil { + ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err) + } + metricsFiles[i] = dst + } + // For platform builds, the branch and target name is hardcoded to specific // values for later extraction of the metrics in the data metrics pipeline. data, err := proto.Marshal(&upload_proto.Upload{ @@ -64,17 +91,23 @@ func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files .. BranchName: proto.String("developer-metrics"), TargetName: proto.String("platform-build-systems-metrics"), MetricsFiles: metricsFiles, + DirectoriesToDelete: []string{tmpDir}, }) if err != nil { ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err) } - pbFile := filepath.Join(config.OutDir(), uploadPbFilename) + pbFile := filepath.Join(tmpDir, uploadPbFilename) if err := ioutil.WriteFile(pbFile, data, 0644); err != nil { ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err) } - // Remove the upload file as it's not longer needed after it has been processed by the uploader. - defer os.Remove(pbFile) - Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile).RunAndStreamOrFatal() + // Start the uploader in the background as it takes several milliseconds to start the uploader + // and prepare the metrics for upload. This affects small commands like "lunch". + cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile) + if forceDumbOutput { + cmd.RunOrFatal() + } else { + cmd.RunAndStreamOrFatal() + } } diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go index adaa08d61..c812730a8 100644 --- a/ui/build/upload_test.go +++ b/ui/build/upload_test.go @@ -15,6 +15,7 @@ package build import ( + "errors" "io/ioutil" "os" "path/filepath" @@ -61,6 +62,22 @@ func TestUploadMetrics(t *testing.T) { } defer os.RemoveAll(outDir) + // Supply our own getTmpDir to delete the temp dir once the test is done. + orgGetTmpDir := getTmpDir + getTmpDir = func(string, string) (string, error) { + retDir := filepath.Join(outDir, "tmp_upload_dir") + if err := os.Mkdir(retDir, 0755); err != nil { + t.Fatalf("failed to create temporary directory %q: %v", retDir, err) + } + return retDir, nil + } + defer func() { getTmpDir = orgGetTmpDir }() + + metricsUploadDir := filepath.Join(outDir, ".metrics_uploader") + if err := os.Mkdir(metricsUploadDir, 0755); err != nil { + t.Fatalf("failed to create %q directory for oauth valid check: %v", metricsUploadDir, err) + } + var metricsFiles []string if tt.createFiles { for _, f := range tt.files { @@ -80,41 +97,62 @@ func TestUploadMetrics(t *testing.T) { buildDateTime: strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10), }} - UploadMetrics(ctx, config, 1591031903, metricsFiles...) - - if _, err := os.Stat(filepath.Join(outDir, uploadPbFilename)); err == nil { - t.Error("got true, want false for upload protobuf file to exist") - } + UploadMetrics(ctx, config, false, 1591031903, metricsFiles...) }) } } func TestUploadMetricsErrors(t *testing.T) { - expectedErr := "failed to write the marshaled" - defer logger.Recover(func(err error) { - got := err.Error() - if !strings.Contains(got, expectedErr) { - t.Errorf("got %q, want %q to be contained in error", got, expectedErr) - } - }) + ctx := testContext() + tests := []struct { + description string + tmpDir string + tmpDirErr error + expectedErr string + }{{ + description: "getTmpDir returned error", + tmpDirErr: errors.New("getTmpDir failed"), + expectedErr: "getTmpDir failed", + }, { + description: "copyFile operation error", + tmpDir: "/fake_dir", + expectedErr: "failed to copy", + }} - outDir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("failed to create out directory: %v", outDir) + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + got := err.Error() + if !strings.Contains(got, tt.expectedErr) { + t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr) + } + }) + + outDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create out directory: %v", outDir) + } + defer os.RemoveAll(outDir) + + orgGetTmpDir := getTmpDir + getTmpDir = func(string, string) (string, error) { + return tt.tmpDir, tt.tmpDirErr + } + defer func() { getTmpDir = orgGetTmpDir }() + + metricsFile := filepath.Join(outDir, "metrics_file_1") + if err := ioutil.WriteFile(metricsFile, []byte("test file"), 0644); err != nil { + t.Fatalf("failed to create a fake metrics file %q for uploading: %v", metricsFile, err) + } + + config := Config{&configImpl{ + environ: &Environment{ + "ANDROID_ENABLE_METRICS_UPLOAD=fake", + "OUT_DIR=/bad", + }}} + + UploadMetrics(ctx, config, true, 1591031903, metricsFile) + t.Errorf("got nil, expecting %q as a failure", tt.expectedErr) + }) } - defer os.RemoveAll(outDir) - - metricsFile := filepath.Join(outDir, "metrics_file_1") - if err := ioutil.WriteFile(metricsFile, []byte("test file"), 0644); err != nil { - t.Fatalf("failed to create a fake metrics file %q for uploading: %v", metricsFile, err) - } - - config := Config{&configImpl{ - environ: &Environment{ - "ANDROID_ENABLE_METRICS_UPLOAD=fake", - "OUT_DIR=/bad", - }}} - - UploadMetrics(testContext(), config, 1591031903, metricsFile) - t.Errorf("got nil, expecting %q as a failure", expectedErr) } diff --git a/ui/build/util.go b/ui/build/util.go index 75e6753b7..d44cd6daf 100644 --- a/ui/build/util.go +++ b/ui/build/util.go @@ -15,6 +15,7 @@ package build import ( + "io" "os" "path/filepath" "strings" @@ -124,3 +125,20 @@ func decodeKeyValue(str string) (string, string, bool) { } return str[:idx], str[idx+1:], true } + +// copyFile copies a file from src to dst. filepath.Dir(dst) must exist. +func copyFile(src, dst string) (int64, error) { + source, err := os.Open(src) + if err != nil { + return 0, err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return 0, err + } + defer destination.Close() + + return io.Copy(destination, source) +} diff --git a/ui/build/util_test.go b/ui/build/util_test.go index 89bfc7786..b22e9977f 100644 --- a/ui/build/util_test.go +++ b/ui/build/util_test.go @@ -15,6 +15,7 @@ package build import ( + "bytes" "io/ioutil" "os" "path/filepath" @@ -49,3 +50,72 @@ func TestEnsureEmptyDirs(t *testing.T) { ensureEmptyDirectoriesExist(ctx, filepath.Join(tmpDir, "a")) } + +func TestCopyFile(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "test_copy_file") + if err != nil { + t.Fatalf("failed to create temporary directory to hold test text files: %v", err) + } + defer os.Remove(tmpDir) + + data := []byte("fake data") + src := filepath.Join(tmpDir, "src.txt") + if err := ioutil.WriteFile(src, data, 0755); err != nil { + t.Fatalf("failed to create a src file %q for copying: %v", src, err) + } + + dst := filepath.Join(tmpDir, "dst.txt") + + l, err := copyFile(src, dst) + if err != nil { + t.Fatalf("got %v, expecting nil error on copyFile operation", err) + } + + if l != int64(len(data)) { + t.Errorf("got %d, expecting %d for copied bytes", l, len(data)) + } + + dstData, err := ioutil.ReadFile(dst) + if err != nil { + t.Fatalf("got %v, expecting nil error reading dst %q file", err, dst) + } + + if bytes.Compare(data, dstData) != 0 { + t.Errorf("got %q, expecting data %q from dst %q text file", string(data), string(dstData), dst) + } +} + +func TestCopyFileErrors(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "test_copy_file_errors") + if err != nil { + t.Fatalf("failed to create temporary directory to hold test text files: %v", err) + } + defer os.Remove(tmpDir) + + srcExists := filepath.Join(tmpDir, "src_exist.txt") + if err := ioutil.WriteFile(srcExists, []byte("fake data"), 0755); err != nil { + t.Fatalf("failed to create a src file %q for copying: %v", srcExists, err) + } + + tests := []struct { + description string + src string + dst string + }{{ + description: "src file does not exist", + src: "/src/not/exist", + dst: "/dst/not/exist", + }, { + description: "dst directory does not exist", + src: srcExists, + dst: "/dst/not/exist", + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + if _, err := copyFile(tt.src, tt.dst); err == nil { + t.Errorf("got nil, expecting error") + } + }) + } +} diff --git a/ui/metrics/upload_proto/upload.pb.go b/ui/metrics/upload_proto/upload.pb.go index 1b1e5e843..614d4c7a4 100644 --- a/ui/metrics/upload_proto/upload.pb.go +++ b/ui/metrics/upload_proto/upload.pb.go @@ -30,7 +30,10 @@ type Upload struct { // The target name. TargetName *string `protobuf:"bytes,4,opt,name=target_name,json=targetName" json:"target_name,omitempty"` // A list of metrics filepaths to upload. - MetricsFiles []string `protobuf:"bytes,5,rep,name=metrics_files,json=metricsFiles" json:"metrics_files,omitempty"` + MetricsFiles []string `protobuf:"bytes,5,rep,name=metrics_files,json=metricsFiles" json:"metrics_files,omitempty"` + // A list of directories to delete after the copy of metrics files + // is completed for uploading. + DirectoriesToDelete []string `protobuf:"bytes,6,rep,name=directories_to_delete,json=directoriesToDelete" json:"directories_to_delete,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -96,6 +99,13 @@ func (m *Upload) GetMetricsFiles() []string { return nil } +func (m *Upload) GetDirectoriesToDelete() []string { + if m != nil { + return m.DirectoriesToDelete + } + return nil +} + func init() { proto.RegisterType((*Upload)(nil), "soong_metrics_upload.Upload") } @@ -105,18 +115,20 @@ func init() { } var fileDescriptor_91b94b655bd2a7e5 = []byte{ - // 201 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x2d, 0xc8, 0xc9, - 0x4f, 0x4c, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x29, 0xce, 0xcf, 0xcf, 0x4b, 0x8f, - 0xcf, 0x4d, 0x2d, 0x29, 0xca, 0x4c, 0x2e, 0x8e, 0x87, 0xc8, 0x29, 0xdd, 0x66, 0xe4, 0x62, 0x0b, - 0x05, 0x33, 0x85, 0x8c, 0xb8, 0x44, 0x93, 0x8b, 0x52, 0x13, 0x4b, 0x32, 0xf3, 0xf3, 0xe2, 0x4b, - 0x32, 0x73, 0x53, 0x8b, 0x4b, 0x12, 0x73, 0x0b, 0xe2, 0x73, 0x8b, 0x25, 0x18, 0x15, 0x18, 0x35, - 0x58, 0x82, 0x84, 0x61, 0x92, 0x21, 0x30, 0x39, 0xdf, 0x62, 0x21, 0x33, 0x2e, 0xf1, 0xe4, 0xfc, - 0xdc, 0x82, 0x9c, 0x54, 0x4c, 0x5d, 0x4c, 0x60, 0x5d, 0xa2, 0x08, 0x69, 0x64, 0x7d, 0xf2, 0x5c, - 0xdc, 0x49, 0x45, 0x89, 0x79, 0xc9, 0x19, 0xf1, 0x79, 0x89, 0xb9, 0xa9, 0x12, 0xcc, 0x0a, 0x8c, - 0x1a, 0x9c, 0x41, 0x5c, 0x10, 0x21, 0xbf, 0xc4, 0xdc, 0x54, 0x90, 0x82, 0x92, 0xc4, 0xa2, 0xf4, - 0xd4, 0x12, 0x88, 0x02, 0x16, 0x88, 0x02, 0x88, 0x10, 0x58, 0x81, 0x32, 0x17, 0x2f, 0xcc, 0x2b, - 0x69, 0x99, 0x39, 0xa9, 0xc5, 0x12, 0xac, 0x0a, 0xcc, 0x1a, 0x9c, 0x41, 0x3c, 0x50, 0x41, 0x37, - 0x90, 0x98, 0x93, 0x4c, 0x94, 0x14, 0x36, 0x5f, 0xc7, 0x83, 0x43, 0x04, 0x10, 0x00, 0x00, 0xff, - 0xff, 0xe2, 0x01, 0x74, 0x65, 0x20, 0x01, 0x00, 0x00, + // 230 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0xb1, 0x4a, 0x04, 0x31, + 0x10, 0x86, 0xd9, 0xbb, 0xf3, 0xe0, 0xe2, 0xd9, 0xec, 0x79, 0x18, 0x44, 0x70, 0xd1, 0x66, 0x2b, + 0x0b, 0x0b, 0x1f, 0x40, 0xc4, 0x4e, 0x8b, 0xe5, 0x6c, 0x6c, 0x86, 0x98, 0x1d, 0xd7, 0x40, 0x92, + 0x09, 0xc9, 0xf8, 0x1c, 0xbe, 0xb2, 0x6c, 0xe2, 0xe2, 0x82, 0x76, 0xc3, 0xff, 0x7d, 0x7f, 0x31, + 0xbf, 0xd8, 0x7e, 0x06, 0x4b, 0xaa, 0xbf, 0x09, 0x91, 0x98, 0xea, 0xd3, 0x44, 0xe4, 0x07, 0x70, + 0xc8, 0xd1, 0xe8, 0x04, 0x85, 0x5d, 0x7d, 0x2d, 0xc4, 0xfa, 0x25, 0x9f, 0xf5, 0xad, 0xd8, 0xeb, + 0x88, 0x8a, 0x0d, 0x79, 0x60, 0xe3, 0x30, 0xb1, 0x72, 0x01, 0x5c, 0x92, 0x55, 0x53, 0xb5, 0xab, + 0x6e, 0x37, 0xc1, 0xc3, 0xc4, 0x9e, 0x52, 0x7d, 0x27, 0xce, 0x34, 0xb9, 0x60, 0xf1, 0x6f, 0x6b, + 0x91, 0x5b, 0xfb, 0x5f, 0x3c, 0xef, 0x5d, 0x8a, 0xe3, 0xb7, 0xa8, 0xbc, 0xfe, 0x00, 0xaf, 0x1c, + 0xca, 0x65, 0x53, 0xb5, 0x9b, 0x4e, 0x94, 0xe8, 0x59, 0x39, 0x1c, 0x05, 0x56, 0x71, 0x40, 0x2e, + 0xc2, 0xaa, 0x08, 0x25, 0xca, 0xc2, 0xb5, 0x38, 0x99, 0x5e, 0x79, 0x37, 0x16, 0x93, 0x3c, 0x6a, + 0x96, 0xed, 0xa6, 0xdb, 0xfe, 0x84, 0x8f, 0x63, 0x36, 0xbe, 0xd4, 0x9b, 0x88, 0x9a, 0x29, 0x1a, + 0x4c, 0xc0, 0x04, 0x3d, 0x5a, 0x64, 0x94, 0xeb, 0x2c, 0xef, 0x66, 0xf0, 0x40, 0x0f, 0x19, 0xdd, + 0x5f, 0xbc, 0x9e, 0xff, 0xb7, 0x14, 0xe4, 0x15, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x64, 0x04, + 0xa8, 0xf4, 0x54, 0x01, 0x00, 0x00, } diff --git a/ui/metrics/upload_proto/upload.proto b/ui/metrics/upload_proto/upload.proto index 7a9f080e0..bcd0ab204 100644 --- a/ui/metrics/upload_proto/upload.proto +++ b/ui/metrics/upload_proto/upload.proto @@ -32,4 +32,8 @@ message Upload { // A list of metrics filepaths to upload. repeated string metrics_files = 5; + + // A list of directories to delete after the copy of metrics files + // is completed for uploading. + repeated string directories_to_delete = 6; } From e7836594eceb09c5921a7a79f7454c0967aaf38c Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Tue, 7 Jul 2020 12:48:26 +0000 Subject: [PATCH 16/53] Add RBE metrics dump in Soong UI. From aosp/1329396, the RBE metrics protobuf file is part of the metrics uploading process. The RBE metrics protobuf file is generated by running the bootstrap shutdown command. A new function named DumpRBEMetrics was written in order to generate the RBE metrics protobuf file before sending to the uploading process. Bug: b/140638454 Test: * Unit test cases * Ran RBE build on my local host and verified the metrics protobuf file is created. * Ran non-RBE build after RBE build and verified that the previous metrics protobuf file was deleted. Change-Id: I4b8068905cb67c4b8c2d94793917b98974fed707 Merged-In: I4b8068905cb67c4b8c2d94793917b98974fed707 --- cmd/soong_ui/main.go | 1 + ui/build/Android.bp | 1 + ui/build/config.go | 9 +++ ui/build/rbe.go | 83 ++++++++++++++++++---- ui/build/rbe_test.go | 162 ++++++++++++++++++++++++++++++++++++++++++ ui/metrics/metrics.go | 13 ++-- 6 files changed, 250 insertions(+), 19 deletions(-) create mode 100644 ui/build/rbe_test.go diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 0471a6ef2..1a4d1747e 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -186,6 +186,7 @@ func main() { config.Parallel(), config.RemoteParallel(), config.HighmemParallel()) defer met.Dump(soongMetricsFile) + defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile) if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { if !strings.HasSuffix(start, "N") { diff --git a/ui/build/Android.bp b/ui/build/Android.bp index 0a0bb1614..4ef27212a 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -63,6 +63,7 @@ bootstrap_go_package { "cleanbuild_test.go", "config_test.go", "environment_test.go", + "rbe_test.go", "upload_test.go", "util_test.go", "proc_sync_test.go", diff --git a/ui/build/config.go b/ui/build/config.go index a85b38755..e701854d0 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -804,6 +804,15 @@ func (c *configImpl) StartRBE() bool { return true } +func (c *configImpl) RBEStatsOutputDir() string { + for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + return "" +} + func (c *configImpl) UseRemoteBuild() bool { return c.UseGoma() || c.UseRBE() } diff --git a/ui/build/rbe.go b/ui/build/rbe.go index ceea4bf2b..fcdab3b03 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -15,14 +15,39 @@ package build import ( + "os" "path/filepath" "android/soong/ui/metrics" ) -const bootstrapCmd = "bootstrap" -const rbeLeastNProcs = 2500 -const rbeLeastNFiles = 16000 +const ( + rbeLeastNProcs = 2500 + rbeLeastNFiles = 16000 + + // prebuilt RBE binaries + bootstrapCmd = "bootstrap" + + // RBE metrics proto buffer file + rbeMetricsPBFilename = "rbe_metrics.pb" +) + +func rbeCommand(ctx Context, config Config, rbeCmd string) string { + var cmdPath string + if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok { + cmdPath = filepath.Join(rbeDir, rbeCmd) + } else if home, ok := config.Environment().Get("HOME"); ok { + cmdPath = filepath.Join(home, "rbe", rbeCmd) + } else { + ctx.Fatalf("rbe command path not found") + } + + if _, err := os.Stat(cmdPath); err != nil && os.IsNotExist(err) { + ctx.Fatalf("rbe command %q not found", rbeCmd) + } + + return cmdPath +} func startRBE(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSetupTool, "rbe_bootstrap") @@ -35,18 +60,50 @@ func startRBE(ctx Context, config Config) { ctx.Fatalf("max open files is insufficient: %d; want >= %d.\n", n, rbeLeastNFiles) } - var rbeBootstrap string - if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok { - rbeBootstrap = filepath.Join(rbeDir, bootstrapCmd) - } else if home, ok := config.Environment().Get("HOME"); ok { - rbeBootstrap = filepath.Join(home, "rbe", bootstrapCmd) - } else { - ctx.Fatalln("rbe bootstrap not found") - } - - cmd := Command(ctx, config, "boostrap", rbeBootstrap) + cmd := Command(ctx, config, "startRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd)) if output, err := cmd.CombinedOutput(); err != nil { ctx.Fatalf("rbe bootstrap failed with: %v\n%s\n", err, output) } } + +func stopRBE(ctx Context, config Config) { + cmd := Command(ctx, config, "stopRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd), "-shutdown") + if output, err := cmd.CombinedOutput(); err != nil { + ctx.Fatalf("rbe bootstrap with shutdown failed with: %v\n%s\n", err, output) + } +} + +// DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics. +// The protobuf file is created if RBE is enabled and the proxy service has +// started. The proxy service is shutdown in order to dump the RBE metrics to the +// protobuf file. +func DumpRBEMetrics(ctx Context, config Config, filename string) { + ctx.BeginTrace(metrics.RunShutdownTool, "dump_rbe_metrics") + defer ctx.EndTrace() + + // Remove the previous metrics file in case there is a failure or RBE has been + // disable for this run. + os.Remove(filename) + + // If RBE is not enabled then there are no metrics to generate. + // If RBE does not require to start, the RBE proxy maybe started + // manually for debugging purpose and can generate the metrics + // afterwards. + if !config.StartRBE() { + return + } + + outputDir := config.RBEStatsOutputDir() + if outputDir == "" { + ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.") + } + metricsFile := filepath.Join(outputDir, rbeMetricsPBFilename) + + // Stop the proxy first in order to generate the RBE metrics protobuf file. + stopRBE(ctx, config) + + if _, err := copyFile(metricsFile, filename); err != nil { + ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err) + } +} diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go new file mode 100644 index 000000000..acbca650c --- /dev/null +++ b/ui/build/rbe_test.go @@ -0,0 +1,162 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "android/soong/ui/logger" +) + +func TestDumpRBEMetrics(t *testing.T) { + ctx := testContext() + tests := []struct { + description string + env []string + generated bool + }{{ + description: "RBE disabled", + env: []string{ + "NOSTART_RBE=true", + }, + }, { + description: "rbe metrics generated", + env: []string{ + "USE_RBE=true", + }, + generated: true, + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create a temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd) + if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(rbeBootstrapProgram), 0755); err != nil { + t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err) + } + + env := Environment(tt.env) + env.Set("OUT_DIR", tmpDir) + env.Set("RBE_DIR", tmpDir) + + tmpRBEDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create a temp directory for RBE: %v", err) + } + defer os.RemoveAll(tmpRBEDir) + env.Set("RBE_output_dir", tmpRBEDir) + + config := Config{&configImpl{ + environ: &env, + }} + + rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename) + DumpRBEMetrics(ctx, config, rbeMetricsFilename) + + // Validate that the rbe metrics file exists if RBE is enabled. + if _, err := os.Stat(rbeMetricsFilename); err == nil { + if !tt.generated { + t.Errorf("got true, want false for rbe metrics file %s to exist.", rbeMetricsFilename) + } + } else if os.IsNotExist(err) { + if tt.generated { + t.Errorf("got false, want true for rbe metrics file %s to exist.", rbeMetricsFilename) + } + } else { + t.Errorf("unknown error found on checking %s exists: %v", rbeMetricsFilename, err) + } + }) + } +} + +func TestDumpRBEMetricsErrors(t *testing.T) { + ctx := testContext() + tests := []struct { + description string + rbeOutputDirDefined bool + bootstrapProgram string + expectedErr string + }{{ + description: "output_dir not defined", + bootstrapProgram: rbeBootstrapProgram, + expectedErr: "RBE output dir variable not defined", + }, { + description: "stopRBE failed", + rbeOutputDirDefined: true, + bootstrapProgram: "#!/bin/bash\nexit 1", + expectedErr: "shutdown failed", + }, { + description: "failed to copy metrics file", + rbeOutputDirDefined: true, + bootstrapProgram: "#!/bin/bash", + expectedErr: "failed to copy", + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + got := err.Error() + if !strings.Contains(got, tt.expectedErr) { + t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr) + } + }) + + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create a temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd) + if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(tt.bootstrapProgram), 0755); err != nil { + t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err) + } + + env := &Environment{} + env.Set("USE_RBE", "true") + env.Set("OUT_DIR", tmpDir) + env.Set("RBE_DIR", tmpDir) + + if tt.rbeOutputDirDefined { + tmpRBEDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create a temp directory for RBE: %v", err) + } + defer os.RemoveAll(tmpRBEDir) + env.Set("RBE_output_dir", tmpRBEDir) + } + + config := Config{&configImpl{ + environ: env, + }} + + rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename) + DumpRBEMetrics(ctx, config, rbeMetricsFilename) + t.Errorf("got nil, expecting %q as a failure", tt.expectedErr) + }) + } +} + +var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s", rbeMetricsPBFilename) diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 8254e4a2a..7f104c879 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -25,12 +25,13 @@ import ( ) const ( - RunSetupTool = "setup" - RunKati = "kati" - RunSoong = "soong" - PrimaryNinja = "ninja" - TestRun = "test" - Total = "total" + PrimaryNinja = "ninja" + RunKati = "kati" + RunSetupTool = "setup" + RunShutdownTool = "shutdown" + RunSoong = "soong" + TestRun = "test" + Total = "total" ) type Metrics struct { From 70ca506da3f31d4c3311d56bcb20075d297d9a88 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Sat, 1 Aug 2020 16:49:35 +0000 Subject: [PATCH 17/53] Add newline to each generated bash script in rbe_test.go. "exec format error" showed up on Mac when running the rbe_test.go. The generated bash scripts did not have a newline at the end of the file. Bug: b/162600461 Fixes: b/162600461 Test: "m blueprint_tools" on Android Mac builder. Change-Id: I425fa5f428733b6569c3fcdaa312023f9761d020 Merged-In: I425fa5f428733b6569c3fcdaa312023f9761d020 --- ui/build/rbe_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go index acbca650c..92ef77921 100644 --- a/ui/build/rbe_test.go +++ b/ui/build/rbe_test.go @@ -105,12 +105,12 @@ func TestDumpRBEMetricsErrors(t *testing.T) { }, { description: "stopRBE failed", rbeOutputDirDefined: true, - bootstrapProgram: "#!/bin/bash\nexit 1", + bootstrapProgram: "#!/bin/bash\nexit 1\n", expectedErr: "shutdown failed", }, { description: "failed to copy metrics file", rbeOutputDirDefined: true, - bootstrapProgram: "#!/bin/bash", + bootstrapProgram: "#!/bin/bash\n", expectedErr: "failed to copy", }} @@ -159,4 +159,4 @@ func TestDumpRBEMetricsErrors(t *testing.T) { } } -var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s", rbeMetricsPBFilename) +var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s\n", rbeMetricsPBFilename) From ac91c3f79654ec7daceed0e54132473b2f12f0e5 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Mon, 13 Jul 2020 18:21:14 +0000 Subject: [PATCH 18/53] Use monotonic clock instead of wall time clock for metrics. From several dashboards, there are high spikes from several builds where the real time of executing a part of the build system took many many hours. This is due to using the wall time clock when calculating the duration. For metrics, the monotonic clock should be used for more accurate measurement. Also, cleaned up the time.go file to use the proper proto functions when marshalling perf protobuf and deleted unnecessary functions. Bug: b/143492984 Test: Wrote and ran unit test case. Change-Id: I1540cfbf37915e3045ecedaa2595c1f9429bcd62 Merged-In: I1540cfbf37915e3045ecedaa2595c1f9429bcd62 --- ui/metrics/Android.bp | 3 +++ ui/metrics/time.go | 36 +++++++++++++++++------------------ ui/metrics/time_test.go | 42 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 19 deletions(-) create mode 100644 ui/metrics/time_test.go diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp index 3596e1019..8188a69e9 100644 --- a/ui/metrics/Android.bp +++ b/ui/metrics/Android.bp @@ -25,6 +25,9 @@ bootstrap_go_package { "metrics.go", "time.go", ], + testSrcs: [ + "time_test.go", + ], } bootstrap_go_package { diff --git a/ui/metrics/time.go b/ui/metrics/time.go index b8baf16fd..401656337 100644 --- a/ui/metrics/time.go +++ b/ui/metrics/time.go @@ -19,13 +19,18 @@ import ( "android/soong/ui/metrics/metrics_proto" "android/soong/ui/tracer" + "github.com/golang/protobuf/proto" ) +// for testing purpose only +var _now = now + type timeEvent struct { desc string name string - atNanos uint64 // timestamp measured in nanoseconds since the reference date + // the time that the event started to occur. + start time.Time } type TimeTracer interface { @@ -39,33 +44,26 @@ type timeTracerImpl struct { var _ TimeTracer = &timeTracerImpl{} -func (t *timeTracerImpl) now() uint64 { - return uint64(time.Now().UnixNano()) +func now() time.Time { + return time.Now() } -func (t *timeTracerImpl) Begin(name, desc string, thread tracer.Thread) { - t.beginAt(name, desc, t.now()) +func (t *timeTracerImpl) Begin(name, desc string, _ tracer.Thread) { + t.activeEvents = append(t.activeEvents, timeEvent{name: name, desc: desc, start: _now()}) } -func (t *timeTracerImpl) beginAt(name, desc string, atNanos uint64) { - t.activeEvents = append(t.activeEvents, timeEvent{name: name, desc: desc, atNanos: atNanos}) -} - -func (t *timeTracerImpl) End(thread tracer.Thread) soong_metrics_proto.PerfInfo { - return t.endAt(t.now()) -} - -func (t *timeTracerImpl) endAt(atNanos uint64) soong_metrics_proto.PerfInfo { +func (t *timeTracerImpl) End(tracer.Thread) soong_metrics_proto.PerfInfo { if len(t.activeEvents) < 1 { panic("Internal error: No pending events for endAt to end!") } lastEvent := t.activeEvents[len(t.activeEvents)-1] t.activeEvents = t.activeEvents[:len(t.activeEvents)-1] - realTime := atNanos - lastEvent.atNanos + realTime := uint64(_now().Sub(lastEvent.start).Nanoseconds()) return soong_metrics_proto.PerfInfo{ - Desc: &lastEvent.desc, - Name: &lastEvent.name, - StartTime: &lastEvent.atNanos, - RealTime: &realTime} + Desc: proto.String(lastEvent.desc), + Name: proto.String(lastEvent.name), + StartTime: proto.Uint64(uint64(lastEvent.start.UnixNano())), + RealTime: proto.Uint64(realTime), + } } diff --git a/ui/metrics/time_test.go b/ui/metrics/time_test.go new file mode 100644 index 000000000..d73080ab2 --- /dev/null +++ b/ui/metrics/time_test.go @@ -0,0 +1,42 @@ +// Copyright 2020 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "testing" + "time" + + "android/soong/ui/tracer" +) + +func TestEnd(t *testing.T) { + startTime := time.Date(2020, time.July, 13, 13, 0, 0, 0, time.UTC) + dur := time.Nanosecond * 10 + initialNow := _now + _now = func() time.Time { return startTime.Add(dur) } + defer func() { _now = initialNow }() + + timeTracer := &timeTracerImpl{} + timeTracer.activeEvents = append(timeTracer.activeEvents, timeEvent{ + desc: "test", + name: "test", + start: startTime, + }) + + perf := timeTracer.End(tracer.Thread(0)) + if perf.GetRealTime() != uint64(dur.Nanoseconds()) { + t.Errorf("got %d, want %d nanoseconds for event duration", perf.GetRealTime(), dur.Nanoseconds()) + } +} From dd3020f35141341aab85a94d55d5a236c10f7fd5 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Mon, 13 Jul 2020 23:01:18 +0000 Subject: [PATCH 19/53] Use the build start time from Soong main UI for metrics build timestamp. Currently, the build timestamp saved to the build_date_timestamp field in the MetricsBase metrics protobuf usually comes from a text file named build_date.txt. It may not be accurate for metrics purposes. Instead, use the build start time from Soong main UI for a better timestamp. Bug: b/140638454 Test: Ran and checked the build_date_timestamp by running the printproto command. Change-Id: I4d893fbbf9830ba21911e56ae13bc5272be47ae6 Merged-In: I4d893fbbf9830ba21911e56ae13bc5272be47ae6 --- cmd/soong_ui/main.go | 5 +++-- ui/build/config.go | 3 --- ui/build/upload.go | 4 ++-- ui/build/upload_test.go | 4 ++-- ui/metrics/metrics.go | 12 +++--------- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 1a4d1747e..78e636cf0 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -117,7 +117,7 @@ func inList(s string, list []string) bool { // Command is the type of soong_ui execution. Only one type of // execution is specified. The args are specific to the command. func main() { - buildStartedMilli := time.Now().UnixNano() / int64(time.Millisecond) + buildStarted := time.Now() c, args := getCommand(os.Args) if c == nil { @@ -138,6 +138,7 @@ func main() { defer trace.Close() met := metrics.New() + met.SetBuildDateTime(buildStarted) stat := &status.Status{} defer stat.Finish() @@ -171,7 +172,7 @@ func main() { buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error") rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb") soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics") - defer build.UploadMetrics(buildCtx, config, c.forceDumbOutput, buildStartedMilli, buildErrorFile, rbeMetricsFile, soongMetricsFile) + defer build.UploadMetrics(buildCtx, config, c.forceDumbOutput, buildStarted, buildErrorFile, rbeMetricsFile, soongMetricsFile) os.MkdirAll(logsDir, 0777) log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log")) diff --git a/ui/build/config.go b/ui/build/config.go index e701854d0..b208a45a9 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -256,9 +256,6 @@ func NewConfig(ctx Context, args ...string) Config { ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10) } - if ctx.Metrics != nil { - ctx.Metrics.SetBuildDateTime(ret.buildDateTime) - } ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) return Config{ret} diff --git a/ui/build/upload.go b/ui/build/upload.go index 3a23a8013..1cc2e940c 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -44,7 +44,7 @@ var ( // environment variable. The metrics files are copied to a temporary directory // and the uploader is then executed in the background to allow the user to continue // working. -func UploadMetrics(ctx Context, config Config, forceDumbOutput bool, buildStartedMilli int64, files ...string) { +func UploadMetrics(ctx Context, config Config, forceDumbOutput bool, buildStarted time.Time, files ...string) { ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics") defer ctx.EndTrace() @@ -86,7 +86,7 @@ func UploadMetrics(ctx Context, config Config, forceDumbOutput bool, buildStarte // For platform builds, the branch and target name is hardcoded to specific // values for later extraction of the metrics in the data metrics pipeline. data, err := proto.Marshal(&upload_proto.Upload{ - CreationTimestampMs: proto.Uint64(uint64(buildStartedMilli)), + CreationTimestampMs: proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))), CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))), BranchName: proto.String("developer-metrics"), TargetName: proto.String("platform-build-systems-metrics"), diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go index c812730a8..dccf156a2 100644 --- a/ui/build/upload_test.go +++ b/ui/build/upload_test.go @@ -97,7 +97,7 @@ func TestUploadMetrics(t *testing.T) { buildDateTime: strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10), }} - UploadMetrics(ctx, config, false, 1591031903, metricsFiles...) + UploadMetrics(ctx, config, false, time.Now(), metricsFiles...) }) } } @@ -151,7 +151,7 @@ func TestUploadMetricsErrors(t *testing.T) { "OUT_DIR=/bad", }}} - UploadMetrics(ctx, config, true, 1591031903, metricsFile) + UploadMetrics(ctx, config, true, time.Now(), metricsFile) t.Errorf("got nil, expecting %q as a failure", tt.expectedErr) }) } diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 7f104c879..95c9129bc 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -17,7 +17,7 @@ package metrics import ( "io/ioutil" "os" - "strconv" + "time" "github.com/golang/protobuf/proto" @@ -131,14 +131,8 @@ func (m *Metrics) getArch(arch string) *soong_metrics_proto.MetricsBase_Arch { } } -func (m *Metrics) SetBuildDateTime(date_time string) { - if date_time != "" { - date_time_timestamp, err := strconv.ParseInt(date_time, 10, 64) - if err != nil { - panic(err) - } - m.metrics.BuildDateTimestamp = &date_time_timestamp - } +func (m *Metrics) SetBuildDateTime(buildTimestamp time.Time) { + m.metrics.BuildDateTimestamp = proto.Int64(buildTimestamp.UnixNano() / int64(time.Second)) } // exports the output to the file at outputPath From 0db3d35c51d8f780c24dfb1fb764b026aefd4f19 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Fri, 7 Aug 2020 17:55:23 +0000 Subject: [PATCH 20/53] Set the OS name in the metrics protobuf file There are cases where the developer is not building a product so the metadata information is not being set. For now, set the OS name so we can track which OS the user is using it. Fixes: b/163118212 Test: m nothing and checked the blueprint Change-Id: I9b521eee2d266b50625aa8e916ddaacb4e2feaea Merged-In: I9b521eee2d266b50625aa8e916ddaacb4e2feaea --- ui/metrics/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 95c9129bc..1b284aa61 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -17,6 +17,7 @@ package metrics import ( "io/ioutil" "os" + "runtime" "time" "github.com/golang/protobuf/proto" @@ -98,8 +99,6 @@ func (m *Metrics) SetMetadataMetrics(metadata map[string]string) { m.metrics.HostArch = m.getArch(v) case "HOST_2ND_ARCH": m.metrics.Host_2NdArch = m.getArch(v) - case "HOST_OS": - m.metrics.HostOs = proto.String(v) case "HOST_OS_EXTRA": m.metrics.HostOsExtra = proto.String(v) case "HOST_CROSS_OS": @@ -137,6 +136,7 @@ func (m *Metrics) SetBuildDateTime(buildTimestamp time.Time) { // exports the output to the file at outputPath func (m *Metrics) Dump(outputPath string) (err error) { + m.metrics.HostOs = proto.String(runtime.GOOS) return writeMessageToFile(&m.metrics, outputPath) } From 09a832cb112618fa760505c1245820be26d13981 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Tue, 11 Aug 2020 20:41:11 +0000 Subject: [PATCH 21/53] Start collecting build configuration metrics. This is an initial CL that collects the environment variables USE_GOMA and USE_RBE. This is a needed metric to determine how many users are currently using GOMA so we can help them migrate to RBE. Bug: b/163598127 Test: m nothing and checked soong_metrics Change-Id: I39536eabad01aafbb3bd7182515d72230b36f658 Merged-In: I39536eabad01aafbb3bd7182515d72230b36f658 --- ui/build/config.go | 21 ++- ui/metrics/metrics.go | 4 + ui/metrics/metrics_proto/metrics.pb.go | 186 ++++++++++++++++--------- ui/metrics/metrics_proto/metrics.proto | 8 ++ 4 files changed, 155 insertions(+), 64 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index b208a45a9..f0310d5a7 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -23,6 +23,9 @@ import ( "time" "android/soong/shared" + "github.com/golang/protobuf/proto" + + smpb "android/soong/ui/metrics/metrics_proto" ) type Config struct{ *configImpl } @@ -258,7 +261,9 @@ func NewConfig(ctx Context, args ...string) Config { ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) - return Config{ret} + c := Config{ret} + storeConfigMetrics(ctx, c) + return c } // NewBuildActionConfig returns a build configuration based on the build action. The arguments are @@ -267,6 +272,20 @@ func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...s return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...) } +// storeConfigMetrics selects a set of configuration information and store in +// the metrics system for further analysis. +func storeConfigMetrics(ctx Context, config Config) { + if ctx.Metrics == nil { + return + } + + b := &smpb.BuildConfig{ + UseGoma: proto.Bool(config.UseGoma()), + UseRbe: proto.Bool(config.UseRBE()), + } + ctx.Metrics.BuildConfig(b) +} + // getConfigArgs processes the command arguments based on the build action and creates a set of new // arguments to be accepted by Config. func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string { diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 1b284aa61..65cae4a1b 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -66,6 +66,10 @@ func (m *Metrics) SetTimeMetrics(perf soong_metrics_proto.PerfInfo) { } } +func (m *Metrics) BuildConfig(b *soong_metrics_proto.BuildConfig) { + m.metrics.BuildConfig = b +} + func (m *Metrics) SetMetadataMetrics(metadata map[string]string) { for k, v := range metadata { switch k { diff --git a/ui/metrics/metrics_proto/metrics.pb.go b/ui/metrics/metrics_proto/metrics.pb.go index 3986d0efd..aaa858765 100644 --- a/ui/metrics/metrics_proto/metrics.pb.go +++ b/ui/metrics/metrics_proto/metrics.pb.go @@ -152,7 +152,7 @@ func (x *ModuleTypeInfo_BuildSystem) UnmarshalJSON(data []byte) error { } func (ModuleTypeInfo_BuildSystem) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6039342a2ba47b72, []int{2, 0} + return fileDescriptor_6039342a2ba47b72, []int{3, 0} } type MetricsBase struct { @@ -197,10 +197,11 @@ type MetricsBase struct { // The metrics for calling Ninja. NinjaRuns []*PerfInfo `protobuf:"bytes,20,rep,name=ninja_runs,json=ninjaRuns" json:"ninja_runs,omitempty"` // The metrics for the whole build - Total *PerfInfo `protobuf:"bytes,21,opt,name=total" json:"total,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Total *PerfInfo `protobuf:"bytes,21,opt,name=total" json:"total,omitempty"` + BuildConfig *BuildConfig `protobuf:"bytes,23,opt,name=build_config,json=buildConfig" json:"build_config,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *MetricsBase) Reset() { *m = MetricsBase{} } @@ -380,6 +381,60 @@ func (m *MetricsBase) GetTotal() *PerfInfo { return nil } +func (m *MetricsBase) GetBuildConfig() *BuildConfig { + if m != nil { + return m.BuildConfig + } + return nil +} + +type BuildConfig struct { + UseGoma *bool `protobuf:"varint,1,opt,name=use_goma,json=useGoma" json:"use_goma,omitempty"` + UseRbe *bool `protobuf:"varint,2,opt,name=use_rbe,json=useRbe" json:"use_rbe,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *BuildConfig) Reset() { *m = BuildConfig{} } +func (m *BuildConfig) String() string { return proto.CompactTextString(m) } +func (*BuildConfig) ProtoMessage() {} +func (*BuildConfig) Descriptor() ([]byte, []int) { + return fileDescriptor_6039342a2ba47b72, []int{1} +} + +func (m *BuildConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_BuildConfig.Unmarshal(m, b) +} +func (m *BuildConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_BuildConfig.Marshal(b, m, deterministic) +} +func (m *BuildConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_BuildConfig.Merge(m, src) +} +func (m *BuildConfig) XXX_Size() int { + return xxx_messageInfo_BuildConfig.Size(m) +} +func (m *BuildConfig) XXX_DiscardUnknown() { + xxx_messageInfo_BuildConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_BuildConfig proto.InternalMessageInfo + +func (m *BuildConfig) GetUseGoma() bool { + if m != nil && m.UseGoma != nil { + return *m.UseGoma + } + return false +} + +func (m *BuildConfig) GetUseRbe() bool { + if m != nil && m.UseRbe != nil { + return *m.UseRbe + } + return false +} + type PerfInfo struct { // The description for the phase/action/part while the tool running. Desc *string `protobuf:"bytes,1,opt,name=desc" json:"desc,omitempty"` @@ -402,7 +457,7 @@ func (m *PerfInfo) Reset() { *m = PerfInfo{} } func (m *PerfInfo) String() string { return proto.CompactTextString(m) } func (*PerfInfo) ProtoMessage() {} func (*PerfInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_6039342a2ba47b72, []int{1} + return fileDescriptor_6039342a2ba47b72, []int{2} } func (m *PerfInfo) XXX_Unmarshal(b []byte) error { @@ -474,7 +529,7 @@ func (m *ModuleTypeInfo) Reset() { *m = ModuleTypeInfo{} } func (m *ModuleTypeInfo) String() string { return proto.CompactTextString(m) } func (*ModuleTypeInfo) ProtoMessage() {} func (*ModuleTypeInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_6039342a2ba47b72, []int{2} + return fileDescriptor_6039342a2ba47b72, []int{3} } func (m *ModuleTypeInfo) XXX_Unmarshal(b []byte) error { @@ -532,7 +587,7 @@ func (m *CriticalUserJourneyMetrics) Reset() { *m = CriticalUserJourneyM func (m *CriticalUserJourneyMetrics) String() string { return proto.CompactTextString(m) } func (*CriticalUserJourneyMetrics) ProtoMessage() {} func (*CriticalUserJourneyMetrics) Descriptor() ([]byte, []int) { - return fileDescriptor_6039342a2ba47b72, []int{3} + return fileDescriptor_6039342a2ba47b72, []int{4} } func (m *CriticalUserJourneyMetrics) XXX_Unmarshal(b []byte) error { @@ -579,7 +634,7 @@ func (m *CriticalUserJourneysMetrics) Reset() { *m = CriticalUserJourney func (m *CriticalUserJourneysMetrics) String() string { return proto.CompactTextString(m) } func (*CriticalUserJourneysMetrics) ProtoMessage() {} func (*CriticalUserJourneysMetrics) Descriptor() ([]byte, []int) { - return fileDescriptor_6039342a2ba47b72, []int{4} + return fileDescriptor_6039342a2ba47b72, []int{5} } func (m *CriticalUserJourneysMetrics) XXX_Unmarshal(b []byte) error { @@ -612,6 +667,7 @@ func init() { proto.RegisterEnum("soong_build_metrics.MetricsBase_Arch", MetricsBase_Arch_name, MetricsBase_Arch_value) proto.RegisterEnum("soong_build_metrics.ModuleTypeInfo_BuildSystem", ModuleTypeInfo_BuildSystem_name, ModuleTypeInfo_BuildSystem_value) proto.RegisterType((*MetricsBase)(nil), "soong_build_metrics.MetricsBase") + proto.RegisterType((*BuildConfig)(nil), "soong_build_metrics.BuildConfig") proto.RegisterType((*PerfInfo)(nil), "soong_build_metrics.PerfInfo") proto.RegisterType((*ModuleTypeInfo)(nil), "soong_build_metrics.ModuleTypeInfo") proto.RegisterType((*CriticalUserJourneyMetrics)(nil), "soong_build_metrics.CriticalUserJourneyMetrics") @@ -621,58 +677,62 @@ func init() { func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) } var fileDescriptor_6039342a2ba47b72 = []byte{ - // 847 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xdd, 0x6e, 0xdb, 0x36, - 0x14, 0xae, 0x12, 0x25, 0xb6, 0x8e, 0x62, 0x57, 0x65, 0x52, 0x54, 0x5d, 0x11, 0xcc, 0x10, 0xd6, - 0x21, 0x17, 0xab, 0x5b, 0x78, 0x45, 0x50, 0x18, 0xc5, 0x80, 0xc4, 0x31, 0x8a, 0x2e, 0xb0, 0x5d, - 0x28, 0x71, 0x57, 0x6c, 0x17, 0x02, 0x23, 0xd1, 0x8d, 0x3a, 0x4b, 0x14, 0x48, 0xaa, 0x98, 0x1f, - 0x62, 0x0f, 0xb9, 0x8b, 0xbd, 0xc7, 0xc0, 0x43, 0xc9, 0x51, 0x00, 0x0f, 0x09, 0x7a, 0x47, 0x9d, - 0xef, 0x87, 0xdf, 0xa1, 0xc4, 0x23, 0xe8, 0x64, 0x4c, 0x89, 0x34, 0x96, 0xfd, 0x42, 0x70, 0xc5, - 0xc9, 0xbe, 0xe4, 0x3c, 0xff, 0x1c, 0x5d, 0x95, 0xe9, 0x32, 0x89, 0x2a, 0x28, 0xf8, 0xc7, 0x01, - 0x77, 0x62, 0xd6, 0xa7, 0x54, 0x32, 0xf2, 0x0a, 0x0e, 0x0c, 0x21, 0xa1, 0x8a, 0x45, 0x2a, 0xcd, - 0x98, 0x54, 0x34, 0x2b, 0x7c, 0xab, 0x67, 0x1d, 0x6d, 0x87, 0x04, 0xb1, 0x33, 0xaa, 0xd8, 0x65, - 0x8d, 0x90, 0xa7, 0xd0, 0x36, 0x8a, 0x34, 0xf1, 0xb7, 0x7a, 0xd6, 0x91, 0x13, 0xb6, 0xf0, 0xf9, - 0x7d, 0x42, 0x86, 0xf0, 0xb4, 0x58, 0x52, 0xb5, 0xe0, 0x22, 0x8b, 0xbe, 0x32, 0x21, 0x53, 0x9e, - 0x47, 0x31, 0x4f, 0x58, 0x4e, 0x33, 0xe6, 0x6f, 0x23, 0xf7, 0x49, 0x4d, 0xf8, 0x68, 0xf0, 0x51, - 0x05, 0x93, 0xe7, 0xd0, 0x55, 0x54, 0x7c, 0x66, 0x2a, 0x2a, 0x04, 0x4f, 0xca, 0x58, 0xf9, 0x36, - 0x0a, 0x3a, 0xa6, 0xfa, 0xc1, 0x14, 0x49, 0x02, 0x07, 0x15, 0xcd, 0x84, 0xf8, 0x4a, 0x45, 0x4a, - 0x73, 0xe5, 0xef, 0xf4, 0xac, 0xa3, 0xee, 0xe0, 0x45, 0x7f, 0x43, 0xcf, 0xfd, 0x46, 0xbf, 0xfd, - 0x53, 0x8d, 0x7c, 0x34, 0xa2, 0xe1, 0xf6, 0x78, 0xfa, 0x2e, 0x24, 0xc6, 0xaf, 0x09, 0x90, 0x19, - 0xb8, 0xd5, 0x2e, 0x54, 0xc4, 0xd7, 0xfe, 0x2e, 0x9a, 0x3f, 0xbf, 0xd3, 0xfc, 0x44, 0xc4, 0xd7, - 0xc3, 0xd6, 0x7c, 0x7a, 0x3e, 0x9d, 0xfd, 0x36, 0x0d, 0xc1, 0x58, 0xe8, 0x22, 0xe9, 0xc3, 0x7e, - 0xc3, 0x70, 0x9d, 0xba, 0x85, 0x2d, 0x3e, 0xba, 0x21, 0xd6, 0x01, 0x7e, 0x82, 0x2a, 0x56, 0x14, - 0x17, 0xe5, 0x9a, 0xde, 0x46, 0xba, 0x67, 0x90, 0x51, 0x51, 0xd6, 0xec, 0x73, 0x70, 0xae, 0xb9, - 0xac, 0xc2, 0x3a, 0xdf, 0x14, 0xb6, 0xad, 0x0d, 0x30, 0x6a, 0x08, 0x1d, 0x34, 0x1b, 0xe4, 0x89, - 0x31, 0x84, 0x6f, 0x32, 0x74, 0xb5, 0xc9, 0x20, 0x4f, 0xd0, 0xf3, 0x09, 0xb4, 0xd0, 0x93, 0x4b, - 0xdf, 0xc5, 0x1e, 0x76, 0xf5, 0xe3, 0x4c, 0x92, 0xa0, 0xda, 0x8c, 0xcb, 0x88, 0xfd, 0xa5, 0x04, - 0xf5, 0xf7, 0x10, 0x76, 0x0d, 0x3c, 0xd6, 0xa5, 0x35, 0x27, 0x16, 0x5c, 0x4a, 0x6d, 0xd1, 0xb9, - 0xe1, 0x8c, 0x74, 0x6d, 0x26, 0xc9, 0x8f, 0xf0, 0xb0, 0xc1, 0xc1, 0xd8, 0x5d, 0xf3, 0xf9, 0xac, - 0x59, 0x18, 0xe4, 0x05, 0xec, 0x37, 0x78, 0xeb, 0x16, 0x1f, 0x9a, 0x83, 0x5d, 0x73, 0x1b, 0xb9, - 0x79, 0xa9, 0xa2, 0x24, 0x15, 0xbe, 0x67, 0x72, 0xf3, 0x52, 0x9d, 0xa5, 0x82, 0xfc, 0x02, 0xae, - 0x64, 0xaa, 0x2c, 0x22, 0xc5, 0xf9, 0x52, 0xfa, 0x8f, 0x7a, 0xdb, 0x47, 0xee, 0xe0, 0x70, 0xe3, - 0x11, 0x7d, 0x60, 0x62, 0xf1, 0x3e, 0x5f, 0xf0, 0x10, 0x50, 0x71, 0xa9, 0x05, 0x64, 0x08, 0xce, - 0x9f, 0x54, 0xa5, 0x91, 0x28, 0x73, 0xe9, 0x93, 0xfb, 0xa8, 0xdb, 0x9a, 0x1f, 0x96, 0xb9, 0x24, - 0x6f, 0x01, 0x0c, 0x13, 0xc5, 0xfb, 0xf7, 0x11, 0x3b, 0x88, 0xd6, 0xea, 0x3c, 0xcd, 0xbf, 0x50, - 0xa3, 0x3e, 0xb8, 0x97, 0x1a, 0x05, 0xa8, 0xfe, 0x19, 0x76, 0x14, 0x57, 0x74, 0xe9, 0x3f, 0xee, - 0x59, 0x77, 0x0b, 0x0d, 0x37, 0x78, 0x05, 0x7b, 0xb7, 0x6e, 0x57, 0x1b, 0xec, 0xf9, 0xc5, 0x38, - 0xf4, 0x1e, 0x90, 0x0e, 0x38, 0x7a, 0x75, 0x36, 0x3e, 0x9d, 0xbf, 0xf3, 0x2c, 0xd2, 0x02, 0x7d, - 0x23, 0xbd, 0xad, 0xe0, 0x2d, 0xd8, 0x78, 0xfe, 0x2e, 0xd4, 0xdf, 0x93, 0xf7, 0x40, 0xa3, 0x27, - 0xe1, 0xc4, 0xb3, 0x88, 0x03, 0x3b, 0x27, 0xe1, 0xe4, 0xf8, 0xb5, 0xb7, 0xa5, 0x6b, 0x9f, 0xde, - 0x1c, 0x7b, 0xdb, 0x04, 0x60, 0xf7, 0xd3, 0x9b, 0xe3, 0xe8, 0xf8, 0xb5, 0x67, 0x07, 0x7f, 0x5b, - 0xd0, 0xae, 0x33, 0x10, 0x02, 0x76, 0xc2, 0x64, 0x8c, 0x03, 0xcd, 0x09, 0x71, 0xad, 0x6b, 0x38, - 0x92, 0xcc, 0xf8, 0xc2, 0x35, 0x39, 0x04, 0x90, 0x8a, 0x0a, 0x85, 0x33, 0x10, 0x87, 0x95, 0x1d, - 0x3a, 0x58, 0xd1, 0xa3, 0x8f, 0x3c, 0x03, 0x47, 0x30, 0xba, 0x34, 0xa8, 0x8d, 0x68, 0x5b, 0x17, - 0x10, 0x3c, 0x04, 0xc8, 0x58, 0xc6, 0xc5, 0x2a, 0x2a, 0x25, 0xc3, 0x51, 0x64, 0x87, 0x8e, 0xa9, - 0xcc, 0x25, 0x0b, 0xfe, 0xb5, 0xa0, 0x3b, 0xe1, 0x49, 0xb9, 0x64, 0x97, 0xab, 0x82, 0x61, 0xaa, - 0x3f, 0x60, 0xcf, 0x9c, 0x99, 0x5c, 0x49, 0xc5, 0x32, 0x4c, 0xd7, 0x1d, 0xbc, 0xdc, 0x7c, 0xc7, - 0x6e, 0x49, 0xcd, 0x04, 0xbb, 0x40, 0x59, 0xe3, 0xb6, 0x5d, 0xdd, 0x54, 0xc9, 0xf7, 0xe0, 0x66, - 0xa8, 0x89, 0xd4, 0xaa, 0xa8, 0xbb, 0x84, 0x6c, 0x6d, 0x43, 0x7e, 0x80, 0x6e, 0x5e, 0x66, 0x11, - 0x5f, 0x44, 0xa6, 0x28, 0xb1, 0xdf, 0x4e, 0xb8, 0x97, 0x97, 0xd9, 0x6c, 0x61, 0xf6, 0x93, 0xc1, - 0x4b, 0x70, 0x1b, 0x7b, 0xdd, 0x7e, 0x17, 0x0e, 0xec, 0x5c, 0xcc, 0x66, 0x53, 0xfd, 0xd2, 0xda, - 0x60, 0x4f, 0x4e, 0xce, 0xc7, 0xde, 0x56, 0xb0, 0x84, 0xef, 0x46, 0x22, 0x55, 0x69, 0x4c, 0x97, - 0x73, 0xc9, 0xc4, 0xaf, 0xbc, 0x14, 0x39, 0x5b, 0x55, 0x23, 0x62, 0x7d, 0xe8, 0x56, 0xe3, 0xd0, - 0x87, 0xd0, 0xaa, 0xba, 0xc4, 0x94, 0xee, 0xa0, 0x77, 0xd7, 0x94, 0x09, 0x6b, 0x41, 0x70, 0x05, - 0xcf, 0x36, 0xec, 0x26, 0xeb, 0xed, 0x46, 0x60, 0xc7, 0xe5, 0x17, 0xe9, 0x5b, 0xf8, 0x85, 0x6f, - 0x3e, 0xd9, 0xff, 0x4f, 0x1b, 0xa2, 0xf8, 0xf4, 0xf1, 0xef, 0xd5, 0x4f, 0xb4, 0x52, 0x44, 0xf8, - 0x67, 0xfd, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xc4, 0xbd, 0xe2, 0xb1, 0x69, 0x07, 0x00, 0x00, + // 910 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xdf, 0x6e, 0xdb, 0x36, + 0x17, 0xaf, 0x12, 0x25, 0x96, 0x8e, 0x62, 0x57, 0x65, 0x52, 0x44, 0xfd, 0x8a, 0xe0, 0x33, 0x84, + 0x75, 0xc8, 0xc5, 0x9a, 0x16, 0x59, 0x11, 0x14, 0x41, 0x31, 0xc0, 0x71, 0x8c, 0xa0, 0x0b, 0x6c, + 0x17, 0x4c, 0xdc, 0x15, 0xdb, 0x85, 0x20, 0x4b, 0x74, 0xa2, 0xce, 0x12, 0x0d, 0x92, 0x2a, 0xe6, + 0x87, 0xd8, 0x6b, 0xec, 0xcd, 0xf6, 0x1e, 0x03, 0x0f, 0x25, 0x47, 0x01, 0x5c, 0x34, 0xe8, 0x9d, + 0x74, 0x7e, 0x7f, 0xf8, 0x3b, 0xa4, 0x78, 0x04, 0xed, 0x9c, 0x29, 0x91, 0x25, 0xf2, 0x68, 0x21, + 0xb8, 0xe2, 0x64, 0x57, 0x72, 0x5e, 0xdc, 0x44, 0xd3, 0x32, 0x9b, 0xa7, 0x51, 0x05, 0x85, 0xff, + 0x00, 0x78, 0x43, 0xf3, 0x7c, 0x16, 0x4b, 0x46, 0x5e, 0xc3, 0x9e, 0x21, 0xa4, 0xb1, 0x62, 0x91, + 0xca, 0x72, 0x26, 0x55, 0x9c, 0x2f, 0x02, 0xab, 0x6b, 0x1d, 0x6e, 0x52, 0x82, 0xd8, 0x79, 0xac, + 0xd8, 0x75, 0x8d, 0x90, 0x67, 0xe0, 0x18, 0x45, 0x96, 0x06, 0x1b, 0x5d, 0xeb, 0xd0, 0xa5, 0x2d, + 0x7c, 0x7f, 0x9f, 0x92, 0x53, 0x78, 0xb6, 0x98, 0xc7, 0x6a, 0xc6, 0x45, 0x1e, 0x7d, 0x61, 0x42, + 0x66, 0xbc, 0x88, 0x12, 0x9e, 0xb2, 0x22, 0xce, 0x59, 0xb0, 0x89, 0xdc, 0xfd, 0x9a, 0xf0, 0xd1, + 0xe0, 0xfd, 0x0a, 0x26, 0x2f, 0xa0, 0xa3, 0x62, 0x71, 0xc3, 0x54, 0xb4, 0x10, 0x3c, 0x2d, 0x13, + 0x15, 0xd8, 0x28, 0x68, 0x9b, 0xea, 0x07, 0x53, 0x24, 0x29, 0xec, 0x55, 0x34, 0x13, 0xe2, 0x4b, + 0x2c, 0xb2, 0xb8, 0x50, 0xc1, 0x56, 0xd7, 0x3a, 0xec, 0x1c, 0xbf, 0x3c, 0x5a, 0xd3, 0xf3, 0x51, + 0xa3, 0xdf, 0xa3, 0x33, 0x8d, 0x7c, 0x34, 0xa2, 0xd3, 0xcd, 0xc1, 0xe8, 0x82, 0x12, 0xe3, 0xd7, + 0x04, 0xc8, 0x18, 0xbc, 0x6a, 0x95, 0x58, 0x24, 0xb7, 0xc1, 0x36, 0x9a, 0xbf, 0xf8, 0xa6, 0x79, + 0x4f, 0x24, 0xb7, 0xa7, 0xad, 0xc9, 0xe8, 0x72, 0x34, 0xfe, 0x6d, 0x44, 0xc1, 0x58, 0xe8, 0x22, + 0x39, 0x82, 0xdd, 0x86, 0xe1, 0x2a, 0x75, 0x0b, 0x5b, 0x7c, 0x72, 0x47, 0xac, 0x03, 0xfc, 0x04, + 0x55, 0xac, 0x28, 0x59, 0x94, 0x2b, 0xba, 0x83, 0x74, 0xdf, 0x20, 0xfd, 0x45, 0x59, 0xb3, 0x2f, + 0xc1, 0xbd, 0xe5, 0xb2, 0x0a, 0xeb, 0x7e, 0x57, 0x58, 0x47, 0x1b, 0x60, 0x54, 0x0a, 0x6d, 0x34, + 0x3b, 0x2e, 0x52, 0x63, 0x08, 0xdf, 0x65, 0xe8, 0x69, 0x93, 0xe3, 0x22, 0x45, 0xcf, 0x7d, 0x68, + 0xa1, 0x27, 0x97, 0x81, 0x87, 0x3d, 0x6c, 0xeb, 0xd7, 0xb1, 0x24, 0x61, 0xb5, 0x18, 0x97, 0x11, + 0xfb, 0x4b, 0x89, 0x38, 0xd8, 0x41, 0xd8, 0x33, 0xf0, 0x40, 0x97, 0x56, 0x9c, 0x44, 0x70, 0x29, + 0xb5, 0x45, 0xfb, 0x8e, 0xd3, 0xd7, 0xb5, 0xb1, 0x24, 0x3f, 0xc2, 0xe3, 0x06, 0x07, 0x63, 0x77, + 0xcc, 0xe7, 0xb3, 0x62, 0x61, 0x90, 0x97, 0xb0, 0xdb, 0xe0, 0xad, 0x5a, 0x7c, 0x6c, 0x36, 0x76, + 0xc5, 0x6d, 0xe4, 0xe6, 0xa5, 0x8a, 0xd2, 0x4c, 0x04, 0xbe, 0xc9, 0xcd, 0x4b, 0x75, 0x9e, 0x09, + 0xf2, 0x0b, 0x78, 0x92, 0xa9, 0x72, 0x11, 0x29, 0xce, 0xe7, 0x32, 0x78, 0xd2, 0xdd, 0x3c, 0xf4, + 0x8e, 0x0f, 0xd6, 0x6e, 0xd1, 0x07, 0x26, 0x66, 0xef, 0x8b, 0x19, 0xa7, 0x80, 0x8a, 0x6b, 0x2d, + 0x20, 0xa7, 0xe0, 0xfe, 0x19, 0xab, 0x2c, 0x12, 0x65, 0x21, 0x03, 0xf2, 0x10, 0xb5, 0xa3, 0xf9, + 0xb4, 0x2c, 0x24, 0x79, 0x07, 0x60, 0x98, 0x28, 0xde, 0x7d, 0x88, 0xd8, 0x45, 0xb4, 0x56, 0x17, + 0x59, 0xf1, 0x39, 0x36, 0xea, 0xbd, 0x07, 0xa9, 0x51, 0x80, 0xea, 0x9f, 0x61, 0x4b, 0x71, 0x15, + 0xcf, 0x83, 0xa7, 0x5d, 0xeb, 0xdb, 0x42, 0xc3, 0x25, 0x7d, 0xd8, 0x31, 0x84, 0x84, 0x17, 0xb3, + 0xec, 0x26, 0xd8, 0x47, 0x6d, 0x77, 0xad, 0x16, 0xaf, 0x61, 0x1f, 0x79, 0xd4, 0x9b, 0xde, 0xbd, + 0x84, 0xaf, 0x61, 0xe7, 0xde, 0x15, 0x75, 0xc0, 0x9e, 0x5c, 0x0d, 0xa8, 0xff, 0x88, 0xb4, 0xc1, + 0xd5, 0x4f, 0xe7, 0x83, 0xb3, 0xc9, 0x85, 0x6f, 0x91, 0x16, 0xe8, 0x6b, 0xed, 0x6f, 0x84, 0xef, + 0xc0, 0xc6, 0x43, 0xf4, 0xa0, 0xfe, 0x28, 0xfd, 0x47, 0x1a, 0xed, 0xd1, 0xa1, 0x6f, 0x11, 0x17, + 0xb6, 0x7a, 0x74, 0x78, 0xf2, 0xc6, 0xdf, 0xd0, 0xb5, 0x4f, 0x6f, 0x4f, 0xfc, 0x4d, 0x02, 0xb0, + 0xfd, 0xe9, 0xed, 0x49, 0x74, 0xf2, 0xc6, 0xb7, 0xc3, 0x1e, 0x78, 0x8d, 0x2c, 0x7a, 0xea, 0x95, + 0x92, 0x45, 0x37, 0x3c, 0x8f, 0x71, 0x36, 0x3a, 0xb4, 0x55, 0x4a, 0x76, 0xc1, 0xf3, 0x58, 0x7f, + 0x24, 0x1a, 0x12, 0x53, 0x86, 0xf3, 0xd0, 0xa1, 0xdb, 0xa5, 0x64, 0x74, 0xca, 0xc2, 0xbf, 0x2d, + 0x70, 0xea, 0xbd, 0x20, 0x04, 0xec, 0x94, 0xc9, 0x04, 0xc5, 0x2e, 0xc5, 0x67, 0x5d, 0xc3, 0xd1, + 0x68, 0xc6, 0x28, 0x3e, 0x93, 0x03, 0x00, 0xa9, 0x62, 0xa1, 0x70, 0x16, 0xe3, 0xd0, 0xb4, 0xa9, + 0x8b, 0x15, 0x3d, 0x82, 0xc9, 0x73, 0x70, 0x05, 0x8b, 0xe7, 0x06, 0xb5, 0x11, 0x75, 0x74, 0x01, + 0xc1, 0x03, 0x80, 0x9c, 0xe5, 0x5c, 0x2c, 0xa3, 0x52, 0x32, 0x1c, 0x89, 0x36, 0x75, 0x4d, 0x65, + 0x22, 0x59, 0xf8, 0xaf, 0x05, 0x9d, 0x21, 0x4f, 0xcb, 0x39, 0xbb, 0x5e, 0x2e, 0x18, 0xa6, 0xfa, + 0xa3, 0x3e, 0x1a, 0xb9, 0x94, 0x8a, 0xe5, 0x98, 0xae, 0x73, 0xfc, 0x6a, 0xfd, 0x5d, 0xbf, 0x27, + 0x35, 0x27, 0x75, 0x85, 0xb2, 0xc6, 0xad, 0x9f, 0xde, 0x55, 0xc9, 0xff, 0xc1, 0xcb, 0x51, 0x13, + 0xa9, 0xe5, 0xa2, 0xee, 0x12, 0xf2, 0x95, 0x0d, 0xf9, 0x01, 0x3a, 0x45, 0x99, 0x47, 0x7c, 0x16, + 0x99, 0xa2, 0xc4, 0x7e, 0xdb, 0x74, 0xa7, 0x28, 0xf3, 0xf1, 0xcc, 0xac, 0x27, 0xc3, 0x57, 0xd5, + 0x49, 0x54, 0xae, 0xf7, 0x8e, 0xd3, 0x85, 0xad, 0xab, 0xf1, 0x78, 0xa4, 0xcf, 0xdd, 0x01, 0x7b, + 0xd8, 0xbb, 0x1c, 0xf8, 0x1b, 0xe1, 0x1c, 0xfe, 0xd7, 0x17, 0x99, 0xca, 0x92, 0x78, 0x3e, 0x91, + 0x4c, 0xfc, 0xca, 0x4b, 0x51, 0xb0, 0x65, 0x35, 0xaa, 0x56, 0x9b, 0x6e, 0x35, 0x36, 0xfd, 0x14, + 0x5a, 0x55, 0x97, 0x98, 0xf2, 0x6b, 0x1f, 0x67, 0x63, 0xda, 0xd1, 0x5a, 0x10, 0x4e, 0xe1, 0xf9, + 0x9a, 0xd5, 0x64, 0xbd, 0x5c, 0x1f, 0xec, 0xa4, 0xfc, 0x2c, 0x03, 0x0b, 0x6f, 0xda, 0xfa, 0x9d, + 0xfd, 0x7a, 0x5a, 0x8a, 0xe2, 0xb3, 0xa7, 0xbf, 0x57, 0x3f, 0xf3, 0x4a, 0x11, 0xe1, 0x1f, 0xfe, + 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x51, 0x32, 0x87, 0x13, 0xf1, 0x07, 0x00, 0x00, } diff --git a/ui/metrics/metrics_proto/metrics.proto b/ui/metrics/metrics_proto/metrics.proto index 194aa6ba8..62cb4b889 100644 --- a/ui/metrics/metrics_proto/metrics.proto +++ b/ui/metrics/metrics_proto/metrics.proto @@ -92,6 +92,14 @@ message MetricsBase { // The metrics for the whole build optional PerfInfo total = 21; + + optional BuildConfig build_config = 23; +} + +message BuildConfig { + optional bool use_goma = 1; + + optional bool use_rbe = 2; } message PerfInfo { From c1be5ca62959c792ee5f8c6da2b843cafbce7135 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Thu, 7 May 2020 06:56:47 -0400 Subject: [PATCH 22/53] Add support for the remote execution of Abi links. Test: built crosshatch userdebug with RBE_ABI_LINKS=1 Bug: b/166742236 Change-Id: Idb4e15b59878af4a98d9a594127c21deff669e16 Merged-In: Idb4e15b59878af4a98d9a594127c21deff669e16 --- cc/builder.go | 41 +++++++++++++++++++++++++++++++---------- cc/config/global.go | 1 + 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/cc/builder.go b/cc/builder.go index 37fbf4e62..abf7a0019 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -215,15 +215,23 @@ var ( }, []string{"cFlags", "exportDirs"}, nil) _ = pctx.SourcePathVariable("sAbiLinker", "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/header-abi-linker") + _ = pctx.SourcePathVariable("sAbiLinkerLibs", "prebuilts/clang-tools/${config.HostPrebuiltTag}/lib64") - sAbiLink = pctx.AndroidStaticRule("sAbiLink", + sAbiLink, sAbiLinkRE = remoteexec.StaticRules(pctx, "sAbiLink", blueprint.RuleParams{ - Command: "$sAbiLinker -o ${out} $symbolFilter -arch $arch $exportedHeaderFlags @${out}.rsp ", + Command: "$reTemplate$sAbiLinker -o ${out} $symbolFilter -arch $arch $exportedHeaderFlags @${out}.rsp ", CommandDeps: []string{"$sAbiLinker"}, Rspfile: "${out}.rsp", RspfileContent: "${in}", - }, - "symbolFilter", "arch", "exportedHeaderFlags") + }, &remoteexec.REParams{ + Labels: map[string]string{"type": "tool", "name": "abi-linker"}, + ExecStrategy: "${config.REAbiLinkerExecStrategy}", + Inputs: []string{"$sAbiLinkerLibs", "${out}.rsp", "$implicits"}, + RSPFile: "${out}.rsp", + OutputFiles: []string{"$out"}, + ToolchainInputs: []string{"$sAbiLinker"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXPool}"}, + }, []string{"symbolFilter", "arch", "exportedHeaderFlags"}, []string{"implicits"}) _ = pctx.SourcePathVariable("sAbiDiffer", "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/header-abi-diff") @@ -729,17 +737,30 @@ func TransformDumpToLinkedDump(ctx android.ModuleContext, sAbiDumps android.Path for _, tag := range excludedSymbolTags { symbolFilterStr += " --exclude-symbol-tag " + tag } + rule := sAbiLink + args := map[string]string{ + "symbolFilter": symbolFilterStr, + "arch": ctx.Arch().ArchType.Name, + "exportedHeaderFlags": exportedHeaderFlags, + } + if ctx.Config().IsEnvTrue("RBE_ABI_LINKER") { + rule = sAbiLinkRE + rbeImplicits := implicits.Strings() + for _, p := range strings.Split(exportedHeaderFlags, " ") { + if len(p) > 2 { + // Exclude the -I prefix. + rbeImplicits = append(rbeImplicits, p[2:]) + } + } + args["implicits"] = strings.Join(rbeImplicits, ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: sAbiLink, + Rule: rule, Description: "header-abi-linker " + outputFile.Base(), Output: outputFile, Inputs: sAbiDumps, Implicits: implicits, - Args: map[string]string{ - "symbolFilter": symbolFilterStr, - "arch": ctx.Arch().ArchType.Name, - "exportedHeaderFlags": exportedHeaderFlags, - }, + Args: args, }) return android.OptionalPathForPath(outputFile) } diff --git a/cc/config/global.go b/cc/config/global.go index fce03061a..57d8db96b 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -261,6 +261,7 @@ func init() { pctx.VariableFunc("RECXXLinksPool", remoteexec.EnvOverrideFunc("RBE_CXX_LINKS_POOL", remoteexec.DefaultPool)) pctx.VariableFunc("RECXXLinksExecStrategy", remoteexec.EnvOverrideFunc("RBE_CXX_LINKS_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.VariableFunc("REAbiDumperExecStrategy", remoteexec.EnvOverrideFunc("RBE_ABI_DUMPER_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REAbiLinkerExecStrategy", remoteexec.EnvOverrideFunc("RBE_ABI_LINKER_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) } var HostPrebuiltTag = pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS) From c60af159eba9a1c1ef1854db26380d64df2a98a1 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Fri, 28 Aug 2020 14:21:55 -0400 Subject: [PATCH 23/53] Pass cpp linking implicit deps to RBE. Test: build with RBE_CXX=1 RBE_CXX_EXEC_STRATEGY=remote Bug: b/166742236 Change-Id: Ie769b9a99f080b75c1b82e458b5220261e9c1b47 Merged-In: Ie769b9a99f080b75c1b82e458b5220261e9c1b47 --- cc/builder.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cc/builder.go b/cc/builder.go index abf7a0019..5deb1299e 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -76,12 +76,12 @@ var ( &remoteexec.REParams{ Labels: map[string]string{"type": "link", "tool": "clang"}, ExecStrategy: "${config.RECXXLinksExecStrategy}", - Inputs: []string{"${out}.rsp"}, + Inputs: []string{"${out}.rsp", "$implicitInputs"}, RSPFile: "${out}.rsp", OutputFiles: []string{"${out}", "$implicitOutputs"}, ToolchainInputs: []string{"$ldCmd"}, Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"}, - }, []string{"ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags", "extraLibFlags"}, []string{"implicitOutputs"}) + }, []string{"ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags", "extraLibFlags"}, []string{"implicitInputs", "implicitOutputs"}) partialLd, partialLdRE = remoteexec.StaticRules(pctx, "partialLd", blueprint.RuleParams{ @@ -92,11 +92,11 @@ var ( }, &remoteexec.REParams{ Labels: map[string]string{"type": "link", "tool": "clang"}, ExecStrategy: "${config.RECXXLinksExecStrategy}", - Inputs: []string{"$inCommaList"}, + Inputs: []string{"$inCommaList", "$implicitInputs"}, OutputFiles: []string{"${out}", "$implicitOutputs"}, ToolchainInputs: []string{"$ldCmd"}, Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"}, - }, []string{"ldCmd", "ldFlags"}, []string{"inCommaList", "implicitOutputs"}) + }, []string{"ldCmd", "ldFlags"}, []string{"implicitInputs", "inCommaList", "implicitOutputs"}) ar = pctx.AndroidStaticRule("ar", blueprint.RuleParams{ @@ -226,12 +226,12 @@ var ( }, &remoteexec.REParams{ Labels: map[string]string{"type": "tool", "name": "abi-linker"}, ExecStrategy: "${config.REAbiLinkerExecStrategy}", - Inputs: []string{"$sAbiLinkerLibs", "${out}.rsp", "$implicits"}, + Inputs: []string{"$sAbiLinkerLibs", "${out}.rsp", "$implicitInputs"}, RSPFile: "${out}.rsp", OutputFiles: []string{"$out"}, ToolchainInputs: []string{"$sAbiLinker"}, Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXPool}"}, - }, []string{"symbolFilter", "arch", "exportedHeaderFlags"}, []string{"implicits"}) + }, []string{"symbolFilter", "arch", "exportedHeaderFlags"}, []string{"implicitInputs"}) _ = pctx.SourcePathVariable("sAbiDiffer", "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/header-abi-diff") @@ -703,6 +703,7 @@ func TransformObjToDynamicBinary(ctx android.ModuleContext, if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { rule = ldRE args["implicitOutputs"] = strings.Join(implicitOutputs.Strings(), ",") + args["implicitInputs"] = strings.Join(deps.Strings(), ",") } ctx.Build(pctx, android.BuildParams{ @@ -752,7 +753,7 @@ func TransformDumpToLinkedDump(ctx android.ModuleContext, sAbiDumps android.Path rbeImplicits = append(rbeImplicits, p[2:]) } } - args["implicits"] = strings.Join(rbeImplicits, ",") + args["implicitInputs"] = strings.Join(rbeImplicits, ",") } ctx.Build(pctx, android.BuildParams{ Rule: rule, @@ -860,6 +861,7 @@ func TransformObjsToObj(ctx android.ModuleContext, objFiles android.Paths, if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { rule = partialLdRE args["inCommaList"] = strings.Join(objFiles.Strings(), ",") + args["implicitInputs"] = strings.Join(deps.Strings(), ",") } ctx.Build(pctx, android.BuildParams{ Rule: rule, From b48be3a2c1a9b4b1986e7b8e13624d5f54cd19e0 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Fri, 3 Apr 2020 08:40:19 -0700 Subject: [PATCH 24/53] Add a new protobuf message named Upload to upload build metrics. A new protobuf message named Upload is defined to send build metrics to the Android Build Server API after a build is completed. These metrics are collected from googlers only, as described in go/local-developers-metrics-wfh. Bug: b/140638454 Test: m nothing Change-Id: I5e49b5ace053913be2b75673cb1375f4cef37825 Merged-In: I5e49b5ace053913be2b75673cb1375f4cef37825 --- ui/metrics/Android.bp | 9 ++ ui/metrics/upload_proto/regen.sh | 17 ++++ ui/metrics/upload_proto/upload.pb.go | 122 +++++++++++++++++++++++++++ ui/metrics/upload_proto/upload.proto | 35 ++++++++ 4 files changed, 183 insertions(+) create mode 100755 ui/metrics/upload_proto/regen.sh create mode 100644 ui/metrics/upload_proto/upload.pb.go create mode 100644 ui/metrics/upload_proto/upload.proto diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp index 529639de5..3596e1019 100644 --- a/ui/metrics/Android.bp +++ b/ui/metrics/Android.bp @@ -17,6 +17,7 @@ bootstrap_go_package { pkgPath: "android/soong/ui/metrics", deps: [ "golang-protobuf-proto", + "soong-ui-metrics_upload_proto", "soong-ui-metrics_proto", "soong-ui-tracer", ], @@ -35,3 +36,11 @@ bootstrap_go_package { ], } +bootstrap_go_package { + name: "soong-ui-metrics_upload_proto", + pkgPath: "android/soong/ui/metrics/upload_proto", + deps: ["golang-protobuf-proto"], + srcs: [ + "upload_proto/upload.pb.go", + ], +} diff --git a/ui/metrics/upload_proto/regen.sh b/ui/metrics/upload_proto/regen.sh new file mode 100755 index 000000000..4521df7bd --- /dev/null +++ b/ui/metrics/upload_proto/regen.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Generates the golang source file of upload.proto file. + +set -e + +function die() { echo "ERROR: $1" >&2; exit 1; } + +readonly error_msg="Maybe you need to run 'lunch aosp_arm-eng && m aprotoc blueprint_tools'?" + +if ! hash aprotoc &>/dev/null; then + die "could not find aprotoc. ${error_msg}" +fi + +if ! aprotoc --go_out=paths=source_relative:. upload.proto; then + die "build failed. ${error_msg}" +fi diff --git a/ui/metrics/upload_proto/upload.pb.go b/ui/metrics/upload_proto/upload.pb.go new file mode 100644 index 000000000..1b1e5e843 --- /dev/null +++ b/ui/metrics/upload_proto/upload.pb.go @@ -0,0 +1,122 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: upload.proto + +package soong_metrics_upload_proto + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type Upload struct { + // The timestamp in milliseconds that the build was created. + CreationTimestampMs *uint64 `protobuf:"varint,1,opt,name=creation_timestamp_ms,json=creationTimestampMs" json:"creation_timestamp_ms,omitempty"` + // The timestamp in milliseconds when the build was completed. + CompletionTimestampMs *uint64 `protobuf:"varint,2,opt,name=completion_timestamp_ms,json=completionTimestampMs" json:"completion_timestamp_ms,omitempty"` + // The branch name. + BranchName *string `protobuf:"bytes,3,opt,name=branch_name,json=branchName" json:"branch_name,omitempty"` + // The target name. + TargetName *string `protobuf:"bytes,4,opt,name=target_name,json=targetName" json:"target_name,omitempty"` + // A list of metrics filepaths to upload. + MetricsFiles []string `protobuf:"bytes,5,rep,name=metrics_files,json=metricsFiles" json:"metrics_files,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Upload) Reset() { *m = Upload{} } +func (m *Upload) String() string { return proto.CompactTextString(m) } +func (*Upload) ProtoMessage() {} +func (*Upload) Descriptor() ([]byte, []int) { + return fileDescriptor_91b94b655bd2a7e5, []int{0} +} + +func (m *Upload) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Upload.Unmarshal(m, b) +} +func (m *Upload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Upload.Marshal(b, m, deterministic) +} +func (m *Upload) XXX_Merge(src proto.Message) { + xxx_messageInfo_Upload.Merge(m, src) +} +func (m *Upload) XXX_Size() int { + return xxx_messageInfo_Upload.Size(m) +} +func (m *Upload) XXX_DiscardUnknown() { + xxx_messageInfo_Upload.DiscardUnknown(m) +} + +var xxx_messageInfo_Upload proto.InternalMessageInfo + +func (m *Upload) GetCreationTimestampMs() uint64 { + if m != nil && m.CreationTimestampMs != nil { + return *m.CreationTimestampMs + } + return 0 +} + +func (m *Upload) GetCompletionTimestampMs() uint64 { + if m != nil && m.CompletionTimestampMs != nil { + return *m.CompletionTimestampMs + } + return 0 +} + +func (m *Upload) GetBranchName() string { + if m != nil && m.BranchName != nil { + return *m.BranchName + } + return "" +} + +func (m *Upload) GetTargetName() string { + if m != nil && m.TargetName != nil { + return *m.TargetName + } + return "" +} + +func (m *Upload) GetMetricsFiles() []string { + if m != nil { + return m.MetricsFiles + } + return nil +} + +func init() { + proto.RegisterType((*Upload)(nil), "soong_metrics_upload.Upload") +} + +func init() { + proto.RegisterFile("upload.proto", fileDescriptor_91b94b655bd2a7e5) +} + +var fileDescriptor_91b94b655bd2a7e5 = []byte{ + // 201 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x2d, 0xc8, 0xc9, + 0x4f, 0x4c, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x29, 0xce, 0xcf, 0xcf, 0x4b, 0x8f, + 0xcf, 0x4d, 0x2d, 0x29, 0xca, 0x4c, 0x2e, 0x8e, 0x87, 0xc8, 0x29, 0xdd, 0x66, 0xe4, 0x62, 0x0b, + 0x05, 0x33, 0x85, 0x8c, 0xb8, 0x44, 0x93, 0x8b, 0x52, 0x13, 0x4b, 0x32, 0xf3, 0xf3, 0xe2, 0x4b, + 0x32, 0x73, 0x53, 0x8b, 0x4b, 0x12, 0x73, 0x0b, 0xe2, 0x73, 0x8b, 0x25, 0x18, 0x15, 0x18, 0x35, + 0x58, 0x82, 0x84, 0x61, 0x92, 0x21, 0x30, 0x39, 0xdf, 0x62, 0x21, 0x33, 0x2e, 0xf1, 0xe4, 0xfc, + 0xdc, 0x82, 0x9c, 0x54, 0x4c, 0x5d, 0x4c, 0x60, 0x5d, 0xa2, 0x08, 0x69, 0x64, 0x7d, 0xf2, 0x5c, + 0xdc, 0x49, 0x45, 0x89, 0x79, 0xc9, 0x19, 0xf1, 0x79, 0x89, 0xb9, 0xa9, 0x12, 0xcc, 0x0a, 0x8c, + 0x1a, 0x9c, 0x41, 0x5c, 0x10, 0x21, 0xbf, 0xc4, 0xdc, 0x54, 0x90, 0x82, 0x92, 0xc4, 0xa2, 0xf4, + 0xd4, 0x12, 0x88, 0x02, 0x16, 0x88, 0x02, 0x88, 0x10, 0x58, 0x81, 0x32, 0x17, 0x2f, 0xcc, 0x2b, + 0x69, 0x99, 0x39, 0xa9, 0xc5, 0x12, 0xac, 0x0a, 0xcc, 0x1a, 0x9c, 0x41, 0x3c, 0x50, 0x41, 0x37, + 0x90, 0x98, 0x93, 0x4c, 0x94, 0x14, 0x36, 0x5f, 0xc7, 0x83, 0x43, 0x04, 0x10, 0x00, 0x00, 0xff, + 0xff, 0xe2, 0x01, 0x74, 0x65, 0x20, 0x01, 0x00, 0x00, +} diff --git a/ui/metrics/upload_proto/upload.proto b/ui/metrics/upload_proto/upload.proto new file mode 100644 index 000000000..7a9f080e0 --- /dev/null +++ b/ui/metrics/upload_proto/upload.proto @@ -0,0 +1,35 @@ +// Copyright 2020 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package soong_metrics_upload; +option go_package = "soong_metrics_upload_proto"; + +message Upload { + // The timestamp in milliseconds that the build was created. + optional uint64 creation_timestamp_ms = 1; + + // The timestamp in milliseconds when the build was completed. + optional uint64 completion_timestamp_ms = 2; + + // The branch name. + optional string branch_name = 3; + + // The target name. + optional string target_name = 4; + + // A list of metrics filepaths to upload. + repeated string metrics_files = 5; +} From d519a71f97066e47bfea12ddb65c30722e207147 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Mon, 1 Jun 2020 17:29:30 +0000 Subject: [PATCH 25/53] Upload build metrics after a build is completed. Soong now supports the ability to upload metrics to another location by setting the ANDROID_ENABLE_METRICS_UPLOAD to an uploader that accepts the upload.proto proto buffer message. When the environment variable is set, a set of build metrics files (soong_metrics, rbe_metrics.pb and build_error) is uploaded. Bug: 140638454 Test: * Wrote unit test cases * Setup the uploader, built a succcessful and failed aosp_arm-eng target and monitor the uploading of the metrics. Change-Id: I76a65739c557dc90345e098ca03119a950ece2d2 Merged-In: I76a65739c557dc90345e098ca03119a950ece2d2 --- cmd/soong_ui/main.go | 13 ++++- ui/build/Android.bp | 2 + ui/build/config.go | 7 +++ ui/build/config_test.go | 2 + ui/build/upload.go | 80 +++++++++++++++++++++++++++ ui/build/upload_test.go | 117 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 ui/build/upload.go create mode 100644 ui/build/upload_test.go diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 9f40e33ef..8be80f6f0 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -47,11 +47,17 @@ func inList(s string, list []string) bool { } func main() { + buildStartedMilli := time.Now().UnixNano() / int64(time.Millisecond) var stdio terminal.StdioInterface stdio = terminal.StdioImpl{} + logsPrefix := "" // dumpvar uses stdout, everything else should be in stderr if os.Args[1] == "--dumpvar-mode" || os.Args[1] == "--dumpvars-mode" { + // Any metrics files add the prefix to distinguish the type of metrics being + // collected to further aggregate the metrics. For dump-var mode, it is usually + // related to the execution of lunch command. + logsPrefix = "dumpvars-" stdio = terminal.NewCustomStdio(os.Stdin, os.Stderr, os.Stderr) } @@ -110,13 +116,18 @@ func main() { logsDir = filepath.Join(config.DistDir(), "logs") } + rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb") + soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics") + defer build.UploadMetrics(buildCtx, config, buildStartedMilli, rbeMetricsFile, soongMetricsFile) + os.MkdirAll(logsDir, 0777) + log.SetOutput(filepath.Join(logsDir, "soong.log")) trace.SetOutput(filepath.Join(logsDir, "build.trace")) stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log"))) stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log"))) - defer met.Dump(filepath.Join(logsDir, "build_metrics")) + defer met.Dump(soongMetricsFile) if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { if !strings.HasSuffix(start, "N") { diff --git a/ui/build/Android.bp b/ui/build/Android.bp index 6f81c1730..65729821b 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -56,11 +56,13 @@ bootstrap_go_package { "signal.go", "soong.go", "test_build.go", + "upload.go", "util.go", ], testSrcs: [ "config_test.go", "environment_test.go", + "upload_test.go", "util_test.go", "proc_sync_test.go", ], diff --git a/ui/build/config.go b/ui/build/config.go index daefb2f73..fc8737445 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -712,3 +712,10 @@ func (c *configImpl) SetPdkBuild(pdk bool) { func (c *configImpl) IsPdkBuild() bool { return c.pdkBuild } + +func (c *configImpl) MetricsUploaderApp() string { + if p, ok := c.environ.Get("ANDROID_ENABLE_METRICS_UPLOAD"); ok { + return p + } + return "" +} diff --git a/ui/build/config_test.go b/ui/build/config_test.go index 242e3afb0..e1fb27107 100644 --- a/ui/build/config_test.go +++ b/ui/build/config_test.go @@ -22,6 +22,7 @@ import ( "testing" "android/soong/ui/logger" + "android/soong/ui/status" "android/soong/ui/terminal" ) @@ -30,6 +31,7 @@ func testContext() Context { Context: context.Background(), Logger: logger.New(&bytes.Buffer{}), Writer: terminal.NewWriter(terminal.NewCustomStdio(&bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{})), + Status: &status.Status{}, }} } diff --git a/ui/build/upload.go b/ui/build/upload.go new file mode 100644 index 000000000..9863877bf --- /dev/null +++ b/ui/build/upload.go @@ -0,0 +1,80 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +// This file contains the functionality to upload data from one location to +// another. + +import ( + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/golang/protobuf/proto" + + upload_proto "android/soong/ui/metrics/upload_proto" +) + +const ( + uploadPbFilename = ".uploader.pb" +) + +// UploadMetrics uploads a set of metrics files to a server for analysis. An +// uploader full path is required to be specified in order to upload the set +// of metrics files. This is accomplished by defining the ANDROID_ENABLE_METRICS_UPLOAD +// environment variable. +func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files ...string) { + uploader := config.MetricsUploaderApp() + // No metrics to upload if the path to the uploader was not specified. + if uploader == "" { + return + } + + // Some files may not exist. For example, build errors protobuf file + // may not exist since the build was successful. + var metricsFiles []string + for _, f := range files { + if _, err := os.Stat(f); err == nil { + metricsFiles = append(metricsFiles, f) + } + } + + if len(metricsFiles) == 0 { + return + } + + // For platform builds, the branch and target name is hardcoded to specific + // values for later extraction of the metrics in the data metrics pipeline. + data, err := proto.Marshal(&upload_proto.Upload{ + CreationTimestampMs: proto.Uint64(uint64(buildStartedMilli)), + CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))), + BranchName: proto.String("developer-metrics"), + TargetName: proto.String("platform-build-systems-metrics"), + MetricsFiles: metricsFiles, + }) + if err != nil { + ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err) + } + + pbFile := filepath.Join(config.OutDir(), uploadPbFilename) + if err := ioutil.WriteFile(pbFile, data, 0644); err != nil { + ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err) + } + // Remove the upload file as it's not longer needed after it has been processed by the uploader. + defer os.Remove(pbFile) + + Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile).RunAndPrintOrFatal() +} diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go new file mode 100644 index 000000000..ecf487b82 --- /dev/null +++ b/ui/build/upload_test.go @@ -0,0 +1,117 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "android/soong/ui/logger" +) + +func TestUploadMetrics(t *testing.T) { + ctx := testContext() + tests := []struct { + description string + uploader string + createFiles bool + files []string + }{{ + description: "ANDROID_ENABLE_METRICS_UPLOAD not set", + }, { + description: "no metrics files to upload", + uploader: "fake", + }, { + description: "non-existent metrics files no upload", + uploader: "fake", + files: []string{"metrics_file_1", "metrics_file_2", "metrics_file_3"}, + }, { + description: "trigger upload", + uploader: "echo", + createFiles: true, + files: []string{"metrics_file_1", "metrics_file_2"}, + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + t.Fatalf("got unexpected error: %v", err) + }) + + outDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create out directory: %v", outDir) + } + defer os.RemoveAll(outDir) + + var metricsFiles []string + if tt.createFiles { + for _, f := range tt.files { + filename := filepath.Join(outDir, f) + metricsFiles = append(metricsFiles, filename) + if err := ioutil.WriteFile(filename, []byte("test file"), 0644); err != nil { + t.Fatalf("failed to create a fake metrics file %q for uploading: %v", filename, err) + } + } + } + + config := Config{&configImpl{ + environ: &Environment{ + "OUT_DIR=" + outDir, + "ANDROID_ENABLE_METRICS_UPLOAD=" + tt.uploader, + }, + }} + + UploadMetrics(ctx, config, 1591031903, metricsFiles...) + + if _, err := os.Stat(filepath.Join(outDir, uploadPbFilename)); err == nil { + t.Error("got true, want false for upload protobuf file to exist") + } + }) + } +} + +func TestUploadMetricsErrors(t *testing.T) { + expectedErr := "failed to write the marshaled" + defer logger.Recover(func(err error) { + got := err.Error() + if !strings.Contains(got, expectedErr) { + t.Errorf("got %q, want %q to be contained in error", got, expectedErr) + } + }) + + outDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create out directory: %v", outDir) + } + defer os.RemoveAll(outDir) + + metricsFile := filepath.Join(outDir, "metrics_file_1") + if err := ioutil.WriteFile(metricsFile, []byte("test file"), 0644); err != nil { + t.Fatalf("failed to create a fake metrics file %q for uploading: %v", metricsFile, err) + } + + config := Config{&configImpl{ + environ: &Environment{ + "ANDROID_ENABLE_METRICS_UPLOAD=fake", + "OUT_DIR=/bad", + }}} + + UploadMetrics(testContext(), config, 1591031903, metricsFile) + t.Errorf("got nil, expecting %q as a failure", expectedErr) +} From c3cec872755cec776f9c57915ac04a8502bb5f4e Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Wed, 10 Jun 2020 18:48:01 +0000 Subject: [PATCH 26/53] Run the metrics uploader in the background. The metrics uploader was currently running on foreground where it would copy the metrics files in a separate directory and then forked into the background for the upload process. As a result, running the lunch command would take a second longer to run since each metrics uploader run had an average of half a second. Bug: 140638454 Test: * Wrote and updated unit test cases. * Set ANDROID_ENABLE_METRICS_UPLOAD to point to the latest metrics_uploader bash script. Executed the "lunch 1" command and measured the running time. Executed "m nothing" command and checked that the metrics were uploaded. * Ran "lunch 1" and "m nothing" with ANDROID_ENABLE_METRICS_UPLOAD="" * Removed oauth from metrics_uploader and ran "m nothing" and "lunch 1". The oauth Message appeared only to "m nothing" Change-Id: I13c61e666c8f44613dee291a704cef6a27335188 Merged-In: I13c61e666c8f44613dee291a704cef6a27335188 --- cmd/soong_ui/main.go | 4 +- ui/build/upload.go | 45 +++++++++++-- ui/build/upload_test.go | 98 +++++++++++++++++++--------- ui/build/util.go | 18 +++++ ui/build/util_test.go | 70 ++++++++++++++++++++ ui/metrics/upload_proto/upload.pb.go | 42 +++++++----- ui/metrics/upload_proto/upload.proto | 4 ++ 7 files changed, 229 insertions(+), 52 deletions(-) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 8be80f6f0..446c46f3e 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -50,6 +50,7 @@ func main() { buildStartedMilli := time.Now().UnixNano() / int64(time.Millisecond) var stdio terminal.StdioInterface stdio = terminal.StdioImpl{} + simpleOutput := false logsPrefix := "" // dumpvar uses stdout, everything else should be in stderr @@ -58,6 +59,7 @@ func main() { // collected to further aggregate the metrics. For dump-var mode, it is usually // related to the execution of lunch command. logsPrefix = "dumpvars-" + simpleOutput = true stdio = terminal.NewCustomStdio(os.Stdin, os.Stderr, os.Stderr) } @@ -118,7 +120,7 @@ func main() { rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb") soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics") - defer build.UploadMetrics(buildCtx, config, buildStartedMilli, rbeMetricsFile, soongMetricsFile) + defer build.UploadMetrics(buildCtx, config, simpleOutput, buildStartedMilli, rbeMetricsFile, soongMetricsFile) os.MkdirAll(logsDir, 0777) diff --git a/ui/build/upload.go b/ui/build/upload.go index 9863877bf..c7fab6803 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -23,6 +23,7 @@ import ( "path/filepath" "time" + "android/soong/ui/metrics" "github.com/golang/protobuf/proto" upload_proto "android/soong/ui/metrics/upload_proto" @@ -32,11 +33,21 @@ const ( uploadPbFilename = ".uploader.pb" ) +var ( + // For testing purpose + getTmpDir = ioutil.TempDir +) + // UploadMetrics uploads a set of metrics files to a server for analysis. An // uploader full path is required to be specified in order to upload the set // of metrics files. This is accomplished by defining the ANDROID_ENABLE_METRICS_UPLOAD -// environment variable. -func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files ...string) { +// environment variable. The metrics files are copied to a temporary directory +// and the uploader is then executed in the background to allow the user to continue +// working. +func UploadMetrics(ctx Context, config Config, forceDumbOutput bool, buildStartedMilli int64, files ...string) { + ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics") + defer ctx.EndTrace() + uploader := config.MetricsUploaderApp() // No metrics to upload if the path to the uploader was not specified. if uploader == "" { @@ -56,6 +67,22 @@ func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files .. return } + // The temporary directory cannot be deleted as the metrics uploader is started + // in the background and requires to exist until the operation is done. The + // uploader can delete the directory as it is specified in the upload proto. + tmpDir, err := getTmpDir("", "upload_metrics") + if err != nil { + ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err) + } + + for i, src := range metricsFiles { + dst := filepath.Join(tmpDir, filepath.Base(src)) + if _, err := copyFile(src, dst); err != nil { + ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err) + } + metricsFiles[i] = dst + } + // For platform builds, the branch and target name is hardcoded to specific // values for later extraction of the metrics in the data metrics pipeline. data, err := proto.Marshal(&upload_proto.Upload{ @@ -64,17 +91,23 @@ func UploadMetrics(ctx Context, config Config, buildStartedMilli int64, files .. BranchName: proto.String("developer-metrics"), TargetName: proto.String("platform-build-systems-metrics"), MetricsFiles: metricsFiles, + DirectoriesToDelete: []string{tmpDir}, }) if err != nil { ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err) } - pbFile := filepath.Join(config.OutDir(), uploadPbFilename) + pbFile := filepath.Join(tmpDir, uploadPbFilename) if err := ioutil.WriteFile(pbFile, data, 0644); err != nil { ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err) } - // Remove the upload file as it's not longer needed after it has been processed by the uploader. - defer os.Remove(pbFile) - Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile).RunAndPrintOrFatal() + // Start the uploader in the background as it takes several milliseconds to start the uploader + // and prepare the metrics for upload. This affects small commands like "lunch". + cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile) + if forceDumbOutput { + cmd.RunOrFatal() + } else { + cmd.RunAndPrintOrFatal() + } } diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go index ecf487b82..2d42aa859 100644 --- a/ui/build/upload_test.go +++ b/ui/build/upload_test.go @@ -15,6 +15,7 @@ package build import ( + "errors" "io/ioutil" "os" "path/filepath" @@ -59,6 +60,22 @@ func TestUploadMetrics(t *testing.T) { } defer os.RemoveAll(outDir) + // Supply our own getTmpDir to delete the temp dir once the test is done. + orgGetTmpDir := getTmpDir + getTmpDir = func(string, string) (string, error) { + retDir := filepath.Join(outDir, "tmp_upload_dir") + if err := os.Mkdir(retDir, 0755); err != nil { + t.Fatalf("failed to create temporary directory %q: %v", retDir, err) + } + return retDir, nil + } + defer func() { getTmpDir = orgGetTmpDir }() + + metricsUploadDir := filepath.Join(outDir, ".metrics_uploader") + if err := os.Mkdir(metricsUploadDir, 0755); err != nil { + t.Fatalf("failed to create %q directory for oauth valid check: %v", metricsUploadDir, err) + } + var metricsFiles []string if tt.createFiles { for _, f := range tt.files { @@ -77,41 +94,62 @@ func TestUploadMetrics(t *testing.T) { }, }} - UploadMetrics(ctx, config, 1591031903, metricsFiles...) - - if _, err := os.Stat(filepath.Join(outDir, uploadPbFilename)); err == nil { - t.Error("got true, want false for upload protobuf file to exist") - } + UploadMetrics(ctx, config, false, 1591031903, metricsFiles...) }) } } func TestUploadMetricsErrors(t *testing.T) { - expectedErr := "failed to write the marshaled" - defer logger.Recover(func(err error) { - got := err.Error() - if !strings.Contains(got, expectedErr) { - t.Errorf("got %q, want %q to be contained in error", got, expectedErr) - } - }) + ctx := testContext() + tests := []struct { + description string + tmpDir string + tmpDirErr error + expectedErr string + }{{ + description: "getTmpDir returned error", + tmpDirErr: errors.New("getTmpDir failed"), + expectedErr: "getTmpDir failed", + }, { + description: "copyFile operation error", + tmpDir: "/fake_dir", + expectedErr: "failed to copy", + }} - outDir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("failed to create out directory: %v", outDir) + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + got := err.Error() + if !strings.Contains(got, tt.expectedErr) { + t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr) + } + }) + + outDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create out directory: %v", outDir) + } + defer os.RemoveAll(outDir) + + orgGetTmpDir := getTmpDir + getTmpDir = func(string, string) (string, error) { + return tt.tmpDir, tt.tmpDirErr + } + defer func() { getTmpDir = orgGetTmpDir }() + + metricsFile := filepath.Join(outDir, "metrics_file_1") + if err := ioutil.WriteFile(metricsFile, []byte("test file"), 0644); err != nil { + t.Fatalf("failed to create a fake metrics file %q for uploading: %v", metricsFile, err) + } + + config := Config{&configImpl{ + environ: &Environment{ + "ANDROID_ENABLE_METRICS_UPLOAD=fake", + "OUT_DIR=/bad", + }}} + + UploadMetrics(ctx, config, true, 1591031903, metricsFile) + t.Errorf("got nil, expecting %q as a failure", tt.expectedErr) + }) } - defer os.RemoveAll(outDir) - - metricsFile := filepath.Join(outDir, "metrics_file_1") - if err := ioutil.WriteFile(metricsFile, []byte("test file"), 0644); err != nil { - t.Fatalf("failed to create a fake metrics file %q for uploading: %v", metricsFile, err) - } - - config := Config{&configImpl{ - environ: &Environment{ - "ANDROID_ENABLE_METRICS_UPLOAD=fake", - "OUT_DIR=/bad", - }}} - - UploadMetrics(testContext(), config, 1591031903, metricsFile) - t.Errorf("got nil, expecting %q as a failure", expectedErr) } diff --git a/ui/build/util.go b/ui/build/util.go index 0676a860d..14ab6ef07 100644 --- a/ui/build/util.go +++ b/ui/build/util.go @@ -15,6 +15,7 @@ package build import ( + "io" "os" "path/filepath" "strings" @@ -113,3 +114,20 @@ func decodeKeyValue(str string) (string, string, bool) { } return str[:idx], str[idx+1:], true } + +// copyFile copies a file from src to dst. filepath.Dir(dst) must exist. +func copyFile(src, dst string) (int64, error) { + source, err := os.Open(src) + if err != nil { + return 0, err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return 0, err + } + defer destination.Close() + + return io.Copy(destination, source) +} diff --git a/ui/build/util_test.go b/ui/build/util_test.go index 89bfc7786..b22e9977f 100644 --- a/ui/build/util_test.go +++ b/ui/build/util_test.go @@ -15,6 +15,7 @@ package build import ( + "bytes" "io/ioutil" "os" "path/filepath" @@ -49,3 +50,72 @@ func TestEnsureEmptyDirs(t *testing.T) { ensureEmptyDirectoriesExist(ctx, filepath.Join(tmpDir, "a")) } + +func TestCopyFile(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "test_copy_file") + if err != nil { + t.Fatalf("failed to create temporary directory to hold test text files: %v", err) + } + defer os.Remove(tmpDir) + + data := []byte("fake data") + src := filepath.Join(tmpDir, "src.txt") + if err := ioutil.WriteFile(src, data, 0755); err != nil { + t.Fatalf("failed to create a src file %q for copying: %v", src, err) + } + + dst := filepath.Join(tmpDir, "dst.txt") + + l, err := copyFile(src, dst) + if err != nil { + t.Fatalf("got %v, expecting nil error on copyFile operation", err) + } + + if l != int64(len(data)) { + t.Errorf("got %d, expecting %d for copied bytes", l, len(data)) + } + + dstData, err := ioutil.ReadFile(dst) + if err != nil { + t.Fatalf("got %v, expecting nil error reading dst %q file", err, dst) + } + + if bytes.Compare(data, dstData) != 0 { + t.Errorf("got %q, expecting data %q from dst %q text file", string(data), string(dstData), dst) + } +} + +func TestCopyFileErrors(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "test_copy_file_errors") + if err != nil { + t.Fatalf("failed to create temporary directory to hold test text files: %v", err) + } + defer os.Remove(tmpDir) + + srcExists := filepath.Join(tmpDir, "src_exist.txt") + if err := ioutil.WriteFile(srcExists, []byte("fake data"), 0755); err != nil { + t.Fatalf("failed to create a src file %q for copying: %v", srcExists, err) + } + + tests := []struct { + description string + src string + dst string + }{{ + description: "src file does not exist", + src: "/src/not/exist", + dst: "/dst/not/exist", + }, { + description: "dst directory does not exist", + src: srcExists, + dst: "/dst/not/exist", + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + if _, err := copyFile(tt.src, tt.dst); err == nil { + t.Errorf("got nil, expecting error") + } + }) + } +} diff --git a/ui/metrics/upload_proto/upload.pb.go b/ui/metrics/upload_proto/upload.pb.go index 1b1e5e843..614d4c7a4 100644 --- a/ui/metrics/upload_proto/upload.pb.go +++ b/ui/metrics/upload_proto/upload.pb.go @@ -30,7 +30,10 @@ type Upload struct { // The target name. TargetName *string `protobuf:"bytes,4,opt,name=target_name,json=targetName" json:"target_name,omitempty"` // A list of metrics filepaths to upload. - MetricsFiles []string `protobuf:"bytes,5,rep,name=metrics_files,json=metricsFiles" json:"metrics_files,omitempty"` + MetricsFiles []string `protobuf:"bytes,5,rep,name=metrics_files,json=metricsFiles" json:"metrics_files,omitempty"` + // A list of directories to delete after the copy of metrics files + // is completed for uploading. + DirectoriesToDelete []string `protobuf:"bytes,6,rep,name=directories_to_delete,json=directoriesToDelete" json:"directories_to_delete,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -96,6 +99,13 @@ func (m *Upload) GetMetricsFiles() []string { return nil } +func (m *Upload) GetDirectoriesToDelete() []string { + if m != nil { + return m.DirectoriesToDelete + } + return nil +} + func init() { proto.RegisterType((*Upload)(nil), "soong_metrics_upload.Upload") } @@ -105,18 +115,20 @@ func init() { } var fileDescriptor_91b94b655bd2a7e5 = []byte{ - // 201 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x2d, 0xc8, 0xc9, - 0x4f, 0x4c, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x29, 0xce, 0xcf, 0xcf, 0x4b, 0x8f, - 0xcf, 0x4d, 0x2d, 0x29, 0xca, 0x4c, 0x2e, 0x8e, 0x87, 0xc8, 0x29, 0xdd, 0x66, 0xe4, 0x62, 0x0b, - 0x05, 0x33, 0x85, 0x8c, 0xb8, 0x44, 0x93, 0x8b, 0x52, 0x13, 0x4b, 0x32, 0xf3, 0xf3, 0xe2, 0x4b, - 0x32, 0x73, 0x53, 0x8b, 0x4b, 0x12, 0x73, 0x0b, 0xe2, 0x73, 0x8b, 0x25, 0x18, 0x15, 0x18, 0x35, - 0x58, 0x82, 0x84, 0x61, 0x92, 0x21, 0x30, 0x39, 0xdf, 0x62, 0x21, 0x33, 0x2e, 0xf1, 0xe4, 0xfc, - 0xdc, 0x82, 0x9c, 0x54, 0x4c, 0x5d, 0x4c, 0x60, 0x5d, 0xa2, 0x08, 0x69, 0x64, 0x7d, 0xf2, 0x5c, - 0xdc, 0x49, 0x45, 0x89, 0x79, 0xc9, 0x19, 0xf1, 0x79, 0x89, 0xb9, 0xa9, 0x12, 0xcc, 0x0a, 0x8c, - 0x1a, 0x9c, 0x41, 0x5c, 0x10, 0x21, 0xbf, 0xc4, 0xdc, 0x54, 0x90, 0x82, 0x92, 0xc4, 0xa2, 0xf4, - 0xd4, 0x12, 0x88, 0x02, 0x16, 0x88, 0x02, 0x88, 0x10, 0x58, 0x81, 0x32, 0x17, 0x2f, 0xcc, 0x2b, - 0x69, 0x99, 0x39, 0xa9, 0xc5, 0x12, 0xac, 0x0a, 0xcc, 0x1a, 0x9c, 0x41, 0x3c, 0x50, 0x41, 0x37, - 0x90, 0x98, 0x93, 0x4c, 0x94, 0x14, 0x36, 0x5f, 0xc7, 0x83, 0x43, 0x04, 0x10, 0x00, 0x00, 0xff, - 0xff, 0xe2, 0x01, 0x74, 0x65, 0x20, 0x01, 0x00, 0x00, + // 230 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0xb1, 0x4a, 0x04, 0x31, + 0x10, 0x86, 0xd9, 0xbb, 0xf3, 0xe0, 0xe2, 0xd9, 0xec, 0x79, 0x18, 0x44, 0x70, 0xd1, 0x66, 0x2b, + 0x0b, 0x0b, 0x1f, 0x40, 0xc4, 0x4e, 0x8b, 0xe5, 0x6c, 0x6c, 0x86, 0x98, 0x1d, 0xd7, 0x40, 0x92, + 0x09, 0xc9, 0xf8, 0x1c, 0xbe, 0xb2, 0x6c, 0xe2, 0xe2, 0x82, 0x76, 0xc3, 0xff, 0x7d, 0x7f, 0x31, + 0xbf, 0xd8, 0x7e, 0x06, 0x4b, 0xaa, 0xbf, 0x09, 0x91, 0x98, 0xea, 0xd3, 0x44, 0xe4, 0x07, 0x70, + 0xc8, 0xd1, 0xe8, 0x04, 0x85, 0x5d, 0x7d, 0x2d, 0xc4, 0xfa, 0x25, 0x9f, 0xf5, 0xad, 0xd8, 0xeb, + 0x88, 0x8a, 0x0d, 0x79, 0x60, 0xe3, 0x30, 0xb1, 0x72, 0x01, 0x5c, 0x92, 0x55, 0x53, 0xb5, 0xab, + 0x6e, 0x37, 0xc1, 0xc3, 0xc4, 0x9e, 0x52, 0x7d, 0x27, 0xce, 0x34, 0xb9, 0x60, 0xf1, 0x6f, 0x6b, + 0x91, 0x5b, 0xfb, 0x5f, 0x3c, 0xef, 0x5d, 0x8a, 0xe3, 0xb7, 0xa8, 0xbc, 0xfe, 0x00, 0xaf, 0x1c, + 0xca, 0x65, 0x53, 0xb5, 0x9b, 0x4e, 0x94, 0xe8, 0x59, 0x39, 0x1c, 0x05, 0x56, 0x71, 0x40, 0x2e, + 0xc2, 0xaa, 0x08, 0x25, 0xca, 0xc2, 0xb5, 0x38, 0x99, 0x5e, 0x79, 0x37, 0x16, 0x93, 0x3c, 0x6a, + 0x96, 0xed, 0xa6, 0xdb, 0xfe, 0x84, 0x8f, 0x63, 0x36, 0xbe, 0xd4, 0x9b, 0x88, 0x9a, 0x29, 0x1a, + 0x4c, 0xc0, 0x04, 0x3d, 0x5a, 0x64, 0x94, 0xeb, 0x2c, 0xef, 0x66, 0xf0, 0x40, 0x0f, 0x19, 0xdd, + 0x5f, 0xbc, 0x9e, 0xff, 0xb7, 0x14, 0xe4, 0x15, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x64, 0x04, + 0xa8, 0xf4, 0x54, 0x01, 0x00, 0x00, } diff --git a/ui/metrics/upload_proto/upload.proto b/ui/metrics/upload_proto/upload.proto index 7a9f080e0..bcd0ab204 100644 --- a/ui/metrics/upload_proto/upload.proto +++ b/ui/metrics/upload_proto/upload.proto @@ -32,4 +32,8 @@ message Upload { // A list of metrics filepaths to upload. repeated string metrics_files = 5; + + // A list of directories to delete after the copy of metrics files + // is completed for uploading. + repeated string directories_to_delete = 6; } From 6b9cdfa9b23417c64d36ba6e03ce6d2bd9548b04 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Mon, 13 Jul 2020 18:21:14 +0000 Subject: [PATCH 27/53] Use monotonic clock instead of wall time clock for metrics. From several dashboards, there are high spikes from several builds where the real time of executing a part of the build system took many many hours. This is due to using the wall time clock when calculating the duration. For metrics, the monotonic clock should be used for more accurate measurement. Also, cleaned up the time.go file to use the proper proto functions when marshalling perf protobuf and deleted unnecessary functions. Bug: b/143492984 Test: Wrote and ran unit test case. Change-Id: I1540cfbf37915e3045ecedaa2595c1f9429bcd62 Merged-In: I1540cfbf37915e3045ecedaa2595c1f9429bcd62 --- ui/metrics/Android.bp | 3 +++ ui/metrics/time.go | 36 +++++++++++++++++------------------ ui/metrics/time_test.go | 42 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 19 deletions(-) create mode 100644 ui/metrics/time_test.go diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp index 3596e1019..8188a69e9 100644 --- a/ui/metrics/Android.bp +++ b/ui/metrics/Android.bp @@ -25,6 +25,9 @@ bootstrap_go_package { "metrics.go", "time.go", ], + testSrcs: [ + "time_test.go", + ], } bootstrap_go_package { diff --git a/ui/metrics/time.go b/ui/metrics/time.go index 7e8801ac0..c0de35765 100644 --- a/ui/metrics/time.go +++ b/ui/metrics/time.go @@ -19,13 +19,18 @@ import ( "android/soong/ui/metrics/metrics_proto" "android/soong/ui/tracer" + "github.com/golang/protobuf/proto" ) +// for testing purpose only +var _now = now + type timeEvent struct { desc string name string - atNanos uint64 // timestamp measured in nanoseconds since the reference date + // the time that the event started to occur. + start time.Time } type TimeTracer interface { @@ -39,33 +44,26 @@ type timeTracerImpl struct { var _ TimeTracer = &timeTracerImpl{} -func (t *timeTracerImpl) now() uint64 { - return uint64(time.Now().UnixNano()) +func now() time.Time { + return time.Now() } -func (t *timeTracerImpl) Begin(name, desc string, thread tracer.Thread) { - t.beginAt(name, desc, t.now()) +func (t *timeTracerImpl) Begin(name, desc string, _ tracer.Thread) { + t.activeEvents = append(t.activeEvents, timeEvent{name: name, desc: desc, start: _now()}) } -func (t *timeTracerImpl) beginAt(name, desc string, atNanos uint64) { - t.activeEvents = append(t.activeEvents, timeEvent{name: name, desc: desc, atNanos: atNanos}) -} - -func (t *timeTracerImpl) End(thread tracer.Thread) metrics_proto.PerfInfo { - return t.endAt(t.now()) -} - -func (t *timeTracerImpl) endAt(atNanos uint64) metrics_proto.PerfInfo { +func (t *timeTracerImpl) End(tracer.Thread) metrics_proto.PerfInfo { if len(t.activeEvents) < 1 { panic("Internal error: No pending events for endAt to end!") } lastEvent := t.activeEvents[len(t.activeEvents)-1] t.activeEvents = t.activeEvents[:len(t.activeEvents)-1] - realTime := atNanos - lastEvent.atNanos + realTime := uint64(_now().Sub(lastEvent.start).Nanoseconds()) return metrics_proto.PerfInfo{ - Desc: &lastEvent.desc, - Name: &lastEvent.name, - StartTime: &lastEvent.atNanos, - RealTime: &realTime} + Desc: proto.String(lastEvent.desc), + Name: proto.String(lastEvent.name), + StartTime: proto.Uint64(uint64(lastEvent.start.UnixNano())), + RealTime: proto.Uint64(realTime), + } } diff --git a/ui/metrics/time_test.go b/ui/metrics/time_test.go new file mode 100644 index 000000000..d73080ab2 --- /dev/null +++ b/ui/metrics/time_test.go @@ -0,0 +1,42 @@ +// Copyright 2020 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "testing" + "time" + + "android/soong/ui/tracer" +) + +func TestEnd(t *testing.T) { + startTime := time.Date(2020, time.July, 13, 13, 0, 0, 0, time.UTC) + dur := time.Nanosecond * 10 + initialNow := _now + _now = func() time.Time { return startTime.Add(dur) } + defer func() { _now = initialNow }() + + timeTracer := &timeTracerImpl{} + timeTracer.activeEvents = append(timeTracer.activeEvents, timeEvent{ + desc: "test", + name: "test", + start: startTime, + }) + + perf := timeTracer.End(tracer.Thread(0)) + if perf.GetRealTime() != uint64(dur.Nanoseconds()) { + t.Errorf("got %d, want %d nanoseconds for event duration", perf.GetRealTime(), dur.Nanoseconds()) + } +} From f53a655b75c89a844f30847550050f889ae379fd Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Mon, 13 Jul 2020 23:01:18 +0000 Subject: [PATCH 28/53] Use the build start time from Soong main UI for metrics build timestamp. Currently, the build timestamp saved to the build_date_timestamp field in the MetricsBase metrics protobuf usually comes from a text file named build_date.txt. It may not be accurate for metrics purposes. Instead, use the build start time from Soong main UI for a better timestamp. Bug: b/140638454 Test: Ran and checked the build_date_timestamp by running the printproto command. Change-Id: I4d893fbbf9830ba21911e56ae13bc5272be47ae6 Merged-In: I4d893fbbf9830ba21911e56ae13bc5272be47ae6 --- cmd/soong_ui/main.go | 5 +++-- ui/build/config.go | 3 --- ui/build/upload.go | 4 ++-- ui/build/upload_test.go | 5 +++-- ui/metrics/metrics.go | 12 +++--------- 5 files changed, 11 insertions(+), 18 deletions(-) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 446c46f3e..2d938a3b3 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -47,7 +47,7 @@ func inList(s string, list []string) bool { } func main() { - buildStartedMilli := time.Now().UnixNano() / int64(time.Millisecond) + buildStarted := time.Now() var stdio terminal.StdioInterface stdio = terminal.StdioImpl{} simpleOutput := false @@ -83,6 +83,7 @@ func main() { defer trace.Close() met := metrics.New() + met.SetBuildDateTime(buildStarted) stat := &status.Status{} defer stat.Finish() @@ -120,7 +121,7 @@ func main() { rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb") soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics") - defer build.UploadMetrics(buildCtx, config, simpleOutput, buildStartedMilli, rbeMetricsFile, soongMetricsFile) + defer build.UploadMetrics(buildCtx, config, simpleOutput, buildStarted, rbeMetricsFile, soongMetricsFile) os.MkdirAll(logsDir, 0777) diff --git a/ui/build/config.go b/ui/build/config.go index fc8737445..8ba425be5 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -218,9 +218,6 @@ func NewConfig(ctx Context, args ...string) Config { } else { content = strconv.FormatInt(time.Now().Unix(), 10) } - if ctx.Metrics != nil { - ctx.Metrics.SetBuildDateTime(content) - } err := ioutil.WriteFile(buildDateTimeFile, []byte(content), 0777) if err != nil { ctx.Fatalln("Failed to write BUILD_DATETIME to file:", err) diff --git a/ui/build/upload.go b/ui/build/upload.go index c7fab6803..d7c57b8db 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -44,7 +44,7 @@ var ( // environment variable. The metrics files are copied to a temporary directory // and the uploader is then executed in the background to allow the user to continue // working. -func UploadMetrics(ctx Context, config Config, forceDumbOutput bool, buildStartedMilli int64, files ...string) { +func UploadMetrics(ctx Context, config Config, forceDumbOutput bool, buildStarted time.Time, files ...string) { ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics") defer ctx.EndTrace() @@ -86,7 +86,7 @@ func UploadMetrics(ctx Context, config Config, forceDumbOutput bool, buildStarte // For platform builds, the branch and target name is hardcoded to specific // values for later extraction of the metrics in the data metrics pipeline. data, err := proto.Marshal(&upload_proto.Upload{ - CreationTimestampMs: proto.Uint64(uint64(buildStartedMilli)), + CreationTimestampMs: proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))), CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))), BranchName: proto.String("developer-metrics"), TargetName: proto.String("platform-build-systems-metrics"), diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go index 2d42aa859..eb2dafa30 100644 --- a/ui/build/upload_test.go +++ b/ui/build/upload_test.go @@ -21,6 +21,7 @@ import ( "path/filepath" "strings" "testing" + "time" "android/soong/ui/logger" ) @@ -94,7 +95,7 @@ func TestUploadMetrics(t *testing.T) { }, }} - UploadMetrics(ctx, config, false, 1591031903, metricsFiles...) + UploadMetrics(ctx, config, false, time.Now(), metricsFiles...) }) } } @@ -148,7 +149,7 @@ func TestUploadMetricsErrors(t *testing.T) { "OUT_DIR=/bad", }}} - UploadMetrics(ctx, config, true, 1591031903, metricsFile) + UploadMetrics(ctx, config, true, time.Now(), metricsFile) t.Errorf("got nil, expecting %q as a failure", tt.expectedErr) }) } diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 790b67ad5..39b55cb91 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -17,7 +17,7 @@ package metrics import ( "io/ioutil" "os" - "strconv" + "time" "android/soong/ui/metrics/metrics_proto" @@ -127,14 +127,8 @@ func (m *Metrics) getArch(arch string) *metrics_proto.MetricsBase_ARCH { } } -func (m *Metrics) SetBuildDateTime(date_time string) { - if date_time != "" { - date_time_timestamp, err := strconv.ParseInt(date_time, 10, 64) - if err != nil { - panic(err) - } - m.metrics.BuildDateTimestamp = &date_time_timestamp - } +func (m *Metrics) SetBuildDateTime(buildTimestamp time.Time) { + m.metrics.BuildDateTimestamp = proto.Int64(buildTimestamp.UnixNano() / int64(time.Second)) } func (m *Metrics) Serialize() (data []byte, err error) { From 55b5e425818847b410cb29b1694166ce292ec6eb Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Fri, 7 Aug 2020 17:55:23 +0000 Subject: [PATCH 29/53] Set the OS name in the metrics protobuf file There are cases where the developer is not building a product so the metadata information is not being set. For now, set the OS name so we can track which OS the user is using it. Fixes: b/163118212 Test: m nothing and checked the blueprint Change-Id: I9b521eee2d266b50625aa8e916ddaacb4e2feaea Merged-In: I9b521eee2d266b50625aa8e916ddaacb4e2feaea --- ui/metrics/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 39b55cb91..8ae085cdf 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -17,6 +17,7 @@ package metrics import ( "io/ioutil" "os" + "runtime" "time" "android/soong/ui/metrics/metrics_proto" @@ -94,8 +95,6 @@ func (m *Metrics) SetMetadataMetrics(metadata map[string]string) { m.metrics.HostArch = m.getArch(v) case "HOST_2ND_ARCH": m.metrics.Host_2NdArch = m.getArch(v) - case "HOST_OS": - m.metrics.HostOs = proto.String(v) case "HOST_OS_EXTRA": m.metrics.HostOsExtra = proto.String(v) case "HOST_CROSS_OS": @@ -137,6 +136,7 @@ func (m *Metrics) Serialize() (data []byte, err error) { // exports the output to the file at outputPath func (m *Metrics) Dump(outputPath string) (err error) { + m.metrics.HostOs = proto.String(runtime.GOOS) data, err := m.Serialize() if err != nil { return err From c726ab533c4888163926620928dbd4ad700f15e4 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Mon, 13 Apr 2020 13:21:23 -0400 Subject: [PATCH 30/53] [DO NOT MERGE] Allow remote execution of link actions. This CL adds a remoteexec package that allows adding a configurable RBE prefix to the template. Bug: b/166182389 Test: built aosp crosshatch userdebug with and without RBE_CXX_LINKS. Change-Id: Ica920c3d7f79f2996210b9cbd448126451c1707c Merged-In: Ica920c3d7f79f2996210b9cbd448126451c1707c --- Android.bp | 17 ++++ cc/builder.go | 55 ++++++++---- cc/config/global.go | 13 +++ remoteexec/remoteexec.go | 155 ++++++++++++++++++++++++++++++++++ remoteexec/remoteexec_test.go | 83 ++++++++++++++++++ ui/build/dumpvars.go | 1 + 6 files changed, 309 insertions(+), 15 deletions(-) create mode 100644 remoteexec/remoteexec.go create mode 100644 remoteexec/remoteexec_test.go diff --git a/Android.bp b/Android.bp index 670a2ed06..31faa4237 100644 --- a/Android.bp +++ b/Android.bp @@ -99,6 +99,7 @@ bootstrap_go_package { pkgPath: "android/soong/cc/config", deps: [ "soong-android", + "soong-remoteexec", ], srcs: [ "cc/config/clang.go", @@ -417,6 +418,22 @@ bootstrap_go_package { pluginFor: ["soong_build"], } +bootstrap_go_package { + name: "soong-remoteexec", + pkgPath: "android/soong/remoteexec", + deps: [ + "blueprint", + "soong-android", + ], + srcs: [ + "remoteexec/remoteexec.go", + ], + testSrcs: [ + "remoteexec/remoteexec_test.go", + ], + pluginFor: ["soong_build"], +} + // // Defaults to enable various configurations of host bionic // diff --git a/cc/builder.go b/cc/builder.go index 5641b7d15..3dc9a08c2 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -29,6 +29,7 @@ import ( "android/soong/android" "android/soong/cc/config" + "android/soong/remoteexec" ) const ( @@ -62,7 +63,7 @@ var ( }, "ccCmd", "cFlags") - ld = pctx.AndroidStaticRule("ld", + ld, ldRE = remoteexec.StaticRules(pctx, "ld", blueprint.RuleParams{ Command: "$ldCmd ${crtBegin} @${out}.rsp " + "${libFlags} ${crtEnd} -o ${out} ${ldFlags}", @@ -72,16 +73,28 @@ var ( // clang -Wl,--out-implib doesn't update its output file if it hasn't changed. Restat: true, }, - "ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags") + &remoteexec.REParams{Labels: map[string]string{"type": "link", "tool": "clang"}, + ExecStrategy: "${config.RECXXLinksExecStrategy}", + Inputs: []string{"${out}.rsp"}, + RSPFile: "${out}.rsp", + OutputFiles: []string{"${out}"}, + ToolchainInputs: []string{"$ldCmd"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"}, + }, []string{"ldCmd", "crtBegin", "libFlags", "crtEnd", "ldFlags"}, nil) - partialLd = pctx.AndroidStaticRule("partialLd", + partialLd, partialLdRE = remoteexec.StaticRules(pctx, "partialLd", blueprint.RuleParams{ // Without -no-pie, clang 7.0 adds -pie to link Android files, // but -r and -pie cannot be used together. Command: "$ldCmd -nostdlib -no-pie -Wl,-r ${in} -o ${out} ${ldFlags}", CommandDeps: []string{"$ldCmd"}, - }, - "ldCmd", "ldFlags") + }, &remoteexec.REParams{ + Labels: map[string]string{"type": "link", "tool": "clang"}, + ExecStrategy: "${config.RECXXLinksExecStrategy}", Inputs: []string{"$inCommaList"}, + OutputFiles: []string{"${out}"}, + ToolchainInputs: []string{"$ldCmd"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"}, + }, []string{"ldCmd", "ldFlags"}, []string{"inCommaList"}) ar = pctx.AndroidStaticRule("ar", blueprint.RuleParams{ @@ -249,6 +262,7 @@ func init() { } pctx.HostBinToolVariable("SoongZipCmd", "soong_zip") + pctx.Import("android/soong/remoteexec") } type builderFlags struct { @@ -666,12 +680,17 @@ func TransformObjToDynamicBinary(ctx android.ModuleContext, deps = append(deps, crtBegin.Path(), crtEnd.Path()) } + rule := ld + if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { + rule = ldRE + } + ctx.Build(pctx, android.BuildParams{ - Rule: ld, - Description: "link " + outputFile.Base(), - Output: outputFile, - Inputs: objFiles, - Implicits: deps, + Rule: rule, + Description: "link " + outputFile.Base(), + Output: outputFile, + Inputs: objFiles, + Implicits: deps, Args: map[string]string{ "ldCmd": ldCmd, "crtBegin": crtBegin.String(), @@ -803,15 +822,21 @@ func TransformObjsToObj(ctx android.ModuleContext, objFiles android.Paths, ldCmd := "${config.ClangBin}/clang++" + rule := partialLd + args := map[string]string{ + "ldCmd": ldCmd, + "ldFlags": flags.ldFlags, + } + if ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { + rule = partialLdRE + args["inCommaList"] = strings.Join(objFiles.Strings(), ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: partialLd, + Rule: rule, Description: "link " + outputFile.Base(), Output: outputFile, Inputs: objFiles, - Args: map[string]string{ - "ldCmd": ldCmd, - "ldFlags": flags.ldFlags, - }, + Args: args, }) } diff --git a/cc/config/global.go b/cc/config/global.go index 815c31d89..ca4092d51 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -18,6 +18,7 @@ import ( "strings" "android/soong/android" + "android/soong/remoteexec" ) var ( @@ -232,6 +233,9 @@ func init() { } return "" }) + + pctx.VariableFunc("RECXXLinksPool", envOverrideFunc("RBE_CXX_LINKS_POOL", remoteexec.DefaultPool)) + pctx.VariableFunc("RECXXLinksExecStrategy", envOverrideFunc("RBE_CXX_LINKS_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) } var HostPrebuiltTag = pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS) @@ -245,3 +249,12 @@ func bionicHeaders(kernelArch string) string { "-isystem bionic/libc/kernel/android/uapi", }, " ") } + +func envOverrideFunc(envVar, defaultVal string) func(ctx android.PackageVarContext) string { + return func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv(envVar); override != "" { + return override + } + return defaultVal + } +} diff --git a/remoteexec/remoteexec.go b/remoteexec/remoteexec.go new file mode 100644 index 000000000..d43dc6c01 --- /dev/null +++ b/remoteexec/remoteexec.go @@ -0,0 +1,155 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remoteexec + +import ( + "sort" + "strings" + + "android/soong/android" + + "github.com/google/blueprint" +) + +const ( + // ContainerImageKey is the key identifying the container image in the platform spec. + ContainerImageKey = "container-image" + + // PoolKey is the key identifying the pool to use for remote execution. + PoolKey = "Pool" + + // DefaultImage is the default container image used for Android remote execution. The + // image was built with the Dockerfile at + // https://android.googlesource.com/platform/prebuilts/remoteexecution-client/+/refs/heads/master/docker/Dockerfile + DefaultImage = "docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:582efb38f0c229ea39952fff9e132ccbe183e14869b39888010dacf56b360d62" + + // DefaultWrapperPath is the default path to the remote execution wrapper. + DefaultWrapperPath = "prebuilts/remoteexecution-client/live/rewrapper" + + // DefaultPool is the name of the pool to use for remote execution when none is specified. + DefaultPool = "default" + + // LocalExecStrategy is the exec strategy to indicate that the action should be run locally. + LocalExecStrategy = "local" + + // RemoteExecStrategy is the exec strategy to indicate that the action should be run + // remotely. + RemoteExecStrategy = "remote" + + // RemoteLocalFallbackExecStrategy is the exec strategy to indicate that the action should + // be run remotely and fallback to local execution if remote fails. + RemoteLocalFallbackExecStrategy = "remote_local_fallback" +) + +var ( + defaultLabels = map[string]string{"type": "tool"} + defaultExecStrategy = LocalExecStrategy + pctx = android.NewPackageContext("android/soong/remoteexec") +) + +// REParams holds information pertinent to the remote execution of a rule. +type REParams struct { + // Platform is the key value pair used for remotely executing the action. + Platform map[string]string + // Labels is a map of labels that identify the rule. + Labels map[string]string + // ExecStrategy is the remote execution strategy: remote, local, or remote_local_fallback. + ExecStrategy string + // Inputs is a list of input paths or ninja variables. + Inputs []string + // RSPFile is the name of the ninja variable used by the rule as a placeholder for an rsp + // input. + RSPFile string + // OutputFiles is a list of output file paths or ninja variables as placeholders for rule + // outputs. + OutputFiles []string + // ToolchainInputs is a list of paths or ninja variables pointing to the location of + // toolchain binaries used by the rule. + ToolchainInputs []string +} + +func init() { + pctx.VariableFunc("Wrapper", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("RBE_WRAPPER"); override != "" { + return override + } + return DefaultWrapperPath + }) +} + +// Generate the remote execution wrapper template to be added as a prefix to the rule's command. +func (r *REParams) Template() string { + template := "${remoteexec.Wrapper}" + + var kvs []string + labels := r.Labels + if len(labels) == 0 { + labels = defaultLabels + } + for k, v := range labels { + kvs = append(kvs, k+"="+v) + } + sort.Strings(kvs) + template += " --labels=" + strings.Join(kvs, ",") + + var platform []string + for k, v := range r.Platform { + if v == "" { + continue + } + platform = append(platform, k+"="+v) + } + if _, ok := r.Platform[ContainerImageKey]; !ok { + platform = append(platform, ContainerImageKey+"="+DefaultImage) + } + if platform != nil { + sort.Strings(platform) + template += " --platform=\"" + strings.Join(platform, ",") + "\"" + } + + strategy := r.ExecStrategy + if strategy == "" { + strategy = defaultExecStrategy + } + template += " --exec_strategy=" + strategy + + if len(r.Inputs) > 0 { + template += " --inputs=" + strings.Join(r.Inputs, ",") + } + + if r.RSPFile != "" { + template += " --input_list_paths=" + r.RSPFile + } + + if len(r.OutputFiles) > 0 { + template += " --output_files=" + strings.Join(r.OutputFiles, ",") + } + + if len(r.ToolchainInputs) > 0 { + template += " --toolchain_inputs=" + strings.Join(r.ToolchainInputs, ",") + } + + return template + " -- " +} + +// StaticRules returns a pair of rules based on the given RuleParams, where the first rule is a +// locally executable rule and the second rule is a remotely executable rule. +func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.RuleParams, reParams *REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) { + ruleParamsRE := ruleParams + ruleParamsRE.Command = reParams.Template() + ruleParamsRE.Command + + return ctx.AndroidStaticRule(name, ruleParams, commonArgs...), + ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...) +} diff --git a/remoteexec/remoteexec_test.go b/remoteexec/remoteexec_test.go new file mode 100644 index 000000000..30e891ced --- /dev/null +++ b/remoteexec/remoteexec_test.go @@ -0,0 +1,83 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remoteexec + +import ( + "fmt" + "testing" +) + +func TestTemplate(t *testing.T) { + tests := []struct { + name string + params *REParams + want string + }{ + { + name: "basic", + params: &REParams{ + Labels: map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"}, + Inputs: []string{"$in"}, + OutputFiles: []string{"$out"}, + Platform: map[string]string{ + ContainerImageKey: DefaultImage, + PoolKey: "default", + }, + }, + want: fmt.Sprintf("${remoteexec.Wrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=local --inputs=$in --output_files=$out -- ", DefaultImage), + }, + { + name: "all params", + params: &REParams{ + Labels: map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"}, + Inputs: []string{"$in"}, + OutputFiles: []string{"$out"}, + ExecStrategy: "remote", + RSPFile: "$out.rsp", + ToolchainInputs: []string{"clang++"}, + Platform: map[string]string{ + ContainerImageKey: DefaultImage, + PoolKey: "default", + }, + }, + want: fmt.Sprintf("${remoteexec.Wrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=remote --inputs=$in --input_list_paths=$out.rsp --output_files=$out --toolchain_inputs=clang++ -- ", DefaultImage), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.params.Template(); got != test.want { + t.Errorf("Template() returned\n%s\nwant\n%s", got, test.want) + } + }) + } +} + +func TestTemplateDeterminism(t *testing.T) { + r := &REParams{ + Labels: map[string]string{"type": "compile", "lang": "cpp", "compiler": "clang"}, + Inputs: []string{"$in"}, + OutputFiles: []string{"$out"}, + Platform: map[string]string{ + ContainerImageKey: DefaultImage, + PoolKey: "default", + }, + } + want := fmt.Sprintf("${remoteexec.Wrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=local --inputs=$in --output_files=$out -- ", DefaultImage) + for i := 0; i < 1000; i++ { + if got := r.Template(); got != want { + t.Fatalf("Template() returned\n%s\nwant\n%s", got, want) + } + } +} diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index 2a6a9ca7e..2524e274b 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -177,6 +177,7 @@ func runMakeProductConfig(ctx Context, config Config) { // compiler wrappers set up by make "CC_WRAPPER", "CXX_WRAPPER", + "RBE_WRAPPER", "JAVAC_WRAPPER", "R8_WRAPPER", "D8_WRAPPER", From afa3c5f2ed5b707752346d728f708bd2145a5c6e Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Fri, 17 Apr 2020 15:03:58 -0400 Subject: [PATCH 31/53] [DO NOT MERGE] Add support for remotely executing header-abi-dumper actions. Bug: b/166182389 Test: built with and without RBE_ABI_DUMPER set. Change-Id: Ie752c07325453076d191813eaa50da03e3a0c2d8 Merged-In: Ie752c07325453076d191813eaa50da03e3a0c2d8 --- cc/builder.go | 41 ++++++++++++++++++++++++++++++---------- cc/config/global.go | 2 ++ remoteexec/remoteexec.go | 3 ++- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/cc/builder.go b/cc/builder.go index 3dc9a08c2..98c58dd84 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -20,6 +20,7 @@ package cc import ( "fmt" + "os" "path/filepath" "runtime" "strconv" @@ -44,6 +45,14 @@ var ( } ) +func absSrcDir() string { + dir, err := os.Getwd() + if err != nil { + panic(fmt.Errorf("failed to get working directory: %s", err)) + } + return dir +} + var ( pctx = android.NewPackageContext("android/soong/cc") @@ -65,7 +74,7 @@ var ( ld, ldRE = remoteexec.StaticRules(pctx, "ld", blueprint.RuleParams{ - Command: "$ldCmd ${crtBegin} @${out}.rsp " + + Command: "$reTemplate$ldCmd ${crtBegin} @${out}.rsp " + "${libFlags} ${crtEnd} -o ${out} ${ldFlags}", CommandDeps: []string{"$ldCmd"}, Rspfile: "${out}.rsp", @@ -73,7 +82,8 @@ var ( // clang -Wl,--out-implib doesn't update its output file if it hasn't changed. Restat: true, }, - &remoteexec.REParams{Labels: map[string]string{"type": "link", "tool": "clang"}, + &remoteexec.REParams{ + Labels: map[string]string{"type": "link", "tool": "clang"}, ExecStrategy: "${config.RECXXLinksExecStrategy}", Inputs: []string{"${out}.rsp"}, RSPFile: "${out}.rsp", @@ -86,11 +96,12 @@ var ( blueprint.RuleParams{ // Without -no-pie, clang 7.0 adds -pie to link Android files, // but -r and -pie cannot be used together. - Command: "$ldCmd -nostdlib -no-pie -Wl,-r ${in} -o ${out} ${ldFlags}", + Command: "$reTemplate$ldCmd -nostdlib -no-pie -Wl,-r ${in} -o ${out} ${ldFlags}", CommandDeps: []string{"$ldCmd"}, }, &remoteexec.REParams{ - Labels: map[string]string{"type": "link", "tool": "clang"}, - ExecStrategy: "${config.RECXXLinksExecStrategy}", Inputs: []string{"$inCommaList"}, + Labels: map[string]string{"type": "link", "tool": "clang"}, + ExecStrategy: "${config.RECXXLinksExecStrategy}", + Inputs: []string{"$inCommaList"}, OutputFiles: []string{"${out}"}, ToolchainInputs: []string{"$ldCmd"}, Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"}, @@ -202,12 +213,18 @@ var ( _ = pctx.SourcePathVariable("sAbiDumper", "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/header-abi-dumper") // -w has been added since header-abi-dumper does not need to produce any sort of diagnostic information. - sAbiDump = pctx.AndroidStaticRule("sAbiDump", + sAbiDump, sAbiDumpRE = remoteexec.StaticRules(pctx, "sAbiDump", blueprint.RuleParams{ - Command: "rm -f $out && $sAbiDumper -o ${out} $in $exportDirs -- $cFlags -w -isystem prebuilts/clang-tools/${config.HostPrebuiltTag}/clang-headers", + Command: "rm -f $out && $reTemplate$sAbiDumper -o ${out} $in $exportDirs -- $cFlags -w -isystem prebuilts/clang-tools/${config.HostPrebuiltTag}/clang-headers", CommandDeps: []string{"$sAbiDumper"}, - }, - "cFlags", "exportDirs") + }, &remoteexec.REParams{ + Labels: map[string]string{"type": "abi-dump", "tool": "header-abi-dumper"}, + ExecStrategy: "${config.REAbiDumperExecStrategy}", + Platform: map[string]string{ + remoteexec.PoolKey: "${config.RECXXPool}", + "InputRootAbsolutePath": absSrcDir(), + }, + }, []string{"cFlags", "exportDirs"}, nil) _ = pctx.SourcePathVariable("sAbiLinker", "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/header-abi-linker") @@ -499,8 +516,12 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and sAbiDumpFile := android.ObjPathWithExt(ctx, subdir, srcFile, "sdump") sAbiDumpFiles = append(sAbiDumpFiles, sAbiDumpFile) + dumpRule := sAbiDump + if ctx.Config().IsEnvTrue("RBE_ABI_DUMPER") { + dumpRule = sAbiDumpRE + } ctx.Build(pctx, android.BuildParams{ - Rule: sAbiDump, + Rule: dumpRule, Description: "header-abi-dumper " + srcFile.Rel(), Output: sAbiDumpFile, Input: srcFile, diff --git a/cc/config/global.go b/cc/config/global.go index ca4092d51..0799b77a9 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -234,8 +234,10 @@ func init() { return "" }) + pctx.VariableFunc("RECXXPool", envOverrideFunc("RBE_CXX_POOL", remoteexec.DefaultPool)) pctx.VariableFunc("RECXXLinksPool", envOverrideFunc("RBE_CXX_LINKS_POOL", remoteexec.DefaultPool)) pctx.VariableFunc("RECXXLinksExecStrategy", envOverrideFunc("RBE_CXX_LINKS_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REAbiDumperExecStrategy", envOverrideFunc("RBE_ABI_DUMPER_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) } var HostPrebuiltTag = pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS) diff --git a/remoteexec/remoteexec.go b/remoteexec/remoteexec.go index d43dc6c01..f51192263 100644 --- a/remoteexec/remoteexec.go +++ b/remoteexec/remoteexec.go @@ -148,7 +148,8 @@ func (r *REParams) Template() string { // locally executable rule and the second rule is a remotely executable rule. func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.RuleParams, reParams *REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) { ruleParamsRE := ruleParams - ruleParamsRE.Command = reParams.Template() + ruleParamsRE.Command + ruleParams.Command = strings.ReplaceAll(ruleParams.Command, "$reTemplate", "") + ruleParamsRE.Command = strings.ReplaceAll(ruleParamsRE.Command, "$reTemplate", reParams.Template()) return ctx.AndroidStaticRule(name, ruleParams, commonArgs...), ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...) From fc5a03c85c4f83649ef8bfd5fb7f17cb047d78e7 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Tue, 21 Apr 2020 21:36:23 -0400 Subject: [PATCH 32/53] [DO NOT MERGE] Refactor RBE support for Javac/R8/D8 to use the remoteexec package. Bug: b/166182389 Test: build with and without RBE_JAVAC/RBE_R8/RBE_D8 set. Change-Id: I1607f8cfb4d2c3cbb3672152bbeb561d9968bc60 Merged-In: I1607f8cfb4d2c3cbb3672152bbeb561d9968bc60 --- Android.bp | 1 + android/package_ctx.go | 36 ++------------------------- cc/config/global.go | 8 +++--- java/builder.go | 22 ++++++++++++----- java/config/config.go | 27 ++++++-------------- java/dex.go | 53 ++++++++++++++++++++++++++++------------ remoteexec/remoteexec.go | 11 +++++++++ ui/build/config.go | 42 ------------------------------- 8 files changed, 78 insertions(+), 122 deletions(-) diff --git a/Android.bp b/Android.bp index 31faa4237..28cce1c9c 100644 --- a/Android.bp +++ b/Android.bp @@ -303,6 +303,7 @@ bootstrap_go_package { deps: [ "blueprint-proptools", "soong-android", + "soong-remoteexec", ], srcs: [ "java/config/config.go", diff --git a/android/package_ctx.go b/android/package_ctx.go index ff10fe78e..166346930 100644 --- a/android/package_ctx.go +++ b/android/package_ctx.go @@ -254,30 +254,10 @@ func (p PackageContext) StaticRule(name string, params blueprint.RuleParams, }, argNames...) } -// RBEExperimentalFlag indicates which flag should be set for the AndroidRemoteStaticRule -// to use RBE. -type RBEExperimentalFlag int - -const ( - // RBE_NOT_EXPERIMENTAL indicates the rule should use RBE in every build that has - // UseRBE set. - RBE_NOT_EXPERIMENTAL RBEExperimentalFlag = iota - // RBE_JAVAC indicates the rule should use RBE only if the RBE_JAVAC variable is - // set in an RBE enabled build. - RBE_JAVAC - // RBE_R8 indicates the rule should use RBE only if the RBE_R8 variable is set in - // an RBE enabled build. - RBE_R8 - // RBE_D8 indicates the rule should use RBE only if the RBE_D8 variable is set in - // an RBE enabled build. - RBE_D8 -) - // RemoteRuleSupports configures rules with whether they have Goma and/or RBE support. type RemoteRuleSupports struct { - Goma bool - RBE bool - RBEFlag RBEExperimentalFlag + Goma bool + RBE bool } // AndroidRemoteStaticRule wraps blueprint.StaticRule but uses goma or RBE's parallelism if goma or RBE are enabled @@ -299,18 +279,6 @@ func (p PackageContext) AndroidRemoteStaticRule(name string, supports RemoteRule params.Pool = localPool } - if ctx.Config().UseRBE() && supports.RBE { - if supports.RBEFlag == RBE_JAVAC && !ctx.Config().UseRBEJAVAC() { - params.Pool = localPool - } - if supports.RBEFlag == RBE_R8 && !ctx.Config().UseRBER8() { - params.Pool = localPool - } - if supports.RBEFlag == RBE_D8 && !ctx.Config().UseRBED8() { - params.Pool = localPool - } - } - return params, nil }, argNames...) } diff --git a/cc/config/global.go b/cc/config/global.go index 0799b77a9..fde5cab19 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -234,10 +234,10 @@ func init() { return "" }) - pctx.VariableFunc("RECXXPool", envOverrideFunc("RBE_CXX_POOL", remoteexec.DefaultPool)) - pctx.VariableFunc("RECXXLinksPool", envOverrideFunc("RBE_CXX_LINKS_POOL", remoteexec.DefaultPool)) - pctx.VariableFunc("RECXXLinksExecStrategy", envOverrideFunc("RBE_CXX_LINKS_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) - pctx.VariableFunc("REAbiDumperExecStrategy", envOverrideFunc("RBE_ABI_DUMPER_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("RECXXPool", remoteexec.EnvOverrideFunc("RBE_CXX_POOL", remoteexec.DefaultPool)) + pctx.VariableFunc("RECXXLinksPool", remoteexec.EnvOverrideFunc("RBE_CXX_LINKS_POOL", remoteexec.DefaultPool)) + pctx.VariableFunc("RECXXLinksExecStrategy", remoteexec.EnvOverrideFunc("RBE_CXX_LINKS_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REAbiDumperExecStrategy", remoteexec.EnvOverrideFunc("RBE_ABI_DUMPER_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) } var HostPrebuiltTag = pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS) diff --git a/java/builder.go b/java/builder.go index e5ac7b00a..0e7574e63 100644 --- a/java/builder.go +++ b/java/builder.go @@ -27,6 +27,7 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android" + "android/soong/remoteexec" ) var ( @@ -38,12 +39,13 @@ var ( // this, all java rules write into separate directories and then are combined into a .jar file // (if the rule produces .class files) or a .srcjar file (if the rule produces .java files). // .srcjar files are unzipped into a temporary directory when compiled with javac. - javac = pctx.AndroidRemoteStaticRule("javac", android.RemoteRuleSupports{Goma: true, RBE: true, RBEFlag: android.RBE_JAVAC}, + // TODO(b/143658984): goma can't handle the --system argument to javac. + javac, javacRE = remoteexec.StaticRules(pctx, "javac", blueprint.RuleParams{ Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + `(if [ -s $srcJarDir/list ] || [ -s $out.rsp ] ; then ` + - `${config.SoongJavacWrapper} ${config.JavacWrapper}${config.JavacCmd} ${config.JavacHeapFlags} ${config.CommonJdkFlags} ` + + `${config.SoongJavacWrapper} $reTemplate${config.JavacCmd} ${config.JavacHeapFlags} ${config.CommonJdkFlags} ` + `$processorpath $processor $javacFlags $bootClasspath $classpath ` + `-source $javaVersion -target $javaVersion ` + `-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list ; fi ) && ` + @@ -57,9 +59,12 @@ var ( CommandOrderOnly: []string{"${config.SoongJavacWrapper}"}, Rspfile: "$out.rsp", RspfileContent: "$in", - }, - "javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir", - "outDir", "annoDir", "javaVersion") + }, &remoteexec.REParams{ + Labels: map[string]string{"type": "compile", "lang": "java", "compiler": "javac"}, + ExecStrategy: "${config.REJavacExecStrategy}", + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir", + "outDir", "annoDir", "javaVersion"}, nil) extractMatchingApks = pctx.StaticRule( "extractMatchingApks", @@ -157,6 +162,7 @@ var ( func init() { pctx.Import("android/soong/android") pctx.Import("android/soong/java/config") + pctx.Import("android/soong/remoteexec") } type javaBuilderFlags struct { @@ -290,8 +296,12 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab outDir = filepath.Join(shardDir, outDir) annoDir = filepath.Join(shardDir, annoDir) } + rule := javac + if ctx.Config().IsEnvTrue("RBE_JAVAC") { + rule = javacRE + } ctx.Build(pctx, android.BuildParams{ - Rule: javac, + Rule: rule, Description: desc, Output: outputFile, Inputs: srcFiles, diff --git a/java/config/config.go b/java/config/config.go index f76a39315..d36b87847 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -22,6 +22,7 @@ import ( _ "github.com/google/blueprint/bootstrap" "android/soong/android" + "android/soong/remoteexec" ) var ( @@ -118,30 +119,16 @@ func init() { pctx.HostJavaToolVariable("MetalavaJar", "metalava.jar") pctx.HostJavaToolVariable("DokkaJar", "dokka.jar") pctx.HostJavaToolVariable("JetifierJar", "jetifier.jar") + pctx.HostJavaToolVariable("R8Jar", "r8-compat-proguard.jar") + pctx.HostJavaToolVariable("D8Jar", "d8.jar") pctx.HostBinToolVariable("SoongJavacWrapper", "soong_javac_wrapper") pctx.HostBinToolVariable("DexpreoptGen", "dexpreopt_gen") - pctx.VariableFunc("JavacWrapper", func(ctx android.PackageVarContext) string { - if override := ctx.Config().Getenv("JAVAC_WRAPPER"); override != "" { - return override + " " - } - return "" - }) - - pctx.VariableFunc("R8Wrapper", func(ctx android.PackageVarContext) string { - if override := ctx.Config().Getenv("R8_WRAPPER"); override != "" { - return override + " " - } - return "" - }) - - pctx.VariableFunc("D8Wrapper", func(ctx android.PackageVarContext) string { - if override := ctx.Config().Getenv("D8_WRAPPER"); override != "" { - return override + " " - } - return "" - }) + pctx.VariableFunc("REJavaPool", remoteexec.EnvOverrideFunc("RBE_JAVA_POOL", "java16")) + pctx.VariableFunc("REJavacExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAVAC_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("RED8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_D8_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar") diff --git a/java/dex.go b/java/dex.go index 86a28fc18..45fa068d2 100644 --- a/java/dex.go +++ b/java/dex.go @@ -20,12 +20,13 @@ import ( "github.com/google/blueprint" "android/soong/android" + "android/soong/remoteexec" ) -var d8 = pctx.AndroidRemoteStaticRule("d8", android.RemoteRuleSupports{RBE: true, RBEFlag: android.RBE_D8}, +var d8, d8RE = remoteexec.StaticRules(pctx, "d8", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + - `${config.D8Wrapper}${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` + + `$reTemplate${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` + `${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + `${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`, CommandDeps: []string{ @@ -33,14 +34,19 @@ var d8 = pctx.AndroidRemoteStaticRule("d8", android.RemoteRuleSupports{RBE: true "${config.SoongZipCmd}", "${config.MergeZipsCmd}", }, - }, - "outDir", "d8Flags", "zipFlags") + }, &remoteexec.REParams{ + Labels: map[string]string{"type": "compile", "compiler": "d8"}, + Inputs: []string{"${config.D8Jar}"}, + ExecStrategy: "${config.RED8ExecStrategy}", + ToolchainInputs: []string{"${config.JavaCmd}"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"outDir", "d8Flags", "zipFlags"}, nil) -var r8 = pctx.AndroidRemoteStaticRule("r8", android.RemoteRuleSupports{RBE: true, RBEFlag: android.RBE_R8}, +var r8, r8RE = remoteexec.StaticRules(pctx, "r8", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + `rm -f "$outDict" && ` + - `${config.R8Wrapper}${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` + + `$reTemplate${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` + `--force-proguard-compatibility ` + `--no-data-resources ` + `-printmapping $outDict ` + @@ -53,8 +59,13 @@ var r8 = pctx.AndroidRemoteStaticRule("r8", android.RemoteRuleSupports{RBE: true "${config.SoongZipCmd}", "${config.MergeZipsCmd}", }, - }, - "outDir", "outDict", "r8Flags", "zipFlags") + }, &remoteexec.REParams{ + Labels: map[string]string{"type": "compile", "compiler": "r8"}, + Inputs: []string{"$implicits", "${config.R8Jar}"}, + ExecStrategy: "${config.RER8ExecStrategy}", + ToolchainInputs: []string{"${config.JavaCmd}"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"outDir", "outDict", "r8Flags", "zipFlags"}, []string{"implicits"}) func (j *Module) dexCommonFlags(ctx android.ModuleContext) []string { flags := j.deviceProperties.Dxflags @@ -186,24 +197,34 @@ func (j *Module) compileDex(ctx android.ModuleContext, flags javaBuilderFlags, proguardDictionary := android.PathForModuleOut(ctx, "proguard_dictionary") j.proguardDictionary = proguardDictionary r8Flags, r8Deps := j.r8Flags(ctx, flags) + rule := r8 + args := map[string]string{ + "r8Flags": strings.Join(r8Flags, " "), + "zipFlags": zipFlags, + "outDict": j.proguardDictionary.String(), + "outDir": outDir.String(), + } + if ctx.Config().IsEnvTrue("RBE_R8") { + rule = r8RE + args["implicits"] = strings.Join(r8Deps.Strings(), ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: r8, + Rule: rule, Description: "r8", Output: javalibJar, ImplicitOutput: proguardDictionary, Input: classesJar, Implicits: r8Deps, - Args: map[string]string{ - "r8Flags": strings.Join(r8Flags, " "), - "zipFlags": zipFlags, - "outDict": j.proguardDictionary.String(), - "outDir": outDir.String(), - }, + Args: args, }) } else { d8Flags, d8Deps := j.d8Flags(ctx, flags) + rule := d8 + if ctx.Config().IsEnvTrue("RBE_D8") { + rule = d8RE + } ctx.Build(pctx, android.BuildParams{ - Rule: d8, + Rule: rule, Description: "d8", Output: javalibJar, Input: classesJar, diff --git a/remoteexec/remoteexec.go b/remoteexec/remoteexec.go index f51192263..860db5568 100644 --- a/remoteexec/remoteexec.go +++ b/remoteexec/remoteexec.go @@ -154,3 +154,14 @@ func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.R return ctx.AndroidStaticRule(name, ruleParams, commonArgs...), ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...) } + +// EnvOverrideFunc retrieves a variable func that evaluates to the value of the given environment +// variable if set, otherwise the given default. +func EnvOverrideFunc(envVar, defaultVal string) func(ctx android.PackageVarContext) string { + return func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv(envVar); override != "" { + return override + } + return defaultVal + } +} diff --git a/ui/build/config.go b/ui/build/config.go index 8ba425be5..f279c139f 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -500,48 +500,6 @@ func (c *configImpl) UseRBE() bool { return false } -func (c *configImpl) UseRBEJAVAC() bool { - if !c.UseRBE() { - return false - } - - if v, ok := c.environ.Get("RBE_JAVAC"); ok { - v = strings.TrimSpace(v) - if v != "" && v != "false" { - return true - } - } - return false -} - -func (c *configImpl) UseRBER8() bool { - if !c.UseRBE() { - return false - } - - if v, ok := c.environ.Get("RBE_R8"); ok { - v = strings.TrimSpace(v) - if v != "" && v != "false" { - return true - } - } - return false -} - -func (c *configImpl) UseRBED8() bool { - if !c.UseRBE() { - return false - } - - if v, ok := c.environ.Get("RBE_D8"); ok { - v = strings.TrimSpace(v) - if v != "" && v != "false" { - return true - } - } - return false -} - func (c *configImpl) StartRBE() bool { if !c.UseRBE() { return false From 498a732edb3f2a3b72ecb83aeea45fddb7e22084 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Thu, 16 Jul 2020 12:18:37 -0400 Subject: [PATCH 33/53] [DO NOT MERGE] Generate RBE socket address randomly for each invocation of soong. Test: started soong multiple times and saw different socket files in out/soong/.temp Bug: b/166182389 Change-Id: Ib0087dd3867bdbf584c6868196070840cff291aa Merged-In: Ib0087dd3867bdbf584c6868196070840cff291aa --- ui/build/config.go | 10 +++++++++- ui/build/rbe.go | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ui/build/config.go b/ui/build/config.go index f279c139f..915919054 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -152,7 +152,8 @@ func NewConfig(ctx Context, args ...string) Config { // Tell python not to spam the source tree with .pyc files. ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") - ret.environ.Set("TMPDIR", absPath(ctx, ret.TempDir())) + tmpDir := absPath(ctx, ret.TempDir()) + ret.environ.Set("TMPDIR", tmpDir) // Precondition: the current directory is the top of the source tree if _, err := os.Stat(srcDirFileCheck); err != nil { @@ -218,12 +219,19 @@ func NewConfig(ctx Context, args ...string) Config { } else { content = strconv.FormatInt(time.Now().Unix(), 10) } + err := ioutil.WriteFile(buildDateTimeFile, []byte(content), 0777) if err != nil { ctx.Fatalln("Failed to write BUILD_DATETIME to file:", err) } ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) + if ret.UseRBE() { + for k, v := range getRBEVars(ctx, tmpDir) { + ret.environ.Set(k, v) + } + } + return Config{ret} } diff --git a/ui/build/rbe.go b/ui/build/rbe.go index c80b8ea7d..147afcc72 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -15,7 +15,10 @@ package build import ( + "fmt" + "math/rand" "path/filepath" + "time" "android/soong/ui/metrics" ) @@ -24,6 +27,11 @@ const bootstrapCmd = "bootstrap" const rbeLeastNProcs = 2500 const rbeLeastNFiles = 16000 +func getRBEVars(ctx Context, tmpDir string) map[string]string { + rand.Seed(time.Now().UnixNano()) + return map[string]string{"RBE_server_address": fmt.Sprintf("unix://%v/reproxy_%v.sock", tmpDir, rand.Intn(1000))} +} + func startRBE(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSetupTool, "rbe_bootstrap") defer ctx.EndTrace() From fbc4e3f9b60121f9ca47241ea6932d4cc2f19b5e Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Tue, 7 Jul 2020 12:48:26 +0000 Subject: [PATCH 34/53] [DO NOT MERGE] Add RBE metrics dump in Soong UI. From aosp/1329396, the RBE metrics protobuf file is part of the metrics uploading process. The RBE metrics protobuf file is generated by running the bootstrap shutdown command. A new function named DumpRBEMetrics was written in order to generate the RBE metrics protobuf file before sending to the uploading process. Bug: b/140638454 Test: * Unit test cases * Ran RBE build on my local host and verified the metrics protobuf file is created. * Ran non-RBE build after RBE build and verified that the previous metrics protobuf file was deleted. Change-Id: I4b8068905cb67c4b8c2d94793917b98974fed707 Merged-In: I4b8068905cb67c4b8c2d94793917b98974fed707 --- cmd/soong_ui/main.go | 1 + ui/build/Android.bp | 1 + ui/build/config.go | 13 ++++ ui/build/rbe.go | 83 +++++++++++++++++++---- ui/build/rbe_test.go | 150 ++++++++++++++++++++++++++++++++++++++++++ ui/metrics/metrics.go | 12 ++-- 6 files changed, 242 insertions(+), 18 deletions(-) create mode 100644 ui/build/rbe_test.go diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 2d938a3b3..e86246ca8 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -131,6 +131,7 @@ func main() { stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log"))) defer met.Dump(soongMetricsFile) + defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile) if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { if !strings.HasSuffix(start, "N") { diff --git a/ui/build/Android.bp b/ui/build/Android.bp index 65729821b..cb2b76c2e 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -62,6 +62,7 @@ bootstrap_go_package { testSrcs: [ "config_test.go", "environment_test.go", + "rbe_test.go", "upload_test.go", "util_test.go", "proc_sync_test.go", diff --git a/ui/build/config.go b/ui/build/config.go index 915919054..e0df7b819 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -522,6 +522,19 @@ func (c *configImpl) StartRBE() bool { return true } +func (c *configImpl) RBEStatsOutputDir() string { + for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + return "" +} + +func (c *configImpl) UseRemoteBuild() bool { + return c.UseGoma() || c.UseRBE() +} + // RemoteParallel controls how many remote jobs (i.e., commands which contain // gomacc) are run in parallel. Note the parallelism of all other jobs is // still limited by Parallel() diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 147afcc72..fd3b7abb5 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -17,15 +17,40 @@ package build import ( "fmt" "math/rand" + "os" "path/filepath" "time" "android/soong/ui/metrics" ) -const bootstrapCmd = "bootstrap" -const rbeLeastNProcs = 2500 -const rbeLeastNFiles = 16000 +const ( + rbeLeastNProcs = 2500 + rbeLeastNFiles = 16000 + + // prebuilt RBE binaries + bootstrapCmd = "bootstrap" + + // RBE metrics proto buffer file + rbeMetricsPBFilename = "rbe_metrics.pb" +) + +func rbeCommand(ctx Context, config Config, rbeCmd string) string { + var cmdPath string + if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok { + cmdPath = filepath.Join(rbeDir, rbeCmd) + } else if home, ok := config.Environment().Get("HOME"); ok { + cmdPath = filepath.Join(home, "rbe", rbeCmd) + } else { + ctx.Fatalf("rbe command path not found") + } + + if _, err := os.Stat(cmdPath); err != nil && os.IsNotExist(err) { + ctx.Fatalf("rbe command %q not found", rbeCmd) + } + + return cmdPath +} func getRBEVars(ctx Context, tmpDir string) map[string]string { rand.Seed(time.Now().UnixNano()) @@ -43,18 +68,50 @@ func startRBE(ctx Context, config Config) { ctx.Fatalf("max open files is insufficient: %d; want >= %d.\n", n, rbeLeastNFiles) } - var rbeBootstrap string - if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok { - rbeBootstrap = filepath.Join(rbeDir, bootstrapCmd) - } else if home, ok := config.Environment().Get("HOME"); ok { - rbeBootstrap = filepath.Join(home, "rbe", bootstrapCmd) - } else { - ctx.Fatalln("rbe bootstrap not found") - } - - cmd := Command(ctx, config, "bootstrap", rbeBootstrap) + cmd := Command(ctx, config, "startRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd)) if output, err := cmd.CombinedOutput(); err != nil { ctx.Fatalf("rbe bootstrap failed with: %v\n%s\n", err, output) } } + +func stopRBE(ctx Context, config Config) { + cmd := Command(ctx, config, "stopRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd), "-shutdown") + if output, err := cmd.CombinedOutput(); err != nil { + ctx.Fatalf("rbe bootstrap with shutdown failed with: %v\n%s\n", err, output) + } +} + +// DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics. +// The protobuf file is created if RBE is enabled and the proxy service has +// started. The proxy service is shutdown in order to dump the RBE metrics to the +// protobuf file. +func DumpRBEMetrics(ctx Context, config Config, filename string) { + ctx.BeginTrace(metrics.RunShutdownTool, "dump_rbe_metrics") + defer ctx.EndTrace() + + // Remove the previous metrics file in case there is a failure or RBE has been + // disable for this run. + os.Remove(filename) + + // If RBE is not enabled then there are no metrics to generate. + // If RBE does not require to start, the RBE proxy maybe started + // manually for debugging purpose and can generate the metrics + // afterwards. + if !config.StartRBE() { + return + } + + outputDir := config.RBEStatsOutputDir() + if outputDir == "" { + ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.") + } + metricsFile := filepath.Join(outputDir, rbeMetricsPBFilename) + + // Stop the proxy first in order to generate the RBE metrics protobuf file. + stopRBE(ctx, config) + + if _, err := copyFile(metricsFile, filename); err != nil { + ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err) + } +} diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go new file mode 100644 index 000000000..9a3f0e967 --- /dev/null +++ b/ui/build/rbe_test.go @@ -0,0 +1,150 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "android/soong/ui/logger" +) + +func TestDumpRBEMetrics(t *testing.T) { + ctx := testContext() + tests := []struct { + description string + env []string + generated bool + }{{ + description: "RBE disabled", + env: []string{ + "NOSTART_RBE=true", + }, + }, { + description: "rbe metrics generated", + env: []string{ + "USE_RBE=true", + }, + generated: true, + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "tmpdir") + if err != nil { + t.Fatalf("failed to make temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd) + if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(rbeBootstrapProgram), 0755); err != nil { + t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err) + } + + env := Environment(tt.env) + env.Set("OUT_DIR", tmpDir) + env.Set("RBE_DIR", tmpDir) + env.Set("RBE_output_dir", tmpDir) + config := Config{&configImpl{ + environ: &env, + }} + + rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename) + DumpRBEMetrics(ctx, config, rbeMetricsFilename) + + // Validate that the rbe metrics file exists if RBE is enabled. + if _, err := os.Stat(rbeMetricsFilename); err == nil { + if !tt.generated { + t.Errorf("got true, want false for rbe metrics file %s to exist.", rbeMetricsFilename) + } + } else if os.IsNotExist(err) { + if tt.generated { + t.Errorf("got false, want true for rbe metrics file %s to exist.", rbeMetricsFilename) + } + } else { + t.Errorf("unknown error found on checking %s exists: %v", rbeMetricsFilename, err) + } + }) + } +} + +func TestDumpRBEMetricsErrors(t *testing.T) { + ctx := testContext() + tests := []struct { + description string + rbeOutputDirDefined bool + bootstrapProgram string + expectedErr string + }{{ + description: "output_dir not defined", + bootstrapProgram: rbeBootstrapProgram, + expectedErr: "RBE output dir variable not defined", + }, { + description: "stopRBE failed", + rbeOutputDirDefined: true, + bootstrapProgram: "#!/bin/bash\nexit 1", + expectedErr: "shutdown failed", + }, { + description: "failed to copy metrics file", + rbeOutputDirDefined: true, + bootstrapProgram: "#!/bin/bash", + expectedErr: "failed to copy", + }} + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + defer logger.Recover(func(err error) { + got := err.Error() + if !strings.Contains(got, tt.expectedErr) { + t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr) + } + }) + + tmpDir, err := ioutil.TempDir("", "tmpdir") + if err != nil { + t.Fatalf("failed to make temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd) + if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(tt.bootstrapProgram), 0755); err != nil { + t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err) + } + + env := &Environment{} + env.Set("USE_RBE", "true") + env.Set("OUT_DIR", tmpDir) + env.Set("RBE_DIR", tmpDir) + + if tt.rbeOutputDirDefined { + env.Set("RBE_output_dir", tmpDir) + } + + config := Config{&configImpl{ + environ: env, + }} + + rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename) + DumpRBEMetrics(ctx, config, rbeMetricsFilename) + t.Errorf("got nil, expecting %q as a failure", tt.expectedErr) + }) + } +} + +var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s", rbeMetricsPBFilename) diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 8ae085cdf..d550f22c4 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -26,11 +26,13 @@ import ( ) const ( - RunSetupTool = "setup" - RunKati = "kati" - RunSoong = "soong" - PrimaryNinja = "ninja" - TestRun = "test" + PrimaryNinja = "ninja" + RunKati = "kati" + RunSetupTool = "setup" + RunShutdownTool = "shutdown" + RunSoong = "soong" + TestRun = "test" + Total = "total" ) type Metrics struct { From 11c7e806f625dbf9b0e424a751f778880e24ef60 Mon Sep 17 00:00:00 2001 From: Ola Rozenfeld Date: Thu, 19 Mar 2020 20:04:13 -0400 Subject: [PATCH 35/53] Migrating RBE flags to RBE_ variables. Test: N/A, this only adds new variables. Bug: b/167405184 Change-Id: Iad09352090ec255af82e77941e6e050137f1e83b Merged-In: Iad09352090ec255af82e77941e6e050137f1e83b --- ui/build/ninja.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ui/build/ninja.go b/ui/build/ninja.go index 4fc1f01e5..dfc3be1f3 100644 --- a/ui/build/ninja.go +++ b/ui/build/ninja.go @@ -128,6 +128,16 @@ func runNinja(ctx Context, config Config) { "GOMA_USE_LOCAL", // RBE client + "RBE_compare", + "RBE_exec_root", + "RBE_exec_strategy", + "RBE_invocation_id", + "RBE_log_dir", + "RBE_platform", + "RBE_remote_accept_cache", + "RBE_remote_update_cache", + "RBE_server_address", + // TODO: remove old FLAG_ variables. "FLAG_compare", "FLAG_exec_root", "FLAG_exec_strategy", From b7cfc8eabd852e8315cff0cb1c819e48e85d3809 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Thu, 16 Jul 2020 12:18:37 -0400 Subject: [PATCH 36/53] Generate RBE socket address randomly for each invocation of soong. Test: started soong multiple times and saw different socket files in out/soong/.temp Bug: b/140638454 Change-Id: Ib0087dd3867bdbf584c6868196070840cff291aa Merged-In: Ib0087dd3867bdbf584c6868196070840cff291aa --- ui/build/config.go | 9 ++++++++- ui/build/rbe.go | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ui/build/config.go b/ui/build/config.go index f0310d5a7..c504b7d8b 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -187,7 +187,8 @@ func NewConfig(ctx Context, args ...string) Config { // Tell python not to spam the source tree with .pyc files. ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") - ret.environ.Set("TMPDIR", absPath(ctx, ret.TempDir())) + tmpDir := absPath(ctx, ret.TempDir()) + ret.environ.Set("TMPDIR", tmpDir) // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(), @@ -261,6 +262,12 @@ func NewConfig(ctx Context, args ...string) Config { ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) + if ret.UseRBE() { + for k, v := range getRBEVars(ctx, tmpDir) { + ret.environ.Set(k, v) + } + } + c := Config{ret} storeConfigMetrics(ctx, c) return c diff --git a/ui/build/rbe.go b/ui/build/rbe.go index fcdab3b03..fd3b7abb5 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -15,8 +15,11 @@ package build import ( + "fmt" + "math/rand" "os" "path/filepath" + "time" "android/soong/ui/metrics" ) @@ -49,6 +52,11 @@ func rbeCommand(ctx Context, config Config, rbeCmd string) string { return cmdPath } +func getRBEVars(ctx Context, tmpDir string) map[string]string { + rand.Seed(time.Now().UnixNano()) + return map[string]string{"RBE_server_address": fmt.Sprintf("unix://%v/reproxy_%v.sock", tmpDir, rand.Intn(1000))} +} + func startRBE(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSetupTool, "rbe_bootstrap") defer ctx.EndTrace() From b258d9367c878e9946b3e4f37bf7f329a92fe1d0 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Sat, 1 Aug 2020 16:49:35 +0000 Subject: [PATCH 37/53] [DO NOT MERGE] Add newline to each generated bash script in rbe_test.go. "exec format error" showed up on Mac when running the rbe_test.go. The generated bash scripts did not have a newline at the end of the file. Bug: b/162600461 Fixes: b/162600461 Test: "m blueprint_tools" on Android Mac builder. Change-Id: I425fa5f428733b6569c3fcdaa312023f9761d020 Merged-In: I425fa5f428733b6569c3fcdaa312023f9761d020 --- ui/build/rbe_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go index 9a3f0e967..ba56c6bb8 100644 --- a/ui/build/rbe_test.go +++ b/ui/build/rbe_test.go @@ -98,12 +98,12 @@ func TestDumpRBEMetricsErrors(t *testing.T) { }, { description: "stopRBE failed", rbeOutputDirDefined: true, - bootstrapProgram: "#!/bin/bash\nexit 1", + bootstrapProgram: "#!/bin/bash\nexit 1\n", expectedErr: "shutdown failed", }, { description: "failed to copy metrics file", rbeOutputDirDefined: true, - bootstrapProgram: "#!/bin/bash", + bootstrapProgram: "#!/bin/bash\n", expectedErr: "failed to copy", }} @@ -147,4 +147,4 @@ func TestDumpRBEMetricsErrors(t *testing.T) { } } -var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s", rbeMetricsPBFilename) +var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s\n", rbeMetricsPBFilename) From 19c39230e9588a3adc743b2abe843f8926de03f1 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Wed, 12 Aug 2020 01:26:23 -0400 Subject: [PATCH 38/53] [DO NOT MERGE] Add reasonable defaults to RBE configuration parameters. Bug: b/166182389 Test: simple one action build Change-Id: Ic66ad2b89866a67008950035bc3b559dae4e3a3e Merged-In: Ic66ad2b89866a67008950035bc3b559dae4e3a3e --- ui/build/config.go | 67 ++++++++++++++++++++++++++++++++++++++++++-- ui/build/rbe.go | 22 +++++++++++---- ui/build/rbe_test.go | 27 ++++-------------- 3 files changed, 86 insertions(+), 30 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index e0df7b819..889e5ccc5 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -17,6 +17,7 @@ package build import ( "io/ioutil" "log" + "fmt" "os" "path/filepath" "runtime" @@ -227,7 +228,7 @@ func NewConfig(ctx Context, args ...string) Config { ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) if ret.UseRBE() { - for k, v := range getRBEVars(ctx, tmpDir) { + for k, v := range getRBEVars(ctx, Config{ret}) { ret.environ.Set(k, v) } } @@ -522,13 +523,73 @@ func (c *configImpl) StartRBE() bool { return true } -func (c *configImpl) RBEStatsOutputDir() string { +func (c *configImpl) logDir() string { + if c.Dist() { + return filepath.Join(c.DistDir(), "logs") + } + return c.OutDir() +} + +func (c *configImpl) rbeStatsOutputDir() string { for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} { if v, ok := c.environ.Get(f); ok { return v } } - return "" + return c.logDir() +} + +func (c *configImpl) rbeLogPath() string { + for _, f := range []string{"RBE_log_path", "FLAG_log_path"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + return fmt.Sprintf("text://%v/reproxy_log.txt", c.logDir()) +} + +func (c *configImpl) rbeExecRoot() string { + for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + wd, err := os.Getwd() + if err != nil { + return "" + } + return wd +} + +func (c *configImpl) rbeDir() string { + if v, ok := c.environ.Get("RBE_DIR"); ok { + return v + } + return "prebuilts/remoteexecution-client/live/" +} + +func (c *configImpl) rbeReproxy() string { + for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + return filepath.Join(c.rbeDir(), "reproxy") +} + +func (c *configImpl) rbeAuth() (string, string) { + credFlags := []string{"use_application_default_credentials", "use_gce_credentials", "credential_file"} + for _, cf := range credFlags { + for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} { + if v, ok := c.environ.Get(f); ok { + v = strings.TrimSpace(v) + if v != "" && v != "false" && v != "0" { + return "RBE_" + cf, v + } + } + } + } + return "RBE_use_application_default_credentials", "true" } func (c *configImpl) UseRemoteBuild() bool { diff --git a/ui/build/rbe.go b/ui/build/rbe.go index fd3b7abb5..99107e3d9 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -37,10 +37,8 @@ const ( func rbeCommand(ctx Context, config Config, rbeCmd string) string { var cmdPath string - if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok { + if rbeDir := config.rbeDir(); rbeDir != "" { cmdPath = filepath.Join(rbeDir, rbeCmd) - } else if home, ok := config.Environment().Get("HOME"); ok { - cmdPath = filepath.Join(home, "rbe", rbeCmd) } else { ctx.Fatalf("rbe command path not found") } @@ -52,9 +50,18 @@ func rbeCommand(ctx Context, config Config, rbeCmd string) string { return cmdPath } -func getRBEVars(ctx Context, tmpDir string) map[string]string { +func getRBEVars(ctx Context, config Config) map[string]string { rand.Seed(time.Now().UnixNano()) - return map[string]string{"RBE_server_address": fmt.Sprintf("unix://%v/reproxy_%v.sock", tmpDir, rand.Intn(1000))} + vars := map[string]string{ + "RBE_server_address": fmt.Sprintf("unix://%v/reproxy_%v.sock", absPath(ctx, config.TempDir()), rand.Intn(1000)), + "RBE_log_path": config.rbeLogPath(), + "RBE_re_proxy": config.rbeReproxy(), + "RBE_exec_root": config.rbeExecRoot(), + "RBE_output_dir": config.rbeStatsOutputDir(), + } + k, v := config.rbeAuth() + vars[k] = v + return vars } func startRBE(ctx Context, config Config) { @@ -102,7 +109,7 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) { return } - outputDir := config.RBEStatsOutputDir() + outputDir := config.rbeStatsOutputDir() if outputDir == "" { ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.") } @@ -111,6 +118,9 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) { // Stop the proxy first in order to generate the RBE metrics protobuf file. stopRBE(ctx, config) + if metricsFile == filename { + return + } if _, err := copyFile(metricsFile, filename); err != nil { ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err) } diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go index ba56c6bb8..d3a52b7ef 100644 --- a/ui/build/rbe_test.go +++ b/ui/build/rbe_test.go @@ -87,24 +87,13 @@ func TestDumpRBEMetrics(t *testing.T) { func TestDumpRBEMetricsErrors(t *testing.T) { ctx := testContext() tests := []struct { - description string - rbeOutputDirDefined bool - bootstrapProgram string - expectedErr string + description string + bootstrapProgram string + expectedErr string }{{ - description: "output_dir not defined", - bootstrapProgram: rbeBootstrapProgram, - expectedErr: "RBE output dir variable not defined", - }, { - description: "stopRBE failed", - rbeOutputDirDefined: true, - bootstrapProgram: "#!/bin/bash\nexit 1\n", - expectedErr: "shutdown failed", - }, { - description: "failed to copy metrics file", - rbeOutputDirDefined: true, - bootstrapProgram: "#!/bin/bash\n", - expectedErr: "failed to copy", + description: "stopRBE failed", + bootstrapProgram: "#!/bin/bash\nexit 1\n", + expectedErr: "shutdown failed", }} for _, tt := range tests { @@ -132,10 +121,6 @@ func TestDumpRBEMetricsErrors(t *testing.T) { env.Set("OUT_DIR", tmpDir) env.Set("RBE_DIR", tmpDir) - if tt.rbeOutputDirDefined { - env.Set("RBE_output_dir", tmpDir) - } - config := Config{&configImpl{ environ: env, }} From 00ff4cc1efced8bea445c4dce69a9e16bdae72d0 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Thu, 13 Aug 2020 22:53:42 -0400 Subject: [PATCH 39/53] [DO NOT MERGE] Do not generate a random socket address for RBE if NOSTART_RBE is set. Bug: b/166182389 Test: A build with NOSTART_RBE=1 Change-Id: I5872632442c754b597901b7729eed597ee45859f Merged-In: I5872632442c754b597901b7729eed597ee45859f --- ui/build/rbe.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 99107e3d9..73438ccb8 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -53,11 +53,13 @@ func rbeCommand(ctx Context, config Config, rbeCmd string) string { func getRBEVars(ctx Context, config Config) map[string]string { rand.Seed(time.Now().UnixNano()) vars := map[string]string{ - "RBE_server_address": fmt.Sprintf("unix://%v/reproxy_%v.sock", absPath(ctx, config.TempDir()), rand.Intn(1000)), - "RBE_log_path": config.rbeLogPath(), - "RBE_re_proxy": config.rbeReproxy(), - "RBE_exec_root": config.rbeExecRoot(), - "RBE_output_dir": config.rbeStatsOutputDir(), + "RBE_log_path": config.rbeLogPath(), + "RBE_re_proxy": config.rbeReproxy(), + "RBE_exec_root": config.rbeExecRoot(), + "RBE_output_dir": config.rbeStatsOutputDir(), + } + if config.StartRBE() { + vars["RBE_server_address"] = fmt.Sprintf("unix://%v/reproxy_%v.sock", absPath(ctx, config.TempDir()), rand.Intn(1000)) } k, v := config.rbeAuth() vars[k] = v From 8229916e1fcd98ca68e0dc471025367ac22cf777 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Thu, 13 Aug 2020 14:51:51 -0400 Subject: [PATCH 40/53] [DO NOT MERGE] Add a default value for the RBE_log_dir variable. Bug: b/166182389 Test: built simple target. Change-Id: Id93dd80119037b594e5f8c370f4ac2fbc12a5144 Merged-In: Id93dd80119037b594e5f8c370f4ac2fbc12a5144 --- ui/build/rbe.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 73438ccb8..6a26063b0 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -54,6 +54,7 @@ func getRBEVars(ctx Context, config Config) map[string]string { rand.Seed(time.Now().UnixNano()) vars := map[string]string{ "RBE_log_path": config.rbeLogPath(), + "RBE_log_dir": config.logDir(), "RBE_re_proxy": config.rbeReproxy(), "RBE_exec_root": config.rbeExecRoot(), "RBE_output_dir": config.rbeStatsOutputDir(), From 320c0417976904ff84987999ce5a00da93daf3f3 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Thu, 13 Aug 2020 11:41:35 -0400 Subject: [PATCH 41/53] [DO NOT MERGE] Make the default exec strategy for javac/r8/d8 remote local fallback. Bug: b/166182389 Test: presubmits Change-Id: Ia67c2a1e13638db1fb9c93755ce72159767d46b4 Merged-In: Ia67c2a1e13638db1fb9c93755ce72159767d46b4 --- java/config/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/config/config.go b/java/config/config.go index d36b87847..c3523bee4 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -126,9 +126,9 @@ func init() { pctx.HostBinToolVariable("DexpreoptGen", "dexpreopt_gen") pctx.VariableFunc("REJavaPool", remoteexec.EnvOverrideFunc("RBE_JAVA_POOL", "java16")) - pctx.VariableFunc("REJavacExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAVAC_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) - pctx.VariableFunc("RED8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_D8_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) - pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REJavacExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAVAC_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)) + pctx.VariableFunc("RED8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_D8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)) + pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)) pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar") From 54814ca5e6c11be527451c8177a2cf31ac5e22c3 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Wed, 12 Aug 2020 01:26:23 -0400 Subject: [PATCH 42/53] Add reasonable defaults to RBE configuration parameters. Test: simple one action build Bug: b/140638454 Change-Id: Ic66ad2b89866a67008950035bc3b559dae4e3a3e Merged-In: I060e46f228493d1793f727840cf320815a55d3fa --- ui/build/config.go | 67 ++++++++++++++++++++++++++++++++++++++++++-- ui/build/rbe.go | 22 +++++++++++---- ui/build/rbe_test.go | 32 ++++----------------- 3 files changed, 86 insertions(+), 35 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index c504b7d8b..5bf4bafe3 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -15,6 +15,7 @@ package build import ( + "fmt" "os" "path/filepath" "runtime" @@ -263,7 +264,7 @@ func NewConfig(ctx Context, args ...string) Config { ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) if ret.UseRBE() { - for k, v := range getRBEVars(ctx, tmpDir) { + for k, v := range getRBEVars(ctx, Config{ret}) { ret.environ.Set(k, v) } } @@ -827,13 +828,73 @@ func (c *configImpl) StartRBE() bool { return true } -func (c *configImpl) RBEStatsOutputDir() string { +func (c *configImpl) logDir() string { + if c.Dist() { + return filepath.Join(c.DistDir(), "logs") + } + return c.OutDir() +} + +func (c *configImpl) rbeStatsOutputDir() string { for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} { if v, ok := c.environ.Get(f); ok { return v } } - return "" + return c.logDir() +} + +func (c *configImpl) rbeLogPath() string { + for _, f := range []string{"RBE_log_path", "FLAG_log_path"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + return fmt.Sprintf("text://%v/reproxy_log.txt", c.logDir()) +} + +func (c *configImpl) rbeExecRoot() string { + for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + wd, err := os.Getwd() + if err != nil { + return "" + } + return wd +} + +func (c *configImpl) rbeDir() string { + if v, ok := c.environ.Get("RBE_DIR"); ok { + return v + } + return "prebuilts/remoteexecution-client/live/" +} + +func (c *configImpl) rbeReproxy() string { + for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} { + if v, ok := c.environ.Get(f); ok { + return v + } + } + return filepath.Join(c.rbeDir(), "reproxy") +} + +func (c *configImpl) rbeAuth() (string, string) { + credFlags := []string{"use_application_default_credentials", "use_gce_credentials", "credential_file"} + for _, cf := range credFlags { + for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} { + if v, ok := c.environ.Get(f); ok { + v = strings.TrimSpace(v) + if v != "" && v != "false" && v != "0" { + return "RBE_" + cf, v + } + } + } + } + return "RBE_use_application_default_credentials", "true" } func (c *configImpl) UseRemoteBuild() bool { diff --git a/ui/build/rbe.go b/ui/build/rbe.go index fd3b7abb5..99107e3d9 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -37,10 +37,8 @@ const ( func rbeCommand(ctx Context, config Config, rbeCmd string) string { var cmdPath string - if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok { + if rbeDir := config.rbeDir(); rbeDir != "" { cmdPath = filepath.Join(rbeDir, rbeCmd) - } else if home, ok := config.Environment().Get("HOME"); ok { - cmdPath = filepath.Join(home, "rbe", rbeCmd) } else { ctx.Fatalf("rbe command path not found") } @@ -52,9 +50,18 @@ func rbeCommand(ctx Context, config Config, rbeCmd string) string { return cmdPath } -func getRBEVars(ctx Context, tmpDir string) map[string]string { +func getRBEVars(ctx Context, config Config) map[string]string { rand.Seed(time.Now().UnixNano()) - return map[string]string{"RBE_server_address": fmt.Sprintf("unix://%v/reproxy_%v.sock", tmpDir, rand.Intn(1000))} + vars := map[string]string{ + "RBE_server_address": fmt.Sprintf("unix://%v/reproxy_%v.sock", absPath(ctx, config.TempDir()), rand.Intn(1000)), + "RBE_log_path": config.rbeLogPath(), + "RBE_re_proxy": config.rbeReproxy(), + "RBE_exec_root": config.rbeExecRoot(), + "RBE_output_dir": config.rbeStatsOutputDir(), + } + k, v := config.rbeAuth() + vars[k] = v + return vars } func startRBE(ctx Context, config Config) { @@ -102,7 +109,7 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) { return } - outputDir := config.RBEStatsOutputDir() + outputDir := config.rbeStatsOutputDir() if outputDir == "" { ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.") } @@ -111,6 +118,9 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) { // Stop the proxy first in order to generate the RBE metrics protobuf file. stopRBE(ctx, config) + if metricsFile == filename { + return + } if _, err := copyFile(metricsFile, filename); err != nil { ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err) } diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go index 92ef77921..c56a33c0a 100644 --- a/ui/build/rbe_test.go +++ b/ui/build/rbe_test.go @@ -94,24 +94,13 @@ func TestDumpRBEMetrics(t *testing.T) { func TestDumpRBEMetricsErrors(t *testing.T) { ctx := testContext() tests := []struct { - description string - rbeOutputDirDefined bool - bootstrapProgram string - expectedErr string + description string + bootstrapProgram string + expectedErr string }{{ - description: "output_dir not defined", - bootstrapProgram: rbeBootstrapProgram, - expectedErr: "RBE output dir variable not defined", - }, { - description: "stopRBE failed", - rbeOutputDirDefined: true, - bootstrapProgram: "#!/bin/bash\nexit 1\n", - expectedErr: "shutdown failed", - }, { - description: "failed to copy metrics file", - rbeOutputDirDefined: true, - bootstrapProgram: "#!/bin/bash\n", - expectedErr: "failed to copy", + description: "stopRBE failed", + bootstrapProgram: "#!/bin/bash\nexit 1\n", + expectedErr: "shutdown failed", }} for _, tt := range tests { @@ -139,15 +128,6 @@ func TestDumpRBEMetricsErrors(t *testing.T) { env.Set("OUT_DIR", tmpDir) env.Set("RBE_DIR", tmpDir) - if tt.rbeOutputDirDefined { - tmpRBEDir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("failed to create a temp directory for RBE: %v", err) - } - defer os.RemoveAll(tmpRBEDir) - env.Set("RBE_output_dir", tmpRBEDir) - } - config := Config{&configImpl{ environ: env, }} From d2c89f6311bdcc0325b01ef0517f22449b0f5af4 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Thu, 13 Aug 2020 11:41:35 -0400 Subject: [PATCH 43/53] Make the default exec strategy for javac/r8/d8 remote local fallback. Test: presubmits Bug: b/166273085 Change-Id: Ia67c2a1e13638db1fb9c93755ce72159767d46b4 Merged-In: Ia67c2a1e13638db1fb9c93755ce72159767d46b4 --- java/config/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/config/config.go b/java/config/config.go index 95add017d..1d0dd617e 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -148,9 +148,9 @@ func init() { pctx.HostBinToolVariable("DexpreoptGen", "dexpreopt_gen") pctx.VariableFunc("REJavaPool", remoteexec.EnvOverrideFunc("RBE_JAVA_POOL", "java16")) - pctx.VariableFunc("REJavacExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAVAC_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) - pctx.VariableFunc("RED8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_D8_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) - pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("REJavacExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAVAC_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)) + pctx.VariableFunc("RED8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_D8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)) + pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.RemoteLocalFallbackExecStrategy)) pctx.VariableFunc("RETurbineExecStrategy", remoteexec.EnvOverrideFunc("RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.VariableFunc("RESignApkExecStrategy", remoteexec.EnvOverrideFunc("RBE_SIGNAPK_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.VariableFunc("REJarExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAR_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) From ee17f8c02e27ef3fb7956e0177d2c98d3b4aa0a5 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Tue, 18 Aug 2020 17:29:48 -0400 Subject: [PATCH 44/53] Add Goma deprecation PSA to soong. Test: build with Goma Bug: b/166273085 Change-Id: Ie7e146285afd40bc0a1dc17a0b898960f57a118b Merged-In: Ie7e146285afd40bc0a1dc17a0b898960f57a118b --- cmd/soong_ui/main.go | 1 + ui/build/rbe.go | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 78e636cf0..89ece8d16 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -173,6 +173,7 @@ func main() { rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb") soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics") defer build.UploadMetrics(buildCtx, config, c.forceDumbOutput, buildStarted, buildErrorFile, rbeMetricsFile, soongMetricsFile) + defer build.PrintGomaDeprecation(buildCtx, config) os.MkdirAll(logsDir, 0777) log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log")) diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 99107e3d9..d45e38059 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -125,3 +125,13 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) { ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err) } } + +// PrintGomaDeprecation prints a PSA on the deprecation of Goma if it is set for the build. +func PrintGomaDeprecation(ctx Context, config Config) { + if config.UseGoma() { + fmt.Fprintln(ctx.Writer, "") + fmt.Fprintln(ctx.Writer, "Goma for Android is being deprecated and replaced with RBE.") + fmt.Fprintln(ctx.Writer, "See go/goma_android_deprecation for more details.") + fmt.Fprintln(ctx.Writer, "") + } +} From cbe9aa2c2a1e682afd3f8fd974592805606e2178 Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Thu, 13 Aug 2020 22:53:42 -0400 Subject: [PATCH 45/53] Do not generate a random socket address for RBE if NOSTART_RBE is set. Test: A build with NOSTART_RBE=1 Bug: b/166273085 Change-Id: I5872632442c754b597901b7729eed597ee45859f Merged-In: I5872632442c754b597901b7729eed597ee45859f --- ui/build/rbe.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ui/build/rbe.go b/ui/build/rbe.go index d45e38059..52598e9a2 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -53,11 +53,13 @@ func rbeCommand(ctx Context, config Config, rbeCmd string) string { func getRBEVars(ctx Context, config Config) map[string]string { rand.Seed(time.Now().UnixNano()) vars := map[string]string{ - "RBE_server_address": fmt.Sprintf("unix://%v/reproxy_%v.sock", absPath(ctx, config.TempDir()), rand.Intn(1000)), - "RBE_log_path": config.rbeLogPath(), - "RBE_re_proxy": config.rbeReproxy(), - "RBE_exec_root": config.rbeExecRoot(), - "RBE_output_dir": config.rbeStatsOutputDir(), + "RBE_log_path": config.rbeLogPath(), + "RBE_re_proxy": config.rbeReproxy(), + "RBE_exec_root": config.rbeExecRoot(), + "RBE_output_dir": config.rbeStatsOutputDir(), + } + if config.StartRBE() { + vars["RBE_server_address"] = fmt.Sprintf("unix://%v/reproxy_%v.sock", absPath(ctx, config.TempDir()), rand.Intn(1000)) } k, v := config.rbeAuth() vars[k] = v From 26195be91c8d3171f21670a31cedaed3f3d0c92a Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Thu, 13 Aug 2020 14:51:51 -0400 Subject: [PATCH 46/53] Add a default value for the RBE_log_dir variable. Test: built simple target. Bug: b/166273085 Change-Id: Id93dd80119037b594e5f8c370f4ac2fbc12a5144 Merged-In: Id93dd80119037b594e5f8c370f4ac2fbc12a5144 --- ui/build/rbe.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 52598e9a2..67bcebb75 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -54,6 +54,7 @@ func getRBEVars(ctx Context, config Config) map[string]string { rand.Seed(time.Now().UnixNano()) vars := map[string]string{ "RBE_log_path": config.rbeLogPath(), + "RBE_log_dir": config.logDir(), "RBE_re_proxy": config.rbeReproxy(), "RBE_exec_root": config.rbeExecRoot(), "RBE_output_dir": config.rbeStatsOutputDir(), From 486337ea5f67e4216e3c3cf6859f9dd9076556b1 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 5 Jun 2020 20:31:36 +0000 Subject: [PATCH 47/53] Add eakammer and patricearruda to OWNERS Bug: b/168709033 Test: N/A Change-Id: I4209fc5149dcce6bf75aef130afc69c4600c7f3b Merged-In: I4209fc5149dcce6bf75aef130afc69c4600c7f3b --- OWNERS | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/OWNERS b/OWNERS index 85c70df8d..8bb5c304e 100644 --- a/OWNERS +++ b/OWNERS @@ -1,4 +1,11 @@ -per-file * = asmundak@google.com,ccross@android.com,dwillemsen@google.com,jungjw@google.com +per-file * = asmundak@google.com +per-file * = ccross@android.com +per-file * = dwillemsen@google.com +per-file * = eakammer@google.com +per-file * = jungjw@google.com +per-file * = patricearruda@google.com +per-file * = paulduffin@google.com + per-file ndk_*.go, *gen_stub_libs.py = danalbert@google.com per-file clang.go,global.go = srhines@google.com, chh@google.com, pirama@google.com, yikong@google.com per-file tidy.go = srhines@google.com, chh@google.com From ee50d5b5b0ecfa2c90245fbba9eba6bcb3d886ab Mon Sep 17 00:00:00 2001 From: Ramy Medhat Date: Tue, 18 Aug 2020 17:29:48 -0400 Subject: [PATCH 48/53] [DO NOT MERGE] Add Goma deprecation PSA to soong. Test: build with Goma Bug: b/166273085 Change-Id: Ie7e146285afd40bc0a1dc17a0b898960f57a118b Merged-In: Ie7e146285afd40bc0a1dc17a0b898960f57a118b --- cmd/soong_ui/main.go | 4 +++- ui/build/rbe.go | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index e86246ca8..7a19286c8 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -119,9 +119,11 @@ func main() { logsDir = filepath.Join(config.DistDir(), "logs") } + buildErrorFile := filepath.Join(logsDir, logsPrefix+"build_error") rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb") soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics") - defer build.UploadMetrics(buildCtx, config, simpleOutput, buildStarted, rbeMetricsFile, soongMetricsFile) + defer build.UploadMetrics(buildCtx, config, simpleOutput, buildStarted, buildErrorFile, rbeMetricsFile, soongMetricsFile) + defer build.PrintGomaDeprecation(buildCtx, config) os.MkdirAll(logsDir, 0777) diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 6a26063b0..67bcebb75 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -128,3 +128,13 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) { ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err) } } + +// PrintGomaDeprecation prints a PSA on the deprecation of Goma if it is set for the build. +func PrintGomaDeprecation(ctx Context, config Config) { + if config.UseGoma() { + fmt.Fprintln(ctx.Writer, "") + fmt.Fprintln(ctx.Writer, "Goma for Android is being deprecated and replaced with RBE.") + fmt.Fprintln(ctx.Writer, "See go/goma_android_deprecation for more details.") + fmt.Fprintln(ctx.Writer, "") + } +} From 5eae8cdca131e7331e628e079260c11ce426298a Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Tue, 11 Aug 2020 20:41:11 +0000 Subject: [PATCH 49/53] Start collecting build configuration metrics. This is an initial CL that collects the environment variables USE_GOMA and USE_RBE. This is a needed metric to determine how many users are currently using GOMA so we can help them migrate to RBE. Bug: b/163598127 Test: m nothing and checked soong_metrics Change-Id: I39536eabad01aafbb3bd7182515d72230b36f658 Merged-In: I39536eabad01aafbb3bd7182515d72230b36f658 --- ui/build/config.go | 23 ++- ui/metrics/metrics.go | 4 + ui/metrics/metrics_proto/metrics.pb.go | 227 +++++++++++++++++-------- ui/metrics/metrics_proto/metrics.proto | 8 + 4 files changed, 185 insertions(+), 77 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index 889e5ccc5..71b561c2f 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -15,9 +15,9 @@ package build import ( + "fmt" "io/ioutil" "log" - "fmt" "os" "path/filepath" "runtime" @@ -26,6 +26,9 @@ import ( "time" "android/soong/shared" + "github.com/golang/protobuf/proto" + + smpb "android/soong/ui/metrics/metrics_proto" ) type Config struct{ *configImpl } @@ -233,7 +236,23 @@ func NewConfig(ctx Context, args ...string) Config { } } - return Config{ret} + c := Config{ret} + storeConfigMetrics(ctx, c) + return c +} + +// storeConfigMetrics selects a set of configuration information and store in +// the metrics system for further analysis. +func storeConfigMetrics(ctx Context, config Config) { + if ctx.Metrics == nil { + return + } + + b := &smpb.BuildConfig{ + UseGoma: proto.Bool(config.UseGoma()), + UseRbe: proto.Bool(config.UseRBE()), + } + ctx.Metrics.BuildConfig(b) } func (c *configImpl) parseArgs(ctx Context, args []string) { diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index d550f22c4..9929b9eb5 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -64,6 +64,10 @@ func (m *Metrics) SetTimeMetrics(perf metrics_proto.PerfInfo) { } } +func (m *Metrics) BuildConfig(b *metrics_proto.BuildConfig) { + m.metrics.BuildConfig = b +} + func (m *Metrics) SetMetadataMetrics(metadata map[string]string) { for k, v := range metadata { switch k { diff --git a/ui/metrics/metrics_proto/metrics.pb.go b/ui/metrics/metrics_proto/metrics.pb.go index feefc89c2..b138cfa63 100644 --- a/ui/metrics/metrics_proto/metrics.pb.go +++ b/ui/metrics/metrics_proto/metrics.pb.go @@ -3,9 +3,11 @@ package metrics_proto -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -16,7 +18,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type MetricsBase_BUILDVARIANT int32 @@ -31,6 +33,7 @@ var MetricsBase_BUILDVARIANT_name = map[int32]string{ 1: "USERDEBUG", 2: "ENG", } + var MetricsBase_BUILDVARIANT_value = map[string]int32{ "USER": 0, "USERDEBUG": 1, @@ -42,9 +45,11 @@ func (x MetricsBase_BUILDVARIANT) Enum() *MetricsBase_BUILDVARIANT { *p = x return p } + func (x MetricsBase_BUILDVARIANT) String() string { return proto.EnumName(MetricsBase_BUILDVARIANT_name, int32(x)) } + func (x *MetricsBase_BUILDVARIANT) UnmarshalJSON(data []byte) error { value, err := proto.UnmarshalJSONEnum(MetricsBase_BUILDVARIANT_value, data, "MetricsBase_BUILDVARIANT") if err != nil { @@ -53,8 +58,9 @@ func (x *MetricsBase_BUILDVARIANT) UnmarshalJSON(data []byte) error { *x = MetricsBase_BUILDVARIANT(value) return nil } + func (MetricsBase_BUILDVARIANT) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{0, 0} + return fileDescriptor_6039342a2ba47b72, []int{0, 0} } type MetricsBase_ARCH int32 @@ -74,6 +80,7 @@ var MetricsBase_ARCH_name = map[int32]string{ 3: "X86", 4: "X86_64", } + var MetricsBase_ARCH_value = map[string]int32{ "UNKNOWN": 0, "ARM": 1, @@ -87,9 +94,11 @@ func (x MetricsBase_ARCH) Enum() *MetricsBase_ARCH { *p = x return p } + func (x MetricsBase_ARCH) String() string { return proto.EnumName(MetricsBase_ARCH_name, int32(x)) } + func (x *MetricsBase_ARCH) UnmarshalJSON(data []byte) error { value, err := proto.UnmarshalJSONEnum(MetricsBase_ARCH_value, data, "MetricsBase_ARCH") if err != nil { @@ -98,8 +107,9 @@ func (x *MetricsBase_ARCH) UnmarshalJSON(data []byte) error { *x = MetricsBase_ARCH(value) return nil } + func (MetricsBase_ARCH) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{0, 1} + return fileDescriptor_6039342a2ba47b72, []int{0, 1} } type ModuleTypeInfo_BUILDSYSTEM int32 @@ -115,6 +125,7 @@ var ModuleTypeInfo_BUILDSYSTEM_name = map[int32]string{ 1: "SOONG", 2: "MAKE", } + var ModuleTypeInfo_BUILDSYSTEM_value = map[string]int32{ "UNKNOWN": 0, "SOONG": 1, @@ -126,9 +137,11 @@ func (x ModuleTypeInfo_BUILDSYSTEM) Enum() *ModuleTypeInfo_BUILDSYSTEM { *p = x return p } + func (x ModuleTypeInfo_BUILDSYSTEM) String() string { return proto.EnumName(ModuleTypeInfo_BUILDSYSTEM_name, int32(x)) } + func (x *ModuleTypeInfo_BUILDSYSTEM) UnmarshalJSON(data []byte) error { value, err := proto.UnmarshalJSONEnum(ModuleTypeInfo_BUILDSYSTEM_value, data, "ModuleTypeInfo_BUILDSYSTEM") if err != nil { @@ -137,8 +150,9 @@ func (x *ModuleTypeInfo_BUILDSYSTEM) UnmarshalJSON(data []byte) error { *x = ModuleTypeInfo_BUILDSYSTEM(value) return nil } + func (ModuleTypeInfo_BUILDSYSTEM) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{2, 0} + return fileDescriptor_6039342a2ba47b72, []int{3, 0} } type MetricsBase struct { @@ -181,26 +195,28 @@ type MetricsBase struct { // The metrics for calling Soong. SoongRuns []*PerfInfo `protobuf:"bytes,19,rep,name=soong_runs,json=soongRuns" json:"soong_runs,omitempty"` // The metrics for calling Ninja. - NinjaRuns []*PerfInfo `protobuf:"bytes,20,rep,name=ninja_runs,json=ninjaRuns" json:"ninja_runs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + NinjaRuns []*PerfInfo `protobuf:"bytes,20,rep,name=ninja_runs,json=ninjaRuns" json:"ninja_runs,omitempty"` + BuildConfig *BuildConfig `protobuf:"bytes,23,opt,name=build_config,json=buildConfig" json:"build_config,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *MetricsBase) Reset() { *m = MetricsBase{} } func (m *MetricsBase) String() string { return proto.CompactTextString(m) } func (*MetricsBase) ProtoMessage() {} func (*MetricsBase) Descriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{0} + return fileDescriptor_6039342a2ba47b72, []int{0} } + func (m *MetricsBase) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MetricsBase.Unmarshal(m, b) } func (m *MetricsBase) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MetricsBase.Marshal(b, m, deterministic) } -func (dst *MetricsBase) XXX_Merge(src proto.Message) { - xxx_messageInfo_MetricsBase.Merge(dst, src) +func (m *MetricsBase) XXX_Merge(src proto.Message) { + xxx_messageInfo_MetricsBase.Merge(m, src) } func (m *MetricsBase) XXX_Size() int { return xxx_messageInfo_MetricsBase.Size(m) @@ -356,6 +372,60 @@ func (m *MetricsBase) GetNinjaRuns() []*PerfInfo { return nil } +func (m *MetricsBase) GetBuildConfig() *BuildConfig { + if m != nil { + return m.BuildConfig + } + return nil +} + +type BuildConfig struct { + UseGoma *bool `protobuf:"varint,1,opt,name=use_goma,json=useGoma" json:"use_goma,omitempty"` + UseRbe *bool `protobuf:"varint,2,opt,name=use_rbe,json=useRbe" json:"use_rbe,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *BuildConfig) Reset() { *m = BuildConfig{} } +func (m *BuildConfig) String() string { return proto.CompactTextString(m) } +func (*BuildConfig) ProtoMessage() {} +func (*BuildConfig) Descriptor() ([]byte, []int) { + return fileDescriptor_6039342a2ba47b72, []int{1} +} + +func (m *BuildConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_BuildConfig.Unmarshal(m, b) +} +func (m *BuildConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_BuildConfig.Marshal(b, m, deterministic) +} +func (m *BuildConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_BuildConfig.Merge(m, src) +} +func (m *BuildConfig) XXX_Size() int { + return xxx_messageInfo_BuildConfig.Size(m) +} +func (m *BuildConfig) XXX_DiscardUnknown() { + xxx_messageInfo_BuildConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_BuildConfig proto.InternalMessageInfo + +func (m *BuildConfig) GetUseGoma() bool { + if m != nil && m.UseGoma != nil { + return *m.UseGoma + } + return false +} + +func (m *BuildConfig) GetUseRbe() bool { + if m != nil && m.UseRbe != nil { + return *m.UseRbe + } + return false +} + type PerfInfo struct { // The description for the phase/action/part while the tool running. Desc *string `protobuf:"bytes,1,opt,name=desc" json:"desc,omitempty"` @@ -378,16 +448,17 @@ func (m *PerfInfo) Reset() { *m = PerfInfo{} } func (m *PerfInfo) String() string { return proto.CompactTextString(m) } func (*PerfInfo) ProtoMessage() {} func (*PerfInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{1} + return fileDescriptor_6039342a2ba47b72, []int{2} } + func (m *PerfInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PerfInfo.Unmarshal(m, b) } func (m *PerfInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PerfInfo.Marshal(b, m, deterministic) } -func (dst *PerfInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_PerfInfo.Merge(dst, src) +func (m *PerfInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_PerfInfo.Merge(m, src) } func (m *PerfInfo) XXX_Size() int { return xxx_messageInfo_PerfInfo.Size(m) @@ -449,16 +520,17 @@ func (m *ModuleTypeInfo) Reset() { *m = ModuleTypeInfo{} } func (m *ModuleTypeInfo) String() string { return proto.CompactTextString(m) } func (*ModuleTypeInfo) ProtoMessage() {} func (*ModuleTypeInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_metrics_9e7b895801991242, []int{2} + return fileDescriptor_6039342a2ba47b72, []int{3} } + func (m *ModuleTypeInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ModuleTypeInfo.Unmarshal(m, b) } func (m *ModuleTypeInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ModuleTypeInfo.Marshal(b, m, deterministic) } -func (dst *ModuleTypeInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_ModuleTypeInfo.Merge(dst, src) +func (m *ModuleTypeInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_ModuleTypeInfo.Merge(m, src) } func (m *ModuleTypeInfo) XXX_Size() int { return xxx_messageInfo_ModuleTypeInfo.Size(m) @@ -493,65 +565,70 @@ func (m *ModuleTypeInfo) GetNumOfModules() uint32 { } func init() { - proto.RegisterType((*MetricsBase)(nil), "build_metrics.MetricsBase") - proto.RegisterType((*PerfInfo)(nil), "build_metrics.PerfInfo") - proto.RegisterType((*ModuleTypeInfo)(nil), "build_metrics.ModuleTypeInfo") proto.RegisterEnum("build_metrics.MetricsBase_BUILDVARIANT", MetricsBase_BUILDVARIANT_name, MetricsBase_BUILDVARIANT_value) proto.RegisterEnum("build_metrics.MetricsBase_ARCH", MetricsBase_ARCH_name, MetricsBase_ARCH_value) proto.RegisterEnum("build_metrics.ModuleTypeInfo_BUILDSYSTEM", ModuleTypeInfo_BUILDSYSTEM_name, ModuleTypeInfo_BUILDSYSTEM_value) + proto.RegisterType((*MetricsBase)(nil), "build_metrics.MetricsBase") + proto.RegisterType((*BuildConfig)(nil), "build_metrics.BuildConfig") + proto.RegisterType((*PerfInfo)(nil), "build_metrics.PerfInfo") + proto.RegisterType((*ModuleTypeInfo)(nil), "build_metrics.ModuleTypeInfo") } -func init() { proto.RegisterFile("metrics.proto", fileDescriptor_metrics_9e7b895801991242) } +func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) } -var fileDescriptor_metrics_9e7b895801991242 = []byte{ - // 783 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdd, 0x6e, 0xdb, 0x36, - 0x14, 0xae, 0x62, 0x25, 0x96, 0x8e, 0x62, 0x57, 0x61, 0x02, 0x44, 0xc5, 0x50, 0x34, 0x30, 0xf6, - 0x93, 0x01, 0x9b, 0x57, 0x18, 0x81, 0x11, 0x04, 0xbb, 0xb1, 0x13, 0xa3, 0x35, 0x5a, 0xdb, 0x85, - 0x6c, 0x67, 0xdd, 0x2e, 0x46, 0x68, 0x12, 0xdd, 0x68, 0xb3, 0x44, 0x81, 0xa4, 0x8a, 0xf9, 0x21, - 0xf6, 0x8c, 0x7b, 0x91, 0x5d, 0x0c, 0x3c, 0xb4, 0x5c, 0xa5, 0x17, 0x29, 0x72, 0x47, 0x9d, 0xef, - 0x87, 0xdf, 0x91, 0xc8, 0x23, 0x68, 0x65, 0x4c, 0x89, 0x34, 0x96, 0xdd, 0x42, 0x70, 0xc5, 0x49, - 0xeb, 0x8f, 0x32, 0x5d, 0x27, 0x74, 0x5b, 0xec, 0xfc, 0xe7, 0x80, 0x37, 0x31, 0xeb, 0x61, 0x24, - 0x19, 0x79, 0x09, 0x27, 0x86, 0x90, 0x44, 0x8a, 0x51, 0x95, 0x66, 0x4c, 0xaa, 0x28, 0x2b, 0x02, - 0xeb, 0xcc, 0x3a, 0x6f, 0x84, 0x04, 0xb1, 0x9b, 0x48, 0xb1, 0x45, 0x85, 0x90, 0x67, 0xe0, 0x18, - 0x45, 0x9a, 0x04, 0x7b, 0x67, 0xd6, 0xb9, 0x1b, 0x36, 0xf1, 0x79, 0x9c, 0x90, 0x2b, 0x78, 0x56, - 0xac, 0x23, 0xb5, 0xe2, 0x22, 0xa3, 0x1f, 0x99, 0x90, 0x29, 0xcf, 0x69, 0xcc, 0x13, 0x96, 0x47, - 0x19, 0x0b, 0x1a, 0xc8, 0x3d, 0xad, 0x08, 0xb7, 0x06, 0xbf, 0xde, 0xc2, 0xe4, 0x1b, 0x68, 0xab, - 0x48, 0x7c, 0x60, 0x8a, 0x16, 0x82, 0x27, 0x65, 0xac, 0x02, 0x1b, 0x05, 0x2d, 0x53, 0x7d, 0x67, - 0x8a, 0xe4, 0x77, 0x38, 0xd9, 0xd2, 0x4c, 0x88, 0x8f, 0x91, 0x48, 0xa3, 0x5c, 0x05, 0xfb, 0x67, - 0xd6, 0x79, 0xbb, 0xf7, 0x5d, 0xf7, 0x5e, 0xb7, 0xdd, 0x5a, 0xa7, 0xdd, 0xe1, 0x72, 0xfc, 0xf6, - 0xe6, 0x76, 0x10, 0x8e, 0x07, 0xd3, 0xc5, 0x55, 0x63, 0x34, 0x7d, 0x15, 0x12, 0xe3, 0x34, 0xd4, - 0x92, 0x5b, 0xe3, 0x43, 0xc6, 0xe0, 0x6d, 0xfd, 0x23, 0x11, 0xdf, 0x05, 0x07, 0x68, 0xfb, 0xe2, - 0x01, 0xdb, 0x41, 0x78, 0xfd, 0xfa, 0xaa, 0xb9, 0x9c, 0xbe, 0x99, 0xce, 0x7e, 0x99, 0x86, 0x60, - 0xc4, 0x03, 0x11, 0xdf, 0x91, 0x2e, 0x1c, 0xd7, 0xac, 0x76, 0x49, 0x9b, 0xd8, 0xd6, 0xd1, 0x27, - 0x62, 0xb5, 0xf5, 0x0f, 0xb0, 0x0d, 0x44, 0xe3, 0xa2, 0xdc, 0xd1, 0x1d, 0xa4, 0xfb, 0x06, 0xb9, - 0x2e, 0xca, 0x8a, 0x3d, 0x02, 0xf7, 0x8e, 0xcb, 0x6d, 0x4c, 0xf7, 0x91, 0x31, 0x1d, 0x2d, 0xc5, - 0x90, 0x6f, 0xa1, 0x85, 0x36, 0xbd, 0x3c, 0x31, 0x56, 0xf0, 0x48, 0x2b, 0x4f, 0xcb, 0x7b, 0x79, - 0x82, 0x6e, 0xa7, 0xd0, 0x44, 0x37, 0x2e, 0x03, 0x0f, 0x73, 0x1f, 0xe8, 0xc7, 0x99, 0x24, 0x9d, - 0xed, 0x36, 0x5c, 0x52, 0xf6, 0xb7, 0x12, 0x51, 0x70, 0x88, 0xb0, 0x67, 0xe0, 0x91, 0x2e, 0xed, - 0x38, 0xb1, 0xe0, 0x52, 0x6a, 0x8b, 0xd6, 0x27, 0xce, 0xb5, 0xae, 0xcd, 0x24, 0xf9, 0x16, 0x9e, - 0xd6, 0x38, 0x18, 0xb8, 0x6d, 0x8e, 0xc9, 0x8e, 0x85, 0x41, 0x7e, 0x84, 0xe3, 0x1a, 0x6f, 0xd7, - 0xdc, 0x53, 0xf3, 0x32, 0x77, 0xdc, 0x5a, 0x6e, 0x5e, 0x2a, 0x9a, 0xa4, 0x22, 0xf0, 0x4d, 0x6e, - 0x5e, 0xaa, 0x9b, 0x54, 0x90, 0x4b, 0xf0, 0x24, 0x53, 0x65, 0x41, 0x15, 0xe7, 0x6b, 0x19, 0x1c, - 0x9d, 0x35, 0xce, 0xbd, 0xde, 0xe9, 0x67, 0x2f, 0xe7, 0x1d, 0x13, 0xab, 0x71, 0xbe, 0xe2, 0x21, - 0x20, 0x77, 0xa1, 0xa9, 0xe4, 0x02, 0xdc, 0xbf, 0x22, 0x95, 0x52, 0x51, 0xe6, 0x32, 0x20, 0x0f, - 0xeb, 0x1c, 0xcd, 0x0c, 0xcb, 0x5c, 0x92, 0x3e, 0x80, 0xe4, 0x3c, 0xff, 0x60, 0x64, 0xc7, 0x0f, - 0xcb, 0x5c, 0xa4, 0x56, 0xba, 0x3c, 0xcd, 0xff, 0x8c, 0x8c, 0xee, 0xe4, 0x0b, 0x3a, 0xa4, 0x6a, - 0x5d, 0xe7, 0x25, 0x1c, 0xd6, 0xef, 0x05, 0x71, 0xc0, 0x5e, 0xce, 0x47, 0xa1, 0xff, 0x84, 0xb4, - 0xc0, 0xd5, 0xab, 0x9b, 0xd1, 0x70, 0xf9, 0xca, 0xb7, 0x48, 0x13, 0xf4, 0x95, 0xf1, 0xf7, 0x3a, - 0x3f, 0x83, 0xad, 0x0f, 0x00, 0xf1, 0xa0, 0x3a, 0x02, 0xfe, 0x13, 0x8d, 0x0e, 0xc2, 0x89, 0x6f, - 0x11, 0x17, 0xf6, 0x07, 0xe1, 0xa4, 0x7f, 0xe1, 0xef, 0xe9, 0xda, 0xfb, 0xcb, 0xbe, 0xdf, 0x20, - 0x00, 0x07, 0xef, 0x2f, 0xfb, 0xb4, 0x7f, 0xe1, 0xdb, 0x9d, 0x7f, 0x2c, 0x70, 0xaa, 0x1c, 0x84, - 0x80, 0x9d, 0x30, 0x19, 0xe3, 0xac, 0x71, 0x43, 0x5c, 0xeb, 0x1a, 0x4e, 0x0b, 0x33, 0x59, 0x70, - 0x4d, 0x9e, 0x03, 0x48, 0x15, 0x09, 0x85, 0xe3, 0x09, 0xe7, 0x88, 0x1d, 0xba, 0x58, 0xd1, 0x53, - 0x89, 0x7c, 0x05, 0xae, 0x60, 0xd1, 0xda, 0xa0, 0x36, 0xa2, 0x8e, 0x2e, 0x20, 0xf8, 0x1c, 0x20, - 0x63, 0x19, 0x17, 0x1b, 0x5a, 0x4a, 0x86, 0x53, 0xc2, 0x0e, 0x5d, 0x53, 0x59, 0x4a, 0xd6, 0xf9, - 0xd7, 0x82, 0xf6, 0x84, 0x27, 0xe5, 0x9a, 0x2d, 0x36, 0x05, 0xc3, 0x54, 0x4b, 0x38, 0x34, 0xef, - 0x4d, 0x6e, 0xa4, 0x62, 0x19, 0xa6, 0x6b, 0xf7, 0xbe, 0xff, 0xfc, 0x42, 0xdc, 0x13, 0x99, 0xe1, - 0x32, 0xff, 0x75, 0xbe, 0x18, 0x4d, 0x6a, 0x57, 0x03, 0x25, 0x73, 0xb4, 0x21, 0x2f, 0xc0, 0xcb, - 0x50, 0x43, 0xd5, 0xa6, 0xa8, 0xfa, 0x83, 0x6c, 0x67, 0x43, 0xbe, 0x86, 0x76, 0x5e, 0x66, 0x94, - 0xaf, 0xa8, 0x29, 0x4a, 0xec, 0xb4, 0x15, 0x1e, 0xe6, 0x65, 0x36, 0x5b, 0x99, 0xfd, 0x64, 0xe7, - 0x27, 0xf0, 0x6a, 0x7b, 0xdd, 0xff, 0x0a, 0x2e, 0xec, 0xcf, 0x67, 0xb3, 0xa9, 0xfe, 0x5c, 0x0e, - 0xd8, 0x93, 0xc1, 0x9b, 0x91, 0xbf, 0x37, 0x3c, 0x7a, 0xdd, 0xf8, 0xad, 0xfa, 0x25, 0x50, 0xfc, - 0x25, 0xfc, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xd4, 0x8d, 0x19, 0x89, 0x22, 0x06, 0x00, 0x00, +var fileDescriptor_6039342a2ba47b72 = []byte{ + // 847 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdd, 0x6e, 0xe2, 0x46, + 0x14, 0x5e, 0x02, 0x1b, 0xec, 0xe3, 0xc0, 0x3a, 0x93, 0x48, 0xf1, 0xb6, 0x5a, 0x2d, 0x42, 0xfd, + 0xa1, 0x52, 0x4b, 0x57, 0x28, 0x42, 0x51, 0xd4, 0x5e, 0x00, 0x41, 0x59, 0xb4, 0x0b, 0xac, 0x06, + 0x48, 0xb7, 0xbd, 0xa8, 0xe5, 0xd8, 0x43, 0xe2, 0x16, 0x7b, 0xac, 0x99, 0xf1, 0xaa, 0x79, 0x88, + 0x3e, 0x51, 0x1f, 0xa6, 0xaf, 0x52, 0xcd, 0x19, 0x4c, 0x9c, 0x5c, 0xa4, 0xda, 0xbb, 0xf1, 0xf9, + 0x7e, 0xe6, 0x3b, 0x1e, 0xcf, 0x31, 0x34, 0x12, 0xa6, 0x44, 0x1c, 0xca, 0x6e, 0x26, 0xb8, 0xe2, + 0xa4, 0x71, 0x9d, 0xc7, 0x9b, 0xc8, 0xdf, 0x16, 0xdb, 0xff, 0xd8, 0xe0, 0x4c, 0xcd, 0x7a, 0x18, + 0x48, 0x46, 0xde, 0xc0, 0xb1, 0x21, 0x44, 0x81, 0x62, 0xbe, 0x8a, 0x13, 0x26, 0x55, 0x90, 0x64, + 0x5e, 0xa5, 0x55, 0xe9, 0x54, 0x29, 0x41, 0xec, 0x22, 0x50, 0x6c, 0x59, 0x20, 0xe4, 0x25, 0x58, + 0x46, 0x11, 0x47, 0xde, 0x5e, 0xab, 0xd2, 0xb1, 0x69, 0x1d, 0x9f, 0x27, 0x11, 0x39, 0x87, 0x97, + 0xd9, 0x26, 0x50, 0x6b, 0x2e, 0x12, 0xff, 0x13, 0x13, 0x32, 0xe6, 0xa9, 0x1f, 0xf2, 0x88, 0xa5, + 0x41, 0xc2, 0xbc, 0x2a, 0x72, 0x4f, 0x0a, 0xc2, 0x95, 0xc1, 0x47, 0x5b, 0x98, 0x7c, 0x0d, 0x4d, + 0x15, 0x88, 0x1b, 0xa6, 0xfc, 0x4c, 0xf0, 0x28, 0x0f, 0x95, 0x57, 0x43, 0x41, 0xc3, 0x54, 0x3f, + 0x98, 0x22, 0xf9, 0x1d, 0x8e, 0xb7, 0x34, 0x13, 0xe2, 0x53, 0x20, 0xe2, 0x20, 0x55, 0xde, 0xf3, + 0x56, 0xa5, 0xd3, 0xec, 0x7d, 0xdb, 0x7d, 0xd0, 0x6d, 0xb7, 0xd4, 0x69, 0x77, 0xb8, 0x9a, 0xbc, + 0xbf, 0xb8, 0x1a, 0xd0, 0xc9, 0x60, 0xb6, 0x3c, 0xaf, 0x8e, 0x67, 0x97, 0x94, 0x18, 0xa7, 0xa1, + 0x96, 0x5c, 0x19, 0x1f, 0x32, 0x01, 0x67, 0xeb, 0x1f, 0x88, 0xf0, 0xd6, 0xdb, 0x47, 0xdb, 0xd7, + 0x4f, 0xd8, 0x0e, 0xe8, 0xe8, 0xed, 0x79, 0x7d, 0x35, 0x7b, 0x37, 0x9b, 0xff, 0x32, 0xa3, 0x60, + 0xc4, 0x03, 0x11, 0xde, 0x92, 0x2e, 0x1c, 0x95, 0xac, 0x76, 0x49, 0xeb, 0xd8, 0xd6, 0xe1, 0x3d, + 0xb1, 0xd8, 0xfa, 0x7b, 0xd8, 0x06, 0xf2, 0xc3, 0x2c, 0xdf, 0xd1, 0x2d, 0xa4, 0xbb, 0x06, 0x19, + 0x65, 0x79, 0xc1, 0x1e, 0x83, 0x7d, 0xcb, 0xe5, 0x36, 0xa6, 0xfd, 0x99, 0x31, 0x2d, 0x2d, 0xc5, + 0x90, 0xef, 0xa1, 0x81, 0x36, 0xbd, 0x34, 0x32, 0x56, 0xf0, 0x99, 0x56, 0x8e, 0x96, 0xf7, 0xd2, + 0x08, 0xdd, 0x4e, 0xa0, 0x8e, 0x6e, 0x5c, 0x7a, 0x0e, 0xe6, 0xde, 0xd7, 0x8f, 0x73, 0x49, 0xda, + 0xdb, 0x6d, 0xb8, 0xf4, 0xd9, 0x5f, 0x4a, 0x04, 0xde, 0x01, 0xc2, 0x8e, 0x81, 0xc7, 0xba, 0xb4, + 0xe3, 0x84, 0x82, 0x4b, 0xa9, 0x2d, 0x1a, 0xf7, 0x9c, 0x91, 0xae, 0xcd, 0x25, 0xf9, 0x06, 0x5e, + 0x94, 0x38, 0x18, 0xb8, 0x69, 0x3e, 0x93, 0x1d, 0x0b, 0x83, 0xfc, 0x00, 0x47, 0x25, 0xde, 0xae, + 0xb9, 0x17, 0xe6, 0x65, 0xee, 0xb8, 0xa5, 0xdc, 0x3c, 0x57, 0x7e, 0x14, 0x0b, 0xcf, 0x35, 0xb9, + 0x79, 0xae, 0x2e, 0x62, 0x41, 0xce, 0xc0, 0x91, 0x4c, 0xe5, 0x99, 0xaf, 0x38, 0xdf, 0x48, 0xef, + 0xb0, 0x55, 0xed, 0x38, 0xbd, 0x93, 0x47, 0x2f, 0xe7, 0x03, 0x13, 0xeb, 0x49, 0xba, 0xe6, 0x14, + 0x90, 0xbb, 0xd4, 0x54, 0x72, 0x0a, 0xf6, 0x9f, 0x81, 0x8a, 0x7d, 0x91, 0xa7, 0xd2, 0x23, 0x4f, + 0xeb, 0x2c, 0xcd, 0xa4, 0x79, 0x2a, 0x49, 0x1f, 0x40, 0x72, 0x9e, 0xde, 0x18, 0xd9, 0xd1, 0xd3, + 0x32, 0x1b, 0xa9, 0x85, 0x2e, 0x8d, 0xd3, 0x3f, 0x02, 0xa3, 0x3b, 0xfe, 0x1f, 0x1d, 0x52, 0x51, + 0xf7, 0x33, 0x1c, 0x18, 0x52, 0xc8, 0xd3, 0x75, 0x7c, 0xe3, 0x9d, 0xb4, 0x2a, 0x1d, 0xa7, 0xf7, + 0xc5, 0x23, 0x25, 0xde, 0x90, 0x11, 0x32, 0xa8, 0x73, 0x7d, 0xff, 0xd0, 0x7e, 0x03, 0x07, 0xe5, + 0x6b, 0x45, 0x2c, 0xa8, 0xad, 0x16, 0x63, 0xea, 0x3e, 0x23, 0x0d, 0xb0, 0xf5, 0xea, 0x62, 0x3c, + 0x5c, 0x5d, 0xba, 0x15, 0x52, 0x07, 0x7d, 0xe3, 0xdc, 0xbd, 0xf6, 0x4f, 0x50, 0xd3, 0xdf, 0x0f, + 0x71, 0xa0, 0xf8, 0x82, 0xdc, 0x67, 0x1a, 0x1d, 0xd0, 0xa9, 0x5b, 0x21, 0x36, 0x3c, 0x1f, 0xd0, + 0x69, 0xff, 0xd4, 0xdd, 0xd3, 0xb5, 0x8f, 0x67, 0x7d, 0xb7, 0x4a, 0x00, 0xf6, 0x3f, 0x9e, 0xf5, + 0xfd, 0xfe, 0xa9, 0x5b, 0x6b, 0x0f, 0xc0, 0x29, 0x65, 0xd1, 0xa3, 0x28, 0x97, 0xcc, 0xbf, 0xe1, + 0x49, 0x80, 0x03, 0xcb, 0xa2, 0xf5, 0x5c, 0xb2, 0x4b, 0x9e, 0x04, 0xfa, 0x44, 0x35, 0x24, 0xae, + 0x19, 0x0e, 0x29, 0x8b, 0xee, 0xe7, 0x92, 0xd1, 0x6b, 0xd6, 0xfe, 0xbb, 0x02, 0x56, 0xf1, 0x26, + 0x08, 0x81, 0x5a, 0xc4, 0x64, 0x88, 0x62, 0x9b, 0xe2, 0x5a, 0xd7, 0x70, 0x5e, 0x99, 0xd9, 0x86, + 0x6b, 0xf2, 0x0a, 0x40, 0xaa, 0x40, 0x28, 0x1c, 0x90, 0x38, 0xc9, 0x6a, 0xd4, 0xc6, 0x8a, 0x9e, + 0x8b, 0xe4, 0x4b, 0xb0, 0x05, 0x0b, 0x36, 0x06, 0xad, 0x21, 0x6a, 0xe9, 0x02, 0x82, 0xaf, 0x00, + 0x12, 0x96, 0x70, 0x71, 0xe7, 0xe7, 0x92, 0xe1, 0x9c, 0xaa, 0x51, 0xdb, 0x54, 0x56, 0x92, 0xb5, + 0xff, 0xad, 0x40, 0x73, 0xca, 0xa3, 0x7c, 0xc3, 0x96, 0x77, 0x19, 0xc3, 0x54, 0xab, 0xe2, 0x50, + 0xe4, 0x9d, 0x54, 0x2c, 0xc1, 0x74, 0xcd, 0xde, 0x77, 0x8f, 0xaf, 0xe4, 0x03, 0x91, 0x19, 0x6f, + 0x8b, 0x5f, 0x17, 0xcb, 0xf1, 0xb4, 0x74, 0x39, 0x51, 0xb2, 0x40, 0x1b, 0xf2, 0x1a, 0x9c, 0x04, + 0x35, 0xbe, 0xba, 0xcb, 0x8a, 0xfe, 0x20, 0xd9, 0xd9, 0x90, 0xaf, 0xa0, 0x99, 0xe6, 0x89, 0xcf, + 0xd7, 0xbe, 0x29, 0x4a, 0xec, 0xb4, 0x41, 0x0f, 0xd2, 0x3c, 0x99, 0xaf, 0xcd, 0x7e, 0xb2, 0xfd, + 0x23, 0x38, 0xa5, 0xbd, 0x1e, 0x1e, 0xa4, 0x0d, 0xcf, 0x17, 0xf3, 0xf9, 0x4c, 0x9f, 0xb8, 0x05, + 0xb5, 0xe9, 0xe0, 0xdd, 0xd8, 0xdd, 0x1b, 0x1e, 0xbe, 0xad, 0xfe, 0x56, 0xfc, 0x94, 0x7c, 0xfc, + 0x29, 0xfd, 0x17, 0x00, 0x00, 0xff, 0xff, 0x76, 0x9a, 0xf7, 0xbd, 0xa4, 0x06, 0x00, 0x00, } diff --git a/ui/metrics/metrics_proto/metrics.proto b/ui/metrics/metrics_proto/metrics.proto index b3de2f46f..6c5c321d6 100644 --- a/ui/metrics/metrics_proto/metrics.proto +++ b/ui/metrics/metrics_proto/metrics.proto @@ -91,6 +91,14 @@ message MetricsBase { // The metrics for calling Ninja. repeated PerfInfo ninja_runs = 20; + + optional BuildConfig build_config = 23; +} + +message BuildConfig { + optional bool use_goma = 1; + + optional bool use_rbe = 2; } message PerfInfo { From 17fbc895ec3e02d1044ece619696c67a8e7d28d1 Mon Sep 17 00:00:00 2001 From: Victor Khimenko Date: Thu, 17 Sep 2020 03:07:31 +0200 Subject: [PATCH 50/53] Add -D__ANDROID_NATIVE_BRIDGE__ to native_bridge targets And remove dual meaning from arm_on_x86 target. It was set for x86 targets with arm support and for arm targets with x86 support - and that 2nd one was abused for native_bridge version of RenderScript. It's much safer and cleaner to rely on __ANDROID_NATIVE_BRIDGE__ there while leaving arm_on_x86 to mean "x86 binary built in the image with arm support". It's only used by bcc compiler which is in canadian-cross situation here: it's pure x86-64 binary, but it needs to know about ALL supported architectures on device because it needs to include appropriate codegen module. Bug: http://b/153609531 Test: cts-tradefed run cts --abi armeabi-v7a -m CtsRsCppTestCases Test: cts-tradefed run cts --abi arm64-v8a -m CtsRsCppTestCases Change-Id: I869212e0b82eeaa30361a4e183d5c862ab40ef12 Merged-In: I869212e0b82eeaa30361a4e183d5c862ab40ef12 --- android/arch.go | 35 +++++------------------------------ cc/compiler.go | 4 ++++ 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/android/arch.go b/android/arch.go index d14221f7c..18ff13e9c 100644 --- a/android/arch.go +++ b/android/arch.go @@ -1440,20 +1440,15 @@ func (m *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { // key: value, // }, // }, - // TODO(ccross): is this still necessary with native bridge? if os.Class == Device { - if (arch.ArchType == X86 && (hasArmAbi(arch) || - hasArmAndroidArch(ctx.Config().Targets[Android]))) || - (arch.ArchType == Arm && - hasX86AndroidArch(ctx.Config().Targets[Android])) { + if arch.ArchType == X86 && (hasArmAbi(arch) || + hasArmAndroidArch(ctx.Config().Targets[Android])) { field := "Arm_on_x86" prefix := "target.arm_on_x86" m.appendProperties(ctx, genProps, targetProp, field, prefix) } - if (arch.ArchType == X86_64 && (hasArmAbi(arch) || - hasArmAndroidArch(ctx.Config().Targets[Android]))) || - (arch.ArchType == Arm && - hasX8664AndroidArch(ctx.Config().Targets[Android])) { + if arch.ArchType == X86_64 && (hasArmAbi(arch) || + hasArmAndroidArch(ctx.Config().Targets[Android])) { field := "Arm_on_x86_64" prefix := "target.arm_on_x86_64" m.appendProperties(ctx, genProps, targetProp, field, prefix) @@ -1600,27 +1595,7 @@ func hasArmAbi(arch Arch) bool { // hasArmArch returns true if targets has at least non-native_bridge arm Android arch func hasArmAndroidArch(targets []Target) bool { for _, target := range targets { - if target.Os == Android && target.Arch.ArchType == Arm && target.NativeBridge == NativeBridgeDisabled { - return true - } - } - return false -} - -// hasX86Arch returns true if targets has at least x86 Android arch -func hasX86AndroidArch(targets []Target) bool { - for _, target := range targets { - if target.Os == Android && target.Arch.ArchType == X86 { - return true - } - } - return false -} - -// hasX8664Arch returns true if targets has at least x86_64 Android arch -func hasX8664AndroidArch(targets []Target) bool { - for _, target := range targets { - if target.Os == Android && target.Arch.ArchType == X86_64 { + if target.Os == Android && target.Arch.ArchType == Arm { return true } } diff --git a/cc/compiler.go b/cc/compiler.go index 08ce13392..f6c358775 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -333,6 +333,10 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps } } + if ctx.Target().NativeBridge == android.NativeBridgeEnabled { + flags.Global.CommonFlags = append(flags.Global.CommonFlags, "-D__ANDROID_NATIVE_BRIDGE__") + } + instructionSet := String(compiler.Properties.Instruction_set) if flags.RequiredInstructionSet != "" { instructionSet = flags.RequiredInstructionSet From dbe59cdd81995fa52a3c07288336a166360b4665 Mon Sep 17 00:00:00 2001 From: Kousik Kumar Date: Mon, 21 Sep 2020 13:39:24 -0400 Subject: [PATCH 51/53] Deprecate USE_GOMA and replace with FORCE_USE_GOMA Bug: b/163361527 Test: Ran builds with USE_GOMA / FORCE_USE_GOMA flags Change-Id: I67b229fbf52e913b0c4f5e7932021132ddc0598f Merged-In: I67b229fbf52e913b0c4f5e7932021132ddc0598f --- cmd/soong_ui/main.go | 1 - ui/build/config.go | 24 ++++++++++++++++++++++++ ui/build/rbe.go | 10 ---------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 7a19286c8..a4b7d0f3d 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -123,7 +123,6 @@ func main() { rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb") soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics") defer build.UploadMetrics(buildCtx, config, simpleOutput, buildStarted, buildErrorFile, rbeMetricsFile, soongMetricsFile) - defer build.PrintGomaDeprecation(buildCtx, config) os.MkdirAll(logsDir, 0777) diff --git a/ui/build/config.go b/ui/build/config.go index 71b561c2f..10e89c042 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -26,6 +26,7 @@ import ( "time" "android/soong/shared" + "github.com/golang/protobuf/proto" smpb "android/soong/ui/metrics/metrics_proto" @@ -153,6 +154,17 @@ func NewConfig(ctx Context, args ...string) Config { "EMPTY_NINJA_FILE", ) + if ret.UseGoma() { + ctx.Println("Goma for Android is being deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.") + ctx.Println() + ctx.Println("See go/goma_android_exceptions for exceptions.") + ctx.Fatalln("USE_GOMA flag is no longer supported.") + } + + if ret.ForceUseGoma() { + ret.environ.Set("USE_GOMA", "true") + } + // Tell python not to spam the source tree with .pyc files. ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") @@ -494,6 +506,18 @@ func (c *configImpl) Parallel() int { return c.parallel } +// ForceUseGoma determines whether we should override Goma deprecation +// and use Goma for the current build or not. +func (c *configImpl) ForceUseGoma() bool { + if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok { + v = strings.TrimSpace(v) + if v != "" && v != "false" { + return true + } + } + return false +} + func (c *configImpl) UseGoma() bool { if v, ok := c.environ.Get("USE_GOMA"); ok { v = strings.TrimSpace(v) diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 67bcebb75..6a26063b0 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -128,13 +128,3 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) { ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err) } } - -// PrintGomaDeprecation prints a PSA on the deprecation of Goma if it is set for the build. -func PrintGomaDeprecation(ctx Context, config Config) { - if config.UseGoma() { - fmt.Fprintln(ctx.Writer, "") - fmt.Fprintln(ctx.Writer, "Goma for Android is being deprecated and replaced with RBE.") - fmt.Fprintln(ctx.Writer, "See go/goma_android_deprecation for more details.") - fmt.Fprintln(ctx.Writer, "") - } -} From 3fc4c98bd9945204eef2a6957ba5a829929a67bc Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Mon, 28 Sep 2020 18:22:07 +0000 Subject: [PATCH 52/53] Collect FORCE_USE_GOMA environment variable as a metric. FORCE_USE_GOMA is needed to be collected as a metric for the deprecation of GOMA on Android builds. Bug: b/169423400 Test: m nothing, ran printproto command Change-Id: I12300389f1f27239d8f21875d1b6e4ad069d95d7 Merged-In: I12300389f1f27239d8f21875d1b6e4ad069d95d7 --- ui/build/config.go | 5 +- ui/metrics/metrics_proto/metrics.pb.go | 115 +++++++++++++------------ ui/metrics/metrics_proto/metrics.proto | 2 + 3 files changed, 67 insertions(+), 55 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index 10e89c042..ef46bec06 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -261,8 +261,9 @@ func storeConfigMetrics(ctx Context, config Config) { } b := &smpb.BuildConfig{ - UseGoma: proto.Bool(config.UseGoma()), - UseRbe: proto.Bool(config.UseRBE()), + ForceUseGoma: proto.Bool(config.ForceUseGoma()), + UseGoma: proto.Bool(config.UseGoma()), + UseRbe: proto.Bool(config.UseRBE()), } ctx.Metrics.BuildConfig(b) } diff --git a/ui/metrics/metrics_proto/metrics.pb.go b/ui/metrics/metrics_proto/metrics.pb.go index b138cfa63..7f7024f79 100644 --- a/ui/metrics/metrics_proto/metrics.pb.go +++ b/ui/metrics/metrics_proto/metrics.pb.go @@ -382,6 +382,7 @@ func (m *MetricsBase) GetBuildConfig() *BuildConfig { type BuildConfig struct { UseGoma *bool `protobuf:"varint,1,opt,name=use_goma,json=useGoma" json:"use_goma,omitempty"` UseRbe *bool `protobuf:"varint,2,opt,name=use_rbe,json=useRbe" json:"use_rbe,omitempty"` + ForceUseGoma *bool `protobuf:"varint,3,opt,name=force_use_goma,json=forceUseGoma" json:"force_use_goma,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -426,6 +427,13 @@ func (m *BuildConfig) GetUseRbe() bool { return false } +func (m *BuildConfig) GetForceUseGoma() bool { + if m != nil && m.ForceUseGoma != nil { + return *m.ForceUseGoma + } + return false +} + type PerfInfo struct { // The description for the phase/action/part while the tool running. Desc *string `protobuf:"bytes,1,opt,name=desc" json:"desc,omitempty"` @@ -577,58 +585,59 @@ func init() { func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) } var fileDescriptor_6039342a2ba47b72 = []byte{ - // 847 bytes of a gzipped FileDescriptorProto + // 860 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdd, 0x6e, 0xe2, 0x46, - 0x14, 0x5e, 0x02, 0x1b, 0xec, 0xe3, 0xc0, 0x3a, 0x93, 0x48, 0xf1, 0xb6, 0x5a, 0x2d, 0x42, 0xfd, - 0xa1, 0x52, 0x4b, 0x57, 0x28, 0x42, 0x51, 0xd4, 0x5e, 0x00, 0x41, 0x59, 0xb4, 0x0b, 0xac, 0x06, - 0x48, 0xb7, 0xbd, 0xa8, 0xe5, 0xd8, 0x43, 0xe2, 0x16, 0x7b, 0xac, 0x99, 0xf1, 0xaa, 0x79, 0x88, - 0x3e, 0x51, 0x1f, 0xa6, 0xaf, 0x52, 0xcd, 0x19, 0x4c, 0x9c, 0x5c, 0xa4, 0xda, 0xbb, 0xf1, 0xf9, - 0x7e, 0xe6, 0x3b, 0x1e, 0xcf, 0x31, 0x34, 0x12, 0xa6, 0x44, 0x1c, 0xca, 0x6e, 0x26, 0xb8, 0xe2, - 0xa4, 0x71, 0x9d, 0xc7, 0x9b, 0xc8, 0xdf, 0x16, 0xdb, 0xff, 0xd8, 0xe0, 0x4c, 0xcd, 0x7a, 0x18, - 0x48, 0x46, 0xde, 0xc0, 0xb1, 0x21, 0x44, 0x81, 0x62, 0xbe, 0x8a, 0x13, 0x26, 0x55, 0x90, 0x64, - 0x5e, 0xa5, 0x55, 0xe9, 0x54, 0x29, 0x41, 0xec, 0x22, 0x50, 0x6c, 0x59, 0x20, 0xe4, 0x25, 0x58, - 0x46, 0x11, 0x47, 0xde, 0x5e, 0xab, 0xd2, 0xb1, 0x69, 0x1d, 0x9f, 0x27, 0x11, 0x39, 0x87, 0x97, - 0xd9, 0x26, 0x50, 0x6b, 0x2e, 0x12, 0xff, 0x13, 0x13, 0x32, 0xe6, 0xa9, 0x1f, 0xf2, 0x88, 0xa5, - 0x41, 0xc2, 0xbc, 0x2a, 0x72, 0x4f, 0x0a, 0xc2, 0x95, 0xc1, 0x47, 0x5b, 0x98, 0x7c, 0x0d, 0x4d, - 0x15, 0x88, 0x1b, 0xa6, 0xfc, 0x4c, 0xf0, 0x28, 0x0f, 0x95, 0x57, 0x43, 0x41, 0xc3, 0x54, 0x3f, - 0x98, 0x22, 0xf9, 0x1d, 0x8e, 0xb7, 0x34, 0x13, 0xe2, 0x53, 0x20, 0xe2, 0x20, 0x55, 0xde, 0xf3, - 0x56, 0xa5, 0xd3, 0xec, 0x7d, 0xdb, 0x7d, 0xd0, 0x6d, 0xb7, 0xd4, 0x69, 0x77, 0xb8, 0x9a, 0xbc, - 0xbf, 0xb8, 0x1a, 0xd0, 0xc9, 0x60, 0xb6, 0x3c, 0xaf, 0x8e, 0x67, 0x97, 0x94, 0x18, 0xa7, 0xa1, - 0x96, 0x5c, 0x19, 0x1f, 0x32, 0x01, 0x67, 0xeb, 0x1f, 0x88, 0xf0, 0xd6, 0xdb, 0x47, 0xdb, 0xd7, - 0x4f, 0xd8, 0x0e, 0xe8, 0xe8, 0xed, 0x79, 0x7d, 0x35, 0x7b, 0x37, 0x9b, 0xff, 0x32, 0xa3, 0x60, - 0xc4, 0x03, 0x11, 0xde, 0x92, 0x2e, 0x1c, 0x95, 0xac, 0x76, 0x49, 0xeb, 0xd8, 0xd6, 0xe1, 0x3d, - 0xb1, 0xd8, 0xfa, 0x7b, 0xd8, 0x06, 0xf2, 0xc3, 0x2c, 0xdf, 0xd1, 0x2d, 0xa4, 0xbb, 0x06, 0x19, - 0x65, 0x79, 0xc1, 0x1e, 0x83, 0x7d, 0xcb, 0xe5, 0x36, 0xa6, 0xfd, 0x99, 0x31, 0x2d, 0x2d, 0xc5, - 0x90, 0xef, 0xa1, 0x81, 0x36, 0xbd, 0x34, 0x32, 0x56, 0xf0, 0x99, 0x56, 0x8e, 0x96, 0xf7, 0xd2, - 0x08, 0xdd, 0x4e, 0xa0, 0x8e, 0x6e, 0x5c, 0x7a, 0x0e, 0xe6, 0xde, 0xd7, 0x8f, 0x73, 0x49, 0xda, - 0xdb, 0x6d, 0xb8, 0xf4, 0xd9, 0x5f, 0x4a, 0x04, 0xde, 0x01, 0xc2, 0x8e, 0x81, 0xc7, 0xba, 0xb4, - 0xe3, 0x84, 0x82, 0x4b, 0xa9, 0x2d, 0x1a, 0xf7, 0x9c, 0x91, 0xae, 0xcd, 0x25, 0xf9, 0x06, 0x5e, - 0x94, 0x38, 0x18, 0xb8, 0x69, 0x3e, 0x93, 0x1d, 0x0b, 0x83, 0xfc, 0x00, 0x47, 0x25, 0xde, 0xae, - 0xb9, 0x17, 0xe6, 0x65, 0xee, 0xb8, 0xa5, 0xdc, 0x3c, 0x57, 0x7e, 0x14, 0x0b, 0xcf, 0x35, 0xb9, - 0x79, 0xae, 0x2e, 0x62, 0x41, 0xce, 0xc0, 0x91, 0x4c, 0xe5, 0x99, 0xaf, 0x38, 0xdf, 0x48, 0xef, - 0xb0, 0x55, 0xed, 0x38, 0xbd, 0x93, 0x47, 0x2f, 0xe7, 0x03, 0x13, 0xeb, 0x49, 0xba, 0xe6, 0x14, - 0x90, 0xbb, 0xd4, 0x54, 0x72, 0x0a, 0xf6, 0x9f, 0x81, 0x8a, 0x7d, 0x91, 0xa7, 0xd2, 0x23, 0x4f, - 0xeb, 0x2c, 0xcd, 0xa4, 0x79, 0x2a, 0x49, 0x1f, 0x40, 0x72, 0x9e, 0xde, 0x18, 0xd9, 0xd1, 0xd3, - 0x32, 0x1b, 0xa9, 0x85, 0x2e, 0x8d, 0xd3, 0x3f, 0x02, 0xa3, 0x3b, 0xfe, 0x1f, 0x1d, 0x52, 0x51, - 0xf7, 0x33, 0x1c, 0x18, 0x52, 0xc8, 0xd3, 0x75, 0x7c, 0xe3, 0x9d, 0xb4, 0x2a, 0x1d, 0xa7, 0xf7, - 0xc5, 0x23, 0x25, 0xde, 0x90, 0x11, 0x32, 0xa8, 0x73, 0x7d, 0xff, 0xd0, 0x7e, 0x03, 0x07, 0xe5, - 0x6b, 0x45, 0x2c, 0xa8, 0xad, 0x16, 0x63, 0xea, 0x3e, 0x23, 0x0d, 0xb0, 0xf5, 0xea, 0x62, 0x3c, - 0x5c, 0x5d, 0xba, 0x15, 0x52, 0x07, 0x7d, 0xe3, 0xdc, 0xbd, 0xf6, 0x4f, 0x50, 0xd3, 0xdf, 0x0f, - 0x71, 0xa0, 0xf8, 0x82, 0xdc, 0x67, 0x1a, 0x1d, 0xd0, 0xa9, 0x5b, 0x21, 0x36, 0x3c, 0x1f, 0xd0, - 0x69, 0xff, 0xd4, 0xdd, 0xd3, 0xb5, 0x8f, 0x67, 0x7d, 0xb7, 0x4a, 0x00, 0xf6, 0x3f, 0x9e, 0xf5, - 0xfd, 0xfe, 0xa9, 0x5b, 0x6b, 0x0f, 0xc0, 0x29, 0x65, 0xd1, 0xa3, 0x28, 0x97, 0xcc, 0xbf, 0xe1, - 0x49, 0x80, 0x03, 0xcb, 0xa2, 0xf5, 0x5c, 0xb2, 0x4b, 0x9e, 0x04, 0xfa, 0x44, 0x35, 0x24, 0xae, - 0x19, 0x0e, 0x29, 0x8b, 0xee, 0xe7, 0x92, 0xd1, 0x6b, 0xd6, 0xfe, 0xbb, 0x02, 0x56, 0xf1, 0x26, - 0x08, 0x81, 0x5a, 0xc4, 0x64, 0x88, 0x62, 0x9b, 0xe2, 0x5a, 0xd7, 0x70, 0x5e, 0x99, 0xd9, 0x86, - 0x6b, 0xf2, 0x0a, 0x40, 0xaa, 0x40, 0x28, 0x1c, 0x90, 0x38, 0xc9, 0x6a, 0xd4, 0xc6, 0x8a, 0x9e, - 0x8b, 0xe4, 0x4b, 0xb0, 0x05, 0x0b, 0x36, 0x06, 0xad, 0x21, 0x6a, 0xe9, 0x02, 0x82, 0xaf, 0x00, - 0x12, 0x96, 0x70, 0x71, 0xe7, 0xe7, 0x92, 0xe1, 0x9c, 0xaa, 0x51, 0xdb, 0x54, 0x56, 0x92, 0xb5, - 0xff, 0xad, 0x40, 0x73, 0xca, 0xa3, 0x7c, 0xc3, 0x96, 0x77, 0x19, 0xc3, 0x54, 0xab, 0xe2, 0x50, - 0xe4, 0x9d, 0x54, 0x2c, 0xc1, 0x74, 0xcd, 0xde, 0x77, 0x8f, 0xaf, 0xe4, 0x03, 0x91, 0x19, 0x6f, - 0x8b, 0x5f, 0x17, 0xcb, 0xf1, 0xb4, 0x74, 0x39, 0x51, 0xb2, 0x40, 0x1b, 0xf2, 0x1a, 0x9c, 0x04, - 0x35, 0xbe, 0xba, 0xcb, 0x8a, 0xfe, 0x20, 0xd9, 0xd9, 0x90, 0xaf, 0xa0, 0x99, 0xe6, 0x89, 0xcf, - 0xd7, 0xbe, 0x29, 0x4a, 0xec, 0xb4, 0x41, 0x0f, 0xd2, 0x3c, 0x99, 0xaf, 0xcd, 0x7e, 0xb2, 0xfd, - 0x23, 0x38, 0xa5, 0xbd, 0x1e, 0x1e, 0xa4, 0x0d, 0xcf, 0x17, 0xf3, 0xf9, 0x4c, 0x9f, 0xb8, 0x05, - 0xb5, 0xe9, 0xe0, 0xdd, 0xd8, 0xdd, 0x1b, 0x1e, 0xbe, 0xad, 0xfe, 0x56, 0xfc, 0x94, 0x7c, 0xfc, - 0x29, 0xfd, 0x17, 0x00, 0x00, 0xff, 0xff, 0x76, 0x9a, 0xf7, 0xbd, 0xa4, 0x06, 0x00, 0x00, + 0x14, 0x5e, 0x02, 0x1b, 0xec, 0x63, 0x60, 0x9d, 0x49, 0xa4, 0x78, 0x5b, 0xad, 0x16, 0xa1, 0xfe, + 0x50, 0xa9, 0xa5, 0x2b, 0x14, 0xa1, 0x28, 0x6a, 0x2f, 0x80, 0xa0, 0x2c, 0xda, 0x05, 0x56, 0x03, + 0xa4, 0xdb, 0x5e, 0xd4, 0x72, 0xec, 0x81, 0xb8, 0xc5, 0x1e, 0x34, 0x33, 0x5e, 0x35, 0x0f, 0xd1, + 0x27, 0xea, 0xc3, 0xf4, 0x55, 0xaa, 0x39, 0x83, 0x89, 0x93, 0x8b, 0x54, 0x7b, 0x37, 0x3e, 0xdf, + 0xcf, 0x7c, 0xf3, 0x77, 0x0c, 0xf5, 0x84, 0x29, 0x11, 0x87, 0xb2, 0xb3, 0x15, 0x5c, 0x71, 0x52, + 0xbf, 0xc9, 0xe2, 0x4d, 0xe4, 0xef, 0x8a, 0xad, 0x7f, 0x6c, 0x70, 0x26, 0x66, 0x3c, 0x08, 0x24, + 0x23, 0x6f, 0xe0, 0xc4, 0x10, 0xa2, 0x40, 0x31, 0x5f, 0xc5, 0x09, 0x93, 0x2a, 0x48, 0xb6, 0x5e, + 0xa9, 0x59, 0x6a, 0x97, 0x29, 0x41, 0xec, 0x32, 0x50, 0x6c, 0x91, 0x23, 0xe4, 0x25, 0x58, 0x46, + 0x11, 0x47, 0xde, 0x41, 0xb3, 0xd4, 0xb6, 0x69, 0x15, 0xbf, 0xc7, 0x11, 0xb9, 0x80, 0x97, 0xdb, + 0x4d, 0xa0, 0x56, 0x5c, 0x24, 0xfe, 0x27, 0x26, 0x64, 0xcc, 0x53, 0x3f, 0xe4, 0x11, 0x4b, 0x83, + 0x84, 0x79, 0x65, 0xe4, 0x9e, 0xe6, 0x84, 0x6b, 0x83, 0x0f, 0x77, 0x30, 0xf9, 0x1a, 0x1a, 0x2a, + 0x10, 0x6b, 0xa6, 0xfc, 0xad, 0xe0, 0x51, 0x16, 0x2a, 0xaf, 0x82, 0x82, 0xba, 0xa9, 0x7e, 0x30, + 0x45, 0xf2, 0x3b, 0x9c, 0xec, 0x68, 0x26, 0xc4, 0xa7, 0x40, 0xc4, 0x41, 0xaa, 0xbc, 0xe7, 0xcd, + 0x52, 0xbb, 0xd1, 0xfd, 0xb6, 0xf3, 0x60, 0xb5, 0x9d, 0xc2, 0x4a, 0x3b, 0x83, 0xe5, 0xf8, 0xfd, + 0xe5, 0x75, 0x9f, 0x8e, 0xfb, 0xd3, 0xc5, 0x45, 0x79, 0x34, 0xbd, 0xa2, 0xc4, 0x38, 0x0d, 0xb4, + 0xe4, 0xda, 0xf8, 0x90, 0x31, 0x38, 0x3b, 0xff, 0x40, 0x84, 0xb7, 0xde, 0x21, 0xda, 0xbe, 0x7e, + 0xc2, 0xb6, 0x4f, 0x87, 0x6f, 0x2f, 0xaa, 0xcb, 0xe9, 0xbb, 0xe9, 0xec, 0x97, 0x29, 0x05, 0x23, + 0xee, 0x8b, 0xf0, 0x96, 0x74, 0xe0, 0xb8, 0x60, 0xb5, 0x4f, 0x5a, 0xc5, 0x65, 0x1d, 0xdd, 0x13, + 0xf3, 0xa9, 0xbf, 0x87, 0x5d, 0x20, 0x3f, 0xdc, 0x66, 0x7b, 0xba, 0x85, 0x74, 0xd7, 0x20, 0xc3, + 0x6d, 0x96, 0xb3, 0x47, 0x60, 0xdf, 0x72, 0xb9, 0x8b, 0x69, 0x7f, 0x66, 0x4c, 0x4b, 0x4b, 0x31, + 0xe4, 0x7b, 0xa8, 0xa3, 0x4d, 0x37, 0x8d, 0x8c, 0x15, 0x7c, 0xa6, 0x95, 0xa3, 0xe5, 0xdd, 0x34, + 0x42, 0xb7, 0x53, 0xa8, 0xa2, 0x1b, 0x97, 0x9e, 0x83, 0xb9, 0x0f, 0xf5, 0xe7, 0x4c, 0x92, 0xd6, + 0x6e, 0x1a, 0x2e, 0x7d, 0xf6, 0x97, 0x12, 0x81, 0x57, 0x43, 0xd8, 0x31, 0xf0, 0x48, 0x97, 0xf6, + 0x9c, 0x50, 0x70, 0x29, 0xb5, 0x45, 0xfd, 0x9e, 0x33, 0xd4, 0xb5, 0x99, 0x24, 0xdf, 0xc0, 0x8b, + 0x02, 0x07, 0x03, 0x37, 0xcc, 0x35, 0xd9, 0xb3, 0x30, 0xc8, 0x0f, 0x70, 0x5c, 0xe0, 0xed, 0x17, + 0xf7, 0xc2, 0x6c, 0xe6, 0x9e, 0x5b, 0xc8, 0xcd, 0x33, 0xe5, 0x47, 0xb1, 0xf0, 0x5c, 0x93, 0x9b, + 0x67, 0xea, 0x32, 0x16, 0xe4, 0x1c, 0x1c, 0xc9, 0x54, 0xb6, 0xf5, 0x15, 0xe7, 0x1b, 0xe9, 0x1d, + 0x35, 0xcb, 0x6d, 0xa7, 0x7b, 0xfa, 0x68, 0x73, 0x3e, 0x30, 0xb1, 0x1a, 0xa7, 0x2b, 0x4e, 0x01, + 0xb9, 0x0b, 0x4d, 0x25, 0x67, 0x60, 0xff, 0x19, 0xa8, 0xd8, 0x17, 0x59, 0x2a, 0x3d, 0xf2, 0xb4, + 0xce, 0xd2, 0x4c, 0x9a, 0xa5, 0x92, 0xf4, 0x00, 0x24, 0xe7, 0xe9, 0xda, 0xc8, 0x8e, 0x9f, 0x96, + 0xd9, 0x48, 0xcd, 0x75, 0x69, 0x9c, 0xfe, 0x11, 0x18, 0xdd, 0xc9, 0xff, 0xe8, 0x90, 0x8a, 0xba, + 0x9f, 0xa1, 0x66, 0x48, 0x21, 0x4f, 0x57, 0xf1, 0xda, 0x3b, 0x6d, 0x96, 0xda, 0x4e, 0xf7, 0x8b, + 0x47, 0x4a, 0x7c, 0x21, 0x43, 0x64, 0x50, 0xe7, 0xe6, 0xfe, 0xa3, 0xf5, 0x06, 0x6a, 0xc5, 0x67, + 0x45, 0x2c, 0xa8, 0x2c, 0xe7, 0x23, 0xea, 0x3e, 0x23, 0x75, 0xb0, 0xf5, 0xe8, 0x72, 0x34, 0x58, + 0x5e, 0xb9, 0x25, 0x52, 0x05, 0xfd, 0xe2, 0xdc, 0x83, 0xd6, 0x4f, 0x50, 0xd1, 0xf7, 0x87, 0x38, + 0x90, 0xdf, 0x20, 0xf7, 0x99, 0x46, 0xfb, 0x74, 0xe2, 0x96, 0x88, 0x0d, 0xcf, 0xfb, 0x74, 0xd2, + 0x3b, 0x73, 0x0f, 0x74, 0xed, 0xe3, 0x79, 0xcf, 0x2d, 0x13, 0x80, 0xc3, 0x8f, 0xe7, 0x3d, 0xbf, + 0x77, 0xe6, 0x56, 0x5a, 0x6b, 0x70, 0x0a, 0x59, 0x74, 0x2b, 0xca, 0x24, 0xf3, 0xd7, 0x3c, 0x09, + 0xb0, 0x61, 0x59, 0xb4, 0x9a, 0x49, 0x76, 0xc5, 0x93, 0x40, 0x9f, 0xa8, 0x86, 0xc4, 0x0d, 0xc3, + 0x26, 0x65, 0xd1, 0xc3, 0x4c, 0x32, 0x7a, 0xc3, 0xc8, 0x57, 0xd0, 0x58, 0x71, 0x11, 0x32, 0x7f, + 0xaf, 0x2c, 0x23, 0x5e, 0xc3, 0xea, 0xd2, 0xc8, 0x5b, 0x7f, 0x97, 0xc0, 0xca, 0xf7, 0x8b, 0x10, + 0xa8, 0x44, 0x4c, 0x86, 0x38, 0x85, 0x4d, 0x71, 0xac, 0x6b, 0xd8, 0xd5, 0x4c, 0x07, 0xc4, 0x31, + 0x79, 0x05, 0x20, 0x55, 0x20, 0x14, 0xb6, 0x51, 0xb4, 0xad, 0x50, 0x1b, 0x2b, 0xba, 0x7b, 0x92, + 0x2f, 0xc1, 0x16, 0x2c, 0xd8, 0x18, 0xb4, 0x82, 0xa8, 0xa5, 0x0b, 0x08, 0xbe, 0x02, 0x48, 0x58, + 0xc2, 0xc5, 0x9d, 0xce, 0x85, 0xdd, 0xac, 0x42, 0x6d, 0x53, 0x59, 0x4a, 0xd6, 0xfa, 0xb7, 0x04, + 0x8d, 0x09, 0x8f, 0xb2, 0x0d, 0x5b, 0xdc, 0x6d, 0x19, 0xa6, 0x5a, 0xe6, 0x47, 0x27, 0xef, 0xa4, + 0x62, 0x09, 0xa6, 0x6b, 0x74, 0xbf, 0x7b, 0xfc, 0x70, 0x1f, 0x88, 0x4c, 0x13, 0x9c, 0xff, 0x3a, + 0x5f, 0x8c, 0x26, 0x85, 0x27, 0x8c, 0x92, 0x39, 0xda, 0x90, 0xd7, 0xe0, 0x24, 0xa8, 0xf1, 0xd5, + 0xdd, 0x36, 0x5f, 0x1f, 0x24, 0x7b, 0x1b, 0xbd, 0x81, 0x69, 0x96, 0xf8, 0x7c, 0xe5, 0x9b, 0xa2, + 0xc4, 0x95, 0xd6, 0x69, 0x2d, 0xcd, 0x92, 0xd9, 0xca, 0xcc, 0x27, 0x5b, 0x3f, 0x82, 0x53, 0x98, + 0xeb, 0xe1, 0x71, 0xdb, 0xf0, 0x7c, 0x3e, 0x9b, 0x4d, 0xf5, 0xbd, 0xb0, 0xa0, 0x32, 0xe9, 0xbf, + 0x1b, 0xb9, 0x07, 0x83, 0xa3, 0xb7, 0xe5, 0xdf, 0xf2, 0x5f, 0x97, 0x8f, 0xbf, 0xae, 0xff, 0x02, + 0x00, 0x00, 0xff, 0xff, 0x44, 0x56, 0xd0, 0xb0, 0xca, 0x06, 0x00, 0x00, } diff --git a/ui/metrics/metrics_proto/metrics.proto b/ui/metrics/metrics_proto/metrics.proto index 6c5c321d6..adecaf31f 100644 --- a/ui/metrics/metrics_proto/metrics.proto +++ b/ui/metrics/metrics_proto/metrics.proto @@ -99,6 +99,8 @@ message BuildConfig { optional bool use_goma = 1; optional bool use_rbe = 2; + + optional bool force_use_goma = 3; } message PerfInfo { From 8cb9f3f5efc9d6979b9869edc70b09c5e78a6671 Mon Sep 17 00:00:00 2001 From: Saeid Farivar Asanjan Date: Wed, 9 Sep 2020 04:18:55 +0000 Subject: [PATCH 53/53] Add car-ui-lib-overlayable to apex allowed_deps Bug: 168062639 Test: manual Change-Id: If31714ad0ab00af65e6f880fb8f7224e2c46d0a3 Merged-In: If31714ad0ab00af65e6f880fb8f7224e2c46d0a3 (cherry picked from commit 0b94aab5ef726b083777d1f196361d5878dad7c1) --- apex/allowed_deps.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/apex/allowed_deps.txt b/apex/allowed_deps.txt index 5b8563dc2..0af690745 100644 --- a/apex/allowed_deps.txt +++ b/apex/allowed_deps.txt @@ -132,6 +132,7 @@ bouncycastle_ike_digests(minSdkVersion:current) brotli-java(minSdkVersion:current) captiveportal-lib(minSdkVersion:29) car-ui-lib(minSdkVersion:28) +car-ui-lib-overlayable(minSdkVersion:28) CellBroadcastApp(minSdkVersion:29) CellBroadcastServiceModule(minSdkVersion:29) codecs_g711dec(minSdkVersion:29)