From 05ab2d0731495fed0e9ee7d4158cf70697728f77 Mon Sep 17 00:00:00 2001 From: Patrice Arruda Date: Sat, 12 Dec 2020 06:24:26 +0000 Subject: [PATCH] Enable bazel profiling in soong_build. Bazel is executed several times during the execution of soong_build. For each bazel execution, generate a profile and save under the BAZEL_METRICS_DIR which is defined in soong_ui. Bug: b/174479924 Test: * USE_BAZEL_ANALYSIS=1 USE_BAZEL=1 m nothing and checked if the cquery and graph build bazel profiles were generated. * Verified that the generated bazel profiles were uploaded to the local dev metrics pipeline. Change-Id: I3d20204484dc6c5a1525a5d3eec1d62cfb33535b --- android/bazel_handler.go | 19 ++++++++++++++++--- bazel/Android.bp | 1 + bazel/constants.go | 26 ++++++++++++++++++++++++++ shared/Android.bp | 3 +++ shared/paths.go | 6 ++++-- ui/build/bazel.go | 5 +++-- ui/build/soong.go | 1 + 7 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 bazel/constants.go diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 7d8d12f19..d810726eb 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -26,6 +26,8 @@ import ( "strings" "sync" + "android/soong/bazel" + "android/soong/shared" "github.com/google/blueprint/bootstrap" ) @@ -68,6 +70,7 @@ type bazelContext struct { outputBase string workspaceDir string buildDir string + metricsDir string requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel requestMutex sync.Mutex // requests can be written in parallel @@ -153,6 +156,11 @@ func NewBazelContext(c *config) (BazelContext, error) { } else { missingEnvVars = append(missingEnvVars, "BAZEL_WORKSPACE") } + if len(c.Getenv("BAZEL_METRICS_DIR")) > 1 { + bazelCtx.metricsDir = c.Getenv("BAZEL_METRICS_DIR") + } else { + missingEnvVars = append(missingEnvVars, "BAZEL_METRICS_DIR") + } if len(missingEnvVars) > 0 { return nil, errors.New(fmt.Sprintf("missing required env vars to use bazel: %s", missingEnvVars)) } else { @@ -160,6 +168,10 @@ func NewBazelContext(c *config) (BazelContext, error) { } } +func (context *bazelContext) BazelMetricsDir() string { + return context.metricsDir +} + func (context *bazelContext) BazelEnabled() bool { return true } @@ -189,12 +201,13 @@ func pwdPrefix() string { return "" } -func (context *bazelContext) issueBazelCommand(command string, labels []string, +func (context *bazelContext) issueBazelCommand(runName bazel.RunName, command string, labels []string, extraFlags ...string) (string, error) { cmdFlags := []string{"--output_base=" + context.outputBase, command} cmdFlags = append(cmdFlags, labels...) cmdFlags = append(cmdFlags, "--package_path=%workspace%/"+context.intermediatesDir()) + cmdFlags = append(cmdFlags, "--profile="+shared.BazelMetricsFilename(context, runName)) cmdFlags = append(cmdFlags, extraFlags...) bazelCmd := exec.Command(context.bazelPath, cmdFlags...) @@ -341,7 +354,7 @@ func (context *bazelContext) InvokeBazel() error { return err } buildroot_label := "//:buildroot" - cqueryOutput, err = context.issueBazelCommand("cquery", + cqueryOutput, err = context.issueBazelCommand(bazel.CqueryBuildRootRunName, "cquery", []string{fmt.Sprintf("deps(%s)", buildroot_label)}, "--output=starlark", "--starlark:file="+cquery_file_relpath) @@ -371,7 +384,7 @@ func (context *bazelContext) InvokeBazel() error { // bazel actions should either be added to the Ninja file and executed later, // or bazel should handle execution. // TODO(cparsons): Use --target_pattern_file to avoid command line limits. - _, err = context.issueBazelCommand("build", []string{buildroot_label}) + _, err = context.issueBazelCommand(bazel.BazelBuildPhonyRootRunName, "build", []string{buildroot_label}) if err != nil { return err diff --git a/bazel/Android.bp b/bazel/Android.bp index 0113726c7..d557be557 100644 --- a/bazel/Android.bp +++ b/bazel/Android.bp @@ -2,6 +2,7 @@ bootstrap_go_package { name: "soong-bazel", pkgPath: "android/soong/bazel", srcs: [ + "constants.go", "properties.go", ], pluginFor: [ diff --git a/bazel/constants.go b/bazel/constants.go new file mode 100644 index 000000000..15c75cf4a --- /dev/null +++ b/bazel/constants.go @@ -0,0 +1,26 @@ +package bazel + +type RunName string + +// Below is a list bazel execution run names used through out the +// Platform Build systems. Each run name represents an unique key +// to query the bazel metrics. +const ( + // Perform a bazel build of the phony root to generate symlink forests + // for dependencies of the bazel build. + BazelBuildPhonyRootRunName = RunName("bazel-build-phony-root") + + // Perform aquery of the bazel build root to retrieve action information. + AqueryBuildRootRunName = RunName("aquery-buildroot") + + // Perform cquery of the Bazel build root and its dependencies. + CqueryBuildRootRunName = RunName("cquery-buildroot") + + // Run bazel as a ninja executer + BazelNinjaExecRunName = RunName("bazel-ninja-exec") +) + +// String returns the name of the run. +func (c RunName) String() string { + return string(c) +} diff --git a/shared/Android.bp b/shared/Android.bp index 07dfe11d3..2a4f56f77 100644 --- a/shared/Android.bp +++ b/shared/Android.bp @@ -4,4 +4,7 @@ bootstrap_go_package { srcs: [ "paths.go", ], + deps: [ + "soong-bazel", + ], } diff --git a/shared/paths.go b/shared/paths.go index f5dc5dd2e..1b9ff6098 100644 --- a/shared/paths.go +++ b/shared/paths.go @@ -18,6 +18,8 @@ package shared import ( "path/filepath" + + "android/soong/bazel" ) // A SharedPaths represents a list of paths that are shared between @@ -37,6 +39,6 @@ func TempDirForOutDir(outDir string) (tempPath string) { // on the action name. This is to help to store a set of bazel // profiles since bazel may execute multiple times during a single // build. -func BazelMetricsFilename(s SharedPaths, actionName string) string { - return filepath.Join(s.BazelMetricsDir(), actionName+"_bazel_profile.gz") +func BazelMetricsFilename(s SharedPaths, actionName bazel.RunName) string { + return filepath.Join(s.BazelMetricsDir(), actionName.String()+"_bazel_profile.gz") } diff --git a/ui/build/bazel.go b/ui/build/bazel.go index 4f2d6455c..81ce9397a 100644 --- a/ui/build/bazel.go +++ b/ui/build/bazel.go @@ -22,6 +22,7 @@ import ( "path/filepath" "strings" + "android/soong/bazel" "android/soong/shared" "android/soong/ui/metrics" ) @@ -97,9 +98,9 @@ func runBazel(ctx Context, config Config) { } // Start constructing the `build` command. - actionName := "build" + actionName := bazel.BazelNinjaExecRunName cmd.Args = append(cmd.Args, - actionName, + "build", // Use output_groups to select the set of outputs to produce from a // ninja_build target. "--output_groups="+outputGroups, diff --git a/ui/build/soong.go b/ui/build/soong.go index 08c2ee18e..6a9367259 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -156,6 +156,7 @@ func runSoong(ctx Context, config Config) { cmd.Environment.Set("BAZEL_HOME", filepath.Join(config.BazelOutDir(), "bazelhome")) cmd.Environment.Set("BAZEL_OUTPUT_BASE", filepath.Join(config.BazelOutDir(), "output")) cmd.Environment.Set("BAZEL_WORKSPACE", absPath(ctx, ".")) + cmd.Environment.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir()) cmd.Environment.Set("SOONG_SANDBOX_SOONG_BUILD", "true") cmd.Sandbox = soongSandbox