From 715b08fd828d0f53af0a3a26e7fdc5495757b821 Mon Sep 17 00:00:00 2001 From: Chris Parsons Date: Tue, 22 Mar 2022 19:23:40 -0400 Subject: [PATCH] Add metrics collection to soong_build Metrics are added to a materialized protobuf soong_build_metrics.pb Recorded events have scoped names and recorded runtimes. For example, mixedbuilds includes the following events: mixedbuilds.prepare mixedbuilds.analyze.write_files mixedbuild (this is a total of all mixedbuild events) Test: Manually verified materialized protos for mixedbuild, bp2build, and legacy build. Change-Id: I2e72d4502c0373bee0279bb1320b2e4dd586129c --- android/Android.bp | 1 + android/metrics.go | 17 +- bp2build/metrics.go | 3 + cmd/soong_build/main.go | 119 ++++++++----- ui/build/soong.go | 2 +- .../bp2build_metrics.pb.go | 163 ++++++++++++++---- .../bp2build_metrics.proto | 18 ++ ui/metrics/metrics_proto/metrics.pb.go | 32 +++- ui/metrics/metrics_proto/metrics.proto | 3 + 9 files changed, 275 insertions(+), 83 deletions(-) diff --git a/android/Android.bp b/android/Android.bp index d3540b211..c072ac230 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -8,6 +8,7 @@ bootstrap_go_package { deps: [ "blueprint", "blueprint-bootstrap", + "blueprint-metrics", "sbox_proto", "soong", "soong-android-soongconfig", diff --git a/android/metrics.go b/android/metrics.go index 2cd5efae3..9038bde9f 100644 --- a/android/metrics.go +++ b/android/metrics.go @@ -18,6 +18,7 @@ import ( "io/ioutil" "runtime" + "github.com/google/blueprint/metrics" "google.golang.org/protobuf/proto" soong_metrics_proto "android/soong/ui/metrics/metrics_proto" @@ -55,7 +56,7 @@ func (soongMetricsSingleton) GenerateBuildActions(ctx SingletonContext) { }) } -func collectMetrics(config Config) *soong_metrics_proto.SoongBuildMetrics { +func collectMetrics(config Config, eventHandler metrics.EventHandler) *soong_metrics_proto.SoongBuildMetrics { metrics := &soong_metrics_proto.SoongBuildMetrics{} soongMetrics := ReadSoongMetrics(config) @@ -68,11 +69,21 @@ func collectMetrics(config Config) *soong_metrics_proto.SoongBuildMetrics { metrics.TotalAllocCount = proto.Uint64(memStats.Mallocs) metrics.TotalAllocSize = proto.Uint64(memStats.TotalAlloc) + for _, event := range eventHandler.CompletedEvents() { + perfInfo := soong_metrics_proto.PerfInfo{ + Description: proto.String(event.Id), + Name: proto.String("soong_build"), + StartTime: proto.Uint64(uint64(event.Start.UnixNano())), + RealTime: proto.Uint64(event.RuntimeNanoseconds()), + } + metrics.Events = append(metrics.Events, &perfInfo) + } + return metrics } -func WriteMetrics(config Config, metricsFile string) error { - metrics := collectMetrics(config) +func WriteMetrics(config Config, eventHandler metrics.EventHandler, metricsFile string) error { + metrics := collectMetrics(config, eventHandler) buf, err := proto.Marshal(metrics) if err != nil { diff --git a/bp2build/metrics.go b/bp2build/metrics.go index 8a0b1c96e..04fac441e 100644 --- a/bp2build/metrics.go +++ b/bp2build/metrics.go @@ -43,6 +43,8 @@ type CodegenMetrics struct { // Counts of total modules by module type. totalModuleTypeCount map[string]uint64 + + Events []*bp2build_metrics_proto.Event } // Serialize returns the protoized version of CodegenMetrics: bp2build_metrics_proto.Bp2BuildMetrics @@ -55,6 +57,7 @@ func (metrics *CodegenMetrics) Serialize() bp2build_metrics_proto.Bp2BuildMetric ConvertedModules: metrics.convertedModules, ConvertedModuleTypeCount: metrics.convertedModuleTypeCount, TotalModuleTypeCount: metrics.totalModuleTypeCount, + Events: metrics.Events, } } diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index b3a6ee07d..4b3161b33 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -26,10 +26,11 @@ import ( "android/soong/android" "android/soong/bp2build" "android/soong/shared" + "android/soong/ui/metrics/bp2build_metrics_proto" "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/deptools" - "github.com/google/blueprint/pathtools" + "github.com/google/blueprint/metrics" androidProtobuf "google.golang.org/protobuf/android" ) @@ -134,8 +135,14 @@ func newConfig(availableEnv map[string]string) android.Config { // TODO(cparsons): Don't output any ninja file, as the second pass will overwrite // the incorrect results from the first pass, and file I/O is expensive. func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) { - bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration) + firstCtx.EventHandler.Begin("mixed_build") + defer firstCtx.EventHandler.End("mixed_build") + firstCtx.EventHandler.Begin("prepare") + bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration) + firstCtx.EventHandler.End("prepare") + + firstCtx.EventHandler.Begin("bazel") // Invoke bazel commands and save results for second pass. if err := configuration.BazelContext.InvokeBazel(); err != nil { fmt.Fprintf(os.Stderr, "%s", err) @@ -147,18 +154,25 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) } + firstCtx.EventHandler.End("bazel") + secondCtx := newContext(secondConfig) + secondCtx.EventHandler = firstCtx.EventHandler + secondCtx.EventHandler.Begin("analyze") ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig) ninjaDeps = append(ninjaDeps, extraNinjaDeps...) + secondCtx.EventHandler.End("analyze") - globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration) + globListFiles := writeBuildGlobsNinjaFile(secondCtx, configuration.SoongOutDir(), configuration) ninjaDeps = append(ninjaDeps, globListFiles...) - writeDepFile(cmdlineArgs.OutFile, ninjaDeps) + writeDepFile(cmdlineArgs.OutFile, *secondCtx.EventHandler, ninjaDeps) } // Run the code-generation phase to convert BazelTargetModules to BUILD files. func runQueryView(queryviewDir, queryviewMarker string, configuration android.Config, ctx *android.Context) { + ctx.EventHandler.Begin("queryview") + defer ctx.EventHandler.End("queryview") codegenContext := bp2build.NewCodegenContext(configuration, *ctx, bp2build.QueryView) absoluteQueryViewDir := shared.JoinPath(topDir, queryviewDir) if err := createBazelQueryView(codegenContext, absoluteQueryViewDir); err != nil { @@ -169,9 +183,14 @@ func runQueryView(queryviewDir, queryviewMarker string, configuration android.Co touch(shared.JoinPath(topDir, queryviewMarker)) } -func writeMetrics(configuration android.Config) { - metricsFile := filepath.Join(configuration.SoongOutDir(), "soong_build_metrics.pb") - err := android.WriteMetrics(configuration, metricsFile) +func writeMetrics(configuration android.Config, eventHandler metrics.EventHandler) { + metricsDir := configuration.Getenv("LOG_DIR") + if len(metricsDir) < 1 { + fmt.Fprintf(os.Stderr, "\nMissing required env var for generating soong metrics: LOG_DIR\n") + os.Exit(1) + } + metricsFile := filepath.Join(metricsDir, "soong_build_metrics.pb") + err := android.WriteMetrics(configuration, eventHandler, metricsFile) if err != nil { fmt.Fprintf(os.Stderr, "error writing soong_build metrics %s: %s", metricsFile, err) os.Exit(1) @@ -191,18 +210,23 @@ func writeJsonModuleGraphAndActions(ctx *android.Context, graphPath string, acti ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile) } -func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.MultipleGlobResults, config interface{}) []string { +func writeBuildGlobsNinjaFile(ctx *android.Context, buildDir string, config interface{}) []string { + ctx.EventHandler.Begin("globs_ninja_file") + defer ctx.EventHandler.End("globs_ninja_file") + globDir := bootstrap.GlobDirectory(buildDir, globListDir) bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{ - GlobLister: globs, + GlobLister: ctx.Globs, GlobFile: globFile, GlobDir: globDir, - SrcDir: srcDir, + SrcDir: ctx.SrcDir(), }, config) return bootstrap.GlobFileListFiles(globDir) } -func writeDepFile(outputFile string, ninjaDeps []string) { +func writeDepFile(outputFile string, eventHandler metrics.EventHandler, ninjaDeps []string) { + eventHandler.Begin("ninja_deps") + defer eventHandler.End("ninja_deps") depFile := shared.JoinPath(topDir, outputFile+".d") err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps) if err != nil { @@ -230,36 +254,36 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str blueprintArgs := cmdlineArgs - var stopBefore bootstrap.StopBefore - if generateModuleGraphFile { - stopBefore = bootstrap.StopBeforeWriteNinja - } else if generateQueryView { - stopBefore = bootstrap.StopBeforePrepareBuildActions - } else if generateDocFile { - stopBefore = bootstrap.StopBeforePrepareBuildActions - } else { - stopBefore = bootstrap.DoEverything - } - ctx := newContext(configuration) if mixedModeBuild { runMixedModeBuild(configuration, ctx, extraNinjaDeps) } else { + var stopBefore bootstrap.StopBefore + if generateModuleGraphFile { + stopBefore = bootstrap.StopBeforeWriteNinja + } else if generateQueryView { + stopBefore = bootstrap.StopBeforePrepareBuildActions + } else if generateDocFile { + stopBefore = bootstrap.StopBeforePrepareBuildActions + } else { + stopBefore = bootstrap.DoEverything + } + ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, stopBefore, ctx.Context, configuration) ninjaDeps = append(ninjaDeps, extraNinjaDeps...) - globListFiles := writeBuildGlobsNinjaFile(ctx.SrcDir(), configuration.SoongOutDir(), ctx.Globs, configuration) + globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration) ninjaDeps = append(ninjaDeps, globListFiles...) // Convert the Soong module graph into Bazel BUILD files. if generateQueryView { queryviewMarkerFile := bazelQueryViewDir + ".marker" runQueryView(bazelQueryViewDir, queryviewMarkerFile, configuration, ctx) - writeDepFile(queryviewMarkerFile, ninjaDeps) + writeDepFile(queryviewMarkerFile, *ctx.EventHandler, ninjaDeps) return queryviewMarkerFile } else if generateModuleGraphFile { writeJsonModuleGraphAndActions(ctx, moduleGraphFile, moduleActionsFile) - writeDepFile(moduleGraphFile, ninjaDeps) + writeDepFile(moduleGraphFile, *ctx.EventHandler, ninjaDeps) return moduleGraphFile } else if generateDocFile { // TODO: we could make writeDocs() return the list of documentation files @@ -269,16 +293,16 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str fmt.Fprintf(os.Stderr, "error building Soong documentation: %s\n", err) os.Exit(1) } - writeDepFile(docFile, ninjaDeps) + writeDepFile(docFile, *ctx.EventHandler, ninjaDeps) return docFile } else { // The actual output (build.ninja) was written in the RunBlueprint() call // above - writeDepFile(cmdlineArgs.OutFile, ninjaDeps) + writeDepFile(cmdlineArgs.OutFile, *ctx.EventHandler, ninjaDeps) } } - writeMetrics(configuration) + writeMetrics(configuration, *ctx.EventHandler) return cmdlineArgs.OutFile } @@ -335,6 +359,7 @@ func main() { } finalOutputFile := doChosenActivity(configuration, extraNinjaDeps) + writeUsedEnvironmentFile(configuration, finalOutputFile) } @@ -466,6 +491,9 @@ func getExistingBazelRelatedFiles(topDir string) ([]string, error) { // an alternate pipeline of mutators and singletons specifically for generating // Bazel BUILD files instead of Ninja files. func runBp2Build(configuration android.Config, extraNinjaDeps []string) { + eventHandler := metrics.EventHandler{} + eventHandler.Begin("bp2build") + // Register an alternate set of singletons and mutators for bazel // conversion for Bazel conversion. bp2buildCtx := android.NewContext(configuration) @@ -500,7 +528,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.StopBeforePrepareBuildActions, bp2buildCtx.Context, configuration) ninjaDeps = append(ninjaDeps, extraNinjaDeps...) - globListFiles := writeBuildGlobsNinjaFile(bp2buildCtx.SrcDir(), configuration.SoongOutDir(), bp2buildCtx.Globs, configuration) + globListFiles := writeBuildGlobsNinjaFile(bp2buildCtx, configuration.SoongOutDir(), configuration) ninjaDeps = append(ninjaDeps, globListFiles...) // Run the code-generation phase to convert BazelTargetModules to BUILD files @@ -537,27 +565,38 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { symlinkForestDeps := bp2build.PlantSymlinkForest( topDir, workspaceRoot, generatedRoot, ".", excludes) + ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) + ninjaDeps = append(ninjaDeps, symlinkForestDeps...) + + writeDepFile(bp2buildMarker, eventHandler, ninjaDeps) + + // Create an empty bp2build marker file. + touch(shared.JoinPath(topDir, bp2buildMarker)) + + eventHandler.End("bp2build") + // Only report metrics when in bp2build mode. The metrics aren't relevant // for queryview, since that's a total repo-wide conversion and there's a // 1:1 mapping for each module. metrics.Print() - writeBp2BuildMetrics(&metrics, configuration) - - ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) - ninjaDeps = append(ninjaDeps, symlinkForestDeps...) - - writeDepFile(bp2buildMarker, ninjaDeps) - - // Create an empty bp2build marker file. - touch(shared.JoinPath(topDir, bp2buildMarker)) + writeBp2BuildMetrics(&metrics, configuration, eventHandler) } // Write Bp2Build metrics into $LOG_DIR -func writeBp2BuildMetrics(metrics *bp2build.CodegenMetrics, configuration android.Config) { +func writeBp2BuildMetrics(codegenMetrics *bp2build.CodegenMetrics, + configuration android.Config, eventHandler metrics.EventHandler) { + for _, event := range eventHandler.CompletedEvents() { + codegenMetrics.Events = append(codegenMetrics.Events, + &bp2build_metrics_proto.Event{ + Name: event.Id, + StartTime: uint64(event.Start.UnixNano()), + RealTime: event.RuntimeNanoseconds(), + }) + } metricsDir := configuration.Getenv("LOG_DIR") if len(metricsDir) < 1 { fmt.Fprintf(os.Stderr, "\nMissing required env var for generating bp2build metrics: LOG_DIR\n") os.Exit(1) } - metrics.Write(metricsDir) + codegenMetrics.Write(metricsDir) } diff --git a/ui/build/soong.go b/ui/build/soong.go index 1b993e14f..c7f22f946 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -537,7 +537,7 @@ func shouldCollectBuildSoongMetrics(config Config) bool { } func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics { - soongBuildMetricsFile := filepath.Join(config.OutDir(), "soong", "soong_build_metrics.pb") + soongBuildMetricsFile := filepath.Join(config.LogsDir(), "soong_build_metrics.pb") buf, err := ioutil.ReadFile(soongBuildMetricsFile) if err != nil { ctx.Fatalf("Failed to load %s: %s", soongBuildMetricsFile, err) diff --git a/ui/metrics/bp2build_metrics_proto/bp2build_metrics.pb.go b/ui/metrics/bp2build_metrics_proto/bp2build_metrics.pb.go index 95f02caba..93f3471a5 100644 --- a/ui/metrics/bp2build_metrics_proto/bp2build_metrics.pb.go +++ b/ui/metrics/bp2build_metrics_proto/bp2build_metrics.pb.go @@ -53,6 +53,9 @@ type Bp2BuildMetrics struct { ConvertedModuleTypeCount map[string]uint64 `protobuf:"bytes,6,rep,name=convertedModuleTypeCount,proto3" json:"convertedModuleTypeCount,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` // Counts of total modules by module type. TotalModuleTypeCount map[string]uint64 `protobuf:"bytes,7,rep,name=totalModuleTypeCount,proto3" json:"totalModuleTypeCount,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + // List of traced runtime events of bp2build, useful for tracking bp2build + // runtime. + Events []*Event `protobuf:"bytes,8,rep,name=events,proto3" json:"events,omitempty"` } func (x *Bp2BuildMetrics) Reset() { @@ -136,13 +139,89 @@ func (x *Bp2BuildMetrics) GetTotalModuleTypeCount() map[string]uint64 { return nil } +func (x *Bp2BuildMetrics) GetEvents() []*Event { + if x != nil { + return x.Events + } + return nil +} + +// Traced runtime event of bp2build. +type Event struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The event name. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The absolute start time of the event + // The number of nanoseconds elapsed since January 1, 1970 UTC. + StartTime uint64 `protobuf:"varint,2,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + // The real running time. + // The number of nanoseconds elapsed since start_time. + RealTime uint64 `protobuf:"varint,3,opt,name=real_time,json=realTime,proto3" json:"real_time,omitempty"` +} + +func (x *Event) Reset() { + *x = Event{} + if protoimpl.UnsafeEnabled { + mi := &file_bp2build_metrics_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Event) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Event) ProtoMessage() {} + +func (x *Event) ProtoReflect() protoreflect.Message { + mi := &file_bp2build_metrics_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Event.ProtoReflect.Descriptor instead. +func (*Event) Descriptor() ([]byte, []int) { + return file_bp2build_metrics_proto_rawDescGZIP(), []int{1} +} + +func (x *Event) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Event) GetStartTime() uint64 { + if x != nil { + return x.StartTime + } + return 0 +} + +func (x *Event) GetRealTime() uint64 { + if x != nil { + return x.RealTime + } + return 0 +} + var File_bp2build_metrics_proto protoreflect.FileDescriptor var file_bp2build_metrics_proto_rawDesc = []byte{ 0x0a, 0x16, 0x62, 0x70, 0x32, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1c, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x62, 0x70, 0x32, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, 0xac, 0x06, 0x0a, 0x0f, 0x42, 0x70, 0x32, 0x42, 0x75, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, 0xe9, 0x06, 0x0a, 0x0f, 0x42, 0x70, 0x32, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x14, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, @@ -179,24 +258,34 @@ var file_bp2build_metrics_proto_rawDesc = []byte{ 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x6f, 0x64, - 0x75, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x1a, 0x41, 0x0a, 0x13, - 0x52, 0x75, 0x6c, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0x4b, 0x0a, 0x1d, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x4d, 0x6f, 0x64, 0x75, - 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x75, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x06, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x73, + 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x62, 0x70, 0x32, 0x62, 0x75, + 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0x41, 0x0a, 0x13, 0x52, 0x75, 0x6c, + 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x47, 0x0a, 0x19, - 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x43, - 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x31, 0x5a, 0x2f, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, - 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x2f, 0x62, 0x70, 0x32, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x4b, 0x0a, 0x1d, + 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x47, 0x0a, 0x19, 0x54, 0x6f, 0x74, + 0x61, 0x6c, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x43, 0x6f, 0x75, 0x6e, + 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x57, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, + 0x0a, 0x09, 0x72, 0x65, 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x08, 0x72, 0x65, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x31, 0x5a, 0x2f, 0x61, + 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, 0x69, 0x2f, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x62, 0x70, 0x32, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -211,22 +300,24 @@ func file_bp2build_metrics_proto_rawDescGZIP() []byte { return file_bp2build_metrics_proto_rawDescData } -var file_bp2build_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_bp2build_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_bp2build_metrics_proto_goTypes = []interface{}{ (*Bp2BuildMetrics)(nil), // 0: soong_build_bp2build_metrics.Bp2BuildMetrics - nil, // 1: soong_build_bp2build_metrics.Bp2BuildMetrics.RuleClassCountEntry - nil, // 2: soong_build_bp2build_metrics.Bp2BuildMetrics.ConvertedModuleTypeCountEntry - nil, // 3: soong_build_bp2build_metrics.Bp2BuildMetrics.TotalModuleTypeCountEntry + (*Event)(nil), // 1: soong_build_bp2build_metrics.Event + nil, // 2: soong_build_bp2build_metrics.Bp2BuildMetrics.RuleClassCountEntry + nil, // 3: soong_build_bp2build_metrics.Bp2BuildMetrics.ConvertedModuleTypeCountEntry + nil, // 4: soong_build_bp2build_metrics.Bp2BuildMetrics.TotalModuleTypeCountEntry } var file_bp2build_metrics_proto_depIdxs = []int32{ - 1, // 0: soong_build_bp2build_metrics.Bp2BuildMetrics.ruleClassCount:type_name -> soong_build_bp2build_metrics.Bp2BuildMetrics.RuleClassCountEntry - 2, // 1: soong_build_bp2build_metrics.Bp2BuildMetrics.convertedModuleTypeCount:type_name -> soong_build_bp2build_metrics.Bp2BuildMetrics.ConvertedModuleTypeCountEntry - 3, // 2: soong_build_bp2build_metrics.Bp2BuildMetrics.totalModuleTypeCount:type_name -> soong_build_bp2build_metrics.Bp2BuildMetrics.TotalModuleTypeCountEntry - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 2, // 0: soong_build_bp2build_metrics.Bp2BuildMetrics.ruleClassCount:type_name -> soong_build_bp2build_metrics.Bp2BuildMetrics.RuleClassCountEntry + 3, // 1: soong_build_bp2build_metrics.Bp2BuildMetrics.convertedModuleTypeCount:type_name -> soong_build_bp2build_metrics.Bp2BuildMetrics.ConvertedModuleTypeCountEntry + 4, // 2: soong_build_bp2build_metrics.Bp2BuildMetrics.totalModuleTypeCount:type_name -> soong_build_bp2build_metrics.Bp2BuildMetrics.TotalModuleTypeCountEntry + 1, // 3: soong_build_bp2build_metrics.Bp2BuildMetrics.events:type_name -> soong_build_bp2build_metrics.Event + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_bp2build_metrics_proto_init() } @@ -247,6 +338,18 @@ func file_bp2build_metrics_proto_init() { return nil } } + file_bp2build_metrics_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Event); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -254,7 +357,7 @@ func file_bp2build_metrics_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_bp2build_metrics_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/ui/metrics/bp2build_metrics_proto/bp2build_metrics.proto b/ui/metrics/bp2build_metrics_proto/bp2build_metrics.proto index 6d98a3df5..19a78277e 100644 --- a/ui/metrics/bp2build_metrics_proto/bp2build_metrics.proto +++ b/ui/metrics/bp2build_metrics_proto/bp2build_metrics.proto @@ -38,4 +38,22 @@ message Bp2BuildMetrics { // Counts of total modules by module type. map totalModuleTypeCount = 7; + + // List of traced runtime events of bp2build, useful for tracking bp2build + // runtime. + repeated Event events = 8; +} + +// Traced runtime event of bp2build. +message Event { + // The event name. + string name = 1; + + // The absolute start time of the event + // The number of nanoseconds elapsed since January 1, 1970 UTC. + uint64 start_time = 2; + + // The real running time. + // The number of nanoseconds elapsed since start_time. + uint64 real_time = 3; } diff --git a/ui/metrics/metrics_proto/metrics.pb.go b/ui/metrics/metrics_proto/metrics.pb.go index 2e530b0f4..26229c683 100644 --- a/ui/metrics/metrics_proto/metrics.pb.go +++ b/ui/metrics/metrics_proto/metrics.pb.go @@ -1072,6 +1072,8 @@ type SoongBuildMetrics struct { TotalAllocSize *uint64 `protobuf:"varint,4,opt,name=total_alloc_size,json=totalAllocSize" json:"total_alloc_size,omitempty"` // The approximate maximum size of the heap in soong_build in bytes. MaxHeapSize *uint64 `protobuf:"varint,5,opt,name=max_heap_size,json=maxHeapSize" json:"max_heap_size,omitempty"` + // Runtime metrics for soong_build execution. + Events []*PerfInfo `protobuf:"bytes,6,rep,name=events" json:"events,omitempty"` } func (x *SoongBuildMetrics) Reset() { @@ -1141,6 +1143,13 @@ func (x *SoongBuildMetrics) GetMaxHeapSize() uint64 { return 0 } +func (x *SoongBuildMetrics) GetEvents() []*PerfInfo { + if x != nil { + return x.Events + } + return nil +} + var File_metrics_proto protoreflect.FileDescriptor var file_metrics_proto_rawDesc = []byte{ @@ -1340,7 +1349,7 @@ var file_metrics_proto_rawDesc = []byte{ 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x43, 0x72, 0x69, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x6f, 0x75, 0x72, 0x6e, 0x65, 0x79, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x04, 0x63, 0x75, 0x6a, 0x73, - 0x22, 0xc3, 0x01, 0x0a, 0x11, 0x53, 0x6f, 0x6f, 0x6e, 0x67, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, + 0x22, 0xfa, 0x01, 0x0a, 0x11, 0x53, 0x6f, 0x6f, 0x6e, 0x67, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, @@ -1352,9 +1361,13 @@ var file_metrics_proto_rawDesc = []byte{ 0x28, 0x04, 0x52, 0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x6d, 0x61, 0x78, 0x5f, 0x68, 0x65, 0x61, 0x70, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x48, 0x65, - 0x61, 0x70, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x28, 0x5a, 0x26, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, - 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x61, 0x70, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x35, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, + 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x50, 0x65, 0x72, + 0x66, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x28, 0x5a, + 0x26, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, + 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -1403,11 +1416,12 @@ var file_metrics_proto_depIdxs = []int32{ 2, // 14: soong_build_metrics.ModuleTypeInfo.build_system:type_name -> soong_build_metrics.ModuleTypeInfo.BuildSystem 3, // 15: soong_build_metrics.CriticalUserJourneyMetrics.metrics:type_name -> soong_build_metrics.MetricsBase 9, // 16: soong_build_metrics.CriticalUserJourneysMetrics.cujs:type_name -> soong_build_metrics.CriticalUserJourneyMetrics - 17, // [17:17] is the sub-list for method output_type - 17, // [17:17] is the sub-list for method input_type - 17, // [17:17] is the sub-list for extension type_name - 17, // [17:17] is the sub-list for extension extendee - 0, // [0:17] is the sub-list for field type_name + 6, // 17: soong_build_metrics.SoongBuildMetrics.events:type_name -> soong_build_metrics.PerfInfo + 18, // [18:18] is the sub-list for method output_type + 18, // [18:18] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_metrics_proto_init() } diff --git a/ui/metrics/metrics_proto/metrics.proto b/ui/metrics/metrics_proto/metrics.proto index db0a14ae1..26e4d73d7 100644 --- a/ui/metrics/metrics_proto/metrics.proto +++ b/ui/metrics/metrics_proto/metrics.proto @@ -235,4 +235,7 @@ message SoongBuildMetrics { // The approximate maximum size of the heap in soong_build in bytes. optional uint64 max_heap_size = 5; + + // Runtime metrics for soong_build execution. + repeated PerfInfo events = 6; }