diff --git a/ui/build/config.go b/ui/build/config.go index 1dd948ce5..077a4d199 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -35,10 +35,10 @@ import ( ) const ( - envConfigDir = "vendor/google/tools/soong_config" - jsonSuffix = "json" + envConfigDir = "vendor/google/tools/soong_config" + jsonSuffix = "json" - configFetcher = "vendor/google/tools/soong/expconfigfetcher" + configFetcher = "vendor/google/tools/soong/expconfigfetcher" envConfigFetchTimeout = 10 * time.Second ) @@ -62,6 +62,7 @@ type configImpl struct { jsonModuleGraph bool bp2build bool queryview bool + reportMkMetrics bool // Collect and report mk2bp migration progress metrics. soongDocs bool skipConfig bool skipKati bool @@ -155,7 +156,7 @@ func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error } configExists := false - outConfigFilePath := filepath.Join(config.OutDir(), envConfigName + jsonSuffix) + outConfigFilePath := filepath.Join(config.OutDir(), envConfigName+jsonSuffix) if _, err := os.Stat(outConfigFilePath); err == nil { configExists = true } @@ -711,6 +712,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { c.skipConfig = true } else if arg == "--skip-soong-tests" { c.skipSoongTests = true + } else if arg == "--mk-metrics" { + c.reportMkMetrics = true } else if len(arg) > 0 && arg[0] == '-' { parseArgNum := func(def int) int { if len(arg) > 2 { @@ -1381,6 +1384,11 @@ func (c *configImpl) BazelMetricsDir() string { return filepath.Join(c.LogsDir(), "bazel_metrics") } +// MkFileMetrics returns the file path for make-related metrics. +func (c *configImpl) MkMetrics() string { + return filepath.Join(c.LogsDir(), "mk_metrics.pb") +} + func (c *configImpl) SetEmptyNinjaFile(v bool) { c.emptyNinjaFile = v } diff --git a/ui/build/finder.go b/ui/build/finder.go index 68efe2150..262de3de7 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -138,6 +138,17 @@ func FindSources(ctx Context, config Config, f *finder.Finder) { ctx.Fatalf("Could not export module list: %v", err) } + // Gate collecting/reporting mk metrics on builds that specifically request + // it, as identifying the total number of mk files adds 4-5ms onto null + // builds. + if config.reportMkMetrics { + androidMksTotal := f.FindNamedAt(".", "Android.mk") + + ctx.Metrics.SetToplevelMakefiles(len(androidMks)) + ctx.Metrics.SetTotalMakefiles(len(androidMksTotal)) + ctx.Metrics.DumpMkMetrics(config.MkMetrics()) + } + // Stop searching a subdirectory recursively after finding a CleanSpec.mk. cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk") err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list")) diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp index 3ba3907a6..05db1d753 100644 --- a/ui/metrics/Android.bp +++ b/ui/metrics/Android.bp @@ -21,9 +21,10 @@ bootstrap_go_package { pkgPath: "android/soong/ui/metrics", deps: [ "golang-protobuf-proto", + "soong-ui-bp2build_metrics_proto", "soong-ui-metrics_upload_proto", "soong-ui-metrics_proto", - "soong-ui-bp2build_metrics_proto", + "soong-ui-mk_metrics_proto", "soong-ui-tracer", "soong-shared", ], @@ -71,3 +72,15 @@ bootstrap_go_package { "bp2build_metrics_proto/bp2build_metrics.pb.go", ], } + +bootstrap_go_package { + name: "soong-ui-mk_metrics_proto", + pkgPath: "android/soong/ui/metrics/mk_metrics_proto", + deps: [ + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + ], + srcs: [ + "mk_metrics_proto/mk_metrics.pb.go", + ], +} diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 80f8c1ad0..6f1ed6032 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -41,6 +41,7 @@ import ( "google.golang.org/protobuf/proto" soong_metrics_proto "android/soong/ui/metrics/metrics_proto" + mk_metrics_proto "android/soong/ui/metrics/mk_metrics_proto" ) const ( @@ -62,14 +63,22 @@ const ( Total = "total" ) -// Metrics is a struct that stores collected metrics during the course -// of a build which later is dumped to a MetricsBase protobuf file. -// See ui/metrics/metrics_proto/metrics.proto for further details -// on what information is collected. +// Metrics is a struct that stores collected metrics during the course of a +// build. It is later dumped to protobuf files. See underlying metrics protos +// for further details on what information is collected. type Metrics struct { - // The protobuf message that is later written to the file. + // Protobuf containing various top-level build metrics. These include: + // 1. Build identifiers (ex: branch ID, requested product, hostname, + // originating command) + // 2. Per-subprocess top-level metrics (ex: ninja process IO and runtime). + // Note that, since these metrics are reported by soong_ui, there is little + // insight that can be provided into performance breakdowns of individual + // subprocesses. metrics soong_metrics_proto.MetricsBase + // Protobuf containing metrics pertaining to number of makefiles in a build. + mkMetrics mk_metrics_proto.MkMetrics + // A list of pending build events. EventTracer *EventTracer } @@ -78,11 +87,24 @@ type Metrics struct { func New() (metrics *Metrics) { m := &Metrics{ metrics: soong_metrics_proto.MetricsBase{}, + mkMetrics: mk_metrics_proto.MkMetrics{}, EventTracer: &EventTracer{}, } return m } +func (m *Metrics) SetTotalMakefiles(total int) { + m.mkMetrics.TotalMakefiles = uint32(total) +} + +func (m *Metrics) SetToplevelMakefiles(total int) { + m.mkMetrics.ToplevelMakefiles = uint32(total) +} + +func (m *Metrics) DumpMkMetrics(outPath string) { + shared.Save(&m.mkMetrics, outPath) +} + // SetTimeMetrics stores performance information from an executed block of // code. func (m *Metrics) SetTimeMetrics(perf soong_metrics_proto.PerfInfo) { diff --git a/ui/metrics/mk_metrics_proto/mk_metrics.pb.go b/ui/metrics/mk_metrics_proto/mk_metrics.pb.go new file mode 100644 index 000000000..32e136ab8 --- /dev/null +++ b/ui/metrics/mk_metrics_proto/mk_metrics.pb.go @@ -0,0 +1,177 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc v3.9.1 +// source: mk_metrics.proto + +package mk_metrics_proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Contains metrics pertaining to makefiles. +type MkMetrics struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Total number of mk files present in the workspace. + TotalMakefiles uint32 `protobuf:"varint,1,opt,name=totalMakefiles,proto3" json:"totalMakefiles,omitempty"` + // Number of top-level mk files present in the workspace. + // A mk file is "top level" if there are no mk files in its parent + // direrctories. + // This value is equivalent to the number of entries in Android.mk.list. + ToplevelMakefiles uint32 `protobuf:"varint,2,opt,name=toplevelMakefiles,proto3" json:"toplevelMakefiles,omitempty"` +} + +func (x *MkMetrics) Reset() { + *x = MkMetrics{} + if protoimpl.UnsafeEnabled { + mi := &file_mk_metrics_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MkMetrics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MkMetrics) ProtoMessage() {} + +func (x *MkMetrics) ProtoReflect() protoreflect.Message { + mi := &file_mk_metrics_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MkMetrics.ProtoReflect.Descriptor instead. +func (*MkMetrics) Descriptor() ([]byte, []int) { + return file_mk_metrics_proto_rawDescGZIP(), []int{0} +} + +func (x *MkMetrics) GetTotalMakefiles() uint32 { + if x != nil { + return x.TotalMakefiles + } + return 0 +} + +func (x *MkMetrics) GetToplevelMakefiles() uint32 { + if x != nil { + return x.ToplevelMakefiles + } + return 0 +} + +var File_mk_metrics_proto protoreflect.FileDescriptor + +var file_mk_metrics_proto_rawDesc = []byte{ + 0x0a, 0x10, 0x6d, 0x6b, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x16, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, + 0x6d, 0x6b, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, 0x61, 0x0a, 0x09, 0x4d, 0x6b, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c, + 0x4d, 0x61, 0x6b, 0x65, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x61, 0x6b, 0x65, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, + 0x2c, 0x0a, 0x11, 0x74, 0x6f, 0x70, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x4d, 0x61, 0x6b, 0x65, 0x66, + 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x74, 0x6f, 0x70, 0x6c, + 0x65, 0x76, 0x65, 0x6c, 0x4d, 0x61, 0x6b, 0x65, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x42, 0x2b, 0x5a, + 0x29, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, + 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x6d, 0x6b, 0x5f, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_mk_metrics_proto_rawDescOnce sync.Once + file_mk_metrics_proto_rawDescData = file_mk_metrics_proto_rawDesc +) + +func file_mk_metrics_proto_rawDescGZIP() []byte { + file_mk_metrics_proto_rawDescOnce.Do(func() { + file_mk_metrics_proto_rawDescData = protoimpl.X.CompressGZIP(file_mk_metrics_proto_rawDescData) + }) + return file_mk_metrics_proto_rawDescData +} + +var file_mk_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_mk_metrics_proto_goTypes = []interface{}{ + (*MkMetrics)(nil), // 0: soong_build_mk_metrics.MkMetrics +} +var file_mk_metrics_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_mk_metrics_proto_init() } +func file_mk_metrics_proto_init() { + if File_mk_metrics_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_mk_metrics_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MkMetrics); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_mk_metrics_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_mk_metrics_proto_goTypes, + DependencyIndexes: file_mk_metrics_proto_depIdxs, + MessageInfos: file_mk_metrics_proto_msgTypes, + }.Build() + File_mk_metrics_proto = out.File + file_mk_metrics_proto_rawDesc = nil + file_mk_metrics_proto_goTypes = nil + file_mk_metrics_proto_depIdxs = nil +} diff --git a/ui/metrics/mk_metrics_proto/mk_metrics.proto b/ui/metrics/mk_metrics_proto/mk_metrics.proto new file mode 100644 index 000000000..df7bca3da --- /dev/null +++ b/ui/metrics/mk_metrics_proto/mk_metrics.proto @@ -0,0 +1,30 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package soong_build_mk_metrics; +option go_package = "android/soong/ui/metrics/mk_metrics_proto"; + +// Contains metrics pertaining to makefiles. +message MkMetrics { + // Total number of mk files present in the workspace. + uint32 totalMakefiles = 1; + + // Number of top-level mk files present in the workspace. + // A mk file is "top level" if there are no mk files in its parent + // direrctories. + // This value is equivalent to the number of entries in Android.mk.list. + uint32 toplevelMakefiles = 2; +} diff --git a/ui/metrics/mk_metrics_proto/regen.sh b/ui/metrics/mk_metrics_proto/regen.sh new file mode 100755 index 000000000..64018d42f --- /dev/null +++ b/ui/metrics/mk_metrics_proto/regen.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e + +# Copyright 2022 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates the golang source file of the mk_metrics.proto protobuf file. + +function die() { echo "ERROR: $1" >&2; exit 1; } + +readonly error_msg="Maybe you need to run 'lunch aosp_arm-eng && m aprotoc blueprint_tools'?" + +if ! hash aprotoc &>/dev/null; then + die "could not find aprotoc. ${error_msg}" +fi + +if ! aprotoc --go_out=paths=source_relative:. mk_metrics.proto; then + die "build failed. ${error_msg}" +fi