StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add performance counter metrics to build.trace.gz

Browse Source 
Start a background goroutine at the beginning of soong_build that
captures the CPU usage, heap size, and total system memory every
second.  Propagate the values through soong_build_metrics.pb back
to soong_ui, and then into build.trace.gz.

Test: m nothing, examine build.trace.gz
Change-Id: Iad99f8f1f088f4f7f7d5f76566a38c0c4f4d0daa
This commit is contained in:
Colin Cross
2023-10-27 14:56:12 -07:00
parent aa9a273b40
commit 46b0c75204
6 changed files with 655 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
android/metrics.go
View File

@@ -15,9 +15,13 @@
package android
import (
"bytes"
"io/ioutil"
"os"
"runtime"
"sort"
"strconv"
"time"
"github.com/google/blueprint/metrics"
"google.golang.org/protobuf/proto"
@@ -27,18 +31,21 @@ import (
var soongMetricsOnceKey = NewOnceKey("soong metrics")
type SoongMetrics struct {
Modules int
Variants int
type soongMetrics struct {
modules int
variants int
perfCollector perfCollector
}
func readSoongMetrics(config Config) (SoongMetrics, bool) {
soongMetrics, ok := config.Peek(soongMetricsOnceKey)
if ok {
return soongMetrics.(SoongMetrics), true
} else {
return SoongMetrics{}, false
}
type perfCollector struct {
events []*soong_metrics_proto.PerfCounters
stop chan<- bool
}
func getSoongMetrics(config Config) *soongMetrics {
return config.Once(soongMetricsOnceKey, func() interface{} {
return &soongMetrics{}
}).(*soongMetrics)
}
func init() {
@@ -50,27 +57,27 @@ func soongMetricsSingletonFactory() Singleton { return soongMetricsSingleton{} }
type soongMetricsSingleton struct{}
func (soongMetricsSingleton) GenerateBuildActions(ctx SingletonContext) {
metrics := SoongMetrics{}
metrics := getSoongMetrics(ctx.Config())
ctx.VisitAllModules(func(m Module) {
if ctx.PrimaryModule(m) == m {
metrics.Modules++
metrics.modules++
}
metrics.Variants++
})
ctx.Config().Once(soongMetricsOnceKey, func() interface{} {
return metrics
metrics.variants++
})
}
func collectMetrics(config Config, eventHandler *metrics.EventHandler) *soong_metrics_proto.SoongBuildMetrics {
metrics := &soong_metrics_proto.SoongBuildMetrics{}
soongMetrics, ok := readSoongMetrics(config)
if ok {
metrics.Modules = proto.Uint32(uint32(soongMetrics.Modules))
metrics.Variants = proto.Uint32(uint32(soongMetrics.Variants))
soongMetrics := getSoongMetrics(config)
if soongMetrics.modules > 0 {
metrics.Modules = proto.Uint32(uint32(soongMetrics.modules))
metrics.Variants = proto.Uint32(uint32(soongMetrics.variants))
}
soongMetrics.perfCollector.stop <- true
metrics.PerfCounters = soongMetrics.perfCollector.events
memStats := runtime.MemStats{}
runtime.ReadMemStats(&memStats)
metrics.MaxHeapSize = proto.Uint64(memStats.HeapSys)
@@ -107,6 +114,113 @@ func collectMetrics(config Config, eventHandler *metrics.EventHandler) *soong_me
return metrics
}
func StartBackgroundMetrics(config Config) {
perfCollector := &getSoongMetrics(config).perfCollector
stop := make(chan bool)
perfCollector.stop = stop
previousTime := time.Now()
previousCpuTime := readCpuTime()
ticker := time.NewTicker(time.Second)
go func() {
for {
select {
case <-stop:
ticker.Stop()
return
case <-ticker.C:
// carry on
}
currentTime := time.Now()
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
currentCpuTime := readCpuTime()
interval := currentTime.Sub(previousTime)
intervalCpuTime := currentCpuTime - previousCpuTime
intervalCpuPercent := intervalCpuTime * 100 / interval
// heapAlloc is the memory that has been allocated on the heap but not yet GC'd. It may be referenced,
// or unrefenced but not yet GC'd.
heapAlloc := memStats.HeapAlloc
// heapUnused is the memory that was previously used by the heap, but is currently not used. It does not
// count memory that was used and then returned to the OS.
heapUnused := memStats.HeapIdle - memStats.HeapReleased
// heapOverhead is the memory used by the allocator and GC
heapOverhead := memStats.MSpanSys + memStats.MCacheSys + memStats.GCSys
// otherMem is the memory used outside of the heap.
otherMem := memStats.Sys - memStats.HeapSys - heapOverhead
perfCollector.events = append(perfCollector.events, &soong_metrics_proto.PerfCounters{
Time: proto.Uint64(uint64(currentTime.UnixNano())),
Groups: []*soong_metrics_proto.PerfCounterGroup{
{
Name: proto.String("cpu"),
Counters: []*soong_metrics_proto.PerfCounter{
{Name: proto.String("cpu_percent"), Value: proto.Int64(int64(intervalCpuPercent))},
},
}, {
Name: proto.String("memory"),
Counters: []*soong_metrics_proto.PerfCounter{
{Name: proto.String("heap_alloc"), Value: proto.Int64(int64(heapAlloc))},
{Name: proto.String("heap_unused"), Value: proto.Int64(int64(heapUnused))},
{Name: proto.String("heap_overhead"), Value: proto.Int64(int64(heapOverhead))},
{Name: proto.String("other"), Value: proto.Int64(int64(otherMem))},
},
},
},
})
previousTime = currentTime
previousCpuTime = currentCpuTime
}
}()
}
func readCpuTime() time.Duration {
if runtime.GOOS != "linux" {
return 0
}
stat, err := os.ReadFile("/proc/self/stat")
if err != nil {
return 0
}
endOfComm := bytes.LastIndexByte(stat, ')')
if endOfComm < 0 || endOfComm > len(stat)-2 {
return 0
}
stat = stat[endOfComm+2:]
statFields := bytes.Split(stat, []byte{' '})
// This should come from sysconf(_SC_CLK_TCK), but there's no way to call that from Go. Assume it's 100,
// which is the value for all platforms we support.
const HZ = 100
const MS_PER_HZ = 1e3 / HZ * time.Millisecond
const STAT_UTIME_FIELD = 14 - 2
const STAT_STIME_FIELD = 15 - 2
if len(statFields) < STAT_STIME_FIELD {
return 0
}
userCpuTicks, err := strconv.ParseUint(string(statFields[STAT_UTIME_FIELD]), 10, 64)
if err != nil {
return 0
}
kernelCpuTicks, _ := strconv.ParseUint(string(statFields[STAT_STIME_FIELD]), 10, 64)
if err != nil {
return 0
}
return time.Duration(userCpuTicks+kernelCpuTicks) * MS_PER_HZ
}
func WriteMetrics(config Config, eventHandler *metrics.EventHandler, metricsFile string) error {
metrics := collectMetrics(config, eventHandler)