From 2c40569db0fe26084e634753586bf5f6b18b36ef Mon Sep 17 00:00:00 2001 From: "Lukacs T. Berki" Date: Wed, 11 Aug 2021 09:19:27 +0200 Subject: [PATCH] Do not create a build.Config in multiproduct_kati. This mostly cuts the dependency from multiproduct_kati to the build package; the remaining logic is SetupSignals which will be dealt with in a followup change. This requires some code duplication (e.g. ensureFileExists() and detectTotalRAM()). This removes a Finder invocation added in aosp/452598; it looks like it's not a measurable performance hit. The quoted half a second does not matter much in build_test{1,2,3,4} which take an amount of time measured in hours. It also makes ANDROID_QUIET_BUILD unsupported for the UI of multiproduct_kati; the function of that variable seems to be to suppress the output from build actions, which don't exist in multiproduct_kati so it should not matter. In addition, I checked whether anyone sets this flag for multiproduct_kati and it doesn't appear to be the case. Test: Presubmits. Change-Id: I1c2901e1ea69c3a35429e6c967a7bba845676ef7 --- cmd/multiproduct_kati/main.go | 170 +++++++++++++++------------------- 1 file changed, 74 insertions(+), 96 deletions(-) diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index fcad1026c..8bfa7723d 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -20,18 +20,17 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "os" "os/exec" "path/filepath" + "regexp" "runtime" "strings" "sync" "syscall" "time" - "android/soong/finder" "android/soong/ui/build" "android/soong/ui/logger" "android/soong/ui/status" @@ -77,36 +76,6 @@ func (m *multipleStringArg) Set(s string) error { return nil } -const errorLeadingLines = 20 -const errorTrailingLines = 20 - -func errMsgFromLog(filename string) string { - if filename == "" { - return "" - } - - data, err := ioutil.ReadFile(filename) - if err != nil { - return "" - } - - lines := strings.Split(strings.TrimSpace(string(data)), "\n") - if len(lines) > errorLeadingLines+errorTrailingLines+1 { - lines[errorLeadingLines] = fmt.Sprintf("... skipping %d lines ...", - len(lines)-errorLeadingLines-errorTrailingLines) - - lines = append(lines[:errorLeadingLines+1], - lines[len(lines)-errorTrailingLines:]...) - } - var buf strings.Builder - for _, line := range lines { - buf.WriteString("> ") - buf.WriteString(line) - buf.WriteString("\n") - } - return buf.String() -} - // TODO(b/70370883): This tool uses a lot of open files -- over the default // soft limit of 1024 on some systems. So bump up to the hard limit until I fix // the algorithm. @@ -162,18 +131,49 @@ type mpContext struct { Logger logger.Logger Status status.ToolStatus Tracer tracer.Tracer - Finder *finder.Finder - Config build.Config LogsDir string } +func detectTotalRAM() uint64 { + var info syscall.Sysinfo_t + err := syscall.Sysinfo(&info) + if err != nil { + panic(err) + } + return info.Totalram * uint64(info.Unit) +} + +func findNamedProducts(soongUi string, log logger.Logger) []string { + cmd := exec.Command(soongUi, "--dumpvars-mode", "--vars=all_named_products") + output, err := cmd.Output() + if err != nil { + log.Fatalf("Cannot determine named products: %v", err) + } + + rx := regexp.MustCompile(`^all_named_products='(.*)'$`) + match := rx.FindStringSubmatch(strings.TrimSpace(string(output))) + return strings.Fields(match[1]) +} + +// ensureEmptyFileExists ensures that the containing directory exists, and the +// specified file exists. If it doesn't exist, it will write an empty file. +func ensureEmptyFileExists(file string, log logger.Logger) { + if _, err := os.Stat(file); os.IsNotExist(err) { + f, err := os.Create(file) + if err != nil { + log.Fatalf("Error creating %s: %q\n", file, err) + } + f.Close() + } else if err != nil { + log.Fatalf("Error checking %s: %q\n", file, err) + } +} + func main() { stdio := terminal.StdioImpl{} - output := terminal.NewStatusOutput(stdio.Stdout(), "", false, - build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD")) - + output := terminal.NewStatusOutput(stdio.Stdout(), "", false, false) log := logger.New(output) defer log.Cleanup() @@ -198,63 +198,53 @@ func main() { stat.Finish() }) - buildCtx := build.Context{ContextImpl: &build.ContextImpl{ - Context: ctx, - Logger: log, - Tracer: trace, - Writer: output, - Status: stat, - }} - - args := "" - if *alternateResultDir { - args = "dist" - } - - originalOutDir := os.Getenv("OUT_DIR") - if originalOutDir == "" { - originalOutDir = "out" - } - soongUi := "build/soong/soong_ui.bash" - config := build.NewConfig(buildCtx, args) - if *outDir == "" { + var outputDir string + if *outDir != "" { + outputDir = *outDir + } else { name := "multiproduct" if !*incremental { name += "-" + time.Now().Format("20060102150405") } - *outDir = filepath.Join(originalOutDir, name) - - // Ensure the empty files exist in the output directory - // containing our output directory too. This is mostly for - // safety, but also triggers the ninja_build file so that our - // build servers know that they can parse the output as if it - // was ninja output. - build.SetupOutDir(buildCtx, config) - - if err := os.MkdirAll(*outDir, 0777); err != nil { - log.Fatalf("Failed to create tempdir: %v", err) + outDirBase := os.Getenv("OUT_DIR") + if outDirBase == "" { + outDirBase = "out" } - } - config.Environment().Set("OUT_DIR", *outDir) - log.Println("Output directory:", *outDir) - logsDir := filepath.Join(config.OutDir(), "logs") + outputDir = filepath.Join(outDirBase, name) + } + + log.Println("Output directory:", outputDir) + + // The ninja_build file is used by our buildbots to understand that the output + // can be parsed as ninja output. + if err := os.MkdirAll(outputDir, 0777); err != nil { + log.Fatalf("Failed to create output directory: %v", err) + } + ensureEmptyFileExists(filepath.Join(outputDir, "ninja_build"), log) + + logsDir := filepath.Join(outputDir, "logs") os.MkdirAll(logsDir, 0777) - build.SetupOutDir(buildCtx, config) + var configLogsDir string + if *alternateResultDir { + configLogsDir = filepath.Join(outputDir, "dist/logs") + } else { + configLogsDir = outputDir + } - os.MkdirAll(config.LogsDir(), 0777) - log.SetOutput(filepath.Join(config.LogsDir(), "soong.log")) - trace.SetOutput(filepath.Join(config.LogsDir(), "build.trace")) + os.MkdirAll(configLogsDir, 0777) + log.SetOutput(filepath.Join(configLogsDir, "soong.log")) + trace.SetOutput(filepath.Join(configLogsDir, "build.trace")) var jobs = *numJobs if jobs < 1 { jobs = runtime.NumCPU() / 4 - ramGb := int(config.TotalRAM() / 1024 / 1024 / 1024) + ramGb := int(detectTotalRAM() / (1024 * 1024 * 1024)) if ramJobs := ramGb / 25; ramGb > 0 && jobs > ramJobs { jobs = ramJobs } @@ -267,17 +257,8 @@ func main() { setMaxFiles(log) - finder := build.NewSourceFinder(buildCtx, config) - defer finder.Shutdown() - - build.FindSources(buildCtx, config, finder) - - vars, err := build.DumpMakeVars(buildCtx, config, nil, []string{"all_named_products"}) - if err != nil { - log.Fatal(err) - } + allProducts := findNamedProducts(soongUi, log) var productsList []string - allProducts := strings.Fields(vars["all_named_products"]) if len(includeProducts) > 0 { var missingProducts []string @@ -325,7 +306,7 @@ func main() { log.Verbose("Got product list: ", finalProductsList) - s := buildCtx.Status.StartTool() + s := stat.StartTool() s.SetTotalActions(len(finalProductsList)) mpCtx := &mpContext{ @@ -334,9 +315,6 @@ func main() { Status: s, Tracer: trace, - Finder: finder, - Config: config, - LogsDir: logsDir, } @@ -359,7 +337,7 @@ func main() { if product == "" { return } - runSoongUiForProduct(mpCtx, product, soongUi) + runSoongUiForProduct(mpCtx, product, soongUi, outputDir) } } }() @@ -371,7 +349,7 @@ func main() { FileArgs: []zip.FileArg{ {GlobDir: logsDir, SourcePrefixToStrip: logsDir}, }, - OutputFilePath: filepath.Join(config.RealDistDir(), "logs.zip"), + OutputFilePath: filepath.Join(outputDir, "dist/logs.zip"), NumParallelJobs: runtime.NumCPU(), CompressionLevel: 5, } @@ -413,10 +391,10 @@ func cleanupAfterProduct(outDir, productZip string) { } } -func runSoongUiForProduct(mpctx *mpContext, product, soongUi string) { - outDir := filepath.Join(mpctx.Config.OutDir(), product) +func runSoongUiForProduct(mpctx *mpContext, product, soongUi, mainOutDir string) { + outDir := filepath.Join(mainOutDir, product) logsDir := filepath.Join(mpctx.LogsDir, product) - productZip := filepath.Join(mpctx.Config.OutDir(), product+".zip") + productZip := filepath.Join(mainOutDir, product+".zip") consoleLogPath := filepath.Join(logsDir, "std.log") if err := os.MkdirAll(outDir, 0777); err != nil {