StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make multiproduct_kati call soong_ui.bash .

Browse Source 
This serves to not link parts of soong_ui (and eventually soong_build)
into a separate, weird binary. This is in turn good because they contain
any number of global variables and no one really thought about what
happens when two instances are executing at the same time in the same
address space.

This comes with a slight performance hit: 5 aosp_* projects build 152
seconds instead of 146. I suppose this is a price worth paying for a
clean design?

Test: presubmits.
Change-Id: I5623dcab2290f0fc392dd2ede597b9794a3d2a4e
This commit is contained in:
Lukacs T. Berki
2021-08-10 15:01:13 +02:00
parent 01cad0cddc
commit cef87b62e4
3 changed files with 93 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
cmd/multiproduct_kati/main.go
View File

@@ -15,12 +15,15 @@
package main
import (
"bufio"
"context"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
@@ -207,6 +210,14 @@ func main() {
if *alternateResultDir {
args = "dist"
}
originalOutDir := os.Getenv("OUT_DIR")
if originalOutDir == "" {
originalOutDir = "out"
}
soongUi := "build/soong/soong_ui.bash"
config := build.NewConfig(buildCtx, args)
if *outDir == "" {
name := "multiproduct"
@@ -214,7 +225,7 @@ func main() {
name += "-" + time.Now().Format("20060102150405")
}
*outDir = filepath.Join(config.OutDir(), name)
*outDir = filepath.Join(originalOutDir, name)
// Ensure the empty files exist in the output directory
// containing our output directory too. This is mostly for
@@ -348,7 +359,7 @@ func main() {
if product == "" {
return
}
buildProduct(mpCtx, product)
runSoongUiForProduct(mpCtx, product, soongUi)
}
}
}()
@@ -380,11 +391,33 @@ func main() {
}
}
func buildProduct(mpctx *mpContext, product string) {
var stdLog string
func cleanupAfterProduct(outDir, productZip string) {
if *keepArtifacts {
args := zip.ZipArgs{
FileArgs: []zip.FileArg{
{
GlobDir: outDir,
SourcePrefixToStrip: outDir,
},
},
OutputFilePath: productZip,
NumParallelJobs: runtime.NumCPU(),
CompressionLevel: 5,
}
if err := zip.Zip(args); err != nil {
log.Fatalf("Error zipping artifacts: %v", err)
}
}
if !*incremental {
os.RemoveAll(outDir)
}
}
func runSoongUiForProduct(mpctx *mpContext, product, soongUi string) {
outDir := filepath.Join(mpctx.Config.OutDir(), product)
logsDir := filepath.Join(mpctx.LogsDir, product)
productZip := filepath.Join(mpctx.Config.OutDir(), product+".zip")
consoleLogPath := filepath.Join(logsDir, "std.log")
if err := os.MkdirAll(outDir, 0777); err != nil {
mpctx.Logger.Fatalf("Error creating out directory: %v", err)
@@ -393,98 +426,65 @@ func buildProduct(mpctx *mpContext, product string) {
mpctx.Logger.Fatalf("Error creating log directory: %v", err)
}
stdLog = filepath.Join(logsDir, "std.log")
f, err := os.Create(stdLog)
consoleLogFile, err := os.Create(consoleLogPath)
if err != nil {
mpctx.Logger.Fatalf("Error creating std.log: %v", err)
mpctx.Logger.Fatalf("Error creating console log file: %v", err)
}
defer f.Close()
defer consoleLogFile.Close()
log := logger.New(f)
defer log.Cleanup()
log.SetOutput(filepath.Join(logsDir, "soong.log"))
consoleLogWriter := bufio.NewWriter(consoleLogFile)
defer consoleLogWriter.Flush()
args := []string{"--make-mode", "--skip-soong-tests", "--skip-ninja"}
if !*keepArtifacts {
args = append(args, "--empty-ninja-file")
}
if *onlyConfig {
args = append(args, "--config-only")
} else if *onlySoong {
args = append(args, "--soong-only")
}
if *alternateResultDir {
args = append(args, "dist")
}
cmd := exec.Command(soongUi, args...)
cmd.Stdout = consoleLogWriter
cmd.Stderr = consoleLogWriter
cmd.Env = append(os.Environ(),
"OUT_DIR="+outDir,
"TARGET_PRODUCT="+product,
"TARGET_BUILD_VARIANT="+*buildVariant,
"TARGET_BUILD_TYPE=release",
"TARGET_BUILD_APPS=",
"TARGET_BUILD_UNBUNDLED=")
action := &status.Action{
Description: product,
Outputs: []string{product},
}
mpctx.Status.StartAction(action)
defer logger.Recover(func(err error) {
mpctx.Status.FinishAction(status.ActionResult{
Action: action,
Error: err,
Output: errMsgFromLog(stdLog),
})
})
ctx := build.Context{ContextImpl: &build.ContextImpl{
Context: mpctx.Context,
Logger: log,
Tracer: mpctx.Tracer,
Writer: f,
Thread: mpctx.Tracer.NewThread(product),
Status: &status.Status{},
}}
ctx.Status.AddOutput(terminal.NewStatusOutput(ctx.Writer, "", false,
build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD")))
args := append([]string(nil), flag.Args()...)
args = append(args, "--skip-soong-tests")
config := build.NewConfig(ctx, args...)
config.Environment().Set("OUT_DIR", outDir)
if !*keepArtifacts {
config.SetEmptyNinjaFile(true)
}
build.FindSources(ctx, config, mpctx.Finder)
config.Lunch(ctx, product, *buildVariant)
defer func() {
if *keepArtifacts {
args := zip.ZipArgs{
FileArgs: []zip.FileArg{
{
GlobDir: outDir,
SourcePrefixToStrip: outDir,
},
},
OutputFilePath: filepath.Join(mpctx.Config.OutDir(), product+".zip"),
NumParallelJobs: runtime.NumCPU(),
CompressionLevel: 5,
}
if err := zip.Zip(args); err != nil {
log.Fatalf("Error zipping artifacts: %v", err)
}
}
if !*incremental {
os.RemoveAll(outDir)
}
}()
config.SetSkipNinja(true)
buildWhat := build.RunProductConfig
if !*onlyConfig {
buildWhat |= build.RunSoong
if !*onlySoong {
buildWhat |= build.RunKati
}
}
defer cleanupAfterProduct(outDir, productZip)
before := time.Now()
build.Build(ctx, config)
err = cmd.Run()
// Save std_full.log if Kati re-read the makefiles
if buildWhat&build.RunKati != 0 {
if after, err := os.Stat(config.KatiBuildNinjaFile()); err == nil && after.ModTime().After(before) {
err := copyFile(stdLog, filepath.Join(filepath.Dir(stdLog), "std_full.log"))
if !*onlyConfig && !*onlySoong {
katiBuildNinjaFile := filepath.Join(outDir, "build-"+product+".ninja")
if after, err := os.Stat(katiBuildNinjaFile); err == nil && after.ModTime().After(before) {
err := copyFile(consoleLogPath, filepath.Join(filepath.Dir(consoleLogPath), "std_full.log"))
if err != nil {
log.Fatalf("Error copying log file: %s", err)
}
}
}
mpctx.Status.FinishAction(status.ActionResult{
Action: action,
Error: err,
})
}

5
ui/build/build.go
View File

@@ -238,6 +238,11 @@ func Build(ctx Context, config Config) {
ctx.Verboseln("Skipping use of Kati ninja as requested")
what = what &^ RunKatiNinja
}
if config.SkipSoong() {
ctx.Verboseln("Skipping use of Soong as requested")
what = what &^ RunSoong
}
if config.SkipNinja() {
ctx.Verboseln("Skipping Ninja as requested")
what = what &^ RunNinja

59
ui/build/config.go
View File

@@ -49,6 +49,7 @@ type configImpl struct {
skipConfig bool
skipKati bool
skipKatiNinja bool
skipSoong bool
skipNinja bool
skipSoongTests bool
@@ -582,6 +583,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
arg := strings.TrimSpace(args[i])
if arg == "showcommands" {
c.verbose = true
} else if arg == "--empty-ninja-file" {
c.emptyNinjaFile = true
} else if arg == "--skip-ninja" {
c.skipNinja = true
} else if arg == "--skip-make" {
@@ -596,6 +599,10 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
} else if arg == "--soong-only" {
c.skipKati = true
c.skipKatiNinja = true
} else if arg == "--config-only" {
c.skipKati = true
c.skipKatiNinja = true
c.skipSoong = true
} else if arg == "--skip-config" {
c.skipConfig = true
} else if arg == "--skip-soong-tests" {
@@ -690,54 +697,6 @@ func (c *configImpl) configureLocale(ctx Context) {
}
}
// Lunch configures the environment for a specific product similarly to the
// `lunch` bash function.
func (c *configImpl) Lunch(ctx Context, product, variant string) {
if variant != "eng" && variant != "userdebug" && variant != "user" {
ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)
}
c.environ.Set("TARGET_PRODUCT", product)
c.environ.Set("TARGET_BUILD_VARIANT", variant)
c.environ.Set("TARGET_BUILD_TYPE", "release")
c.environ.Unset("TARGET_BUILD_APPS")
c.environ.Unset("TARGET_BUILD_UNBUNDLED")
}
// Tapas configures the environment to build one or more unbundled apps,
// similarly to the `tapas` bash function.
func (c *configImpl) Tapas(ctx Context, apps []string, arch, variant string) {
if len(apps) == 0 {
apps = []string{"all"}
}
if variant == "" {
variant = "eng"
}
if variant != "eng" && variant != "userdebug" && variant != "user" {
ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)
}
var product string
switch arch {
case "arm", "":
product = "aosp_arm"
case "arm64":
product = "aosm_arm64"
case "x86":
product = "aosp_x86"
case "x86_64":
product = "aosp_x86_64"
default:
ctx.Fatalf("Invalid architecture: %q", arch)
}
c.environ.Set("TARGET_PRODUCT", product)
c.environ.Set("TARGET_BUILD_VARIANT", variant)
c.environ.Set("TARGET_BUILD_TYPE", "release")
c.environ.Set("TARGET_BUILD_APPS", strings.Join(apps, " "))
}
func (c *configImpl) Environment() *Environment {
return c.environ
}
@@ -817,6 +776,10 @@ func (c *configImpl) SkipKatiNinja() bool {
return c.skipKatiNinja
}
func (c *configImpl) SkipSoong() bool {
return c.skipSoong
}
func (c *configImpl) SkipNinja() bool {
return c.skipNinja
}