From bee030d36b9f7f9ed6b1a5915a391b578543af78 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 3 Jan 2024 13:45:48 -0800 Subject: [PATCH] Allow n2 as a replacement for ninja in builds If `SOONG_USE_N2=true` is set in the environment, then n2 is used in place of ninja. Some ninja features are not available in n2 at this time, but this enables bringup efforts to happen in parallel. Bug: 352368206 Test: manual Change-Id: I8455cb24eb640a4651782ee76e48a7d3a9932b93 --- ui/build/config.go | 11 +++++++++++ ui/build/ninja.go | 48 +++++++++++++++++++++++++++++++++++----------- ui/build/soong.go | 23 +++++++++++++++++++++- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index 52c5e233c..2470f843d 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -121,6 +121,10 @@ type configImpl struct { // There's quite a bit of overlap with module-info.json and soong module graph. We // could consider merging them. moduleDebugFile string + + // Whether to use n2 instead of ninja. This is controlled with the + // environment variable SOONG_USE_N2 + useN2 bool } type NinjaWeightListSource uint @@ -283,6 +287,10 @@ func NewConfig(ctx Context, args ...string) Config { ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json")) } + if os.Getenv("SOONG_USE_N2") == "true" { + ret.useN2 = true + } + ret.environ.Unset( // We're already using it "USE_SOONG_UI", @@ -339,6 +347,9 @@ func NewConfig(ctx Context, args ...string) Config { // We read it here already, don't let others share in the fun "GENERATE_SOONG_DEBUG", + + // Use config.useN2 instead. + "SOONG_USE_N2", ) if ret.UseGoma() || ret.ForceUseGoma() { diff --git a/ui/build/ninja.go b/ui/build/ninja.go index ae27330a9..1935e7210 100644 --- a/ui/build/ninja.go +++ b/ui/build/ninja.go @@ -56,6 +56,17 @@ func runNinjaForBuild(ctx Context, config Config) { "-d", "stats", "--frontend_file", fifo, } + if config.useN2 { + executable = config.PrebuiltBuildTool("n2") + args = []string{ + "-d", "trace", + // TODO: implement these features, or remove them. + //"-d", "keepdepfile", + //"-d", "keeprsp", + //"-d", "stats", + "--frontend-file", fifo, + } + } args = append(args, config.NinjaArgs()...) @@ -72,17 +83,21 @@ func runNinjaForBuild(ctx Context, config Config) { args = append(args, "-f", config.CombinedNinjaFile()) - args = append(args, - "-o", "usesphonyoutputs=yes", - "-w", "dupbuild=err", - "-w", "missingdepfile=err") + if !config.useN2 { + args = append(args, + "-o", "usesphonyoutputs=yes", + "-w", "dupbuild=err", + "-w", "missingdepfile=err") + } if !config.BuildBrokenMissingOutputs() { // Missing outputs will be treated as errors. // BUILD_BROKEN_MISSING_OUTPUTS can be used to bypass this check. - args = append(args, - "-w", "missingoutfile=err", - ) + if !config.useN2 { + args = append(args, + "-w", "missingoutfile=err", + ) + } } cmd := Command(ctx, config, "ninja", executable, args...) @@ -97,16 +112,22 @@ func runNinjaForBuild(ctx Context, config Config) { switch config.NinjaWeightListSource() { case NINJA_LOG: - cmd.Args = append(cmd.Args, "-o", "usesninjalogasweightlist=yes") + if !config.useN2 { + cmd.Args = append(cmd.Args, "-o", "usesninjalogasweightlist=yes") + } case EVENLY_DISTRIBUTED: // pass empty weight list means ninja considers every tasks's weight as 1(default value). - cmd.Args = append(cmd.Args, "-o", "usesweightlist=/dev/null") + if !config.useN2 { + cmd.Args = append(cmd.Args, "-o", "usesweightlist=/dev/null") + } case EXTERNAL_FILE: fallthrough case HINT_FROM_SOONG: // The weight list is already copied/generated. - ninjaWeightListPath := filepath.Join(config.OutDir(), ninjaWeightListFileName) - cmd.Args = append(cmd.Args, "-o", "usesweightlist="+ninjaWeightListPath) + if !config.useN2 { + ninjaWeightListPath := filepath.Join(config.OutDir(), ninjaWeightListFileName) + cmd.Args = append(cmd.Args, "-o", "usesweightlist="+ninjaWeightListPath) + } } // Allow both NINJA_ARGS and NINJA_EXTRA_ARGS, since both have been @@ -206,11 +227,16 @@ func runNinjaForBuild(ctx Context, config Config) { // We don't want this build broken flag to cause reanalysis, so allow it through to the // actions. "BUILD_BROKEN_INCORRECT_PARTITION_IMAGES", + "SOONG_USE_N2", + "RUST_BACKTRACE", }, config.BuildBrokenNinjaUsesEnvVars()...)...) } cmd.Environment.Set("DIST_DIR", config.DistDir()) cmd.Environment.Set("SHELL", "/bin/bash") + if config.useN2 { + cmd.Environment.Set("RUST_BACKTRACE", "1") + } // Print the environment variables that Ninja is operating in. ctx.Verboseln("Ninja environment: ") diff --git a/ui/build/soong.go b/ui/build/soong.go index 77fee0ac6..e18cc2579 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -638,6 +638,22 @@ func runSoong(ctx Context, config Config) { "--frontend_file", fifo, "-f", filepath.Join(config.SoongOutDir(), "bootstrap.ninja"), } + if config.useN2 { + ninjaArgs = []string{ + // TODO: implement these features, or remove them. + //"-d", "keepdepfile", + //"-d", "stats", + //"-o", "usesphonyoutputs=yes", + //"-o", "preremoveoutputs=yes", + //"-w", "dupbuild=err", + //"-w", "outputdir=err", + //"-w", "missingoutfile=err", + "-v", + "-j", strconv.Itoa(config.Parallel()), + "--frontend-file", fifo, + "-f", filepath.Join(config.SoongOutDir(), "bootstrap.ninja"), + } + } if extra, ok := config.Environment().Get("SOONG_UI_NINJA_ARGS"); ok { ctx.Printf(`CAUTION: arguments in $SOONG_UI_NINJA_ARGS=%q, e.g. "-n", can make soong_build FAIL or INCORRECT`, extra) @@ -645,8 +661,13 @@ func runSoong(ctx Context, config Config) { } ninjaArgs = append(ninjaArgs, targets...) + ninjaCmd := config.PrebuiltBuildTool("ninja") + if config.useN2 { + ninjaCmd = config.PrebuiltBuildTool("n2") + } + cmd := Command(ctx, config, "soong bootstrap", - config.PrebuiltBuildTool("ninja"), ninjaArgs...) + ninjaCmd, ninjaArgs...) var ninjaEnv Environment