StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Embed minibp into soong_ui.

Browse Source 
This requires linking Blueprint into soong_ui. It lets us avoid the
complicated dance of Ninja files and shell scripts: now the information
as to how soong_build is built is passed directly to Blueprint using a
struct that contains all the information the command line arguments used
to contain.

The ability to run Blueprint from the command line is kept (for now).

Some variables in bootstrap/command.go needed public accessor functions
because soong_build reads them. This will be disentangled by moving the
flag parsing to soong_build.

The presence of the flag definitions in Blueprint means that soong_ui
now also accepts them. This is not a problem in practice because they
are ignored and because soong_ui itself is hidden behind a few layers of
shell scripts.

Test: Presubmits + the new bootstrap_test.sh .
Change-Id: I6dca478f356f56a8aee1e457d71439272351390b
This commit is contained in:
Lukacs T. Berki
2021-03-16 08:55:23 +01:00
parent 645610e791
commit d1e3f1ff08
10 changed files with 487 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
ui/build/soong.go
View File

@@ -23,6 +23,8 @@ import (
"android/soong/shared"
soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/golang/protobuf/proto"
"github.com/google/blueprint/microfactory"
@@ -42,17 +44,75 @@ func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string
// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
//
// However, the execution of <builddir>/build.ninja happens later in build/soong/ui/build/build.go#Build()
// However, the execution of <builddir>/build.ninja happens later in
// build/soong/ui/build/build.go#Build()
//
// We want to rely on as few prebuilts as possible, so there is some bootstrapping here.
// We want to rely on as few prebuilts as possible, so we need to bootstrap
// Soong. The process is as follows:
//
// "Microfactory" is a tool for compiling Go code. We use it to build two other tools:
// - minibp, used to generate build.ninja files. This is really build/blueprint/bootstrap/command.go#Main()
// - bpglob, used during incremental builds to identify files in a glob that have changed
//
// In reality, several build.ninja files are generated and/or used during the bootstrapping and build process.
// See build/blueprint/bootstrap/doc.go for more information.
// 1. We use "Microfactory", a simple tool to compile Go code, to build
// first itself, then soong_ui from soong_ui.bash. This binary contains
// parts of soong_build that are needed to build itself.
// 2. This simplified version of soong_build then reads the Blueprint files
// that describe itself and emits .bootstrap/build.ninja that describes
// how to build its full version and use that to produce the final Ninja
// file Soong emits.
// 3. soong_ui executes .bootstrap/build.ninja
//
// (After this, Kati is executed to parse the Makefiles, but that's not part of
// bootstrapping Soong)
// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
// probably be nicer to use a flag in bootstrap.Args instead.
type BlueprintConfig struct {
srcDir string
buildDir string
ninjaBuildDir string
}
func (c BlueprintConfig) GeneratingPrimaryBuilder() bool {
return true
}
func (c BlueprintConfig) SrcDir() string {
return "."
}
func (c BlueprintConfig) BuildDir() string {
return c.buildDir
}
func (c BlueprintConfig) NinjaBuildDir() string {
return c.ninjaBuildDir
}
func bootstrapBlueprint(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
var args bootstrap.Args
args.RunGoTests = !config.skipSoongTests
args.UseValidations = true // Use validations to depend on tests
args.BuildDir = config.SoongOutDir()
args.NinjaBuildDir = config.OutDir()
args.TopFile = "Android.bp"
args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
args.GlobFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
blueprintCtx := blueprint.NewContext()
blueprintCtx.SetIgnoreUnknownModuleTypes(true)
blueprintConfig := BlueprintConfig{
srcDir: os.Getenv("TOP"),
buildDir: config.SoongOutDir(),
ninjaBuildDir: config.OutDir(),
}
bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
}
func runSoong(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "soong")
defer ctx.EndTrace()
@@ -63,33 +123,15 @@ func runSoong(ctx Context, config Config) {
// unused variables were changed?
envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available")
// Use an anonymous inline function for tracing purposes (this pattern is used several times below).
func() {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
// Use validations to depend on tests.
args := []string{"-n"}
if !config.skipSoongTests {
// Run tests.
args = append(args, "-t")
for _, n := range []string{".bootstrap", ".minibootstrap"} {
dir := filepath.Join(config.SoongOutDir(), n)
if err := os.MkdirAll(dir, 0755); err != nil {
ctx.Fatalf("Cannot mkdir " + dir)
}
}
cmd := Command(ctx, config, "blueprint bootstrap", "build/blueprint/bootstrap.bash", args...)
cmd.Environment.Set("BLUEPRINTDIR", "./build/blueprint")
cmd.Environment.Set("BOOTSTRAP", "./build/blueprint/bootstrap.bash")
cmd.Environment.Set("BUILDDIR", config.SoongOutDir())
cmd.Environment.Set("GOROOT", "./"+filepath.Join("prebuilts/go", config.HostPrebuiltTag()))
cmd.Environment.Set("BLUEPRINT_LIST_FILE", filepath.Join(config.FileListDir(), "Android.bp.list"))
cmd.Environment.Set("NINJA_BUILDDIR", config.OutDir())
cmd.Environment.Set("SRCDIR", ".")
cmd.Environment.Set("TOPNAME", "Android.bp")
cmd.Sandbox = soongSandbox
cmd.RunAndPrintOrFatal()
}()
// This is done unconditionally, but does not take a measurable amount of time
bootstrapBlueprint(ctx, config)
soongBuildEnv := config.Environment().Copy()
soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@@ -105,6 +147,11 @@ func runSoong(ctx Context, config Config) {
soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
// For Soong bootstrapping tests
if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
}
err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
if err != nil {
ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
@@ -129,16 +176,6 @@ func runSoong(ctx Context, config Config) {
cfg.TrimPath = absPath(ctx, ".")
func() {
ctx.BeginTrace(metrics.RunSoong, "minibp")
defer ctx.EndTrace()
minibp := filepath.Join(config.SoongOutDir(), ".minibootstrap/minibp")
if _, err := microfactory.Build(&cfg, minibp, "github.com/google/blueprint/bootstrap/minibp"); err != nil {
ctx.Fatalln("Failed to build minibp:", err)
}
}()
func() {
ctx.BeginTrace(metrics.RunSoong, "bpglob")
defer ctx.EndTrace()
@@ -187,10 +224,6 @@ func runSoong(ctx Context, config Config) {
cmd.Sandbox = soongSandbox
cmd.RunAndStreamOrFatal()
}
// This build generates .bootstrap/build.ninja, which is used in the next step.
ninja("minibootstrap", ".minibootstrap/build.ninja")
// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
ninja("bootstrap", ".bootstrap/build.ninja")