dd9725c177
This CL adds generateJsonModuleGraph bazelBuildMode and ensures that it returns Soong early, before Kati and Soong metrics collection begin (which causes errors, since they're looking for files that Soong in json dump mode did not write). Test: TH Test: rm -rf out && SOONG_DUMP_JSON_MODULE_GRAPH=/tmp/soong.json m nothing Change-Id: I264eadb3b3b0cd6b6e7a65adc7b39bb1c01ca136
364 lines
12 KiB
Go
364 lines
12 KiB
Go
// Copyright 2017 Google Inc. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package build
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
|
|
"android/soong/shared"
|
|
"github.com/google/blueprint/deptools"
|
|
|
|
soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
|
|
"github.com/google/blueprint"
|
|
"github.com/google/blueprint/bootstrap"
|
|
|
|
"github.com/golang/protobuf/proto"
|
|
"github.com/google/blueprint/microfactory"
|
|
|
|
"android/soong/ui/metrics"
|
|
"android/soong/ui/status"
|
|
)
|
|
|
|
const (
|
|
availableEnvFile = "soong.environment.available"
|
|
usedEnvFile = "soong.environment.used"
|
|
)
|
|
|
|
func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
|
|
data, err := shared.EnvFileContents(envDeps)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return ioutil.WriteFile(envFile, data, 0644)
|
|
}
|
|
|
|
// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
|
|
//
|
|
// However, the execution of <builddir>/build.ninja happens later in
|
|
// build/soong/ui/build/build.go#Build()
|
|
//
|
|
// We want to rely on as few prebuilts as possible, so we need to bootstrap
|
|
// Soong. The process is as follows:
|
|
//
|
|
// 1. We use "Microfactory", a simple tool to compile Go code, to build
|
|
// first itself, then soong_ui from soong_ui.bash. This binary contains
|
|
// parts of soong_build that are needed to build itself.
|
|
// 2. This simplified version of soong_build then reads the Blueprint files
|
|
// that describe itself and emits .bootstrap/build.ninja that describes
|
|
// how to build its full version and use that to produce the final Ninja
|
|
// file Soong emits.
|
|
// 3. soong_ui executes .bootstrap/build.ninja
|
|
//
|
|
// (After this, Kati is executed to parse the Makefiles, but that's not part of
|
|
// bootstrapping Soong)
|
|
|
|
// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
|
|
// probably be nicer to use a flag in bootstrap.Args instead.
|
|
type BlueprintConfig struct {
|
|
srcDir string
|
|
buildDir string
|
|
ninjaBuildDir string
|
|
debugCompilation bool
|
|
}
|
|
|
|
func (c BlueprintConfig) SrcDir() string {
|
|
return "."
|
|
}
|
|
|
|
func (c BlueprintConfig) BuildDir() string {
|
|
return c.buildDir
|
|
}
|
|
|
|
func (c BlueprintConfig) NinjaBuildDir() string {
|
|
return c.ninjaBuildDir
|
|
}
|
|
|
|
func (c BlueprintConfig) DebugCompilation() bool {
|
|
return c.debugCompilation
|
|
}
|
|
|
|
func environmentArgs(config Config, suffix string) []string {
|
|
return []string{
|
|
"--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
|
|
"--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix),
|
|
}
|
|
}
|
|
|
|
func writeEmptyGlobFile(ctx Context, path string) {
|
|
err := os.MkdirAll(filepath.Dir(path), 0777)
|
|
if err != nil {
|
|
ctx.Fatalf("Failed to create parent directories of empty ninja glob file '%s': %s", path, err)
|
|
}
|
|
|
|
if _, err := os.Stat(path); os.IsNotExist(err) {
|
|
err = ioutil.WriteFile(path, nil, 0666)
|
|
if err != nil {
|
|
ctx.Fatalf("Failed to create empty ninja glob file '%s': %s", path, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
|
|
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
|
|
defer ctx.EndTrace()
|
|
|
|
var args bootstrap.Args
|
|
|
|
mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
|
|
bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
|
|
// .bootstrap/build.ninja "includes" .bootstrap/build-globs.ninja for incremental builds
|
|
// generate an empty glob before running any rule in .bootstrap/build.ninja
|
|
writeEmptyGlobFile(ctx, bootstrapGlobFile)
|
|
bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
|
|
|
|
args.RunGoTests = !config.skipSoongTests
|
|
args.UseValidations = true // Use validations to depend on tests
|
|
args.BuildDir = config.SoongOutDir()
|
|
args.NinjaBuildDir = config.OutDir()
|
|
args.TopFile = "Android.bp"
|
|
args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
|
|
args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
|
|
// The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d)
|
|
// Building soong_build does not require a glob file
|
|
// Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary
|
|
args.GlobFile = ""
|
|
args.GeneratingPrimaryBuilder = true
|
|
args.EmptyNinjaFile = config.EmptyNinjaFile()
|
|
|
|
args.DelveListen = os.Getenv("SOONG_DELVE")
|
|
if args.DelveListen != "" {
|
|
args.DelvePath = shared.ResolveDelveBinary()
|
|
}
|
|
|
|
commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
|
|
bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
|
|
mainSoongBuildInputs := []string{"Android.bp"}
|
|
|
|
if integratedBp2Build {
|
|
mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
|
|
}
|
|
|
|
soongBuildArgs := make([]string, 0)
|
|
soongBuildArgs = append(soongBuildArgs, commonArgs...)
|
|
soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
|
|
soongBuildArgs = append(soongBuildArgs, "Android.bp")
|
|
|
|
mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
|
|
Inputs: mainSoongBuildInputs,
|
|
Outputs: []string{mainNinjaFile},
|
|
Args: soongBuildArgs,
|
|
}
|
|
|
|
if integratedBp2Build {
|
|
bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
|
|
bp2buildArgs = append(bp2buildArgs, commonArgs...)
|
|
bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
|
|
bp2buildArgs = append(bp2buildArgs, "Android.bp")
|
|
|
|
bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
|
|
Inputs: []string{"Android.bp"},
|
|
Outputs: []string{bp2BuildMarkerFile},
|
|
Args: bp2buildArgs,
|
|
}
|
|
args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{bp2buildInvocation}
|
|
if config.bazelBuildMode() == mixedBuild {
|
|
args.PrimaryBuilderInvocations = append(args.PrimaryBuilderInvocations, mainSoongBuildInvocation)
|
|
}
|
|
} else {
|
|
args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
|
|
}
|
|
|
|
blueprintCtx := blueprint.NewContext()
|
|
blueprintCtx.SetIgnoreUnknownModuleTypes(true)
|
|
blueprintConfig := BlueprintConfig{
|
|
srcDir: os.Getenv("TOP"),
|
|
buildDir: config.SoongOutDir(),
|
|
ninjaBuildDir: config.OutDir(),
|
|
debugCompilation: os.Getenv("SOONG_DELVE") != "",
|
|
}
|
|
|
|
bootstrapDeps := bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
|
|
err := deptools.WriteDepFile(bootstrapDepFile, args.OutFile, bootstrapDeps)
|
|
if err != nil {
|
|
ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err)
|
|
}
|
|
}
|
|
|
|
func checkEnvironmentFile(currentEnv *Environment, envFile string) {
|
|
getenv := func(k string) string {
|
|
v, _ := currentEnv.Get(k)
|
|
return v
|
|
}
|
|
if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
|
|
os.Remove(envFile)
|
|
}
|
|
}
|
|
|
|
func runSoong(ctx Context, config Config) {
|
|
ctx.BeginTrace(metrics.RunSoong, "soong")
|
|
defer ctx.EndTrace()
|
|
|
|
// We have two environment files: .available is the one with every variable,
|
|
// .used with the ones that were actually used. The latter is used to
|
|
// determine whether Soong needs to be re-run since why re-run it if only
|
|
// unused variables were changed?
|
|
envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
|
|
|
|
for _, n := range []string{".bootstrap", ".minibootstrap"} {
|
|
dir := filepath.Join(config.SoongOutDir(), n)
|
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
|
ctx.Fatalf("Cannot mkdir " + dir)
|
|
}
|
|
}
|
|
|
|
buildMode := config.bazelBuildMode()
|
|
integratedBp2Build := (buildMode == mixedBuild) || (buildMode == generateBuildFiles)
|
|
|
|
// This is done unconditionally, but does not take a measurable amount of time
|
|
bootstrapBlueprint(ctx, config, integratedBp2Build)
|
|
|
|
soongBuildEnv := config.Environment().Copy()
|
|
soongBuildEnv.Set("TOP", os.Getenv("TOP"))
|
|
// For Bazel mixed builds.
|
|
soongBuildEnv.Set("BAZEL_PATH", "./tools/bazel")
|
|
soongBuildEnv.Set("BAZEL_HOME", filepath.Join(config.BazelOutDir(), "bazelhome"))
|
|
soongBuildEnv.Set("BAZEL_OUTPUT_BASE", filepath.Join(config.BazelOutDir(), "output"))
|
|
soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
|
|
soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
|
|
|
|
// For Soong bootstrapping tests
|
|
if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
|
|
soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
|
|
}
|
|
|
|
err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
|
|
if err != nil {
|
|
ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
|
|
}
|
|
|
|
func() {
|
|
ctx.BeginTrace(metrics.RunSoong, "environment check")
|
|
defer ctx.EndTrace()
|
|
|
|
soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile)
|
|
checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile)
|
|
|
|
if integratedBp2Build {
|
|
bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build")
|
|
checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile)
|
|
}
|
|
}()
|
|
|
|
var cfg microfactory.Config
|
|
cfg.Map("github.com/google/blueprint", "build/blueprint")
|
|
|
|
cfg.TrimPath = absPath(ctx, ".")
|
|
|
|
func() {
|
|
ctx.BeginTrace(metrics.RunSoong, "bpglob")
|
|
defer ctx.EndTrace()
|
|
|
|
bpglob := filepath.Join(config.SoongOutDir(), ".minibootstrap/bpglob")
|
|
if _, err := microfactory.Build(&cfg, bpglob, "github.com/google/blueprint/bootstrap/bpglob"); err != nil {
|
|
ctx.Fatalln("Failed to build bpglob:", err)
|
|
}
|
|
}()
|
|
|
|
ninja := func(name, file string) {
|
|
ctx.BeginTrace(metrics.RunSoong, name)
|
|
defer ctx.EndTrace()
|
|
|
|
fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
|
|
nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
|
|
defer nr.Close()
|
|
|
|
cmd := Command(ctx, config, "soong "+name,
|
|
config.PrebuiltBuildTool("ninja"),
|
|
"-d", "keepdepfile",
|
|
"-d", "stats",
|
|
"-o", "usesphonyoutputs=yes",
|
|
"-o", "preremoveoutputs=yes",
|
|
"-w", "dupbuild=err",
|
|
"-w", "outputdir=err",
|
|
"-w", "missingoutfile=err",
|
|
"-j", strconv.Itoa(config.Parallel()),
|
|
"--frontend_file", fifo,
|
|
"-f", filepath.Join(config.SoongOutDir(), file))
|
|
|
|
var ninjaEnv Environment
|
|
|
|
// This is currently how the command line to invoke soong_build finds the
|
|
// root of the source tree and the output root
|
|
ninjaEnv.Set("TOP", os.Getenv("TOP"))
|
|
|
|
cmd.Environment = &ninjaEnv
|
|
cmd.Sandbox = soongSandbox
|
|
cmd.RunAndStreamOrFatal()
|
|
}
|
|
// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
|
|
ninja("bootstrap", ".bootstrap/build.ninja")
|
|
|
|
var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
|
|
if shouldCollectBuildSoongMetrics(config) {
|
|
soongBuildMetrics := loadSoongBuildMetrics(ctx, config)
|
|
logSoongBuildMetrics(ctx, soongBuildMetrics)
|
|
}
|
|
|
|
distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
|
|
|
|
if !config.SkipKati() {
|
|
distGzipFile(ctx, config, config.SoongAndroidMk(), "soong")
|
|
distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
|
|
}
|
|
|
|
if shouldCollectBuildSoongMetrics(config) && ctx.Metrics != nil {
|
|
ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
|
|
}
|
|
}
|
|
|
|
func shouldCollectBuildSoongMetrics(config Config) bool {
|
|
// Do not collect metrics protobuf if the soong_build binary ran as the
|
|
// bp2build converter or the JSON graph dump.
|
|
return config.bazelBuildMode() != generateBuildFiles && config.bazelBuildMode() != generateJsonModuleGraph
|
|
}
|
|
|
|
func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics {
|
|
soongBuildMetricsFile := filepath.Join(config.OutDir(), "soong", "soong_build_metrics.pb")
|
|
buf, err := ioutil.ReadFile(soongBuildMetricsFile)
|
|
if err != nil {
|
|
ctx.Fatalf("Failed to load %s: %s", soongBuildMetricsFile, err)
|
|
}
|
|
soongBuildMetrics := &soong_metrics_proto.SoongBuildMetrics{}
|
|
err = proto.Unmarshal(buf, soongBuildMetrics)
|
|
if err != nil {
|
|
ctx.Fatalf("Failed to unmarshal %s: %s", soongBuildMetricsFile, err)
|
|
}
|
|
return soongBuildMetrics
|
|
}
|
|
|
|
func logSoongBuildMetrics(ctx Context, metrics *soong_metrics_proto.SoongBuildMetrics) {
|
|
ctx.Verbosef("soong_build metrics:")
|
|
ctx.Verbosef(" modules: %v", metrics.GetModules())
|
|
ctx.Verbosef(" variants: %v", metrics.GetVariants())
|
|
ctx.Verbosef(" max heap size: %v MB", metrics.GetMaxHeapSize()/1e6)
|
|
ctx.Verbosef(" total allocation count: %v", metrics.GetTotalAllocCount())
|
|
ctx.Verbosef(" total allocation size: %v MB", metrics.GetTotalAllocSize()/1e6)
|
|
|
|
}
|