diff --git a/README.md b/README.md index d8dd26a05..a67c393aa 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,6 @@ The list of valid module types and their properties can be generated by calling `m soong_docs`. It will be written to `$OUT_DIR/soong/docs/soong_build.html`. This list for the current version of Soong can be found [here](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html). - ### File lists Properties that take a list of files can also take glob patterns and output path diff --git a/android/config.go b/android/config.go index 1587f103c..af1e6c8fb 100644 --- a/android/config.go +++ b/android/config.go @@ -418,8 +418,8 @@ func TestArchConfig(buildDir string, env map[string]string, bp string, fs map[st // bootstrap run. Only per-run data is reset. Data which needs to persist across // multiple runs in the same program execution is carried over (such as Bazel // context or environment deps). -func ConfigForAdditionalRun(cmdlineArgs bootstrap.Args, c Config) (Config, error) { - newConfig, err := NewConfig(cmdlineArgs, c.soongOutDir, c.env) +func ConfigForAdditionalRun(c Config) (Config, error) { + newConfig, err := NewConfig(c.moduleListFile, c.runGoTests, c.outDir, c.soongOutDir, c.env) if err != nil { return Config{}, err } @@ -430,20 +430,20 @@ func ConfigForAdditionalRun(cmdlineArgs bootstrap.Args, c Config) (Config, error // NewConfig creates a new Config object. The srcDir argument specifies the path // to the root source directory. It also loads the config file, if found. -func NewConfig(cmdlineArgs bootstrap.Args, soongOutDir string, availableEnv map[string]string) (Config, error) { +func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir string, availableEnv map[string]string) (Config, error) { // Make a config with default options. config := &config{ ProductVariablesFileName: filepath.Join(soongOutDir, productVariablesFileName), env: availableEnv, - outDir: cmdlineArgs.OutDir, + outDir: outDir, soongOutDir: soongOutDir, - runGoTests: cmdlineArgs.RunGoTests, - useValidationsForGoTests: cmdlineArgs.UseValidations, + runGoTests: runGoTests, + useValidationsForGoTests: runGoTests, multilibConflicts: make(map[ArchType]bool), - moduleListFile: cmdlineArgs.ModuleListFile, + moduleListFile: moduleListFile, fs: pathtools.NewOsFs(absSrcDir), } diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 399efda3a..09a223473 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -35,10 +35,13 @@ import ( var ( topDir string + outDir string soongOutDir string availableEnvFile string usedEnvFile string + runGoTests bool + globFile string globListDir string delveListen string @@ -60,7 +63,7 @@ func init() { flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables") flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output") flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files") - flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory") + flag.StringVar(&outDir, "out", "", "the ninja builddir directory") flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse") // Debug flags @@ -81,8 +84,7 @@ func init() { // Flags that probably shouldn't be flags of soong_build but we haven't found // the time to remove them yet - flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap") - flag.BoolVar(&cmdlineArgs.UseValidations, "use-validations", false, "use validations to depend on go tests") + flag.BoolVar(&runGoTests, "t", false, "build and run go tests during bootstrap") } func newNameResolver(config android.Config) *android.NameResolver { @@ -109,8 +111,8 @@ func newContext(configuration android.Config) *android.Context { return ctx } -func newConfig(cmdlineArgs bootstrap.Args, outDir string, availableEnv map[string]string) android.Config { - configuration, err := android.NewConfig(cmdlineArgs, outDir, availableEnv) +func newConfig(availableEnv map[string]string) android.Config { + configuration, err := android.NewConfig(cmdlineArgs.ModuleListFile, runGoTests, outDir, soongOutDir, availableEnv) if err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) @@ -124,10 +126,7 @@ func newConfig(cmdlineArgs bootstrap.Args, outDir string, availableEnv map[strin // TODO(cparsons): Don't output any ninja file, as the second pass will overwrite // the incorrect results from the first pass, and file I/O is expensive. func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) { - var firstArgs, secondArgs bootstrap.Args - - firstArgs = cmdlineArgs - bootstrap.RunBlueprint(firstArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration) + bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration) // Invoke bazel commands and save results for second pass. if err := configuration.BazelContext.InvokeBazel(); err != nil { @@ -135,20 +134,19 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, os.Exit(1) } // Second pass: Full analysis, using the bazel command results. Output ninja file. - secondArgs = cmdlineArgs - secondConfig, err := android.ConfigForAdditionalRun(secondArgs, configuration) + secondConfig, err := android.ConfigForAdditionalRun(configuration) if err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) } secondCtx := newContext(secondConfig) - ninjaDeps := bootstrap.RunBlueprint(secondArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig) + ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig) ninjaDeps = append(ninjaDeps, extraNinjaDeps...) globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration) ninjaDeps = append(ninjaDeps, globListFiles...) - writeDepFile(secondArgs.OutFile, ninjaDeps) + writeDepFile(cmdlineArgs.OutFile, ninjaDeps) } // Run the code-generation phase to convert BazelTargetModules to BUILD files. @@ -306,7 +304,7 @@ func main() { availableEnv := parseAvailableEnv() - configuration := newConfig(cmdlineArgs, soongOutDir, availableEnv) + configuration := newConfig(availableEnv) extraNinjaDeps := []string{ configuration.ProductVariablesFileName, usedEnvFile, @@ -500,8 +498,8 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { "bazel-" + filepath.Base(topDir), } - if cmdlineArgs.OutDir[0] != '/' { - excludes = append(excludes, cmdlineArgs.OutDir) + if outDir[0] != '/' { + excludes = append(excludes, outDir) } existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir) diff --git a/ui/build/config.go b/ui/build/config.go index d5d03c30e..126a8d45a 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -111,9 +111,6 @@ const ( // Don't use bazel at all. noBazel bazelBuildMode = iota - // Only generate build files (in a subdirectory of the out directory) and exit. - generateBuildFiles - // Generate synthetic build files and incorporate these files into a build which // partially uses Bazel. Build metadata may come from Android.bp or BUILD files. mixedBuild @@ -783,6 +780,10 @@ func (c *configImpl) HostToolDir() string { return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin") } +func (c *configImpl) NamedGlobFile(name string) string { + return shared.JoinPath(c.SoongOutDir(), ".bootstrap/build-globs."+name+".ninja") +} + func (c *configImpl) MainNinjaFile() string { return shared.JoinPath(c.SoongOutDir(), "build.ninja") } diff --git a/ui/build/soong.go b/ui/build/soong.go index ed3af1843..9aa481006 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -37,6 +37,12 @@ import ( const ( availableEnvFile = "soong.environment.available" usedEnvFile = "soong.environment.used" + + soongBuildTag = "build" + bp2buildTag = "bp2build" + jsonModuleGraphTag = "modulegraph" + queryviewTag = "queryview" + soongDocsTag = "soong_docs" ) func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error { @@ -116,7 +122,7 @@ func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderI func environmentArgs(config Config, suffix string) []string { return []string{ "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile), - "--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix), + "--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+"."+suffix), } } @@ -134,156 +140,135 @@ func writeEmptyGlobFile(ctx Context, path string) { } } +func primaryBuilderInvocation(config Config, name string, output string, specificArgs []string) bootstrap.PrimaryBuilderInvocation { + commonArgs := make([]string, 0, 0) + + if !config.skipSoongTests { + commonArgs = append(commonArgs, "-t") + } + + commonArgs = append(commonArgs, "-l", filepath.Join(config.FileListDir(), "Android.bp.list")) + + if os.Getenv("SOONG_DELVE") != "" { + commonArgs = append(commonArgs, "--delve_listen", os.Getenv("SOONG_DELVE")) + commonArgs = append(commonArgs, "--delve_path", shared.ResolveDelveBinary()) + } + + allArgs := make([]string, 0, 0) + allArgs = append(allArgs, specificArgs...) + allArgs = append(allArgs, + "--globListDir", name, + "--globFile", config.NamedGlobFile(name)) + + allArgs = append(allArgs, commonArgs...) + allArgs = append(allArgs, environmentArgs(config, name)...) + allArgs = append(allArgs, "Android.bp") + + return bootstrap.PrimaryBuilderInvocation{ + Inputs: []string{"Android.bp"}, + Outputs: []string{output}, + Args: allArgs, + } +} + func bootstrapBlueprint(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap") defer ctx.EndTrace() - var args bootstrap.Args - - bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja") - bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja") - queryviewGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.queryview.ninja") - soongDocsGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.soong_docs.ninja") - moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja") - - // The glob .ninja files are subninja'd. However, they are generated during - // the build itself so we write an empty file so that the subninja doesn't - // fail on clean builds - writeEmptyGlobFile(ctx, bootstrapGlobFile) - writeEmptyGlobFile(ctx, bp2buildGlobFile) - writeEmptyGlobFile(ctx, queryviewGlobFile) - writeEmptyGlobFile(ctx, soongDocsGlobFile) - writeEmptyGlobFile(ctx, moduleGraphGlobFile) - - bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d") - - args.RunGoTests = !config.skipSoongTests - args.UseValidations = true // Use validations to depend on tests - args.SoongOutDir = config.SoongOutDir() - args.OutDir = config.OutDir() - args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list") - args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja") - // The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d) - // Building soong_build does not require a glob file - // Using "" instead of ".ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary - args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile, moduleGraphGlobFile, queryviewGlobFile, soongDocsGlobFile} - args.EmptyNinjaFile = config.EmptyNinjaFile() - - args.DelveListen = os.Getenv("SOONG_DELVE") - if args.DelveListen != "" { - args.DelvePath = shared.ResolveDelveBinary() + mainSoongBuildExtraArgs := []string{"-o", config.MainNinjaFile()} + if config.EmptyNinjaFile() { + mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file") } - commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, config.MainNinjaFile()) - mainSoongBuildInputs := []string{"Android.bp"} + mainSoongBuildInvocation := primaryBuilderInvocation( + config, + soongBuildTag, + config.MainNinjaFile(), + mainSoongBuildExtraArgs) if config.bazelBuildMode() == mixedBuild { - mainSoongBuildInputs = append(mainSoongBuildInputs, config.Bp2BuildMarkerFile()) + // Mixed builds call Bazel from soong_build and they therefore need the + // Bazel workspace to be available. Make that so by adding a dependency on + // the bp2build marker file to the action that invokes soong_build . + mainSoongBuildInvocation.Inputs = append(mainSoongBuildInvocation.Inputs, + config.Bp2BuildMarkerFile()) } - soongBuildArgs := []string{ - "--globListDir", "build", - "--globFile", bootstrapGlobFile, + bp2buildInvocation := primaryBuilderInvocation( + config, + bp2buildTag, + config.Bp2BuildMarkerFile(), + []string{ + "--bp2build_marker", config.Bp2BuildMarkerFile(), + }) + + jsonModuleGraphInvocation := primaryBuilderInvocation( + config, + jsonModuleGraphTag, + config.ModuleGraphFile(), + []string{ + "--module_graph_file", config.ModuleGraphFile(), + }) + + queryviewInvocation := primaryBuilderInvocation( + config, + queryviewTag, + config.QueryviewMarkerFile(), + []string{ + "--bazel_queryview_dir", filepath.Join(config.SoongOutDir(), "queryview"), + }) + + soongDocsInvocation := primaryBuilderInvocation( + config, + soongDocsTag, + config.SoongDocsHtml(), + []string{ + "--soong_docs", config.SoongDocsHtml(), + }) + + globFiles := []string{ + config.NamedGlobFile(soongBuildTag), + config.NamedGlobFile(bp2buildTag), + config.NamedGlobFile(jsonModuleGraphTag), + config.NamedGlobFile(queryviewTag), + config.NamedGlobFile(soongDocsTag), } - soongBuildArgs = append(soongBuildArgs, commonArgs...) - soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...) - soongBuildArgs = append(soongBuildArgs, "Android.bp") - - mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{ - Inputs: mainSoongBuildInputs, - Outputs: []string{config.MainNinjaFile()}, - Args: soongBuildArgs, + // The glob .ninja files are subninja'd. However, they are generated during + // the build itself so we write an empty file if the file does not exist yet + // so that the subninja doesn't fail on clean builds + for _, globFile := range globFiles { + writeEmptyGlobFile(ctx, globFile) } - bp2buildArgs := []string{ - "--bp2build_marker", config.Bp2BuildMarkerFile(), - "--globListDir", "bp2build", - "--globFile", bp2buildGlobFile, - } + var blueprintArgs bootstrap.Args - bp2buildArgs = append(bp2buildArgs, commonArgs...) - bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...) - bp2buildArgs = append(bp2buildArgs, "Android.bp") - - bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{ - Inputs: []string{"Android.bp"}, - Outputs: []string{config.Bp2BuildMarkerFile()}, - Args: bp2buildArgs, - } - - queryviewArgs := []string{ - "--bazel_queryview_dir", filepath.Join(config.SoongOutDir(), "queryview"), - "--globListDir", "queryview", - "--globFile", queryviewGlobFile, - } - - queryviewArgs = append(queryviewArgs, commonArgs...) - queryviewArgs = append(queryviewArgs, environmentArgs(config, ".queryview")...) - queryviewArgs = append(queryviewArgs, "Android.bp") - - queryviewInvocation := bootstrap.PrimaryBuilderInvocation{ - Inputs: []string{"Android.bp"}, - Outputs: []string{config.QueryviewMarkerFile()}, - Args: queryviewArgs, - } - - soongDocsArgs := []string{ - "--soong_docs", config.SoongDocsHtml(), - "--globListDir", "soong_docs", - "--globFile", soongDocsGlobFile, - } - - soongDocsArgs = append(soongDocsArgs, commonArgs...) - soongDocsArgs = append(soongDocsArgs, environmentArgs(config, ".soong_docs")...) - soongDocsArgs = append(soongDocsArgs, "Android.bp") - - soongDocsInvocation := bootstrap.PrimaryBuilderInvocation{ - Inputs: []string{"Android.bp"}, - Outputs: []string{config.SoongDocsHtml()}, - Args: soongDocsArgs, - } - - moduleGraphArgs := []string{ - "--module_graph_file", config.ModuleGraphFile(), - "--globListDir", "modulegraph", - "--globFile", moduleGraphGlobFile, - } - - moduleGraphArgs = append(moduleGraphArgs, commonArgs...) - moduleGraphArgs = append(moduleGraphArgs, environmentArgs(config, ".modulegraph")...) - moduleGraphArgs = append(moduleGraphArgs, "Android.bp") - - moduleGraphInvocation := bootstrap.PrimaryBuilderInvocation{ - Inputs: []string{"Android.bp"}, - Outputs: []string{config.ModuleGraphFile()}, - Args: moduleGraphArgs, - } - - args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{ - bp2buildInvocation, - mainSoongBuildInvocation, - moduleGraphInvocation, - queryviewInvocation, - soongDocsInvocation, - } + blueprintArgs.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list") + blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja") + blueprintArgs.EmptyNinjaFile = false blueprintCtx := blueprint.NewContext() blueprintCtx.SetIgnoreUnknownModuleTypes(true) blueprintConfig := BlueprintConfig{ - soongOutDir: config.SoongOutDir(), - toolDir: config.HostToolDir(), - outDir: config.OutDir(), - runGoTests: !config.skipSoongTests, - useValidations: true, - debugCompilation: os.Getenv("SOONG_DELVE") != "", - subninjas: args.Subninjas, - primaryBuilderInvocations: args.PrimaryBuilderInvocations, + soongOutDir: config.SoongOutDir(), + toolDir: config.HostToolDir(), + outDir: config.OutDir(), + runGoTests: !config.skipSoongTests, + useValidations: !config.skipSoongTests, + // If we want to debug soong_build, we need to compile it for debugging + debugCompilation: os.Getenv("SOONG_DELVE") != "", + subninjas: globFiles, + primaryBuilderInvocations: []bootstrap.PrimaryBuilderInvocation{ + mainSoongBuildInvocation, + bp2buildInvocation, + jsonModuleGraphInvocation, + queryviewInvocation, + soongDocsInvocation}, } - args.EmptyNinjaFile = false - bootstrapDeps := bootstrap.RunBlueprint(args, bootstrap.DoEverything, blueprintCtx, blueprintConfig) - err := deptools.WriteDepFile(bootstrapDepFile, args.OutFile, bootstrapDeps) + bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig) + bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d") + err := deptools.WriteDepFile(bootstrapDepFile, blueprintArgs.OutFile, bootstrapDeps) if err != nil { ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err) } @@ -343,12 +328,22 @@ func runSoong(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSoong, "environment check") defer ctx.EndTrace() - soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile) - checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile) + checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".build")) - if integratedBp2Build { - bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build") - checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile) + if integratedBp2Build || config.Bp2Build() { + checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build")) + } + + if config.JsonModuleGraph() { + checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".modulegraph")) + } + + if config.Queryview() { + checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".queryview")) + } + + if config.SoongDocs() { + checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".soong_docs")) } }()