Refactor the creation of soong_build calls.

They were a bit repetitive.

In addition, make the environment checks more correct; they could
probably use an integration test or two to make sure that when the
environment changes, exactly those outputs are rebuilt that need to be,
but for now, this is an improvement already.

Test: Presubmits.
Change-Id: Idd79b81ca6975d57d00e5bf4699d266152505ff8

README.md

@@ -37,7 +37,6 @@ The list of valid module types and their properties can be generated by calling
 `m soong_docs`. It will be written to `$OUT_DIR/soong/docs/soong_build.html`.
 This list for the current version of Soong can be found [here](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html).
 
-
 ### File lists
 
 Properties that take a list of files can also take glob patterns and output path

android/config.go

@@ -418,8 +418,8 @@ func TestArchConfig(buildDir string, env map[string]string, bp string, fs map[st
 // bootstrap run. Only per-run data is reset. Data which needs to persist across
 // multiple runs in the same program execution is carried over (such as Bazel
 // context or environment deps).
-func ConfigForAdditionalRun(cmdlineArgs bootstrap.Args, c Config) (Config, error) {
+func ConfigForAdditionalRun(c Config) (Config, error) {
-	newConfig, err := NewConfig(cmdlineArgs, c.soongOutDir, c.env)
+	newConfig, err := NewConfig(c.moduleListFile, c.runGoTests, c.outDir, c.soongOutDir, c.env)
 	if err != nil {
 		return Config{}, err
 	}
@@ -430,20 +430,20 @@ func ConfigForAdditionalRun(cmdlineArgs bootstrap.Args, c Config) (Config, error
 
 // NewConfig creates a new Config object. The srcDir argument specifies the path
 // to the root source directory. It also loads the config file, if found.
-func NewConfig(cmdlineArgs bootstrap.Args, soongOutDir string, availableEnv map[string]string) (Config, error) {
+func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir string, availableEnv map[string]string) (Config, error) {
 	// Make a config with default options.
 	config := &config{
 		ProductVariablesFileName: filepath.Join(soongOutDir, productVariablesFileName),
 
 		env: availableEnv,
 
-		outDir:                   cmdlineArgs.OutDir,
+		outDir:                   outDir,
 		soongOutDir:              soongOutDir,
-		runGoTests:               cmdlineArgs.RunGoTests,
+		runGoTests:               runGoTests,
-		useValidationsForGoTests: cmdlineArgs.UseValidations,
+		useValidationsForGoTests: runGoTests,
 		multilibConflicts:        make(map[ArchType]bool),
 
-		moduleListFile: cmdlineArgs.ModuleListFile,
+		moduleListFile: moduleListFile,
 		fs:             pathtools.NewOsFs(absSrcDir),
 	}
 

cmd/soong_build/main.go

@@ -35,10 +35,13 @@ import (
 
 var (
 	topDir           string
+	outDir           string
 	soongOutDir      string
 	availableEnvFile string
 	usedEnvFile      string
 
+	runGoTests bool
+
 	globFile    string
 	globListDir string
 	delveListen string
@@ -60,7 +63,7 @@ func init() {
 	flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
 	flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
 	flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
-	flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
+	flag.StringVar(&outDir, "out", "", "the ninja builddir directory")
 	flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
 
 	// Debug flags
@@ -81,8 +84,7 @@ func init() {
 
 	// Flags that probably shouldn't be flags of soong_build but we haven't found
 	// the time to remove them yet
-	flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")
+	flag.BoolVar(&runGoTests, "t", false, "build and run go tests during bootstrap")
-	flag.BoolVar(&cmdlineArgs.UseValidations, "use-validations", false, "use validations to depend on go tests")
 }
 
 func newNameResolver(config android.Config) *android.NameResolver {
@@ -109,8 +111,8 @@ func newContext(configuration android.Config) *android.Context {
 	return ctx
 }
 
-func newConfig(cmdlineArgs bootstrap.Args, outDir string, availableEnv map[string]string) android.Config {
+func newConfig(availableEnv map[string]string) android.Config {
-	configuration, err := android.NewConfig(cmdlineArgs, outDir, availableEnv)
+	configuration, err := android.NewConfig(cmdlineArgs.ModuleListFile, runGoTests, outDir, soongOutDir, availableEnv)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s", err)
 		os.Exit(1)
@@ -124,10 +126,7 @@ func newConfig(cmdlineArgs bootstrap.Args, outDir string, availableEnv map[strin
 // TODO(cparsons): Don't output any ninja file, as the second pass will overwrite
 // the incorrect results from the first pass, and file I/O is expensive.
 func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) {
-	var firstArgs, secondArgs bootstrap.Args
+	bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration)
-
-	firstArgs = cmdlineArgs
-	bootstrap.RunBlueprint(firstArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration)
 
 	// Invoke bazel commands and save results for second pass.
 	if err := configuration.BazelContext.InvokeBazel(); err != nil {
@@ -135,20 +134,19 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context,
 		os.Exit(1)
 	}
 	// Second pass: Full analysis, using the bazel command results. Output ninja file.
-	secondArgs = cmdlineArgs
+	secondConfig, err := android.ConfigForAdditionalRun(configuration)
-	secondConfig, err := android.ConfigForAdditionalRun(secondArgs, configuration)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s", err)
 		os.Exit(1)
 	}
 	secondCtx := newContext(secondConfig)
-	ninjaDeps := bootstrap.RunBlueprint(secondArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig)
+	ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig)
 	ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
 
 	globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration)
 	ninjaDeps = append(ninjaDeps, globListFiles...)
 
-	writeDepFile(secondArgs.OutFile, ninjaDeps)
+	writeDepFile(cmdlineArgs.OutFile, ninjaDeps)
 }
 
 // Run the code-generation phase to convert BazelTargetModules to BUILD files.
@@ -306,7 +304,7 @@ func main() {
 
 	availableEnv := parseAvailableEnv()
 
-	configuration := newConfig(cmdlineArgs, soongOutDir, availableEnv)
+	configuration := newConfig(availableEnv)
 	extraNinjaDeps := []string{
 		configuration.ProductVariablesFileName,
 		usedEnvFile,
@@ -500,8 +498,8 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
 		"bazel-" + filepath.Base(topDir),
 	}
 
-	if cmdlineArgs.OutDir[0] != '/' {
+	if outDir[0] != '/' {
-		excludes = append(excludes, cmdlineArgs.OutDir)
+		excludes = append(excludes, outDir)
 	}
 
 	existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir)

ui/build/config.go

@@ -111,9 +111,6 @@ const (
 	// Don't use bazel at all.
 	noBazel bazelBuildMode = iota
 
-	// Only generate build files (in a subdirectory of the out directory) and exit.
-	generateBuildFiles
-
 	// Generate synthetic build files and incorporate these files into a build which
 	// partially uses Bazel. Build metadata may come from Android.bp or BUILD files.
 	mixedBuild
@@ -783,6 +780,10 @@ func (c *configImpl) HostToolDir() string {
 	return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
 }
 
+func (c *configImpl) NamedGlobFile(name string) string {
+	return shared.JoinPath(c.SoongOutDir(), ".bootstrap/build-globs."+name+".ninja")
+}
+
 func (c *configImpl) MainNinjaFile() string {
 	return shared.JoinPath(c.SoongOutDir(), "build.ninja")
 }

ui/build/soong.go

@@ -37,6 +37,12 @@ import (
 const (
 	availableEnvFile = "soong.environment.available"
 	usedEnvFile      = "soong.environment.used"
+
+	soongBuildTag      = "build"
+	bp2buildTag        = "bp2build"
+	jsonModuleGraphTag = "modulegraph"
+	queryviewTag       = "queryview"
+	soongDocsTag       = "soong_docs"
 )
 
 func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
@@ -116,7 +122,7 @@ func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderI
 func environmentArgs(config Config, suffix string) []string {
 	return []string{
 		"--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
-		"--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix),
+		"--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+"."+suffix),
 	}
 }
 
@@ -134,156 +140,135 @@ func writeEmptyGlobFile(ctx Context, path string) {
 	}
 }
 
+func primaryBuilderInvocation(config Config, name string, output string, specificArgs []string) bootstrap.PrimaryBuilderInvocation {
+	commonArgs := make([]string, 0, 0)
+
+	if !config.skipSoongTests {
+		commonArgs = append(commonArgs, "-t")
+	}
+
+	commonArgs = append(commonArgs, "-l", filepath.Join(config.FileListDir(), "Android.bp.list"))
+
+	if os.Getenv("SOONG_DELVE") != "" {
+		commonArgs = append(commonArgs, "--delve_listen", os.Getenv("SOONG_DELVE"))
+		commonArgs = append(commonArgs, "--delve_path", shared.ResolveDelveBinary())
+	}
+
+	allArgs := make([]string, 0, 0)
+	allArgs = append(allArgs, specificArgs...)
+	allArgs = append(allArgs,
+		"--globListDir", name,
+		"--globFile", config.NamedGlobFile(name))
+
+	allArgs = append(allArgs, commonArgs...)
+	allArgs = append(allArgs, environmentArgs(config, name)...)
+	allArgs = append(allArgs, "Android.bp")
+
+	return bootstrap.PrimaryBuilderInvocation{
+		Inputs:  []string{"Android.bp"},
+		Outputs: []string{output},
+		Args:    allArgs,
+	}
+}
+
 func bootstrapBlueprint(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
 	defer ctx.EndTrace()
 
-	var args bootstrap.Args
+	mainSoongBuildExtraArgs := []string{"-o", config.MainNinjaFile()}
-
+	if config.EmptyNinjaFile() {
-	bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
+		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
-	bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")
-	queryviewGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.queryview.ninja")
-	soongDocsGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.soong_docs.ninja")
-	moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja")
-
-	// The glob .ninja files are subninja'd. However, they are generated during
-	// the build itself so we write an empty file so that the subninja doesn't
-	// fail on clean builds
-	writeEmptyGlobFile(ctx, bootstrapGlobFile)
-	writeEmptyGlobFile(ctx, bp2buildGlobFile)
-	writeEmptyGlobFile(ctx, queryviewGlobFile)
-	writeEmptyGlobFile(ctx, soongDocsGlobFile)
-	writeEmptyGlobFile(ctx, moduleGraphGlobFile)
-
-	bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
-
-	args.RunGoTests = !config.skipSoongTests
-	args.UseValidations = true // Use validations to depend on tests
-	args.SoongOutDir = config.SoongOutDir()
-	args.OutDir = config.OutDir()
-	args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
-	args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
-	// The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d)
-	// Building soong_build does not require a glob file
-	// Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary
-	args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile, moduleGraphGlobFile, queryviewGlobFile, soongDocsGlobFile}
-	args.EmptyNinjaFile = config.EmptyNinjaFile()
-
-	args.DelveListen = os.Getenv("SOONG_DELVE")
-	if args.DelveListen != "" {
-		args.DelvePath = shared.ResolveDelveBinary()
 	}
 
-	commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, config.MainNinjaFile())
+	mainSoongBuildInvocation := primaryBuilderInvocation(
-	mainSoongBuildInputs := []string{"Android.bp"}
+		config,
+		soongBuildTag,
+		config.MainNinjaFile(),
+		mainSoongBuildExtraArgs)
 
 	if config.bazelBuildMode() == mixedBuild {
-		mainSoongBuildInputs = append(mainSoongBuildInputs, config.Bp2BuildMarkerFile())
+		// Mixed builds call Bazel from soong_build and they therefore need the
+		// Bazel workspace to be available. Make that so by adding a dependency on
+		// the bp2build marker file to the action that invokes soong_build .
+		mainSoongBuildInvocation.Inputs = append(mainSoongBuildInvocation.Inputs,
+			config.Bp2BuildMarkerFile())
 	}
 
-	soongBuildArgs := []string{
+	bp2buildInvocation := primaryBuilderInvocation(
-		"--globListDir", "build",
+		config,
-		"--globFile", bootstrapGlobFile,
+		bp2buildTag,
+		config.Bp2BuildMarkerFile(),
+		[]string{
+			"--bp2build_marker", config.Bp2BuildMarkerFile(),
+		})
+
+	jsonModuleGraphInvocation := primaryBuilderInvocation(
+		config,
+		jsonModuleGraphTag,
+		config.ModuleGraphFile(),
+		[]string{
+			"--module_graph_file", config.ModuleGraphFile(),
+		})
+
+	queryviewInvocation := primaryBuilderInvocation(
+		config,
+		queryviewTag,
+		config.QueryviewMarkerFile(),
+		[]string{
+			"--bazel_queryview_dir", filepath.Join(config.SoongOutDir(), "queryview"),
+		})
+
+	soongDocsInvocation := primaryBuilderInvocation(
+		config,
+		soongDocsTag,
+		config.SoongDocsHtml(),
+		[]string{
+			"--soong_docs", config.SoongDocsHtml(),
+		})
+
+	globFiles := []string{
+		config.NamedGlobFile(soongBuildTag),
+		config.NamedGlobFile(bp2buildTag),
+		config.NamedGlobFile(jsonModuleGraphTag),
+		config.NamedGlobFile(queryviewTag),
+		config.NamedGlobFile(soongDocsTag),
 	}
 
-	soongBuildArgs = append(soongBuildArgs, commonArgs...)
+	// The glob .ninja files are subninja'd. However, they are generated during
-	soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
+	// the build itself so we write an empty file if the file does not exist yet
-	soongBuildArgs = append(soongBuildArgs, "Android.bp")
+	// so that the subninja doesn't fail on clean builds
-
+	for _, globFile := range globFiles {
-	mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
+		writeEmptyGlobFile(ctx, globFile)
-		Inputs:  mainSoongBuildInputs,
-		Outputs: []string{config.MainNinjaFile()},
-		Args:    soongBuildArgs,
 	}
 
-	bp2buildArgs := []string{
+	var blueprintArgs bootstrap.Args
-		"--bp2build_marker", config.Bp2BuildMarkerFile(),
-		"--globListDir", "bp2build",
-		"--globFile", bp2buildGlobFile,
-	}
 
-	bp2buildArgs = append(bp2buildArgs, commonArgs...)
+	blueprintArgs.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
-	bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
+	blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
-	bp2buildArgs = append(bp2buildArgs, "Android.bp")
+	blueprintArgs.EmptyNinjaFile = false
-
-	bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
-		Inputs:  []string{"Android.bp"},
-		Outputs: []string{config.Bp2BuildMarkerFile()},
-		Args:    bp2buildArgs,
-	}
-
-	queryviewArgs := []string{
-		"--bazel_queryview_dir", filepath.Join(config.SoongOutDir(), "queryview"),
-		"--globListDir", "queryview",
-		"--globFile", queryviewGlobFile,
-	}
-
-	queryviewArgs = append(queryviewArgs, commonArgs...)
-	queryviewArgs = append(queryviewArgs, environmentArgs(config, ".queryview")...)
-	queryviewArgs = append(queryviewArgs, "Android.bp")
-
-	queryviewInvocation := bootstrap.PrimaryBuilderInvocation{
-		Inputs:  []string{"Android.bp"},
-		Outputs: []string{config.QueryviewMarkerFile()},
-		Args:    queryviewArgs,
-	}
-
-	soongDocsArgs := []string{
-		"--soong_docs", config.SoongDocsHtml(),
-		"--globListDir", "soong_docs",
-		"--globFile", soongDocsGlobFile,
-	}
-
-	soongDocsArgs = append(soongDocsArgs, commonArgs...)
-	soongDocsArgs = append(soongDocsArgs, environmentArgs(config, ".soong_docs")...)
-	soongDocsArgs = append(soongDocsArgs, "Android.bp")
-
-	soongDocsInvocation := bootstrap.PrimaryBuilderInvocation{
-		Inputs:  []string{"Android.bp"},
-		Outputs: []string{config.SoongDocsHtml()},
-		Args:    soongDocsArgs,
-	}
-
-	moduleGraphArgs := []string{
-		"--module_graph_file", config.ModuleGraphFile(),
-		"--globListDir", "modulegraph",
-		"--globFile", moduleGraphGlobFile,
-	}
-
-	moduleGraphArgs = append(moduleGraphArgs, commonArgs...)
-	moduleGraphArgs = append(moduleGraphArgs, environmentArgs(config, ".modulegraph")...)
-	moduleGraphArgs = append(moduleGraphArgs, "Android.bp")
-
-	moduleGraphInvocation := bootstrap.PrimaryBuilderInvocation{
-		Inputs:  []string{"Android.bp"},
-		Outputs: []string{config.ModuleGraphFile()},
-		Args:    moduleGraphArgs,
-	}
-
-	args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
-		bp2buildInvocation,
-		mainSoongBuildInvocation,
-		moduleGraphInvocation,
-		queryviewInvocation,
-		soongDocsInvocation,
-	}
 
 	blueprintCtx := blueprint.NewContext()
 	blueprintCtx.SetIgnoreUnknownModuleTypes(true)
 	blueprintConfig := BlueprintConfig{
-		soongOutDir:               config.SoongOutDir(),
+		soongOutDir:    config.SoongOutDir(),
-		toolDir:                   config.HostToolDir(),
+		toolDir:        config.HostToolDir(),
-		outDir:                    config.OutDir(),
+		outDir:         config.OutDir(),
-		runGoTests:                !config.skipSoongTests,
+		runGoTests:     !config.skipSoongTests,
-		useValidations:            true,
+		useValidations: !config.skipSoongTests,
-		debugCompilation:          os.Getenv("SOONG_DELVE") != "",
+		// If we want to debug soong_build, we need to compile it for debugging
-		subninjas:                 args.Subninjas,
+		debugCompilation: os.Getenv("SOONG_DELVE") != "",
-		primaryBuilderInvocations: args.PrimaryBuilderInvocations,
+		subninjas:        globFiles,
+		primaryBuilderInvocations: []bootstrap.PrimaryBuilderInvocation{
+			mainSoongBuildInvocation,
+			bp2buildInvocation,
+			jsonModuleGraphInvocation,
+			queryviewInvocation,
+			soongDocsInvocation},
 	}
 
-	args.EmptyNinjaFile = false
+	bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
-	bootstrapDeps := bootstrap.RunBlueprint(args, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
+	bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
-	err := deptools.WriteDepFile(bootstrapDepFile, args.OutFile, bootstrapDeps)
+	err := deptools.WriteDepFile(bootstrapDepFile, blueprintArgs.OutFile, bootstrapDeps)
 	if err != nil {
 		ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err)
 	}
@@ -343,12 +328,22 @@ func runSoong(ctx Context, config Config) {
 		ctx.BeginTrace(metrics.RunSoong, "environment check")
 		defer ctx.EndTrace()
 
-		soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile)
+		checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".build"))
-		checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile)
 
-		if integratedBp2Build {
+		if integratedBp2Build || config.Bp2Build() {
-			bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build")
+			checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build"))
-			checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile)
+		}
+
+		if config.JsonModuleGraph() {
+			checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".modulegraph"))
+		}
+
+		if config.Queryview() {
+			checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".queryview"))
+		}
+
+		if config.SoongDocs() {
+			checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".soong_docs"))
 		}
 	}()