Refactor the creation of soong_build calls. am: 89fcdcb788

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1818245 Change-Id: I224e6b7417ba9f6c9a7f2f2e4804c98d5faa3a52
2021-09-08 13:39:23 +00:00
parent e5d3519643 89fcdcb788
commit b42d232951
5 changed files with 155 additions and 162 deletions
--- a/README.md
+++ b/README.md
@@ -37,7 +37,6 @@ The list of valid module types and their properties can be generated by calling
 `m soong_docs`. It will be written to `$OUT_DIR/soong/docs/soong_build.html`.
 This list for the current version of Soong can be found [here](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html).

-
 ### File lists

 Properties that take a list of files can also take glob patterns and output path
--- a/android/config.go
+++ b/android/config.go
@@ -418,8 +418,8 @@ func TestArchConfig(buildDir string, env map[string]string, bp string, fs map[st
 // bootstrap run. Only per-run data is reset. Data which needs to persist across
 // multiple runs in the same program execution is carried over (such as Bazel
 // context or environment deps).
-func ConfigForAdditionalRun(cmdlineArgs bootstrap.Args, c Config) (Config, error) {
-	newConfig, err := NewConfig(cmdlineArgs, c.soongOutDir, c.env)
+func ConfigForAdditionalRun(c Config) (Config, error) {
+	newConfig, err := NewConfig(c.moduleListFile, c.runGoTests, c.outDir, c.soongOutDir, c.env)
 	if err != nil {
 		return Config{}, err
 	}
@@ -430,20 +430,20 @@ func ConfigForAdditionalRun(cmdlineArgs bootstrap.Args, c Config) (Config, error

 // NewConfig creates a new Config object. The srcDir argument specifies the path
 // to the root source directory. It also loads the config file, if found.
-func NewConfig(cmdlineArgs bootstrap.Args, soongOutDir string, availableEnv map[string]string) (Config, error) {
+func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir string, availableEnv map[string]string) (Config, error) {
 	// Make a config with default options.
 	config := &config{
 		ProductVariablesFileName: filepath.Join(soongOutDir, productVariablesFileName),

 		env: availableEnv,

-		outDir:                   cmdlineArgs.OutDir,
+		outDir:                   outDir,
 		soongOutDir:              soongOutDir,
-		runGoTests:               cmdlineArgs.RunGoTests,
-		useValidationsForGoTests: cmdlineArgs.UseValidations,
+		runGoTests:               runGoTests,
+		useValidationsForGoTests: runGoTests,
 		multilibConflicts:        make(map[ArchType]bool),

-		moduleListFile: cmdlineArgs.ModuleListFile,
+		moduleListFile: moduleListFile,
 		fs:             pathtools.NewOsFs(absSrcDir),
 	}

--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -35,10 +35,13 @@ import (

 var (
 	topDir           string
+	outDir           string
 	soongOutDir      string
 	availableEnvFile string
 	usedEnvFile      string

+	runGoTests bool
+
 	globFile    string
 	globListDir string
 	delveListen string
@@ -60,7 +63,7 @@ func init() {
 	flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
 	flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
 	flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
-	flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
+	flag.StringVar(&outDir, "out", "", "the ninja builddir directory")
 	flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")

 	// Debug flags
@@ -81,8 +84,7 @@ func init() {

 	// Flags that probably shouldn't be flags of soong_build but we haven't found
 	// the time to remove them yet
-	flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")
-	flag.BoolVar(&cmdlineArgs.UseValidations, "use-validations", false, "use validations to depend on go tests")
+	flag.BoolVar(&runGoTests, "t", false, "build and run go tests during bootstrap")
 }

 func newNameResolver(config android.Config) *android.NameResolver {
@@ -109,8 +111,8 @@ func newContext(configuration android.Config) *android.Context {
 	return ctx
 }

-func newConfig(cmdlineArgs bootstrap.Args, outDir string, availableEnv map[string]string) android.Config {
-	configuration, err := android.NewConfig(cmdlineArgs, outDir, availableEnv)
+func newConfig(availableEnv map[string]string) android.Config {
+	configuration, err := android.NewConfig(cmdlineArgs.ModuleListFile, runGoTests, outDir, soongOutDir, availableEnv)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s", err)
 		os.Exit(1)
@@ -124,10 +126,7 @@ func newConfig(cmdlineArgs bootstrap.Args, outDir string, availableEnv map[strin
 // TODO(cparsons): Don't output any ninja file, as the second pass will overwrite
 // the incorrect results from the first pass, and file I/O is expensive.
 func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) {
-	var firstArgs, secondArgs bootstrap.Args
-
-	firstArgs = cmdlineArgs
-	bootstrap.RunBlueprint(firstArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration)
+	bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration)

 	// Invoke bazel commands and save results for second pass.
 	if err := configuration.BazelContext.InvokeBazel(); err != nil {
@@ -135,20 +134,19 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context,
 		os.Exit(1)
 	}
 	// Second pass: Full analysis, using the bazel command results. Output ninja file.
-	secondArgs = cmdlineArgs
-	secondConfig, err := android.ConfigForAdditionalRun(secondArgs, configuration)
+	secondConfig, err := android.ConfigForAdditionalRun(configuration)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s", err)
 		os.Exit(1)
 	}
 	secondCtx := newContext(secondConfig)
-	ninjaDeps := bootstrap.RunBlueprint(secondArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig)
+	ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig)
 	ninjaDeps = append(ninjaDeps, extraNinjaDeps...)

 	globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration)
 	ninjaDeps = append(ninjaDeps, globListFiles...)

-	writeDepFile(secondArgs.OutFile, ninjaDeps)
+	writeDepFile(cmdlineArgs.OutFile, ninjaDeps)
 }

 // Run the code-generation phase to convert BazelTargetModules to BUILD files.
@@ -306,7 +304,7 @@ func main() {

 	availableEnv := parseAvailableEnv()

-	configuration := newConfig(cmdlineArgs, soongOutDir, availableEnv)
+	configuration := newConfig(availableEnv)
 	extraNinjaDeps := []string{
 		configuration.ProductVariablesFileName,
 		usedEnvFile,
@@ -500,8 +498,8 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
 		"bazel-" + filepath.Base(topDir),
 	}

-	if cmdlineArgs.OutDir[0] != '/' {
-		excludes = append(excludes, cmdlineArgs.OutDir)
+	if outDir[0] != '/' {
+		excludes = append(excludes, outDir)
 	}

 	existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir)
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -111,9 +111,6 @@ const (
 	// Don't use bazel at all.
 	noBazel bazelBuildMode = iota

-	// Only generate build files (in a subdirectory of the out directory) and exit.
-	generateBuildFiles
-
 	// Generate synthetic build files and incorporate these files into a build which
 	// partially uses Bazel. Build metadata may come from Android.bp or BUILD files.
 	mixedBuild
@@ -783,6 +780,10 @@ func (c *configImpl) HostToolDir() string {
 	return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
 }

+func (c *configImpl) NamedGlobFile(name string) string {
+	return shared.JoinPath(c.SoongOutDir(), ".bootstrap/build-globs."+name+".ninja")
+}
+
 func (c *configImpl) MainNinjaFile() string {
 	return shared.JoinPath(c.SoongOutDir(), "build.ninja")
 }
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -37,6 +37,12 @@ import (
 const (
 	availableEnvFile = "soong.environment.available"
 	usedEnvFile      = "soong.environment.used"
+
+	soongBuildTag      = "build"
+	bp2buildTag        = "bp2build"
+	jsonModuleGraphTag = "modulegraph"
+	queryviewTag       = "queryview"
+	soongDocsTag       = "soong_docs"
 )

 func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
@@ -116,7 +122,7 @@ func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderI
 func environmentArgs(config Config, suffix string) []string {
 	return []string{
 		"--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
-		"--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix),
+		"--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+"."+suffix),
 	}
 }

@@ -134,156 +140,135 @@ func writeEmptyGlobFile(ctx Context, path string) {
 	}
 }

+func primaryBuilderInvocation(config Config, name string, output string, specificArgs []string) bootstrap.PrimaryBuilderInvocation {
+	commonArgs := make([]string, 0, 0)
+
+	if !config.skipSoongTests {
+		commonArgs = append(commonArgs, "-t")
+	}
+
+	commonArgs = append(commonArgs, "-l", filepath.Join(config.FileListDir(), "Android.bp.list"))
+
+	if os.Getenv("SOONG_DELVE") != "" {
+		commonArgs = append(commonArgs, "--delve_listen", os.Getenv("SOONG_DELVE"))
+		commonArgs = append(commonArgs, "--delve_path", shared.ResolveDelveBinary())
+	}
+
+	allArgs := make([]string, 0, 0)
+	allArgs = append(allArgs, specificArgs...)
+	allArgs = append(allArgs,
+		"--globListDir", name,
+		"--globFile", config.NamedGlobFile(name))
+
+	allArgs = append(allArgs, commonArgs...)
+	allArgs = append(allArgs, environmentArgs(config, name)...)
+	allArgs = append(allArgs, "Android.bp")
+
+	return bootstrap.PrimaryBuilderInvocation{
+		Inputs:  []string{"Android.bp"},
+		Outputs: []string{output},
+		Args:    allArgs,
+	}
+}
+
 func bootstrapBlueprint(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
 	defer ctx.EndTrace()

-	var args bootstrap.Args
-
-	bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
-	bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")
-	queryviewGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.queryview.ninja")
-	soongDocsGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.soong_docs.ninja")
-	moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja")
-
-	// The glob .ninja files are subninja'd. However, they are generated during
-	// the build itself so we write an empty file so that the subninja doesn't
-	// fail on clean builds
-	writeEmptyGlobFile(ctx, bootstrapGlobFile)
-	writeEmptyGlobFile(ctx, bp2buildGlobFile)
-	writeEmptyGlobFile(ctx, queryviewGlobFile)
-	writeEmptyGlobFile(ctx, soongDocsGlobFile)
-	writeEmptyGlobFile(ctx, moduleGraphGlobFile)
-
-	bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
-
-	args.RunGoTests = !config.skipSoongTests
-	args.UseValidations = true // Use validations to depend on tests
-	args.SoongOutDir = config.SoongOutDir()
-	args.OutDir = config.OutDir()
-	args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
-	args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
-	// The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d)
-	// Building soong_build does not require a glob file
-	// Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary
-	args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile, moduleGraphGlobFile, queryviewGlobFile, soongDocsGlobFile}
-	args.EmptyNinjaFile = config.EmptyNinjaFile()
-
-	args.DelveListen = os.Getenv("SOONG_DELVE")
-	if args.DelveListen != "" {
-		args.DelvePath = shared.ResolveDelveBinary()
+	mainSoongBuildExtraArgs := []string{"-o", config.MainNinjaFile()}
+	if config.EmptyNinjaFile() {
+		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
 	}

-	commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, config.MainNinjaFile())
-	mainSoongBuildInputs := []string{"Android.bp"}
+	mainSoongBuildInvocation := primaryBuilderInvocation(
+		config,
+		soongBuildTag,
+		config.MainNinjaFile(),
+		mainSoongBuildExtraArgs)

 	if config.bazelBuildMode() == mixedBuild {
-		mainSoongBuildInputs = append(mainSoongBuildInputs, config.Bp2BuildMarkerFile())
+		// Mixed builds call Bazel from soong_build and they therefore need the
+		// Bazel workspace to be available. Make that so by adding a dependency on
+		// the bp2build marker file to the action that invokes soong_build .
+		mainSoongBuildInvocation.Inputs = append(mainSoongBuildInvocation.Inputs,
+			config.Bp2BuildMarkerFile())
 	}

-	soongBuildArgs := []string{
-		"--globListDir", "build",
-		"--globFile", bootstrapGlobFile,
+	bp2buildInvocation := primaryBuilderInvocation(
+		config,
+		bp2buildTag,
+		config.Bp2BuildMarkerFile(),
+		[]string{
+			"--bp2build_marker", config.Bp2BuildMarkerFile(),
+		})
+
+	jsonModuleGraphInvocation := primaryBuilderInvocation(
+		config,
+		jsonModuleGraphTag,
+		config.ModuleGraphFile(),
+		[]string{
+			"--module_graph_file", config.ModuleGraphFile(),
+		})
+
+	queryviewInvocation := primaryBuilderInvocation(
+		config,
+		queryviewTag,
+		config.QueryviewMarkerFile(),
+		[]string{
+			"--bazel_queryview_dir", filepath.Join(config.SoongOutDir(), "queryview"),
+		})
+
+	soongDocsInvocation := primaryBuilderInvocation(
+		config,
+		soongDocsTag,
+		config.SoongDocsHtml(),
+		[]string{
+			"--soong_docs", config.SoongDocsHtml(),
+		})
+
+	globFiles := []string{
+		config.NamedGlobFile(soongBuildTag),
+		config.NamedGlobFile(bp2buildTag),
+		config.NamedGlobFile(jsonModuleGraphTag),
+		config.NamedGlobFile(queryviewTag),
+		config.NamedGlobFile(soongDocsTag),
 	}

-	soongBuildArgs = append(soongBuildArgs, commonArgs...)
-	soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
-	soongBuildArgs = append(soongBuildArgs, "Android.bp")
-
-	mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
-		Inputs:  mainSoongBuildInputs,
-		Outputs: []string{config.MainNinjaFile()},
-		Args:    soongBuildArgs,
+	// The glob .ninja files are subninja'd. However, they are generated during
+	// the build itself so we write an empty file if the file does not exist yet
+	// so that the subninja doesn't fail on clean builds
+	for _, globFile := range globFiles {
+		writeEmptyGlobFile(ctx, globFile)
 	}

-	bp2buildArgs := []string{
-		"--bp2build_marker", config.Bp2BuildMarkerFile(),
-		"--globListDir", "bp2build",
-		"--globFile", bp2buildGlobFile,
-	}
+	var blueprintArgs bootstrap.Args

-	bp2buildArgs = append(bp2buildArgs, commonArgs...)
-	bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
-	bp2buildArgs = append(bp2buildArgs, "Android.bp")
-
-	bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
-		Inputs:  []string{"Android.bp"},
-		Outputs: []string{config.Bp2BuildMarkerFile()},
-		Args:    bp2buildArgs,
-	}
-
-	queryviewArgs := []string{
-		"--bazel_queryview_dir", filepath.Join(config.SoongOutDir(), "queryview"),
-		"--globListDir", "queryview",
-		"--globFile", queryviewGlobFile,
-	}
-
-	queryviewArgs = append(queryviewArgs, commonArgs...)
-	queryviewArgs = append(queryviewArgs, environmentArgs(config, ".queryview")...)
-	queryviewArgs = append(queryviewArgs, "Android.bp")
-
-	queryviewInvocation := bootstrap.PrimaryBuilderInvocation{
-		Inputs:  []string{"Android.bp"},
-		Outputs: []string{config.QueryviewMarkerFile()},
-		Args:    queryviewArgs,
-	}
-
-	soongDocsArgs := []string{
-		"--soong_docs", config.SoongDocsHtml(),
-		"--globListDir", "soong_docs",
-		"--globFile", soongDocsGlobFile,
-	}
-
-	soongDocsArgs = append(soongDocsArgs, commonArgs...)
-	soongDocsArgs = append(soongDocsArgs, environmentArgs(config, ".soong_docs")...)
-	soongDocsArgs = append(soongDocsArgs, "Android.bp")
-
-	soongDocsInvocation := bootstrap.PrimaryBuilderInvocation{
-		Inputs:  []string{"Android.bp"},
-		Outputs: []string{config.SoongDocsHtml()},
-		Args:    soongDocsArgs,
-	}
-
-	moduleGraphArgs := []string{
-		"--module_graph_file", config.ModuleGraphFile(),
-		"--globListDir", "modulegraph",
-		"--globFile", moduleGraphGlobFile,
-	}
-
-	moduleGraphArgs = append(moduleGraphArgs, commonArgs...)
-	moduleGraphArgs = append(moduleGraphArgs, environmentArgs(config, ".modulegraph")...)
-	moduleGraphArgs = append(moduleGraphArgs, "Android.bp")
-
-	moduleGraphInvocation := bootstrap.PrimaryBuilderInvocation{
-		Inputs:  []string{"Android.bp"},
-		Outputs: []string{config.ModuleGraphFile()},
-		Args:    moduleGraphArgs,
-	}
-
-	args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
-		bp2buildInvocation,
-		mainSoongBuildInvocation,
-		moduleGraphInvocation,
-		queryviewInvocation,
-		soongDocsInvocation,
-	}
+	blueprintArgs.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
+	blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
+	blueprintArgs.EmptyNinjaFile = false

 	blueprintCtx := blueprint.NewContext()
 	blueprintCtx.SetIgnoreUnknownModuleTypes(true)
 	blueprintConfig := BlueprintConfig{
-		soongOutDir:               config.SoongOutDir(),
-		toolDir:                   config.HostToolDir(),
-		outDir:                    config.OutDir(),
-		runGoTests:                !config.skipSoongTests,
-		useValidations:            true,
-		debugCompilation:          os.Getenv("SOONG_DELVE") != "",
-		subninjas:                 args.Subninjas,
-		primaryBuilderInvocations: args.PrimaryBuilderInvocations,
+		soongOutDir:    config.SoongOutDir(),
+		toolDir:        config.HostToolDir(),
+		outDir:         config.OutDir(),
+		runGoTests:     !config.skipSoongTests,
+		useValidations: !config.skipSoongTests,
+		// If we want to debug soong_build, we need to compile it for debugging
+		debugCompilation: os.Getenv("SOONG_DELVE") != "",
+		subninjas:        globFiles,
+		primaryBuilderInvocations: []bootstrap.PrimaryBuilderInvocation{
+			mainSoongBuildInvocation,
+			bp2buildInvocation,
+			jsonModuleGraphInvocation,
+			queryviewInvocation,
+			soongDocsInvocation},
 	}

-	args.EmptyNinjaFile = false
-	bootstrapDeps := bootstrap.RunBlueprint(args, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
-	err := deptools.WriteDepFile(bootstrapDepFile, args.OutFile, bootstrapDeps)
+	bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
+	bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
+	err := deptools.WriteDepFile(bootstrapDepFile, blueprintArgs.OutFile, bootstrapDeps)
 	if err != nil {
 		ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err)
 	}
@@ -343,12 +328,22 @@ func runSoong(ctx Context, config Config) {
 		ctx.BeginTrace(metrics.RunSoong, "environment check")
 		defer ctx.EndTrace()

-		soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile)
-		checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile)
+		checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".build"))

-		if integratedBp2Build {
-			bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build")
-			checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile)
+		if integratedBp2Build || config.Bp2Build() {
+			checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build"))
+		}
+
+		if config.JsonModuleGraph() {
+			checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".modulegraph"))
+		}
+
+		if config.Queryview() {
+			checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".queryview"))
+		}
+
+		if config.SoongDocs() {
+			checkEnvironmentFile(soongBuildEnv, filepath.Join(config.SoongOutDir(), usedEnvFile+".soong_docs"))
 		}
 	}()