Make running soong_build in alternate modes nicer.

This includes the JSON graph generator and bp2build.

Before:
GENERATE_BAZEL_FILES=1 m nothing
GENERATE_JSON_MODULE_GRAPH=1 m nothing

Now:
m json-module-graph
m bp2build

They can now also be combined with other targets or each other.

The longer-term goal is to run "m queryview" and "m soong_docs" using
the same infrastructure. There are two alternate approaches:

1. Call soong_build from within the main Ninja invocation. This requires
two sequential soong_build invocations and is thus slower.
2. Do everything requested in the same soong_build invocation. This
would be faster, but one AFAIU can't tell Ninja that multiple possible
actions can build the same output so that doesn't work.

(1) is somewhat more desirable because soong_docs seems to be built
from build/make/core/main.mk ; I assume that that can be worked around
although I haven't checked where the output of "m soong_docs" goes.

Test: Presubmits.
Change-Id: If5ba36490d9f3f60733e6d6be9286eb2b67c3ff5

tests/bootstrap_test.sh

@@ -522,7 +522,7 @@ EOF
 
 function test_bp2build_smoke {
   setup
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   [[ -e out/soong/.bootstrap/bp2build_workspace_marker ]] || fail "bp2build marker file not created"
   [[ -e out/soong/workspace ]] || fail "Bazel workspace not created"
 }
@@ -531,7 +531,7 @@ function test_bp2build_generates_marker_file {
   setup
   create_mock_bazel
 
-  run_bp2build
+  run_soong bp2build
 
   if [[ ! -f "./out/soong/.bootstrap/bp2build_workspace_marker" ]]; then
     fail "Marker file was not generated"
@@ -551,7 +551,7 @@ filegroup {
 }
 EOF
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   [[ -e out/soong/bp2build/a/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not created"
   [[ -L out/soong/workspace/a/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not symlinked"
 
@@ -565,7 +565,7 @@ filegroup {
 }
 EOF
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   [[ -e out/soong/bp2build/b/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not created"
   [[ -L out/soong/workspace/b/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not symlinked"
 }
@@ -573,10 +573,10 @@ EOF
 function test_bp2build_null_build {
   setup
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   local mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   local mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
 
   if [[ "$mtime1" != "$mtime2" ]]; then
@@ -597,18 +597,35 @@ filegroup {
 }
 EOF
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   grep -q a1.txt "out/soong/bp2build/a/${GENERATED_BUILD_FILE_NAME}" || fail "a1.txt not in ${GENERATED_BUILD_FILE_NAME} file"
 
   touch a/a2.txt
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   grep -q a2.txt "out/soong/bp2build/a/${GENERATED_BUILD_FILE_NAME}" || fail "a2.txt not in ${GENERATED_BUILD_FILE_NAME} file"
 }
 
+function test_multiple_soong_build_modes() {
+  setup
+  run_soong json-module-graph bp2build nothing
+  if [[ ! -f "out/soong/.bootstrap/bp2build_workspace_marker" ]]; then
+    fail "bp2build marker file was not generated"
+  fi
+
+
+  if [[ ! -f "out/soong/module-graph.json" ]]; then
+    fail "JSON file was not created"
+  fi
+
+  if [[ ! -f "out/soong/build.ninja" ]]; then
+    fail "Main build.ninja file was not created"
+  fi
+}
+
 function test_dump_json_module_graph() {
   setup
-  GENERATE_JSON_MODULE_GRAPH=1 run_soong
+  run_soong json-module-graph
-  if [[ ! -r "out/soong//module-graph.json" ]]; then
+  if [[ ! -r "out/soong/module-graph.json" ]]; then
     fail "JSON file was not created"
   fi
 }
@@ -619,7 +636,7 @@ function test_json_module_graph_back_and_forth_null_build() {
   run_soong
   local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)
 
-  GENERATE_JSON_MODULE_GRAPH=1 run_soong
+  run_soong json-module-graph
   local json_mtime1=$(stat -c "%y" out/soong/module-graph.json)
 
   run_soong
@@ -628,7 +645,7 @@ function test_json_module_graph_back_and_forth_null_build() {
     fail "Output Ninja file changed after writing JSON module graph"
   fi
 
-  GENERATE_JSON_MODULE_GRAPH=1 run_soong
+  run_soong json-module-graph
   local json_mtime2=$(stat -c "%y" out/soong/module-graph.json)
   if [[ "$json_mtime1" != "$json_mtime2" ]]; then
     fail "JSON module graph file changed after writing Ninja file"
@@ -651,7 +668,7 @@ filegroup {
 }
 EOF
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   [[ -e out/soong/workspace ]] || fail "Bazel workspace not created"
   [[ -d out/soong/workspace/a/b ]] || fail "module directory not a directory"
   [[ -L "out/soong/workspace/a/b/${GENERATED_BUILD_FILE_NAME}" ]] || fail "${GENERATED_BUILD_FILE_NAME} file not symlinked"
@@ -675,10 +692,10 @@ filegroup {
 }
 EOF
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
 
   touch a/a2.txt  # No reference in the .bp file needed
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   [[ -L out/soong/workspace/a/a2.txt ]] || fail "a/a2.txt not symlinked"
 }
 
@@ -696,7 +713,7 @@ filegroup {
 }
 EOF
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   [[ -L "out/soong/workspace/a/${GENERATED_BUILD_FILE_NAME}" ]] || fail "${GENERATED_BUILD_FILE_NAME} file not symlinked"
   [[ "$(readlink -f out/soong/workspace/a/${GENERATED_BUILD_FILE_NAME})" =~ "bp2build/a/${GENERATED_BUILD_FILE_NAME}"$ ]] \
     || fail "${GENERATED_BUILD_FILE_NAME} files symlinked to the wrong place"
@@ -725,7 +742,7 @@ filegroup {
 }
 EOF
 
-  if GENERATE_BAZEL_FILES=1 run_soong >& "$MOCK_TOP/errors"; then
+  if run_soong bp2build >& "$MOCK_TOP/errors"; then
     fail "Build should have failed"
   fi
 
@@ -739,7 +756,7 @@ function test_bp2build_back_and_forth_null_build {
   run_soong
   local output_mtime1=$(stat -c "%y" out/soong/build.ninja)
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   local output_mtime2=$(stat -c "%y" out/soong/build.ninja)
   if [[ "$output_mtime1" != "$output_mtime2" ]]; then
     fail "Output Ninja file changed when switching to bp2build"
@@ -757,7 +774,7 @@ function test_bp2build_back_and_forth_null_build {
     fail "bp2build marker file changed when switching to regular build from bp2build"
   fi
 
-  GENERATE_BAZEL_FILES=1 run_soong
+  run_soong bp2build
   local output_mtime4=$(stat -c "%y" out/soong/build.ninja)
   local marker_mtime3=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
   if [[ "$output_mtime1" != "$output_mtime4" ]]; then
@@ -780,6 +797,7 @@ test_delete_android_bp
 test_add_file_to_soong_build
 test_glob_during_bootstrapping
 test_soong_build_rerun_iff_environment_changes
+test_multiple_soong_build_modes
 test_dump_json_module_graph
 test_json_module_graph_back_and_forth_null_build
 test_write_to_source_tree

tests/bp2build_bazel_test.sh

@@ -10,10 +10,10 @@ readonly GENERATED_BUILD_FILE_NAME="BUILD.bazel"
 
 function test_bp2build_null_build() {
   setup
-  run_bp2build
+  run_soong bp2build
   local output_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
 
-  run_bp2build
+  run_soong bp2build
   local output_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
 
   if [[ "$output_mtime1" != "$output_mtime2" ]]; then
@@ -35,10 +35,10 @@ filegroup {
 EOF
   touch foo/bar/a.txt foo/bar/b.txt
 
-  run_bp2build
+  run_soong bp2build
   local output_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
 
-  run_bp2build
+  run_soong bp2build
   local output_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
 
   if [[ "$output_mtime1" != "$output_mtime2" ]]; then
@@ -80,7 +80,7 @@ genrule {
   }
 EOF
 
-  run_bp2build
+  run_soong bp2build
 
   if [[ ! -f "./out/soong/workspace/foo/convertible_soong_module/${GENERATED_BUILD_FILE_NAME}" ]]; then
     fail "./out/soong/workspace/foo/convertible_soong_module/${GENERATED_BUILD_FILE_NAME} was not generated"

tests/lib.sh

@@ -124,10 +124,6 @@ run_bazel() {
   tools/bazel "$@"
 }
 
-run_bp2build() {
-  GENERATE_BAZEL_FILES=true build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests nothing
-}
-
 run_ninja() {
   build/soong/soong_ui.bash --make-mode --skip-make --skip-soong-tests "$@"
 }

ui/build/build.go

@@ -248,6 +248,16 @@ func Build(ctx Context, config Config) {
 		what = what &^ RunNinja
 	}
 
+	if !config.SoongBuildInvocationNeeded() {
+		// This means that the output of soong_build is not needed and thus it would
+		// run unnecessarily. In addition, if this code wasn't there invocations
+		// with only special-cased target names like "m bp2build" would result in
+		// passing Ninja the empty target list and it would then build the default
+		// targets which is not what the user asked for.
+		what = what &^ RunNinja
+		what = what &^ RunKati
+	}
+
 	if config.StartGoma() {
 		startGoma(ctx, config)
 	}
@@ -278,16 +288,6 @@ func Build(ctx Context, config Config) {
 
 	if what&RunSoong != 0 {
 		runSoong(ctx, config)
-
-		if config.bazelBuildMode() == generateBuildFiles {
-			// Return early, if we're using Soong as solely the generator of BUILD files.
-			return
-		}
-
-		if config.bazelBuildMode() == generateJsonModuleGraph {
-			// Return early, if we're using Soong as solely the generator of the JSON module graph
-			return
-		}
 	}
 
 	if what&RunKati != 0 {

ui/build/config.go

@@ -33,7 +33,8 @@ import (
 type Config struct{ *configImpl }
 
 type configImpl struct {
-	// From the environment
+	// Some targets that are implemented in soong_build
+	// (bp2build, json-module-graph) are not here and have their own bits below.
 	arguments     []string
 	goma          bool
 	environ       *Environment
@@ -41,17 +42,19 @@ type configImpl struct {
 	buildDateTime string
 
 	// From the arguments
-	parallel       int
+	parallel        int
-	keepGoing      int
+	keepGoing       int
-	verbose        bool
+	verbose         bool
-	checkbuild     bool
+	checkbuild      bool
-	dist           bool
+	dist            bool
-	skipConfig     bool
+	jsonModuleGraph bool
-	skipKati       bool
+	bp2build        bool
-	skipKatiNinja  bool
+	skipConfig      bool
-	skipSoong      bool
+	skipKati        bool
-	skipNinja      bool
+	skipKatiNinja   bool
-	skipSoongTests bool
+	skipSoong       bool
+	skipNinja       bool
+	skipSoongTests  bool
 
 	// From the product config
 	katiArgs        []string
@@ -109,9 +112,6 @@ const (
 	// Only generate build files (in a subdirectory of the out directory) and exit.
 	generateBuildFiles
 
-	// Only generate the Soong json module graph for use with jq, and exit.
-	generateJsonModuleGraph
-
 	// Generate synthetic build files and incorporate these files into a build which
 	// partially uses Bazel. Build metadata may come from Android.bp or BUILD files.
 	mixedBuild
@@ -639,6 +639,10 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
 			c.environ.Set(k, v)
 		} else if arg == "dist" {
 			c.dist = true
+		} else if arg == "json-module-graph" {
+			c.jsonModuleGraph = true
+		} else if arg == "bp2build" {
+			c.bp2build = true
 		} else {
 			if arg == "checkbuild" {
 				c.checkbuild = true
@@ -705,6 +709,26 @@ func (c *configImpl) Arguments() []string {
 	return c.arguments
 }
 
+func (c *configImpl) SoongBuildInvocationNeeded() bool {
+	if c.Dist() {
+		return true
+	}
+
+	if len(c.Arguments()) > 0 {
+		// Explicit targets requested that are not special targets like b2pbuild
+		// or the JSON module graph
+		return true
+	}
+
+	if !c.JsonModuleGraph() && !c.Bp2Build() {
+		// Command line was empty, the default Ninja target is built
+		return true
+	}
+
+	// build.ninja doesn't need to be generated
+	return false
+}
+
 func (c *configImpl) OutDir() string {
 	if outDir, ok := c.environ.Get("OUT_DIR"); ok {
 		return outDir
@@ -790,6 +814,14 @@ func (c *configImpl) Dist() bool {
 	return c.dist
 }
 
+func (c *configImpl) JsonModuleGraph() bool {
+	return c.jsonModuleGraph
+}
+
+func (c *configImpl) Bp2Build() bool {
+	return c.bp2build
+}
+
 func (c *configImpl) IsVerbose() bool {
 	return c.verbose
 }
@@ -935,10 +967,6 @@ func (c *configImpl) UseBazel() bool {
 func (c *configImpl) bazelBuildMode() bazelBuildMode {
 	if c.Environment().IsEnvTrue("USE_BAZEL_ANALYSIS") {
 		return mixedBuild
-	} else if c.Environment().IsEnvTrue("GENERATE_BAZEL_FILES") {
-		return generateBuildFiles
-	} else if c.Environment().IsEnvTrue("GENERATE_JSON_MODULE_GRAPH") {
-		return generateJsonModuleGraph
 	} else {
 		return noBazel
 	}

ui/build/soong.go

@@ -275,7 +275,7 @@ func runSoong(ctx Context, config Config) {
 	}
 
 	buildMode := config.bazelBuildMode()
-	integratedBp2Build := (buildMode == mixedBuild) || (buildMode == generateBuildFiles)
+	integratedBp2Build := buildMode == mixedBuild
 
 	// This is done unconditionally, but does not take a measurable amount of time
 	bootstrapBlueprint(ctx, config)
@@ -351,18 +351,22 @@ func runSoong(ctx Context, config Config) {
 		cmd.RunAndStreamOrFatal()
 	}
 
-	var target string
+	targets := make([]string, 0, 0)
 
-	if config.bazelBuildMode() == generateBuildFiles {
+	if config.JsonModuleGraph() {
-		target = config.Bp2BuildMarkerFile()
+		targets = append(targets, config.ModuleGraphFile())
-	} else if config.bazelBuildMode() == generateJsonModuleGraph {
-		target = config.ModuleGraphFile()
-	} else {
-		// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
-		target = config.MainNinjaFile()
 	}
 
-	ninja("bootstrap", ".bootstrap/build.ninja", target)
+	if config.Bp2Build() {
+		targets = append(targets, config.Bp2BuildMarkerFile())
+	}
+
+	if config.SoongBuildInvocationNeeded() {
+		// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
+		targets = append(targets, config.MainNinjaFile())
+	}
+
+	ninja("bootstrap", ".bootstrap/build.ninja", targets...)
 
 	var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
 	if shouldCollectBuildSoongMetrics(config) {
@@ -404,7 +408,7 @@ func runMicrofactory(ctx Context, config Config, relExePath string, pkg string,
 func shouldCollectBuildSoongMetrics(config Config) bool {
 	// Do not collect metrics protobuf if the soong_build binary ran as the
 	// bp2build converter or the JSON graph dump.
-	return config.bazelBuildMode() != generateBuildFiles && config.bazelBuildMode() != generateJsonModuleGraph
+	return config.SoongBuildInvocationNeeded()
 }
 
 func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics {