Allow running bp2build as part of a regular build.

This is done by setting the INTEGRATED_BP2BUILD environment variable
when invoking the build.

Even though the name of the marker file insinuates that a Bazel
workspace is already created, this is not the case yet.

An issue that remains is that a .d file is not written for the marker
file so it won't be rebuilt if a .bp file changes. Fixing this requires
delicate surgery because writing the .d file is the result of delicate
interplay between Soong and Blueprint.

There are also a number of semi-related fixes:

- The name of soong.environment.{used,available} is now on the command
  line of soong_build (soong_docs is still special cased because its
  command line in the Ninja file is taken from the os.Args of
  soong_build so it's not trivial to remove the --{available,used}_env
  from it
- bp2build writes a separate soong.environment.used file
- I had to call SetAllowMissingDependencies() separately when creating
  the android.Context for bp2build so that bp2build runs in the
  integration tests (it was not obvious how not to do this)
- Fixed a number of integration tests where a command with an expected
  exit code of 1 was used as the last one in a test case, thereby
  breaking the test suite

Test: Presubmits.
Change-Id: Ibeb61c26022cf801dcb98505b4039151b3409873

cmd/soong_build/main.go

@@ -23,29 +23,42 @@ import (
 	"strings"
 	"time"
 
+	"android/soong/bp2build"
 	"android/soong/shared"
 	"github.com/google/blueprint/bootstrap"
 
 	"android/soong/android"
-	"android/soong/bp2build"
 )
 
 var (
 	topDir           string
 	outDir           string
-	docFile           string
+	availableEnvFile string
-	bazelQueryViewDir string
+	usedEnvFile      string
+
 	delveListen string
 	delvePath   string
+
+	docFile           string
+	bazelQueryViewDir string
+	bp2buildMarker    string
 )
 
 func init() {
+	// Flags that make sense in every mode
 	flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
 	flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)")
+	flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
+	flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
+
+	// Debug flags
 	flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging")
 	flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")
+
+	// Flags representing various modes soong_build can run in
 	flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
 	flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
+	flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
 }
 
 func newNameResolver(config android.Config) *android.NameResolver {
@@ -147,8 +160,8 @@ func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, pa
 	writeFakeNinjaFile(extraNinjaDeps, configuration.BuildDir())
 }
 
-func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
+func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string {
-	bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES")
+	bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES") || bp2buildMarker != ""
 	mixedModeBuild := configuration.BazelContext.BazelEnabled()
 	generateQueryView := bazelQueryViewDir != ""
 	jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")
@@ -159,7 +172,11 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
 		// Run the alternate pipeline of bp2build mutators and singleton to convert
 		// Blueprint to BUILD files before everything else.
 		runBp2Build(configuration, extraNinjaDeps)
-		return
+		if bp2buildMarker != "" {
+			return bp2buildMarker
+		} else {
+			return bootstrap.CmdlineOutFile()
+		}
 	}
 
 	ctx := newContext(configuration, prepareBuildActions)
@@ -172,15 +189,44 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
 	// Convert the Soong module graph into Bazel BUILD files.
 	if generateQueryView {
 		runQueryView(configuration, ctx)
-		return
+		return bootstrap.CmdlineOutFile() // TODO: This is a lie
 	}
 
 	if jsonModuleFile != "" {
 		writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)
-		return
+		return bootstrap.CmdlineOutFile() // TODO: This is a lie
 	}
 
 	writeMetrics(configuration)
+	return bootstrap.CmdlineOutFile()
+}
+
+// soong_ui dumps the available environment variables to
+// soong.environment.available . Then soong_build itself is run with an empty
+// environment so that the only way environment variables can be accessed is
+// using Config, which tracks access to them.
+
+// At the end of the build, a file called soong.environment.used is written
+// containing the current value of all used environment variables. The next
+// time soong_ui is run, it checks whether any environment variables that was
+// used had changed and if so, it deletes soong.environment.used to cause a
+// rebuild.
+//
+// The dependency of build.ninja on soong.environment.used is declared in
+// build.ninja.d
+func parseAvailableEnv() map[string]string {
+	if availableEnvFile == "" {
+		fmt.Fprintf(os.Stderr, "--available_env not set\n")
+		os.Exit(1)
+	}
+
+	result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error reading available environment file '%s': %s\n", availableEnvFile, err)
+		os.Exit(1)
+	}
+
+	return result
 }
 
 func main() {
@@ -189,26 +235,7 @@ func main() {
 	shared.ReexecWithDelveMaybe(delveListen, delvePath)
 	android.InitSandbox(topDir)
 
-	// soong_ui dumps the available environment variables to
+	availableEnv := parseAvailableEnv()
-	// soong.environment.available . Then soong_build itself is run with an empty
-	// environment so that the only way environment variables can be accessed is
-	// using Config, which tracks access to them.
-
-	// At the end of the build, a file called soong.environment.used is written
-	// containing the current value of all used environment variables. The next
-	// time soong_ui is run, it checks whether any environment variables that was
-	// used had changed and if so, it deletes soong.environment.used to cause a
-	// rebuild.
-	//
-	// The dependency of build.ninja on soong.environment.used is declared in
-	// build.ninja.d
-	availableEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.available")
-	usedEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.used")
-	availableEnv, err := shared.EnvFromFile(availableEnvFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "error reading available environment file %s: %s\n", availableEnvFile, err)
-		os.Exit(1)
-	}
 
 	// The top-level Blueprints file is passed as the first argument.
 	srcDir := filepath.Dir(flag.Arg(0))
@@ -233,37 +260,37 @@ func main() {
 		// because that is done from within the actual builds as a Ninja action and
 		// thus it would overwrite the actual used variables file so this is
 		// special-cased.
+		// TODO: Fix this by not passing --used_env to the soong_docs invocation
 		runSoongDocs(configuration, extraNinjaDeps)
 		return
 	}
 
-	doChosenActivity(configuration, extraNinjaDeps)
+	finalOutputFile := doChosenActivity(configuration, extraNinjaDeps)
-	writeUsedEnvironmentFile(usedEnvFile, configuration)
+	writeUsedEnvironmentFile(configuration, finalOutputFile)
 }
 
-func writeUsedEnvironmentFile(path string, configuration android.Config) {
+func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) {
+	if usedEnvFile == "" {
+		return
+	}
+
+	path := shared.JoinPath(topDir, usedEnvFile)
 	data, err := shared.EnvFileContents(configuration.EnvDeps())
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err)
+		fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
 		os.Exit(1)
 	}
 
 	err = ioutil.WriteFile(path, data, 0666)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err)
+		fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
 		os.Exit(1)
 	}
 
-	// Touch the output Ninja file so that it's not older than the file we just
+	// Touch the output file so that it's not older than the file we just
 	// wrote. We can't write the environment file earlier because one an access
 	// new environment variables while writing it.
-	outputNinjaFile := shared.JoinPath(topDir, bootstrap.CmdlineOutFile())
+	touch(shared.JoinPath(topDir, finalOutputFile))
-	currentTime := time.Now().Local()
-	err = os.Chtimes(outputNinjaFile, currentTime, currentTime)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "error touching output file %s: %s\n", outputNinjaFile, err)
-		os.Exit(1)
-	}
 }
 
 // Workarounds to support running bp2build in a clean AOSP checkout with no
@@ -289,6 +316,27 @@ func writeFakeNinjaFile(extraNinjaDeps []string, buildDir string) {
 		0666)
 }
 
+func touch(path string) {
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
+		os.Exit(1)
+	}
+
+	err = f.Close()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
+		os.Exit(1)
+	}
+
+	currentTime := time.Now().Local()
+	err = os.Chtimes(path, currentTime, currentTime)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error touching '%s': %s\n", path, err)
+		os.Exit(1)
+	}
+}
+
 // Run Soong in the bp2build mode. This creates a standalone context that registers
 // an alternate pipeline of mutators and singletons specifically for generating
 // Bazel BUILD files instead of Ninja files.
@@ -296,6 +344,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
 	// Register an alternate set of singletons and mutators for bazel
 	// conversion for Bazel conversion.
 	bp2buildCtx := android.NewContext(configuration)
+	bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
 	bp2buildCtx.RegisterForBazelConversion()
 
 	// No need to generate Ninja build rules/statements from Modules and Singletons.
@@ -330,5 +379,9 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
 	metrics.Print()
 
 	extraNinjaDeps = append(extraNinjaDeps, codegenContext.AdditionalNinjaDeps()...)
+	if bp2buildMarker != "" {
+		touch(shared.JoinPath(topDir, bp2buildMarker))
+	} else {
 		writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
+	}
 }

tests/bootstrap_test.sh

@@ -114,7 +114,9 @@ EOF
   rm a/Android.bp
   run_soong
 
-  grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja && fail "Old module in output"
+  if grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja; then
+    fail "Old module in output"
+  fi
 }
 
 function test_add_file_to_glob() {
@@ -404,7 +406,9 @@ EOF
 
   grep -q "Engage" out/soong/build.ninja || fail "New action not present"
 
-  grep -q "Make it so" out/soong/build.ninja && fail "Original action still present"
+  if grep -q "Make it so" out/soong/build.ninja; then
+    fail "Original action still present"
+  fi
 }
 
 function test_null_build_after_docs {
@@ -421,6 +425,27 @@ function test_null_build_after_docs {
   fi
 }
 
+function test_integrated_bp2build_smoke {
+  setup
+  INTEGRATED_BP2BUILD=1 run_soong
+  if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then
+    fail "b2build marker file not created"
+  fi
+}
+
+function test_integrated_bp2build_null_build {
+  setup
+  INTEGRATED_BP2BUILD=1 run_soong
+  local mtime1=$(stat -c "%y" out/soong/build.ninja)
+
+  INTEGRATED_BP2BUILD=1 run_soong
+  local mtime2=$(stat -c "%y" out/soong/build.ninja)
+
+  if [[ "$mtime1" != "$mtime2" ]]; then
+    fail "Output Ninja file changed on null build"
+  fi
+}
+
 function test_dump_json_module_graph() {
   setup
   SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong
@@ -441,3 +466,5 @@ test_add_file_to_soong_build
 test_glob_during_bootstrapping
 test_soong_build_rerun_iff_environment_changes
 test_dump_json_module_graph
+test_integrated_bp2build_smoke
+test_integrated_bp2build_null_build

ui/build/soong.go

@@ -33,6 +33,11 @@ import (
 	"android/soong/ui/status"
 )
 
+const (
+	availableEnvFile = "soong.environment.available"
+	usedEnvFile      = "soong.environment.used"
+)
+
 func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
 	data, err := shared.EnvFileContents(envDeps)
 	if err != nil {
@@ -87,12 +92,22 @@ func (c BlueprintConfig) DebugCompilation() bool {
 	return c.debugCompilation
 }
 
-func bootstrapBlueprint(ctx Context, config Config) {
+func environmentArgs(config Config, suffix string) []string {
+	return []string{
+		"--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
+		"--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix),
+	}
+}
+func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
 	ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
 	defer ctx.EndTrace()
 
 	var args bootstrap.Args
 
+	mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
+	globFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
+	bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
+
 	args.RunGoTests = !config.skipSoongTests
 	args.UseValidations = true // Use validations to depend on tests
 	args.BuildDir = config.SoongOutDir()
@@ -101,7 +116,7 @@ func bootstrapBlueprint(ctx Context, config Config) {
 	args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
 	args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
 	args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
-	args.GlobFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
+	args.GlobFile = globFile
 	args.GeneratingPrimaryBuilder = true
 
 	args.DelveListen = os.Getenv("SOONG_DELVE")
@@ -109,6 +124,44 @@ func bootstrapBlueprint(ctx Context, config Config) {
 		args.DelvePath = shared.ResolveDelveBinary()
 	}
 
+	commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
+	bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
+	mainSoongBuildInputs := []string{"Android.bp"}
+
+	if integratedBp2Build {
+		mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
+	}
+
+	soongBuildArgs := make([]string, 0)
+	soongBuildArgs = append(soongBuildArgs, commonArgs...)
+	soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
+	soongBuildArgs = append(soongBuildArgs, "Android.bp")
+
+	mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
+		Inputs:  mainSoongBuildInputs,
+		Outputs: []string{mainNinjaFile},
+		Args:    soongBuildArgs,
+	}
+
+	if integratedBp2Build {
+		bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
+		bp2buildArgs = append(bp2buildArgs, commonArgs...)
+		bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
+		bp2buildArgs = append(bp2buildArgs, "Android.bp")
+
+		bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
+			Inputs:  []string{"Android.bp"},
+			Outputs: []string{bp2BuildMarkerFile},
+			Args:    bp2buildArgs,
+		}
+		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
+			bp2buildInvocation,
+			mainSoongBuildInvocation,
+		}
+	} else {
+		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
+	}
+
 	blueprintCtx := blueprint.NewContext()
 	blueprintCtx.SetIgnoreUnknownModuleTypes(true)
 	blueprintConfig := BlueprintConfig{
@@ -121,6 +174,16 @@ func bootstrapBlueprint(ctx Context, config Config) {
 	bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
 }
 
+func checkEnvironmentFile(currentEnv *Environment, envFile string) {
+	getenv := func(k string) string {
+		v, _ := currentEnv.Get(k)
+		return v
+	}
+	if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
+		os.Remove(envFile)
+	}
+}
+
 func runSoong(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSoong, "soong")
 	defer ctx.EndTrace()
@@ -129,7 +192,7 @@ func runSoong(ctx Context, config Config) {
 	// .used with the ones that were actually used. The latter is used to
 	// determine whether Soong needs to be re-run since why re-run it if only
 	// unused variables were changed?
-	envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available")
+	envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
 
 	for _, n := range []string{".bootstrap", ".minibootstrap"} {
 		dir := filepath.Join(config.SoongOutDir(), n)
@@ -138,8 +201,10 @@ func runSoong(ctx Context, config Config) {
 		}
 	}
 
+	integratedBp2Build := config.Environment().IsEnvTrue("INTEGRATED_BP2BUILD")
+
 	// This is done unconditionally, but does not take a measurable amount of time
-	bootstrapBlueprint(ctx, config)
+	bootstrapBlueprint(ctx, config, integratedBp2Build)
 
 	soongBuildEnv := config.Environment().Copy()
 	soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@@ -164,13 +229,12 @@ func runSoong(ctx Context, config Config) {
 		ctx.BeginTrace(metrics.RunSoong, "environment check")
 		defer ctx.EndTrace()
 
-		envFile := filepath.Join(config.SoongOutDir(), "soong.environment.used")
+		soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile)
-		getenv := func(k string) string {
+		checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile)
-			v, _ := soongBuildEnv.Get(k)
+
-			return v
+		if integratedBp2Build {
-		}
+			bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build")
-		if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
+			checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile)
-			os.Remove(envFile)
 		}
 	}()