Separate the files of bp2build and soong_build.

The following files are forked:

- build-globs.ninja
- The glob list files
- .d files for the glob list files
- The output file (build.ninja or the bp2build marker file)

This makes bp2build and soong_build not overwrite each other's files
they need for proper incrementality.

Test: Presubmits.

Change-Id: I69d192cbd36ecd9677f46f3fa095dfce6f872227

cmd/soong_build/main.go

@@ -472,10 +472,10 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
 	// Generate out/soong/.bootstrap/build-globs.ninja with the actions to generate flattened globfiles
 	// containing the globs seen during bp2build conversion
 	if blueprintArgs.GlobFile != "" {
-		bootstrap.WriteBuildGlobsNinjaFile(bootstrap.StageMain, bp2buildCtx.Context, blueprintArgs, configuration)
+		bootstrap.WriteBuildGlobsNinjaFile(blueprintArgs.GlobListDir, bp2buildCtx.Context, blueprintArgs, configuration)
 	}
 	// Add the depfile on the expanded globs in out/soong/.primary/globs
-	ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration)...)
+	ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration, blueprintArgs.GlobListDir)...)
 
 	// Run the code-generation phase to convert BazelTargetModules to BUILD files
 	// and print conversion metrics to the user.
@@ -528,12 +528,4 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
 
 	// Create an empty bp2build marker file.
 	touch(shared.JoinPath(topDir, bp2buildMarker))
-
-	// bp2build *always* writes a fake Ninja file containing just the nothing
-	// phony target if it ever re-runs. This allows bp2build to exit early with
-	// GENERATE_BAZEL_FILES=1 m nothing.
-	//
-	// If bp2build is invoked as part of an integrated mixed build, the fake
-	// build.ninja file will be rewritten later into the real file anyway.
-	writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
 }

tests/bootstrap_test.sh

@@ -144,7 +144,7 @@ EOF
   run_soong
   local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)
 
-  local glob_deps_file=out/soong/.primary/globs/0.d
+  local glob_deps_file=out/soong/.bootstrap/globs/0.d
 
   if [ -e "$glob_deps_file" ]; then
     fail "Glob deps file unexpectedly written on first build"
@@ -526,18 +526,14 @@ function test_bp2build_smoke {
   [[ -e out/soong/workspace ]] || fail "Bazel workspace not created"
 }
 
-function test_bp2build_generates_fake_ninja_file {
+function test_bp2build_generates_marker_file {
   setup
   create_mock_bazel
 
   run_bp2build
 
-  if [[ ! -f "./out/soong/build.ninja" ]]; then
+  if [[ ! -f "./out/soong/.bootstrap/bp2build_workspace_marker" ]]; then
-    fail "./out/soong/build.ninja was not generated"
+    fail "Marker file was not generated"
-  fi
-
-  if ! grep "build nothing: phony" "./out/soong/build.ninja"; then
-    fail "missing phony nothing target in out/soong/build.ninja"
   fi
 }
 
@@ -577,10 +573,10 @@ function test_bp2build_null_build {
   setup
 
   GENERATE_BAZEL_FILES=1 run_soong
-  local mtime1=$(stat -c "%y" out/soong/build.ninja)
+  local mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
 
   GENERATE_BAZEL_FILES=1 run_soong
-  local mtime2=$(stat -c "%y" out/soong/build.ninja)
+  local mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
 
   if [[ "$mtime1" != "$mtime2" ]]; then
     fail "Output Ninja file changed on null build"
@@ -712,6 +708,41 @@ EOF
   grep -q "b/${GENERATED_BUILD_FILE_NAME}' exist" "$MOCK_TOP/errors" || fail "Error for b/${GENERATED_BUILD_FILE_NAME} not found"
 }
 
+function test_bp2build_back_and_forth_null_build {
+  setup
+
+  run_soong
+  local output_mtime1=$(stat -c "%y" out/soong/build.ninja)
+
+  GENERATE_BAZEL_FILES=1 run_soong
+  local output_mtime2=$(stat -c "%y" out/soong/build.ninja)
+  if [[ "$output_mtime1" != "$output_mtime2" ]]; then
+    fail "Output Ninja file changed when switching to bp2build"
+  fi
+
+  local marker_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
+
+  run_soong
+  local output_mtime3=$(stat -c "%y" out/soong/build.ninja)
+  local marker_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
+  if [[ "$output_mtime1" != "$output_mtime3" ]]; then
+    fail "Output Ninja file changed when switching to regular build from bp2build"
+  fi
+  if [[ "$marker_mtime1" != "$marker_mtime2" ]]; then
+    fail "bp2build marker file changed when switching to regular build from bp2build"
+  fi
+
+  GENERATE_BAZEL_FILES=1 run_soong
+  local output_mtime4=$(stat -c "%y" out/soong/build.ninja)
+  local marker_mtime3=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
+  if [[ "$output_mtime1" != "$output_mtime4" ]]; then
+    fail "Output Ninja file changed when switching back to bp2build"
+  fi
+  if [[ "$marker_mtime1" != "$marker_mtime3" ]]; then
+    fail "bp2build marker file changed when switching back to bp2build"
+  fi
+}
+
 test_smoke
 test_null_build
 test_null_build_after_docs
@@ -727,8 +758,9 @@ test_soong_build_rerun_iff_environment_changes
 test_dump_json_module_graph
 test_write_to_source_tree
 test_bp2build_smoke
-test_bp2build_generates_fake_ninja_file
+test_bp2build_generates_marker_file
 test_bp2build_null_build
+test_bp2build_back_and_forth_null_build
 test_bp2build_add_android_bp
 test_bp2build_add_to_glob
 test_bp2build_bazel_workspace_structure

tests/bp2build_bazel_test.sh

@@ -48,60 +48,6 @@ EOF
 
 test_bp2build_null_build_with_globs
 
-function test_soong_after_bp2build_regenerates_build_globs_ninja() {
-  setup
-
-  mkdir -p foo/bar
-  cat > foo/bar/Android.bp <<'EOF'
-filegroup {
-    name: "bp2build-files",
-    srcs: ["bp2build.*"],
-    bazel_module: { bp2build_available: true },
-}
-
-filegroup {
-    name: "soong-files",
-    srcs: ["soong.*"],
-}
-EOF
-  touch foo/bar/bp2build.txt foo/bar/soong.txt
-
-  build_globs_file="out/soong/.bootstrap/build-globs.ninja"
-
-  # Test: the build-globs file for bp2build should only contain the bp2build-files
-  # glob, whereas the build-globs file for soong should contain both bp2build-files
-  # and soong-files globs.
-
-  run_bp2build
-  local output_mtime1=$(stat -c "%y" "${build_globs_file}")
-
-  if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then
-    fail "bp2build filegroup globs not found in bp2build's globs file"
-  fi
-
-  if grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then
-    fail "soong filegroup globs unexpectedly found in bp2build's globs file"
-  fi
-
-  run_soong
-  local output_mtime2=$(stat -c "%y" "${build_globs_file}")
-
-  if [[ "$output_mtime1" == "$output_mtime2" ]]; then
-    fail "Output build-globs.ninja file did not change"
-  fi
-
-  if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then
-    fail "bp2build filegroup globs not found in bp2build's globs file"
-  fi
-
-  if ! grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then
-    fail "soong filegroup globs not found in bp2build's globs file"
-  fi
-
-}
-
-test_soong_after_bp2build_regenerates_build_globs_ninja
-
 function test_bp2build_generates_all_buildfiles {
   setup
   create_mock_bazel

ui/build/config.go

@@ -739,6 +739,14 @@ func (c *configImpl) SoongOutDir() string {
 	return filepath.Join(c.OutDir(), "soong")
 }
 
+func (c *configImpl) MainNinjaFile() string {
+	return shared.JoinPath(c.SoongOutDir(), "build.ninja")
+}
+
+func (c *configImpl) Bp2BuildMarkerFile() string {
+	return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
+}
+
 func (c *configImpl) TempDir() string {
 	return shared.TempDirForOutDir(c.SoongOutDir())
 }

ui/build/soong.go

@@ -114,17 +114,20 @@ func writeEmptyGlobFile(ctx Context, path string) {
 	}
 }
 
-func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
+func bootstrapBlueprint(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
 	defer ctx.EndTrace()
 
 	var args bootstrap.Args
 
-	mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
 	bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
-	// .bootstrap/build.ninja "includes" .bootstrap/build-globs.ninja for incremental builds
+	bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")
-	// generate an empty glob before running any rule in .bootstrap/build.ninja
+
+	// The glob .ninja files are subninja'd. However, they are generated during
+	// the build itself so we write an empty file so that the subninja doesn't
+	// fail on clean builds
 	writeEmptyGlobFile(ctx, bootstrapGlobFile)
+	writeEmptyGlobFile(ctx, bp2buildGlobFile)
 	bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
 
 	args.RunGoTests = !config.skipSoongTests
@@ -137,7 +140,7 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
 	// The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d)
 	// Building soong_build does not require a glob file
 	// Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary
-	args.GlobFile = ""
+	args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile}
 	args.GeneratingPrimaryBuilder = true
 	args.EmptyNinjaFile = config.EmptyNinjaFile()
 
@@ -146,42 +149,46 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
 		args.DelvePath = shared.ResolveDelveBinary()
 	}
 
-	commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
+	commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, config.MainNinjaFile())
-	bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
 	mainSoongBuildInputs := []string{"Android.bp"}
 
-	if integratedBp2Build {
+	if config.bazelBuildMode() == mixedBuild {
-		mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
+		mainSoongBuildInputs = append(mainSoongBuildInputs, config.Bp2BuildMarkerFile())
+	}
+
+	soongBuildArgs := []string{
+		"--globListDir", "globs",
+		"--globFile", bootstrapGlobFile,
 	}
 
-	soongBuildArgs := make([]string, 0)
 	soongBuildArgs = append(soongBuildArgs, commonArgs...)
 	soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
 	soongBuildArgs = append(soongBuildArgs, "Android.bp")
 
 	mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
 		Inputs:  mainSoongBuildInputs,
-		Outputs: []string{mainNinjaFile},
+		Outputs: []string{config.MainNinjaFile()},
 		Args:    soongBuildArgs,
 	}
 
-	if integratedBp2Build {
+	bp2buildArgs := []string{
-		bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
+		"--bp2build_marker", config.Bp2BuildMarkerFile(),
+		"--globListDir", "globs.bp2build",
+		"--globFile", bp2buildGlobFile,
+	}
+
 	bp2buildArgs = append(bp2buildArgs, commonArgs...)
 	bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
 	bp2buildArgs = append(bp2buildArgs, "Android.bp")
 
 	bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
 		Inputs:  []string{"Android.bp"},
-			Outputs: []string{bp2BuildMarkerFile},
+		Outputs: []string{config.Bp2BuildMarkerFile()},
 		Args:    bp2buildArgs,
 	}
-		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{bp2buildInvocation}
+	args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
-		if config.bazelBuildMode() == mixedBuild {
+		bp2buildInvocation,
-			args.PrimaryBuilderInvocations = append(args.PrimaryBuilderInvocations, mainSoongBuildInvocation)
+		mainSoongBuildInvocation,
-		}
-	} else {
-		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
 	}
 
 	blueprintCtx := blueprint.NewContext()
@@ -231,7 +238,7 @@ func runSoong(ctx Context, config Config) {
 	integratedBp2Build := (buildMode == mixedBuild) || (buildMode == generateBuildFiles)
 
 	// This is done unconditionally, but does not take a measurable amount of time
-	bootstrapBlueprint(ctx, config, integratedBp2Build)
+	bootstrapBlueprint(ctx, config)
 
 	soongBuildEnv := config.Environment().Copy()
 	soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@@ -268,7 +275,7 @@ func runSoong(ctx Context, config Config) {
 	runMicrofactory(ctx, config, ".minibootstrap/bpglob", "github.com/google/blueprint/bootstrap/bpglob",
 		map[string]string{"github.com/google/blueprint": "build/blueprint"})
 
-	ninja := func(name, file string) {
+	ninja := func(name, ninjaFile string, targets ...string) {
 		ctx.BeginTrace(metrics.RunSoong, name)
 		defer ctx.EndTrace()
 
@@ -276,8 +283,7 @@ func runSoong(ctx Context, config Config) {
 		nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
 		defer nr.Close()
 
-		cmd := Command(ctx, config, "soong "+name,
+		ninjaArgs := []string{
-			config.PrebuiltBuildTool("ninja"),
 			"-d", "keepdepfile",
 			"-d", "stats",
 			"-o", "usesphonyoutputs=yes",
@@ -287,7 +293,12 @@ func runSoong(ctx Context, config Config) {
 			"-w", "missingoutfile=err",
 			"-j", strconv.Itoa(config.Parallel()),
 			"--frontend_file", fifo,
-			"-f", filepath.Join(config.SoongOutDir(), file))
+			"-f", filepath.Join(config.SoongOutDir(), ninjaFile),
+		}
+
+		ninjaArgs = append(ninjaArgs, targets...)
+		cmd := Command(ctx, config, "soong "+name,
+			config.PrebuiltBuildTool("ninja"), ninjaArgs...)
 
 		var ninjaEnv Environment
 
@@ -299,8 +310,17 @@ func runSoong(ctx Context, config Config) {
 		cmd.Sandbox = soongSandbox
 		cmd.RunAndStreamOrFatal()
 	}
+
+	var target string
+
+	if config.bazelBuildMode() == generateBuildFiles {
+		target = config.Bp2BuildMarkerFile()
+	} else {
 		// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
-	ninja("bootstrap", ".bootstrap/build.ninja")
+		target = config.MainNinjaFile()
+	}
+
+	ninja("bootstrap", ".bootstrap/build.ninja", target)
 
 	var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
 	if shouldCollectBuildSoongMetrics(config) {