Merge "Move bpglob out of ToolDir" am: fe8ceadabf

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1875758

Change-Id: Idc2b87b2b0dc0a8e4c682d0007d346bfa9c2a69e

ui/build/soong.go

@@ -15,6 +15,7 @@
 package build
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -43,6 +44,12 @@ const (
 	jsonModuleGraphTag = "modulegraph"
 	queryviewTag       = "queryview"
 	soongDocsTag       = "soong_docs"
+
+	// bootstrapEpoch is used to determine if an incremental build is incompatible with the current
+	// version of bootstrap and needs cleaning before continuing the build.  Increment this for
+	// incompatible changes, for example when moving the location of the bpglob binary that is
+	// executed during bootstrap before the primary builder has had a chance to update the path.
+	bootstrapEpoch = 0
 )
 
 func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
@@ -121,20 +128,31 @@ func environmentArgs(config Config, tag string) []string {
 	}
 }
 
-func writeEmptyGlobFile(ctx Context, path string) {
+func writeEmptyFile(ctx Context, path string) {
 	err := os.MkdirAll(filepath.Dir(path), 0777)
 	if err != nil {
-		ctx.Fatalf("Failed to create parent directories of empty ninja glob file '%s': %s", path, err)
+		ctx.Fatalf("Failed to create parent directories of empty file '%s': %s", path, err)
 	}
 
-	if _, err := os.Stat(path); os.IsNotExist(err) {
+	if exists, err := fileExists(path); err != nil {
+		ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
+	} else if !exists {
 		err = ioutil.WriteFile(path, nil, 0666)
 		if err != nil {
-			ctx.Fatalf("Failed to create empty ninja glob file '%s': %s", path, err)
+			ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
 		}
 	}
 }
 
+func fileExists(path string) (bool, error) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
 func primaryBuilderInvocation(config Config, name string, output string, specificArgs []string) bootstrap.PrimaryBuilderInvocation {
 	commonArgs := make([]string, 0, 0)
 
@@ -166,10 +184,45 @@ func primaryBuilderInvocation(config Config, name string, output string, specifi
 	}
 }
 
+// bootstrapEpochCleanup deletes files used by bootstrap during incremental builds across
+// incompatible changes.  Incompatible changes are marked by incrementing the bootstrapEpoch
+// constant.  A tree is considered out of date for the current epoch of the
+// .soong.bootstrap.epoch.<epoch> file doesn't exist.
+func bootstrapEpochCleanup(ctx Context, config Config) {
+	epochFile := fmt.Sprintf(".soong.bootstrap.epoch.%d", bootstrapEpoch)
+	epochPath := filepath.Join(config.SoongOutDir(), epochFile)
+	if exists, err := fileExists(epochPath); err != nil {
+		ctx.Fatalf("failed to check if bootstrap epoch file %q exists: %q", epochPath, err)
+	} else if !exists {
+		// The tree is out of date for the current epoch, delete files used by bootstrap
+		// and force the primary builder to rerun.
+		os.Remove(filepath.Join(config.SoongOutDir(), "build.ninja"))
+		for _, globFile := range bootstrapGlobFileList(config) {
+			os.Remove(globFile)
+		}
+
+		// Mark the tree as up to date with the current epoch by writing the epoch marker file.
+		writeEmptyFile(ctx, epochPath)
+	}
+}
+
+func bootstrapGlobFileList(config Config) []string {
+	return []string{
+		config.NamedGlobFile(soongBuildTag),
+		config.NamedGlobFile(bp2buildTag),
+		config.NamedGlobFile(jsonModuleGraphTag),
+		config.NamedGlobFile(queryviewTag),
+		config.NamedGlobFile(soongDocsTag),
+	}
+}
+
 func bootstrapBlueprint(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
 	defer ctx.EndTrace()
 
+	// Clean up some files for incremental builds across incompatible changes.
+	bootstrapEpochCleanup(ctx, config)
+
 	mainSoongBuildExtraArgs := []string{"-o", config.SoongNinjaFile()}
 	if config.EmptyNinjaFile() {
 		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
@@ -232,8 +285,8 @@ func bootstrapBlueprint(ctx Context, config Config) {
 	// The glob .ninja files are subninja'd. However, they are generated during
 	// the build itself so we write an empty file if the file does not exist yet
 	// so that the subninja doesn't fail on clean builds
-	for _, globFile := range globFiles {
+	for _, globFile := range bootstrapGlobFileList(config) {
-		writeEmptyGlobFile(ctx, globFile)
+		writeEmptyFile(ctx, globFile)
 	}
 
 	var blueprintArgs bootstrap.Args
@@ -342,7 +395,7 @@ func runSoong(ctx Context, config Config) {
 		}
 	}()
 
-	runMicrofactory(ctx, config, filepath.Join(config.HostToolDir(), "bpglob"), "github.com/google/blueprint/bootstrap/bpglob",
+	runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
 		map[string]string{"github.com/google/blueprint": "build/blueprint"})
 
 	ninja := func(name, ninjaFile string, targets ...string) {