Merge "Experimental code to support build action caching." into main am: 88f4e331cc

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/3124233

Change-Id: Iaecbc44d1b368c0807d802d659ee8d2d54a55f30
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

aconfig/aconfig_declarations.go

@@ -32,6 +32,7 @@ type AconfigReleaseConfigValue struct {
 type DeclarationsModule struct {
 	android.ModuleBase
 	android.DefaultableModuleBase
+	blueprint.IncrementalModule
 
 	// Properties for "aconfig_declarations"
 	properties struct {
@@ -217,3 +218,17 @@ func (module *DeclarationsModule) GenerateAndroidBuildActions(ctx android.Module
 	android.SetProvider(ctx, android.AconfigDeclarationsProviderKey, providerData[""])
 	android.SetProvider(ctx, android.AconfigReleaseDeclarationsProviderKey, providerData)
 }
+
+func (module *DeclarationsModule) BuildActionProviderKeys() []blueprint.AnyProviderKey {
+	return []blueprint.AnyProviderKey{android.AconfigDeclarationsProviderKey}
+}
+
+func (module *DeclarationsModule) PackageContextPath() string {
+	return pkgPath
+}
+
+func (module *DeclarationsModule) CachedRules() []blueprint.Rule {
+	return []blueprint.Rule{aconfigRule, aconfigTextRule}
+}
+
+var _ blueprint.Incremental = &DeclarationsModule{}

aconfig/init.go

@@ -15,13 +15,16 @@
 package aconfig
 
 import (
+	"encoding/gob"
+
 	"android/soong/android"
 
 	"github.com/google/blueprint"
 )
 
 var (
-	pctx = android.NewPackageContext("android/soong/aconfig")
+	pkgPath = "android/soong/aconfig"
+	pctx    = android.NewPackageContext(pkgPath)
 
 	// For aconfig_declarations: Generate cache file
 	aconfigRule = pctx.AndroidStaticRule("aconfig",
@@ -106,6 +109,9 @@ func init() {
 	RegisterBuildComponents(android.InitRegistrationContext)
 	pctx.HostBinToolVariable("aconfig", "aconfig")
 	pctx.HostBinToolVariable("soong_zip", "soong_zip")
+
+	gob.Register(android.AconfigDeclarationsProviderData{})
+	gob.Register(android.ModuleOutPath{})
 }
 
 func RegisterBuildComponents(ctx android.RegistrationContext) {

android/module.go

@@ -1913,9 +1913,54 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext)
 			return
 		}
 
-		m.module.GenerateAndroidBuildActions(ctx)
+		incrementalAnalysis := false
-		if ctx.Failed() {
+		incrementalEnabled := false
-			return
+		var cacheKey *blueprint.BuildActionCacheKey = nil
+		var incrementalModule *blueprint.Incremental = nil
+		if ctx.bp.GetIncrementalEnabled() {
+			if im, ok := m.module.(blueprint.Incremental); ok {
+				incrementalModule = &im
+				incrementalEnabled = im.IncrementalSupported()
+				incrementalAnalysis = ctx.bp.GetIncrementalAnalysis() && incrementalEnabled
+			}
+		}
+		if incrementalEnabled {
+			hash, err := proptools.CalculateHash(m.GetProperties())
+			if err != nil {
+				ctx.ModuleErrorf("failed to calculate properties hash: %s", err)
+				return
+			}
+			cacheInput := new(blueprint.BuildActionCacheInput)
+			cacheInput.PropertiesHash = hash
+			ctx.VisitDirectDeps(func(module Module) {
+				cacheInput.ProvidersHash =
+					append(cacheInput.ProvidersHash, ctx.bp.OtherModuleProviderInitialValueHashes(module))
+			})
+			hash, err = proptools.CalculateHash(&cacheInput)
+			if err != nil {
+				ctx.ModuleErrorf("failed to calculate cache input hash: %s", err)
+				return
+			}
+			cacheKey = &blueprint.BuildActionCacheKey{
+				Id:        ctx.bp.ModuleId(),
+				InputHash: hash,
+			}
+		}
+
+		restored := false
+		if incrementalAnalysis && cacheKey != nil {
+			restored = ctx.bp.RestoreBuildActions(cacheKey, incrementalModule)
+		}
+
+		if !restored {
+			m.module.GenerateAndroidBuildActions(ctx)
+			if ctx.Failed() {
+				return
+			}
+		}
+
+		if incrementalEnabled && cacheKey != nil {
+			ctx.bp.CacheBuildActions(cacheKey, incrementalModule)
 		}
 
 		// Create the set of tagged dist files after calling GenerateAndroidBuildActions

android/paths.go

@@ -15,6 +15,9 @@
 package android
 
 import (
+	"bytes"
+	"encoding/gob"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -1068,6 +1071,28 @@ type basePath struct {
 	rel  string
 }
 
+func (p basePath) GobEncode() ([]byte, error) {
+	w := new(bytes.Buffer)
+	encoder := gob.NewEncoder(w)
+	err := errors.Join(encoder.Encode(p.path), encoder.Encode(p.rel))
+	if err != nil {
+		return nil, err
+	}
+
+	return w.Bytes(), nil
+}
+
+func (p *basePath) GobDecode(data []byte) error {
+	r := bytes.NewBuffer(data)
+	decoder := gob.NewDecoder(r)
+	err := errors.Join(decoder.Decode(&p.path), decoder.Decode(&p.rel))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (p basePath) Ext() string {
 	return filepath.Ext(p.path)
 }
@@ -1306,6 +1331,28 @@ type OutputPath struct {
 	fullPath string
 }
 
+func (p OutputPath) GobEncode() ([]byte, error) {
+	w := new(bytes.Buffer)
+	encoder := gob.NewEncoder(w)
+	err := errors.Join(encoder.Encode(p.basePath), encoder.Encode(p.soongOutDir), encoder.Encode(p.fullPath))
+	if err != nil {
+		return nil, err
+	}
+
+	return w.Bytes(), nil
+}
+
+func (p *OutputPath) GobDecode(data []byte) error {
+	r := bytes.NewBuffer(data)
+	decoder := gob.NewDecoder(r)
+	err := errors.Join(decoder.Decode(&p.basePath), decoder.Decode(&p.soongOutDir), decoder.Decode(&p.fullPath))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (p OutputPath) withRel(rel string) OutputPath {
 	p.basePath = p.basePath.withRel(rel)
 	p.fullPath = filepath.Join(p.fullPath, rel)

cmd/soong_build/main.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -28,11 +29,11 @@ import (
 	"android/soong/android/allowlists"
 	"android/soong/bp2build"
 	"android/soong/shared"
-
 	"github.com/google/blueprint"
 	"github.com/google/blueprint/bootstrap"
 	"github.com/google/blueprint/deptools"
 	"github.com/google/blueprint/metrics"
+	"github.com/google/blueprint/proptools"
 	androidProtobuf "google.golang.org/protobuf/android"
 )
 
@@ -49,6 +50,14 @@ var (
 	cmdlineArgs android.CmdArgs
 )
 
+const configCacheFile = "config.cache"
+
+type ConfigCache struct {
+	EnvDepsHash                  uint64
+	ProductVariableFileTimestamp int64
+	SoongBuildFileTimestamp      int64
+}
+
 func init() {
 	// Flags that make sense in every mode
 	flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
@@ -82,6 +91,7 @@ func init() {
 	// Flags that probably shouldn't be flags of soong_build, but we haven't found
 	// the time to remove them yet
 	flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")
+	flag.BoolVar(&cmdlineArgs.IncrementalBuildActions, "incremental-build-actions", false, "generate build actions incrementally")
 
 	// Disable deterministic randomization in the protobuf package, so incremental
 	// builds with unrelated Soong changes don't trigger large rebuilds (since we
@@ -218,6 +228,60 @@ func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDe
 	maybeQuit(err, "error writing depfile '%s'", depFile)
 }
 
+// Check if there are changes to the environment file, product variable file and
+// soong_build binary, in which case no incremental will be performed.
+func incrementalValid(config android.Config, configCacheFile string) (*ConfigCache, bool) {
+	var newConfigCache ConfigCache
+	data, err := os.ReadFile(shared.JoinPath(topDir, usedEnvFile))
+	if err != nil {
+		// Clean build
+		if os.IsNotExist(err) {
+			data = []byte{}
+		} else {
+			maybeQuit(err, "")
+		}
+	}
+
+	newConfigCache.EnvDepsHash, err = proptools.CalculateHash(data)
+	newConfigCache.ProductVariableFileTimestamp = getFileTimestamp(filepath.Join(topDir, cmdlineArgs.SoongVariables))
+	newConfigCache.SoongBuildFileTimestamp = getFileTimestamp(filepath.Join(topDir, config.HostToolDir(), "soong_build"))
+	//TODO(b/344917959): out/soong/dexpreopt.config might need to be checked as well.
+
+	file, err := os.Open(configCacheFile)
+	if err != nil && os.IsNotExist(err) {
+		return &newConfigCache, false
+	}
+	maybeQuit(err, "")
+	defer file.Close()
+
+	var configCache ConfigCache
+	decoder := json.NewDecoder(file)
+	err = decoder.Decode(&configCache)
+	maybeQuit(err, "")
+
+	return &newConfigCache, newConfigCache == configCache
+}
+
+func getFileTimestamp(file string) int64 {
+	stat, err := os.Stat(file)
+	if err == nil {
+		return stat.ModTime().UnixMilli()
+	} else if !os.IsNotExist(err) {
+		maybeQuit(err, "")
+	}
+	return 0
+}
+
+func writeConfigCache(configCache *ConfigCache, configCacheFile string) {
+	file, err := os.Create(configCacheFile)
+	maybeQuit(err, "")
+	defer file.Close()
+
+	encoder := json.NewEncoder(file)
+	err = encoder.Encode(*configCache)
+	maybeQuit(err, "")
+}
+
 // runSoongOnlyBuild runs the standard Soong build in a number of different modes.
 func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
 	ctx.EventHandler.Begin("soong_build")
@@ -319,8 +383,26 @@ func main() {
 	ctx := newContext(configuration)
 	android.StartBackgroundMetrics(configuration)
 
+	var configCache *ConfigCache
+	configFile := filepath.Join(topDir, ctx.Config().OutDir(), configCacheFile)
+	incremental := false
+	ctx.SetIncrementalEnabled(cmdlineArgs.IncrementalBuildActions)
+	if cmdlineArgs.IncrementalBuildActions {
+		configCache, incremental = incrementalValid(ctx.Config(), configFile)
+	}
+	ctx.SetIncrementalAnalysis(incremental)
+
 	ctx.Register()
 	finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps)
+
+	if ctx.GetIncrementalEnabled() {
+		data, err := shared.EnvFileContents(configuration.EnvDeps())
+		maybeQuit(err, "")
+		configCache.EnvDepsHash, err = proptools.CalculateHash(data)
+		maybeQuit(err, "")
+		writeConfigCache(configCache, configFile)
+	}
+
 	writeMetrics(configuration, ctx.EventHandler, metricsDir)
 
 	writeUsedEnvironmentFile(configuration)

ui/build/config.go

@@ -85,6 +85,7 @@ type configImpl struct {
 	skipMetricsUpload        bool
 	buildStartedTime         int64 // For metrics-upload-only - manually specify a build-started time
 	buildFromSourceStub      bool
+	incrementalBuildActions  bool
 	ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
 
 	// From the product config
@@ -811,6 +812,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
 			}
 		} else if arg == "--build-from-source-stub" {
 			c.buildFromSourceStub = true
+		} else if arg == "--incremental-build-actions" {
+			c.incrementalBuildActions = true
 		} else if strings.HasPrefix(arg, "--build-command=") {
 			buildCmd := strings.TrimPrefix(arg, "--build-command=")
 			// remove quotations

ui/build/soong.go

@@ -315,6 +315,9 @@ func bootstrapBlueprint(ctx Context, config Config) {
 	if config.ensureAllowlistIntegrity {
 		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--ensure-allowlist-integrity")
 	}
+	if config.incrementalBuildActions {
+		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--incremental-build-actions")
+	}
 
 	queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")