StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "Use hashed subdir for soong_config modules"

Browse Source 
This reverts commit 81b00a8db7.

Reason for revert:
* select() will supersede Soong config modules.
* A tiny change can make hundreds of gigabytes rebuilt.
* Hashed out/ directories are not cleaned.
* Even without this trace, AB build time is fast enough, thanks to
  product-specific ninja files and so on.

Bug: 348548855
Test: m --no-skip-soong-tests
Change-Id: If9a97df1e161a9ef0fb1b801f9e129b71b11d1ac
This commit is contained in:
Inseob Kim
2024-06-25 17:39:52 +09:00
parent 9687618816
commit b7e9f5f035
6 changed files with 3 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@@ -594,12 +594,6 @@ modules (`cc_defaults`, `java_defaults`, etc.), which can then be referenced
by all of the vendor's other modules using the normal namespace and visibility by all of the vendor's other modules using the normal namespace and visibility
rules. rules.
`soongConfigTraceMutator` enables modules affected by soong config variables to
write outputs into a hashed directory path. It does this by recording accesses
to soong config variables on each module, and then accumulating records of each
module's all dependencies. `m soong_config_trace` builds information about
hashes to `$OUT_DIR/soong/soong_config_trace.json`.
## Build logic ## Build logic
The build logic is written in Go using the The build logic is written in Go using the

89
android/module.go
View File

@@ -15,9 +15,6 @@
package android package android
import ( import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt" "fmt"
"net/url" "net/url"
"path/filepath" "path/filepath"
@@ -249,31 +246,6 @@ func SortedUniqueNamedPaths(l NamedPaths) NamedPaths {
return l[:k+1] return l[:k+1]
} }
// soongConfigTrace holds all references to VendorVars. Uses []string for blueprint:"mutated"
type soongConfigTrace struct {
Bools []string `json:",omitempty"`
Strings []string `json:",omitempty"`
IsSets []string `json:",omitempty"`
}
func (c *soongConfigTrace) isEmpty() bool {
return len(c.Bools) == 0 && len(c.Strings) == 0 && len(c.IsSets) == 0
}
// Returns hash of serialized trace records (empty string if there's no trace recorded)
func (c *soongConfigTrace) hash() string {
// Use MD5 for speed. We don't care collision or preimage attack
if c.isEmpty() {
return ""
}
j, err := json.Marshal(c)
if err != nil {
panic(fmt.Errorf("json marshal of %#v failed: %#v", *c, err))
}
hash := md5.Sum(j)
return hex.EncodeToString(hash[:])
}
type nameProperties struct { type nameProperties struct {
// The name of the module. Must be unique across all modules. // The name of the module. Must be unique across all modules.
Name *string Name *string
@@ -525,14 +497,6 @@ type commonProperties struct {
// constants in image.go, but can also be set to a custom value by individual module types. // constants in image.go, but can also be set to a custom value by individual module types.
ImageVariation string `blueprint:"mutated"` ImageVariation string `blueprint:"mutated"`
// SoongConfigTrace records accesses to VendorVars (soong_config). The trace will be hashed
// and used as a subdir of PathForModuleOut. Note that we mainly focus on incremental
// builds among similar products (e.g. aosp_cf_x86_64_phone and aosp_cf_x86_64_foldable),
// and there are variables other than soong_config, which isn't captured by soong config
// trace, but influence modules among products.
SoongConfigTrace soongConfigTrace `blueprint:"mutated"`
SoongConfigTraceHash string `blueprint:"mutated"`
// The team (defined by the owner/vendor) who owns the property. // The team (defined by the owner/vendor) who owns the property.
Team *string `android:"path"` Team *string `android:"path"`
} }
@@ -2575,8 +2539,6 @@ type HostToolProvider interface {
func init() { func init() {
RegisterParallelSingletonType("buildtarget", BuildTargetSingleton) RegisterParallelSingletonType("buildtarget", BuildTargetSingleton)
RegisterParallelSingletonType("soongconfigtrace", soongConfigTraceSingletonFunc)
FinalDepsMutators(registerSoongConfigTraceMutator)
} }
func BuildTargetSingleton() Singleton { func BuildTargetSingleton() Singleton {
@@ -2738,54 +2700,3 @@ func CheckBlueprintSyntax(ctx BaseModuleContext, filename string, contents strin
bpctx := ctx.blueprintBaseModuleContext() bpctx := ctx.blueprintBaseModuleContext()
return blueprint.CheckBlueprintSyntax(bpctx.ModuleFactories(), filename, contents) return blueprint.CheckBlueprintSyntax(bpctx.ModuleFactories(), filename, contents)
} }
func registerSoongConfigTraceMutator(ctx RegisterMutatorsContext) {
ctx.BottomUp("soongconfigtrace", soongConfigTraceMutator).Parallel()
}
// soongConfigTraceMutator accumulates recorded soong_config trace from children. Also it normalizes
// SoongConfigTrace to make it consistent.
func soongConfigTraceMutator(ctx BottomUpMutatorContext) {
trace := &ctx.Module().base().commonProperties.SoongConfigTrace
ctx.VisitDirectDeps(func(m Module) {
childTrace := &m.base().commonProperties.SoongConfigTrace
trace.Bools = append(trace.Bools, childTrace.Bools...)
trace.Strings = append(trace.Strings, childTrace.Strings...)
trace.IsSets = append(trace.IsSets, childTrace.IsSets...)
})
trace.Bools = SortedUniqueStrings(trace.Bools)
trace.Strings = SortedUniqueStrings(trace.Strings)
trace.IsSets = SortedUniqueStrings(trace.IsSets)
ctx.Module().base().commonProperties.SoongConfigTraceHash = trace.hash()
}
// soongConfigTraceSingleton writes a map from each module's config hash value to trace data.
func soongConfigTraceSingletonFunc() Singleton {
return &soongConfigTraceSingleton{}
}
type soongConfigTraceSingleton struct {
}
func (s *soongConfigTraceSingleton) GenerateBuildActions(ctx SingletonContext) {
outFile := PathForOutput(ctx, "soong_config_trace.json")
traces := make(map[string]*soongConfigTrace)
ctx.VisitAllModules(func(module Module) {
trace := &module.base().commonProperties.SoongConfigTrace
if !trace.isEmpty() {
hash := module.base().commonProperties.SoongConfigTraceHash
traces[hash] = trace
}
})
j, err := json.Marshal(traces)
if err != nil {
ctx.Errorf("json marshal to %q failed: %#v", outFile, err)
return
}
WriteFileRule(ctx, outFile, string(j))
ctx.Phony("soong_config_trace", outFile)
}

5
android/module_context.go
View File

@@ -188,7 +188,6 @@ type ModuleContext interface {
TargetRequiredModuleNames() []string TargetRequiredModuleNames() []string
ModuleSubDir() string ModuleSubDir() string
SoongConfigTraceHash() string
Variable(pctx PackageContext, name, value string) Variable(pctx PackageContext, name, value string)
Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule
@@ -377,10 +376,6 @@ func (m *moduleContext) ModuleSubDir() string {
return m.bp.ModuleSubDir() return m.bp.ModuleSubDir()
} }
func (m *moduleContext) SoongConfigTraceHash() string {
return m.module.base().commonProperties.SoongConfigTraceHash
}
func (m *moduleContext) InstallInData() bool { func (m *moduleContext) InstallInData() bool {
return m.module.InstallInData() return m.module.InstallInData()
} }

3
android/paths.go
View File

@@ -1604,11 +1604,10 @@ type ModuleOutPathContext interface {
ModuleName() string ModuleName() string
ModuleDir() string ModuleDir() string
ModuleSubDir() string ModuleSubDir() string
SoongConfigTraceHash() string
} }
func pathForModuleOut(ctx ModuleOutPathContext) OutputPath { func pathForModuleOut(ctx ModuleOutPathContext) OutputPath {
return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir(), ctx.SoongConfigTraceHash()) return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir())
} }
// PathForModuleOut returns a Path representing the paths... under the module's // PathForModuleOut returns a Path representing the paths... under the module's

57
android/soong_config_modules.go
View File

@@ -463,57 +463,6 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s
}).(map[string]blueprint.ModuleFactory) }).(map[string]blueprint.ModuleFactory)
} }
// tracingConfig is a wrapper to soongconfig.SoongConfig which records all accesses to SoongConfig.
type tracingConfig struct {
config soongconfig.SoongConfig
boolSet map[string]bool
stringSet map[string]string
isSetSet map[string]bool
}
func (c *tracingConfig) Bool(name string) bool {
c.boolSet[name] = c.config.Bool(name)
return c.boolSet[name]
}
func (c *tracingConfig) String(name string) string {
c.stringSet[name] = c.config.String(name)
return c.stringSet[name]
}
func (c *tracingConfig) IsSet(name string) bool {
c.isSetSet[name] = c.config.IsSet(name)
return c.isSetSet[name]
}
func (c *tracingConfig) getTrace() soongConfigTrace {
ret := soongConfigTrace{}
for k, v := range c.boolSet {
ret.Bools = append(ret.Bools, fmt.Sprintf("%q:%t", k, v))
}
for k, v := range c.stringSet {
ret.Strings = append(ret.Strings, fmt.Sprintf("%q:%q", k, v))
}
for k, v := range c.isSetSet {
ret.IsSets = append(ret.IsSets, fmt.Sprintf("%q:%t", k, v))
}
return ret
}
func newTracingConfig(config soongconfig.SoongConfig) *tracingConfig {
c := tracingConfig{
config: config,
boolSet: make(map[string]bool),
stringSet: make(map[string]string),
isSetSet: make(map[string]bool),
}
return &c
}
var _ soongconfig.SoongConfig = (*tracingConfig)(nil)
// configModuleFactory takes an existing soongConfigModuleFactory and a // configModuleFactory takes an existing soongConfigModuleFactory and a
// ModuleType to create a new ModuleFactory that uses a custom loadhook. // ModuleType to create a new ModuleFactory that uses a custom loadhook.
func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfig.ModuleType) blueprint.ModuleFactory { func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfig.ModuleType) blueprint.ModuleFactory {
@@ -561,8 +510,8 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi
// conditional on Soong config variables by reading the product // conditional on Soong config variables by reading the product
// config variables from Make. // config variables from Make.
AddLoadHook(module, func(ctx LoadHookContext) { AddLoadHook(module, func(ctx LoadHookContext) {
tracingConfig := newTracingConfig(ctx.Config().VendorConfig(moduleType.ConfigNamespace)) config := ctx.Config().VendorConfig(moduleType.ConfigNamespace)
newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, tracingConfig) newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, config)
if err != nil { if err != nil {
ctx.ModuleErrorf("%s", err) ctx.ModuleErrorf("%s", err)
return return
@@ -570,8 +519,6 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi
for _, ps := range newProps { for _, ps := range newProps {
ctx.AppendProperties(ps) ctx.AppendProperties(ps)
} }
module.(Module).base().commonProperties.SoongConfigTrace = tracingConfig.getTrace()
}) })
return module, props return module, props
} }

195
android/soong_config_modules_test.go
View File

@@ -16,7 +16,6 @@ package android
import ( import (
"fmt" "fmt"
"path/filepath"
"testing" "testing"
) )
@@ -506,197 +505,3 @@ func TestSoongConfigModuleSingletonModule(t *testing.T) {
}) })
} }
} }
func TestSoongConfigModuleTrace(t *testing.T) {
bp := `
soong_config_module_type {
name: "acme_test",
module_type: "test",
config_namespace: "acme",
variables: ["board", "feature1", "FEATURE3", "unused_string_var"],
bool_variables: ["feature2", "unused_feature", "always_true"],
value_variables: ["size", "unused_size"],
properties: ["cflags", "srcs", "defaults"],
}
soong_config_module_type {
name: "acme_test_defaults",
module_type: "test_defaults",
config_namespace: "acme",
variables: ["board", "feature1", "FEATURE3", "unused_string_var"],
bool_variables: ["feature2", "unused_feature", "always_true"],
value_variables: ["size", "unused_size"],
properties: ["cflags", "srcs", "defaults"],
}
soong_config_string_variable {
name: "board",
values: ["soc_a", "soc_b", "soc_c"],
}
soong_config_string_variable {
name: "unused_string_var",
values: ["a", "b"],
}
soong_config_bool_variable {
name: "feature1",
}
soong_config_bool_variable {
name: "FEATURE3",
}
test_defaults {
name: "test_defaults",
cflags: ["DEFAULT"],
}
test {
name: "normal",
defaults: ["test_defaults"],
}
acme_test {
name: "board_1",
defaults: ["test_defaults"],
soong_config_variables: {
board: {
soc_a: {
cflags: ["-DSOC_A"],
},
},
},
}
acme_test {
name: "board_2",
defaults: ["test_defaults"],
soong_config_variables: {
board: {
soc_a: {
cflags: ["-DSOC_A"],
},
},
},
}
acme_test {
name: "size",
defaults: ["test_defaults"],
soong_config_variables: {
size: {
cflags: ["-DSIZE=%s"],
},
},
}
acme_test {
name: "board_and_size",
defaults: ["test_defaults"],
soong_config_variables: {
board: {
soc_a: {
cflags: ["-DSOC_A"],
},
},
size: {
cflags: ["-DSIZE=%s"],
},
},
}
acme_test_defaults {
name: "board_defaults",
soong_config_variables: {
board: {
soc_a: {
cflags: ["-DSOC_A"],
},
},
},
}
acme_test_defaults {
name: "size_defaults",
soong_config_variables: {
size: {
cflags: ["-DSIZE=%s"],
},
},
}
test {
name: "board_and_size_with_defaults",
defaults: ["board_defaults", "size_defaults"],
}
`
fixtureForVendorVars := func(vars map[string]map[string]string) FixturePreparer {
return FixtureModifyProductVariables(func(variables FixtureProductVariables) {
variables.VendorVars = vars
})
}
preparer := fixtureForVendorVars(map[string]map[string]string{
"acme": {
"board": "soc_a",
"size": "42",
"feature1": "true",
"feature2": "false",
// FEATURE3 unset
"unused_feature": "true", // unused
"unused_size": "1", // unused
"unused_string_var": "a", // unused
"always_true": "true",
},
})
t.Run("soong config trace hash", func(t *testing.T) {
result := GroupFixturePreparers(
preparer,
PrepareForTestWithDefaults,
PrepareForTestWithSoongConfigModuleBuildComponents,
prepareForSoongConfigTestModule,
FixtureRegisterWithContext(func(ctx RegistrationContext) {
ctx.FinalDepsMutators(registerSoongConfigTraceMutator)
}),
FixtureWithRootAndroidBp(bp),
).RunTest(t)
// Hashes of modules not using soong config should be empty
normal := result.ModuleForTests("normal", "").Module().(*soongConfigTestModule)
AssertDeepEquals(t, "normal hash", normal.base().commonProperties.SoongConfigTraceHash, "")
AssertDeepEquals(t, "normal hash out", normal.outputPath.RelativeToTop().String(), "out/soong/.intermediates/normal/test")
board1 := result.ModuleForTests("board_1", "").Module().(*soongConfigTestModule)
board2 := result.ModuleForTests("board_2", "").Module().(*soongConfigTestModule)
size := result.ModuleForTests("size", "").Module().(*soongConfigTestModule)
// Trace mutator sets soong config trace hash correctly
board1Hash := board1.base().commonProperties.SoongConfigTrace.hash()
board1Output := board1.outputPath.RelativeToTop().String()
AssertDeepEquals(t, "board hash calc", board1Hash, board1.base().commonProperties.SoongConfigTraceHash)
AssertDeepEquals(t, "board hash path", board1Output, filepath.Join("out/soong/.intermediates/board_1", board1Hash, "test"))
sizeHash := size.base().commonProperties.SoongConfigTrace.hash()
sizeOutput := size.outputPath.RelativeToTop().String()
AssertDeepEquals(t, "size hash calc", sizeHash, size.base().commonProperties.SoongConfigTraceHash)
AssertDeepEquals(t, "size hash path", sizeOutput, filepath.Join("out/soong/.intermediates/size", sizeHash, "test"))
// Trace should be identical for modules using the same set of variables
AssertDeepEquals(t, "board trace", board1.base().commonProperties.SoongConfigTrace, board2.base().commonProperties.SoongConfigTrace)
AssertDeepEquals(t, "board hash", board1.base().commonProperties.SoongConfigTraceHash, board2.base().commonProperties.SoongConfigTraceHash)
// Trace hash should be different for different sets of soong variables
AssertBoolEquals(t, "board hash not equal to size hash", board1.base().commonProperties.SoongConfigTraceHash == size.commonProperties.SoongConfigTraceHash, false)
boardSize := result.ModuleForTests("board_and_size", "").Module().(*soongConfigTestModule)
boardSizeDefaults := result.ModuleForTests("board_and_size_with_defaults", "").Module()
// Trace should propagate
AssertDeepEquals(t, "board_size hash calc", boardSize.base().commonProperties.SoongConfigTrace.hash(), boardSize.base().commonProperties.SoongConfigTraceHash)
AssertDeepEquals(t, "board_size trace", boardSize.base().commonProperties.SoongConfigTrace, boardSizeDefaults.base().commonProperties.SoongConfigTrace)
AssertDeepEquals(t, "board_size hash", boardSize.base().commonProperties.SoongConfigTraceHash, boardSizeDefaults.base().commonProperties.SoongConfigTraceHash)
})
}