StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Revert "Use hashed subdir for soong_config modules"" into main

Browse Source
This commit is contained in:
Inseob Kim
2024-06-27 04:30:30 +00:00
committed by Gerrit Code Review
parent 8eecb134e3 b7e9f5f035
commit f58d7efa29
6 changed files with 3 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@@ -594,12 +594,6 @@ modules (`cc_defaults`, `java_defaults`, etc.), which can then be referenced
by all of the vendor's other modules using the normal namespace and visibility
rules.
`soongConfigTraceMutator` enables modules affected by soong config variables to
write outputs into a hashed directory path. It does this by recording accesses
to soong config variables on each module, and then accumulating records of each
module's all dependencies. `m soong_config_trace` builds information about
hashes to `$OUT_DIR/soong/soong_config_trace.json`.
## Build logic
The build logic is written in Go using the

89
android/module.go
View File

@@ -15,9 +15,6 @@
package android
import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt"
"net/url"
"path/filepath"
@@ -247,31 +244,6 @@ func SortedUniqueNamedPaths(l NamedPaths) NamedPaths {
return l[:k+1]
}
// soongConfigTrace holds all references to VendorVars. Uses []string for blueprint:"mutated"
type soongConfigTrace struct {
Bools []string `json:",omitempty"`
Strings []string `json:",omitempty"`
IsSets []string `json:",omitempty"`
}
func (c *soongConfigTrace) isEmpty() bool {
return len(c.Bools) == 0 && len(c.Strings) == 0 && len(c.IsSets) == 0
}
// Returns hash of serialized trace records (empty string if there's no trace recorded)
func (c *soongConfigTrace) hash() string {
// Use MD5 for speed. We don't care collision or preimage attack
if c.isEmpty() {
return ""
}
j, err := json.Marshal(c)
if err != nil {
panic(fmt.Errorf("json marshal of %#v failed: %#v", *c, err))
}
hash := md5.Sum(j)
return hex.EncodeToString(hash[:])
}
type nameProperties struct {
// The name of the module. Must be unique across all modules.
Name *string
@@ -523,14 +495,6 @@ type commonProperties struct {
// constants in image.go, but can also be set to a custom value by individual module types.
ImageVariation string `blueprint:"mutated"`
// SoongConfigTrace records accesses to VendorVars (soong_config). The trace will be hashed
// and used as a subdir of PathForModuleOut. Note that we mainly focus on incremental
// builds among similar products (e.g. aosp_cf_x86_64_phone and aosp_cf_x86_64_foldable),
// and there are variables other than soong_config, which isn't captured by soong config
// trace, but influence modules among products.
SoongConfigTrace soongConfigTrace `blueprint:"mutated"`
SoongConfigTraceHash string `blueprint:"mutated"`
// The team (defined by the owner/vendor) who owns the property.
Team *string `android:"path"`
}
@@ -2614,8 +2578,6 @@ type HostToolProvider interface {
func init() {
RegisterParallelSingletonType("buildtarget", BuildTargetSingleton)
RegisterParallelSingletonType("soongconfigtrace", soongConfigTraceSingletonFunc)
FinalDepsMutators(registerSoongConfigTraceMutator)
}
func BuildTargetSingleton() Singleton {
@@ -2777,54 +2739,3 @@ func CheckBlueprintSyntax(ctx BaseModuleContext, filename string, contents strin
bpctx := ctx.blueprintBaseModuleContext()
return blueprint.CheckBlueprintSyntax(bpctx.ModuleFactories(), filename, contents)
}
func registerSoongConfigTraceMutator(ctx RegisterMutatorsContext) {
ctx.BottomUp("soongconfigtrace", soongConfigTraceMutator).Parallel()
}
// soongConfigTraceMutator accumulates recorded soong_config trace from children. Also it normalizes
// SoongConfigTrace to make it consistent.
func soongConfigTraceMutator(ctx BottomUpMutatorContext) {
trace := &ctx.Module().base().commonProperties.SoongConfigTrace
ctx.VisitDirectDeps(func(m Module) {
childTrace := &m.base().commonProperties.SoongConfigTrace
trace.Bools = append(trace.Bools, childTrace.Bools...)
trace.Strings = append(trace.Strings, childTrace.Strings...)
trace.IsSets = append(trace.IsSets, childTrace.IsSets...)
})
trace.Bools = SortedUniqueStrings(trace.Bools)
trace.Strings = SortedUniqueStrings(trace.Strings)
trace.IsSets = SortedUniqueStrings(trace.IsSets)
ctx.Module().base().commonProperties.SoongConfigTraceHash = trace.hash()
}
// soongConfigTraceSingleton writes a map from each module's config hash value to trace data.
func soongConfigTraceSingletonFunc() Singleton {
return &soongConfigTraceSingleton{}
}
type soongConfigTraceSingleton struct {
}
func (s *soongConfigTraceSingleton) GenerateBuildActions(ctx SingletonContext) {
outFile := PathForOutput(ctx, "soong_config_trace.json")
traces := make(map[string]*soongConfigTrace)
ctx.VisitAllModules(func(module Module) {
trace := &module.base().commonProperties.SoongConfigTrace
if !trace.isEmpty() {
hash := module.base().commonProperties.SoongConfigTraceHash
traces[hash] = trace
}
})
j, err := json.Marshal(traces)
if err != nil {
ctx.Errorf("json marshal to %q failed: %#v", outFile, err)
return
}
WriteFileRule(ctx, outFile, string(j))
ctx.Phony("soong_config_trace", outFile)
}

5
android/module_context.go
View File

@@ -188,7 +188,6 @@ type ModuleContext interface {
TargetRequiredModuleNames() []string
ModuleSubDir() string
SoongConfigTraceHash() string
Variable(pctx PackageContext, name, value string)
Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule
@@ -382,10 +381,6 @@ func (m *moduleContext) ModuleSubDir() string {
return m.bp.ModuleSubDir()
}
func (m *moduleContext) SoongConfigTraceHash() string {
return m.module.base().commonProperties.SoongConfigTraceHash
}
func (m *moduleContext) InstallInData() bool {
return m.module.InstallInData()
}

3
android/paths.go
View File

@@ -1604,11 +1604,10 @@ type ModuleOutPathContext interface {
ModuleName() string
ModuleDir() string
ModuleSubDir() string
SoongConfigTraceHash() string
}
func pathForModuleOut(ctx ModuleOutPathContext) OutputPath {
return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir(), ctx.SoongConfigTraceHash())
return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir())
}
// PathForModuleOut returns a Path representing the paths... under the module's

57
android/soong_config_modules.go
View File

@@ -463,57 +463,6 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s
}).(map[string]blueprint.ModuleFactory)
}
// tracingConfig is a wrapper to soongconfig.SoongConfig which records all accesses to SoongConfig.
type tracingConfig struct {
config soongconfig.SoongConfig
boolSet map[string]bool
stringSet map[string]string
isSetSet map[string]bool
}
func (c *tracingConfig) Bool(name string) bool {
c.boolSet[name] = c.config.Bool(name)
return c.boolSet[name]
}
func (c *tracingConfig) String(name string) string {
c.stringSet[name] = c.config.String(name)
return c.stringSet[name]
}
func (c *tracingConfig) IsSet(name string) bool {
c.isSetSet[name] = c.config.IsSet(name)
return c.isSetSet[name]
}
func (c *tracingConfig) getTrace() soongConfigTrace {
ret := soongConfigTrace{}
for k, v := range c.boolSet {
ret.Bools = append(ret.Bools, fmt.Sprintf("%q:%t", k, v))
}
for k, v := range c.stringSet {
ret.Strings = append(ret.Strings, fmt.Sprintf("%q:%q", k, v))
}
for k, v := range c.isSetSet {
ret.IsSets = append(ret.IsSets, fmt.Sprintf("%q:%t", k, v))
}
return ret
}
func newTracingConfig(config soongconfig.SoongConfig) *tracingConfig {
c := tracingConfig{
config: config,
boolSet: make(map[string]bool),
stringSet: make(map[string]string),
isSetSet: make(map[string]bool),
}
return &c
}
var _ soongconfig.SoongConfig = (*tracingConfig)(nil)
// configModuleFactory takes an existing soongConfigModuleFactory and a
// ModuleType to create a new ModuleFactory that uses a custom loadhook.
func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfig.ModuleType) blueprint.ModuleFactory {
@@ -561,8 +510,8 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi
// conditional on Soong config variables by reading the product
// config variables from Make.
AddLoadHook(module, func(ctx LoadHookContext) {
tracingConfig := newTracingConfig(ctx.Config().VendorConfig(moduleType.ConfigNamespace))
newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, tracingConfig)
config := ctx.Config().VendorConfig(moduleType.ConfigNamespace)
newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, config)
if err != nil {
ctx.ModuleErrorf("%s", err)
return
@@ -570,8 +519,6 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi
for _, ps := range newProps {
ctx.AppendProperties(ps)
}
module.(Module).base().commonProperties.SoongConfigTrace = tracingConfig.getTrace()
})
return module, props
}

195
android/soong_config_modules_test.go
View File

@@ -16,7 +16,6 @@ package android
import (
"fmt"
"path/filepath"
"testing"
)
@@ -506,197 +505,3 @@ func TestSoongConfigModuleSingletonModule(t *testing.T) {
})
}
}
func TestSoongConfigModuleTrace(t *testing.T) {
bp := `
soong_config_module_type {
name: "acme_test",
module_type: "test",
config_namespace: "acme",
variables: ["board", "feature1", "FEATURE3", "unused_string_var"],
bool_variables: ["feature2", "unused_feature", "always_true"],
value_variables: ["size", "unused_size"],
properties: ["cflags", "srcs", "defaults"],
}
soong_config_module_type {
name: "acme_test_defaults",
module_type: "test_defaults",
config_namespace: "acme",
variables: ["board", "feature1", "FEATURE3", "unused_string_var"],
bool_variables: ["feature2", "unused_feature", "always_true"],
value_variables: ["size", "unused_size"],
properties: ["cflags", "srcs", "defaults"],
}
soong_config_string_variable {
name: "board",
values: ["soc_a", "soc_b", "soc_c"],
}
soong_config_string_variable {
name: "unused_string_var",
values: ["a", "b"],
}
soong_config_bool_variable {
name: "feature1",
}
soong_config_bool_variable {
name: "FEATURE3",
}
test_defaults {
name: "test_defaults",
cflags: ["DEFAULT"],
}
test {
name: "normal",
defaults: ["test_defaults"],
}
acme_test {
name: "board_1",
defaults: ["test_defaults"],
soong_config_variables: {
board: {
soc_a: {
cflags: ["-DSOC_A"],
},
},
},
}
acme_test {
name: "board_2",
defaults: ["test_defaults"],
soong_config_variables: {
board: {
soc_a: {
cflags: ["-DSOC_A"],
},
},
},
}
acme_test {
name: "size",
defaults: ["test_defaults"],
soong_config_variables: {
size: {
cflags: ["-DSIZE=%s"],
},
},
}
acme_test {
name: "board_and_size",
defaults: ["test_defaults"],
soong_config_variables: {
board: {
soc_a: {
cflags: ["-DSOC_A"],
},
},
size: {
cflags: ["-DSIZE=%s"],
},
},
}
acme_test_defaults {
name: "board_defaults",
soong_config_variables: {
board: {
soc_a: {
cflags: ["-DSOC_A"],
},
},
},
}
acme_test_defaults {
name: "size_defaults",
soong_config_variables: {
size: {
cflags: ["-DSIZE=%s"],
},
},
}
test {
name: "board_and_size_with_defaults",
defaults: ["board_defaults", "size_defaults"],
}
`
fixtureForVendorVars := func(vars map[string]map[string]string) FixturePreparer {
return FixtureModifyProductVariables(func(variables FixtureProductVariables) {
variables.VendorVars = vars
})
}
preparer := fixtureForVendorVars(map[string]map[string]string{
"acme": {
"board": "soc_a",
"size": "42",
"feature1": "true",
"feature2": "false",
// FEATURE3 unset
"unused_feature": "true", // unused
"unused_size": "1", // unused
"unused_string_var": "a", // unused
"always_true": "true",
},
})
t.Run("soong config trace hash", func(t *testing.T) {
result := GroupFixturePreparers(
preparer,
PrepareForTestWithDefaults,
PrepareForTestWithSoongConfigModuleBuildComponents,
prepareForSoongConfigTestModule,
FixtureRegisterWithContext(func(ctx RegistrationContext) {
ctx.FinalDepsMutators(registerSoongConfigTraceMutator)
}),
FixtureWithRootAndroidBp(bp),
).RunTest(t)
// Hashes of modules not using soong config should be empty
normal := result.ModuleForTests("normal", "").Module().(*soongConfigTestModule)
AssertDeepEquals(t, "normal hash", normal.base().commonProperties.SoongConfigTraceHash, "")
AssertDeepEquals(t, "normal hash out", normal.outputPath.RelativeToTop().String(), "out/soong/.intermediates/normal/test")
board1 := result.ModuleForTests("board_1", "").Module().(*soongConfigTestModule)
board2 := result.ModuleForTests("board_2", "").Module().(*soongConfigTestModule)
size := result.ModuleForTests("size", "").Module().(*soongConfigTestModule)
// Trace mutator sets soong config trace hash correctly
board1Hash := board1.base().commonProperties.SoongConfigTrace.hash()
board1Output := board1.outputPath.RelativeToTop().String()
AssertDeepEquals(t, "board hash calc", board1Hash, board1.base().commonProperties.SoongConfigTraceHash)
AssertDeepEquals(t, "board hash path", board1Output, filepath.Join("out/soong/.intermediates/board_1", board1Hash, "test"))
sizeHash := size.base().commonProperties.SoongConfigTrace.hash()
sizeOutput := size.outputPath.RelativeToTop().String()
AssertDeepEquals(t, "size hash calc", sizeHash, size.base().commonProperties.SoongConfigTraceHash)
AssertDeepEquals(t, "size hash path", sizeOutput, filepath.Join("out/soong/.intermediates/size", sizeHash, "test"))
// Trace should be identical for modules using the same set of variables
AssertDeepEquals(t, "board trace", board1.base().commonProperties.SoongConfigTrace, board2.base().commonProperties.SoongConfigTrace)
AssertDeepEquals(t, "board hash", board1.base().commonProperties.SoongConfigTraceHash, board2.base().commonProperties.SoongConfigTraceHash)
// Trace hash should be different for different sets of soong variables
AssertBoolEquals(t, "board hash not equal to size hash", board1.base().commonProperties.SoongConfigTraceHash == size.commonProperties.SoongConfigTraceHash, false)
boardSize := result.ModuleForTests("board_and_size", "").Module().(*soongConfigTestModule)
boardSizeDefaults := result.ModuleForTests("board_and_size_with_defaults", "").Module()
// Trace should propagate
AssertDeepEquals(t, "board_size hash calc", boardSize.base().commonProperties.SoongConfigTrace.hash(), boardSize.base().commonProperties.SoongConfigTraceHash)
AssertDeepEquals(t, "board_size trace", boardSize.base().commonProperties.SoongConfigTrace, boardSizeDefaults.base().commonProperties.SoongConfigTrace)
AssertDeepEquals(t, "board_size hash", boardSize.base().commonProperties.SoongConfigTraceHash, boardSizeDefaults.base().commonProperties.SoongConfigTraceHash)
})
}