Merge "Move CLC construction to Ninja phase."

This commit is contained in:
Jiakai Zhang
2023-05-30 14:58:15 +00:00
committed by Gerrit Code Review
9 changed files with 394 additions and 273 deletions

View File

@@ -17,11 +17,11 @@ package dexpreopt
import (
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"android/soong/android"
"github.com/google/blueprint/proptools"
)
// This comment describes the following:
@@ -310,8 +310,8 @@ func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathCont
// Nested class loader context shouldn't have conditional part (it is allowed only at the top level).
for ver, _ := range nestedClcMap {
if ver != AnySdkVersion {
clcStr, _ := ComputeClassLoaderContext(nestedClcMap)
return fmt.Errorf("nested class loader context shouldn't have conditional part: %s", clcStr)
clcPaths := ComputeClassLoaderContextDependencies(nestedClcMap)
return fmt.Errorf("nested class loader context shouldn't have conditional part: %+v", clcPaths)
}
}
subcontexts := nestedClcMap[AnySdkVersion]
@@ -418,6 +418,15 @@ func (clcMap ClassLoaderContextMap) Dump() string {
return string(bytes)
}
func (clcMap ClassLoaderContextMap) DumpForFlag() string {
jsonCLC := toJsonClassLoaderContext(clcMap)
bytes, err := json.Marshal(jsonCLC)
if err != nil {
panic(err)
}
return proptools.ShellEscapeIncludingSpaces(string(bytes))
}
// excludeLibsFromCLCList excludes the libraries from the ClassLoaderContext in this list.
//
// This treats the supplied list as being immutable (as it may come from a dependency). So, it
@@ -544,67 +553,27 @@ func validateClassLoaderContextRec(sdkVer int, clcs []*ClassLoaderContext) (bool
return true, nil
}
// Return the class loader context as a string, and a slice of build paths for all dependencies.
// Returns a slice of build paths for all possible dependencies that the class loader context may
// refer to.
// Perform a depth-first preorder traversal of the class loader context tree for each SDK version.
// Return the resulting string and a slice of on-host build paths to all library dependencies.
func ComputeClassLoaderContext(clcMap ClassLoaderContextMap) (clcStr string, paths android.Paths) {
// CLC for different SDK versions should come in specific order that agrees with PackageManager.
// Since PackageManager processes SDK versions in ascending order and prepends compatibility
// libraries at the front, the required order is descending, except for AnySdkVersion that has
// numerically the largest order, but must be the last one. Example of correct order: [30, 29,
// 28, AnySdkVersion]. There are Soong tests to ensure that someone doesn't change this by
// accident, but there is no way to guard against changes in the PackageManager, except for
// grepping logcat on the first boot for absence of the following messages:
//
// `logcat | grep -E 'ClassLoaderContext [a-z ]+ mismatch`
//
versions := make([]int, 0, len(clcMap))
for ver, _ := range clcMap {
if ver != AnySdkVersion {
versions = append(versions, ver)
}
}
sort.Sort(sort.Reverse(sort.IntSlice(versions))) // descending order
versions = append(versions, AnySdkVersion)
for _, sdkVer := range versions {
sdkVerStr := fmt.Sprintf("%d", sdkVer)
if sdkVer == AnySdkVersion {
sdkVerStr = "any" // a special keyword that means any SDK version
}
hostClc, targetClc, hostPaths := computeClassLoaderContextRec(clcMap[sdkVer])
if hostPaths != nil {
clcStr += fmt.Sprintf(" --host-context-for-sdk %s %s", sdkVerStr, hostClc)
clcStr += fmt.Sprintf(" --target-context-for-sdk %s %s", sdkVerStr, targetClc)
}
func ComputeClassLoaderContextDependencies(clcMap ClassLoaderContextMap) android.Paths {
var paths android.Paths
for _, clcs := range clcMap {
hostPaths := ComputeClassLoaderContextDependenciesRec(clcs)
paths = append(paths, hostPaths...)
}
return clcStr, android.FirstUniquePaths(paths)
return android.FirstUniquePaths(paths)
}
// Helper function for ComputeClassLoaderContext() that handles recursion.
func computeClassLoaderContextRec(clcs []*ClassLoaderContext) (string, string, android.Paths) {
// Helper function for ComputeClassLoaderContextDependencies() that handles recursion.
func ComputeClassLoaderContextDependenciesRec(clcs []*ClassLoaderContext) android.Paths {
var paths android.Paths
var clcsHost, clcsTarget []string
for _, clc := range clcs {
subClcHost, subClcTarget, subPaths := computeClassLoaderContextRec(clc.Subcontexts)
if subPaths != nil {
subClcHost = "{" + subClcHost + "}"
subClcTarget = "{" + subClcTarget + "}"
}
clcsHost = append(clcsHost, "PCL["+clc.Host.String()+"]"+subClcHost)
clcsTarget = append(clcsTarget, "PCL["+clc.Device+"]"+subClcTarget)
subPaths := ComputeClassLoaderContextDependenciesRec(clc.Subcontexts)
paths = append(paths, clc.Host)
paths = append(paths, subPaths...)
}
clcHost := strings.Join(clcsHost, "#")
clcTarget := strings.Join(clcsTarget, "#")
return clcHost, clcTarget, paths
return paths
}
// Class loader contexts that come from Make via JSON dexpreopt.config. JSON CLC representation is

View File

@@ -20,6 +20,7 @@ package dexpreopt
import (
"fmt"
"reflect"
"sort"
"strings"
"testing"
@@ -34,7 +35,7 @@ func TestCLC(t *testing.T) {
// │   └── android.hidl.base
// │
// └── any
// ├── a
// ├── a' (a single quotation mark (') is there to test escaping)
// ├── b
// ├── c
// ├── d
@@ -53,7 +54,7 @@ func TestCLC(t *testing.T) {
m := make(ClassLoaderContextMap)
m.AddContext(ctx, AnySdkVersion, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil)
m.AddContext(ctx, AnySdkVersion, "a'", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil)
m.AddContext(ctx, AnySdkVersion, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil)
m.AddContext(ctx, AnySdkVersion, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil)
@@ -96,11 +97,10 @@ func TestCLC(t *testing.T) {
fixClassLoaderContext(m)
var haveStr string
var havePaths android.Paths
var haveUsesLibsReq, haveUsesLibsOpt []string
if valid && validationError == nil {
haveStr, havePaths = ComputeClassLoaderContext(m)
havePaths = ComputeClassLoaderContextDependencies(m)
haveUsesLibsReq, haveUsesLibsOpt = m.UsesLibs()
}
@@ -111,29 +111,6 @@ func TestCLC(t *testing.T) {
}
})
// Test that class loader context structure is correct.
t.Run("string", func(t *testing.T) {
wantStr := " --host-context-for-sdk 29 " +
"PCL[out/soong/" + AndroidHidlManager + ".jar]#" +
"PCL[out/soong/" + AndroidHidlBase + ".jar]" +
" --target-context-for-sdk 29 " +
"PCL[/system/framework/" + AndroidHidlManager + ".jar]#" +
"PCL[/system/framework/" + AndroidHidlBase + ".jar]" +
" --host-context-for-sdk any " +
"PCL[out/soong/a.jar]#PCL[out/soong/b.jar]#PCL[out/soong/c.jar]#PCL[out/soong/d.jar]" +
"{PCL[out/soong/a2.jar]#PCL[out/soong/b2.jar]#PCL[out/soong/c2.jar]" +
"{PCL[out/soong/a1.jar]#PCL[out/soong/b1.jar]}}#" +
"PCL[out/soong/f.jar]#PCL[out/soong/a3.jar]#PCL[out/soong/b3.jar]" +
" --target-context-for-sdk any " +
"PCL[/system/a.jar]#PCL[/system/b.jar]#PCL[/system/c.jar]#PCL[/system/d.jar]" +
"{PCL[/system/a2.jar]#PCL[/system/b2.jar]#PCL[/system/c2.jar]" +
"{PCL[/system/a1.jar]#PCL[/system/b1.jar]}}#" +
"PCL[/system/f.jar]#PCL[/system/a3.jar]#PCL[/system/b3.jar]"
if wantStr != haveStr {
t.Errorf("\nwant class loader context: %s\nhave class loader context: %s", wantStr, haveStr)
}
})
// Test that all expected build paths are gathered.
t.Run("paths", func(t *testing.T) {
wantPaths := []string{
@@ -143,14 +120,28 @@ func TestCLC(t *testing.T) {
"out/soong/a1.jar", "out/soong/b1.jar",
"out/soong/f.jar", "out/soong/a3.jar", "out/soong/b3.jar",
}
if !reflect.DeepEqual(wantPaths, havePaths.Strings()) {
t.Errorf("\nwant paths: %s\nhave paths: %s", wantPaths, havePaths)
}
actual := havePaths.Strings()
// The order does not matter.
sort.Strings(wantPaths)
sort.Strings(actual)
android.AssertArrayString(t, "", wantPaths, actual)
})
// Test the JSON passed to construct_context.py.
t.Run("json", func(t *testing.T) {
// The tree structure within each SDK version should be kept exactly the same when serialized
// to JSON. The order matters because the Python script keeps the order within each SDK version
// as is.
// The JSON is passed to the Python script as a commandline flag, so quotation ('') and escaping
// must be performed.
android.AssertStringEquals(t, "", strings.TrimSpace(`
'{"29":[{"Name":"android.hidl.manager-V1.0-java","Optional":false,"Host":"out/soong/android.hidl.manager-V1.0-java.jar","Device":"/system/framework/android.hidl.manager-V1.0-java.jar","Subcontexts":[]},{"Name":"android.hidl.base-V1.0-java","Optional":false,"Host":"out/soong/android.hidl.base-V1.0-java.jar","Device":"/system/framework/android.hidl.base-V1.0-java.jar","Subcontexts":[]}],"30":[],"42":[],"any":[{"Name":"a'\''","Optional":false,"Host":"out/soong/a.jar","Device":"/system/a.jar","Subcontexts":[]},{"Name":"b","Optional":false,"Host":"out/soong/b.jar","Device":"/system/b.jar","Subcontexts":[]},{"Name":"c","Optional":false,"Host":"out/soong/c.jar","Device":"/system/c.jar","Subcontexts":[]},{"Name":"d","Optional":false,"Host":"out/soong/d.jar","Device":"/system/d.jar","Subcontexts":[{"Name":"a2","Optional":false,"Host":"out/soong/a2.jar","Device":"/system/a2.jar","Subcontexts":[]},{"Name":"b2","Optional":false,"Host":"out/soong/b2.jar","Device":"/system/b2.jar","Subcontexts":[]},{"Name":"c2","Optional":false,"Host":"out/soong/c2.jar","Device":"/system/c2.jar","Subcontexts":[{"Name":"a1","Optional":false,"Host":"out/soong/a1.jar","Device":"/system/a1.jar","Subcontexts":[]},{"Name":"b1","Optional":false,"Host":"out/soong/b1.jar","Device":"/system/b1.jar","Subcontexts":[]}]}]},{"Name":"f","Optional":false,"Host":"out/soong/f.jar","Device":"/system/f.jar","Subcontexts":[]},{"Name":"a3","Optional":false,"Host":"out/soong/a3.jar","Device":"/system/a3.jar","Subcontexts":[]},{"Name":"b3","Optional":false,"Host":"out/soong/b3.jar","Device":"/system/b3.jar","Subcontexts":[]}]}'
`), m.DumpForFlag())
})
// Test for libraries that are added by the manifest_fixer.
t.Run("uses libs", func(t *testing.T) {
wantUsesLibsReq := []string{"a", "b", "c", "d", "f", "a3", "b3"}
wantUsesLibsReq := []string{"a'", "b", "c", "d", "f", "a3", "b3"}
wantUsesLibsOpt := []string{}
if !reflect.DeepEqual(wantUsesLibsReq, haveUsesLibsReq) {
t.Errorf("\nwant required uses libs: %s\nhave required uses libs: %s", wantUsesLibsReq, haveUsesLibsReq)
@@ -236,49 +227,6 @@ func TestCLCNestedConditional(t *testing.T) {
checkError(t, err, "nested class loader context shouldn't have conditional part")
}
// Test for SDK version order in conditional CLC: no matter in what order the libraries are added,
// they end up in the order that agrees with PackageManager.
func TestCLCSdkVersionOrder(t *testing.T) {
ctx := testContext()
optional := false
m := make(ClassLoaderContextMap)
m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil)
m.AddContext(ctx, 29, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil)
m.AddContext(ctx, 30, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil)
m.AddContext(ctx, AnySdkVersion, "d", optional, buildPath(ctx, "d"), installPath(ctx, "d"), nil)
valid, validationError := validateClassLoaderContext(m)
fixClassLoaderContext(m)
var haveStr string
if valid && validationError == nil {
haveStr, _ = ComputeClassLoaderContext(m)
}
// Test that validation is successful (all paths are known).
t.Run("validate", func(t *testing.T) {
if !(valid && validationError == nil) {
t.Errorf("invalid class loader context")
}
})
// Test that class loader context structure is correct.
t.Run("string", func(t *testing.T) {
wantStr := " --host-context-for-sdk 30 PCL[out/soong/c.jar]" +
" --target-context-for-sdk 30 PCL[/system/c.jar]" +
" --host-context-for-sdk 29 PCL[out/soong/b.jar]" +
" --target-context-for-sdk 29 PCL[/system/b.jar]" +
" --host-context-for-sdk 28 PCL[out/soong/a.jar]" +
" --target-context-for-sdk 28 PCL[/system/a.jar]" +
" --host-context-for-sdk any PCL[out/soong/d.jar]" +
" --target-context-for-sdk any PCL[/system/d.jar]"
if wantStr != haveStr {
t.Errorf("\nwant class loader context: %s\nhave class loader context: %s", wantStr, haveStr)
}
})
}
func TestCLCMExcludeLibs(t *testing.T) {
ctx := testContext()
const optional = false

View File

@@ -52,7 +52,8 @@ var DexpreoptRunningInSoong = false
// GenerateDexpreoptRule generates a set of commands that will preopt a module based on a GlobalConfig and a
// ModuleConfig. The produced files and their install locations will be available through rule.Installs().
func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongConfig,
global *GlobalConfig, module *ModuleConfig) (rule *android.RuleBuilder, err error) {
global *GlobalConfig, module *ModuleConfig, productPackages android.Path) (
rule *android.RuleBuilder, err error) {
defer func() {
if r := recover(); r != nil {
@@ -92,7 +93,8 @@ func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongC
generateDM := shouldGenerateDM(module, global)
for archIdx, _ := range module.Archs {
dexpreoptCommand(ctx, globalSoong, global, module, rule, archIdx, profile, appImage, generateDM)
dexpreoptCommand(ctx, globalSoong, global, module, rule, archIdx, profile, appImage,
generateDM, productPackages)
}
}
}
@@ -232,9 +234,9 @@ func ToOdexPath(path string, arch android.ArchType) string {
pathtools.ReplaceExtension(filepath.Base(path), "odex"))
}
func dexpreoptCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, global *GlobalConfig,
module *ModuleConfig, rule *android.RuleBuilder, archIdx int, profile android.WritablePath,
appImage bool, generateDM bool) {
func dexpreoptCommand(ctx android.BuilderContext, globalSoong *GlobalSoongConfig,
global *GlobalConfig, module *ModuleConfig, rule *android.RuleBuilder, archIdx int,
profile android.WritablePath, appImage bool, generateDM bool, productPackages android.Path) {
arch := module.Archs[archIdx]
@@ -351,11 +353,13 @@ func dexpreoptCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, g
}
// Generate command that saves host and target class loader context in shell variables.
clc, paths := ComputeClassLoaderContext(module.ClassLoaderContexts)
paths := ComputeClassLoaderContextDependencies(module.ClassLoaderContexts)
rule.Command().
Text(`eval "$(`).Tool(globalSoong.ConstructContext).
Text(` --target-sdk-version ${target_sdk_version}`).
Text(clc).Implicits(paths).
FlagWithArg("--context-json=", module.ClassLoaderContexts.DumpForFlag()).
FlagWithInput("--product-packages=", productPackages).
Implicits(paths).
Text(`)"`)
}

View File

@@ -42,7 +42,8 @@ var (
// The flag is useful when running dex2oat on system image and vendor image which are built separately.
usesTargetFiles = flag.Bool("uses_target_files", false, "whether or not dexpreopt is running on target_files")
// basePath indicates the path where target_files.zip is extracted.
basePath = flag.String("base_path", ".", "base path where images and tools are extracted")
basePath = flag.String("base_path", ".", "base path where images and tools are extracted")
productPackagesPath = flag.String("product_packages", "", "path to product_packages.txt")
)
type builderContext struct {
@@ -87,6 +88,10 @@ func main() {
usage("--module configuration file is required")
}
if *productPackagesPath == "" {
usage("--product_packages configuration file is required")
}
// NOTE: duplicating --out_dir here is incorrect (one should be the another
// plus "/soong" but doing so apparently breaks dexpreopt
ctx := &builderContext{android.NullConfig(*outDir, *outDir)}
@@ -159,11 +164,12 @@ func main() {
moduleConfig.DexPreoptImageLocationsOnHost[i] = *basePath + location
}
}
writeScripts(ctx, globalSoongConfig, globalConfig, moduleConfig, *dexpreoptScriptPath)
writeScripts(ctx, globalSoongConfig, globalConfig, moduleConfig, *dexpreoptScriptPath, *productPackagesPath)
}
func writeScripts(ctx android.BuilderContext, globalSoong *dexpreopt.GlobalSoongConfig,
global *dexpreopt.GlobalConfig, module *dexpreopt.ModuleConfig, dexpreoptScriptPath string) {
global *dexpreopt.GlobalConfig, module *dexpreopt.ModuleConfig, dexpreoptScriptPath string,
productPackagesPath string) {
write := func(rule *android.RuleBuilder, file string) {
script := &bytes.Buffer{}
script.WriteString(scriptHeader)
@@ -199,7 +205,8 @@ func writeScripts(ctx android.BuilderContext, globalSoong *dexpreopt.GlobalSoong
panic(err)
}
}
dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule(ctx, globalSoong, global, module)
dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule(
ctx, globalSoong, global, module, android.PathForTesting(productPackagesPath))
if err != nil {
panic(err)
}

View File

@@ -100,8 +100,9 @@ func TestDexPreopt(t *testing.T) {
globalSoong := globalSoongConfigForTests()
global := GlobalConfigForTests(ctx)
module := testSystemModuleConfig(ctx, "test")
productPackages := android.PathForTesting("product_packages.txt")
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module)
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages)
if err != nil {
t.Fatal(err)
}
@@ -124,6 +125,7 @@ func TestDexPreoptSystemOther(t *testing.T) {
systemModule := testSystemModuleConfig(ctx, "Stest")
systemProductModule := testSystemProductModuleConfig(ctx, "SPtest")
productModule := testProductModuleConfig(ctx, "Ptest")
productPackages := android.PathForTesting("product_packages.txt")
global.HasSystemOther = true
@@ -157,7 +159,7 @@ func TestDexPreoptSystemOther(t *testing.T) {
for _, test := range tests {
global.PatternsOnSystemOther = test.patterns
for _, mt := range test.moduleTests {
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, mt.module)
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, mt.module, productPackages)
if err != nil {
t.Fatal(err)
}
@@ -182,11 +184,12 @@ func TestDexPreoptApexSystemServerJars(t *testing.T) {
globalSoong := globalSoongConfigForTests()
global := GlobalConfigForTests(ctx)
module := testApexModuleConfig(ctx, "service-A", "com.android.apex1")
productPackages := android.PathForTesting("product_packages.txt")
global.ApexSystemServerJars = android.CreateTestConfiguredJarList(
[]string{"com.android.apex1:service-A"})
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module)
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages)
if err != nil {
t.Fatal(err)
}
@@ -205,11 +208,12 @@ func TestDexPreoptStandaloneSystemServerJars(t *testing.T) {
globalSoong := globalSoongConfigForTests()
global := GlobalConfigForTests(ctx)
module := testPlatformSystemServerModuleConfig(ctx, "service-A")
productPackages := android.PathForTesting("product_packages.txt")
global.StandaloneSystemServerJars = android.CreateTestConfiguredJarList(
[]string{"platform:service-A"})
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module)
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages)
if err != nil {
t.Fatal(err)
}
@@ -228,11 +232,12 @@ func TestDexPreoptSystemExtSystemServerJars(t *testing.T) {
globalSoong := globalSoongConfigForTests()
global := GlobalConfigForTests(ctx)
module := testSystemExtSystemServerModuleConfig(ctx, "service-A")
productPackages := android.PathForTesting("product_packages.txt")
global.StandaloneSystemServerJars = android.CreateTestConfiguredJarList(
[]string{"system_ext:service-A"})
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module)
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages)
if err != nil {
t.Fatal(err)
}
@@ -251,11 +256,12 @@ func TestDexPreoptApexStandaloneSystemServerJars(t *testing.T) {
globalSoong := globalSoongConfigForTests()
global := GlobalConfigForTests(ctx)
module := testApexModuleConfig(ctx, "service-A", "com.android.apex1")
productPackages := android.PathForTesting("product_packages.txt")
global.ApexStandaloneSystemServerJars = android.CreateTestConfiguredJarList(
[]string{"com.android.apex1:service-A"})
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module)
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages)
if err != nil {
t.Fatal(err)
}
@@ -274,10 +280,11 @@ func TestDexPreoptProfile(t *testing.T) {
globalSoong := globalSoongConfigForTests()
global := GlobalConfigForTests(ctx)
module := testSystemModuleConfig(ctx, "test")
productPackages := android.PathForTesting("product_packages.txt")
module.ProfileClassListing = android.OptionalPathForPath(android.PathForTesting("profile"))
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module)
rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages)
if err != nil {
t.Fatal(err)
}