Merge changes from topics "extdir", "threadvars"

* changes:
  Add support for sdk extensions in prebuilt_apis
  Refactor prebuilt_apis.go
  Add base sdk extension version to the config

android/config.go

@@ -353,6 +353,7 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string
 			DeviceName:                          stringPtr("test_device"),
 			Platform_sdk_version:                intPtr(30),
 			Platform_sdk_codename:               stringPtr("S"),
+			Platform_base_sdk_extension_version: intPtr(1),
 			Platform_version_active_codenames:   []string{"S", "Tiramisu"},
 			DeviceSystemSdkVersions:             []string{"14", "15"},
 			Platform_systemsdk_versions:         []string{"29", "30"},
@@ -742,6 +743,14 @@ func (c *config) PlatformSdkCodename() string {
 	return String(c.productVariables.Platform_sdk_codename)
 }
 
+func (c *config) PlatformSdkExtensionVersion() int {
+	return *c.productVariables.Platform_sdk_extension_version
+}
+
+func (c *config) PlatformBaseSdkExtensionVersion() int {
+	return *c.productVariables.Platform_base_sdk_extension_version
+}
+
 func (c *config) PlatformSecurityPatch() string {
 	return String(c.productVariables.Platform_security_patch)
 }

android/variable.go

@@ -191,6 +191,7 @@ type productVariables struct {
 	Platform_sdk_version_or_codename          *string  `json:",omitempty"`
 	Platform_sdk_final                        *bool    `json:",omitempty"`
 	Platform_sdk_extension_version            *int     `json:",omitempty"`
+	Platform_base_sdk_extension_version       *int     `json:",omitempty"`
 	Platform_version_active_codenames         []string `json:",omitempty"`
 	Platform_vndk_version                     *string  `json:",omitempty"`
 	Platform_systemsdk_versions               []string `json:",omitempty"`

java/prebuilt_apis.go

@@ -16,6 +16,7 @@ package java
 
 import (
 	"fmt"
+	"path"
 	"strconv"
 	"strings"
 
@@ -37,6 +38,11 @@ type prebuiltApisProperties struct {
 	// list of api version directories
 	Api_dirs []string
 
+	// Directory containing finalized api txt files for extension versions.
+	// Extension versions higher than the base sdk extension version will
+	// be assumed to be finalized later than all Api_dirs.
+	Extensions_dir *string
+
 	// The next API directory can optionally point to a directory where
 	// files incompatibility-tracking files are stored for the current
 	// "in progress" API. Each module present in one of the api_dirs will have
@@ -60,36 +66,45 @@ func (module *prebuiltApis) GenerateAndroidBuildActions(ctx android.ModuleContex
 	// no need to implement
 }
 
-func parseJarPath(path string) (module string, apiver string, scope string) {
+// parsePrebuiltPath parses the relevant variables out of a variety of paths, e.g.
-	elements := strings.Split(path, "/")
+// <version>/<scope>/<module>.jar
+// <version>/<scope>/api/<module>.txt
+// extensions/<version>/<scope>/<module>.jar
+// extensions/<version>/<scope>/api/<module>.txt
+func parsePrebuiltPath(ctx android.LoadHookContext, p string) (module string, version string, scope string) {
+	elements := strings.Split(p, "/")
 
-	apiver = elements[0]
+	scopeIdx := len(elements) - 2
-	scope = elements[1]
+	if elements[scopeIdx] == "api" {
+		scopeIdx--
+	}
+	scope = elements[scopeIdx]
+	if scope != "core" && scope != "public" && scope != "system" && scope != "test" && scope != "module-lib" && scope != "system-server" {
+		ctx.ModuleErrorf("invalid scope %q found in path: %q", scope, p)
+		return
+	}
+	version = elements[scopeIdx-1]
 
-	module = strings.TrimSuffix(elements[2], ".jar")
+	module = strings.TrimSuffix(path.Base(p), path.Ext(p))
 	return
 }
 
-func parseApiFilePath(ctx android.LoadHookContext, path string) (module string, apiver string, scope string) {
+// parseFinalizedPrebuiltPath is like parsePrebuiltPath, but verifies the version is numeric (a finalized version).
-	elements := strings.Split(path, "/")
+func parseFinalizedPrebuiltPath(ctx android.LoadHookContext, p string) (module string, version int, scope string) {
-	apiver = elements[0]
+	module, v, scope := parsePrebuiltPath(ctx, p)
-
+	version, err := strconv.Atoi(v)
-	scope = elements[1]
+	if err != nil {
-	if scope != "public" && scope != "system" && scope != "test" && scope != "module-lib" && scope != "system-server" {
+		ctx.ModuleErrorf("Found finalized API files in non-numeric dir '%v'", v)
-		ctx.ModuleErrorf("invalid scope %q found in path: %q", scope, path)
+		return
+	}
 	return
 }
 
-	// elements[2] is string literal "api". skipping.
+func prebuiltApiModuleName(mctx android.LoadHookContext, module, scope, version string) string {
-	module = strings.TrimSuffix(elements[3], ".txt")
+	return fmt.Sprintf("%s_%s_%s_%s", mctx.ModuleName(), scope, version, module)
-	return
 }
 
-func prebuiltApiModuleName(mctx android.LoadHookContext, module string, scope string, apiver string) string {
+func createImport(mctx android.LoadHookContext, module, scope, version, path, sdkVersion string, compileDex bool) {
-	return mctx.ModuleName() + "_" + scope + "_" + apiver + "_" + module
-}
-
-func createImport(mctx android.LoadHookContext, module, scope, apiver, path, sdkVersion string, compileDex bool) {
 	props := struct {
 		Name        *string
 		Jars        []string
@@ -97,7 +112,7 @@ func createImport(mctx android.LoadHookContext, module, scope, apiver, path, sdk
 		Installable *bool
 		Compile_dex *bool
 	}{}
-	props.Name = proptools.StringPtr(prebuiltApiModuleName(mctx, module, scope, apiver))
+	props.Name = proptools.StringPtr(prebuiltApiModuleName(mctx, module, scope, version))
 	props.Jars = append(props.Jars, path)
 	props.Sdk_version = proptools.StringPtr(sdkVersion)
 	props.Installable = proptools.BoolPtr(false)
@@ -132,111 +147,125 @@ func createEmptyFile(mctx android.LoadHookContext, name string) {
 	mctx.CreateModule(genrule.GenRuleFactory, &props)
 }
 
-func getPrebuiltFiles(mctx android.LoadHookContext, p *prebuiltApis, name string) []string {
+// globApiDirs collects all the files in all api_dirs and all scopes that match the given glob, e.g. '*.jar' or 'api/*.txt'.
+// <api-dir>/<scope>/<glob> for all api-dir and scope.
+func globApiDirs(mctx android.LoadHookContext, p *prebuiltApis, api_dir_glob string) []string {
 	var files []string
 	for _, apiver := range p.properties.Api_dirs {
-		files = append(files, getPrebuiltFilesInSubdir(mctx, apiver, name)...)
+		files = append(files, globScopeDir(mctx, apiver, api_dir_glob)...)
 	}
 	return files
 }
 
-func getPrebuiltFilesInSubdir(mctx android.LoadHookContext, subdir string, name string) []string {
+// globExtensionDirs collects all the files under the extension dir (for all versions and scopes) that match the given glob
+// <extension-dir>/<version>/<scope>/<glob> for all version and scope.
+func globExtensionDirs(mctx android.LoadHookContext, p *prebuiltApis, extension_dir_glob string) []string {
+	// <extensions-dir>/<num>/<extension-dir-glob>
+	return globScopeDir(mctx, *p.properties.Extensions_dir+"/*", extension_dir_glob)
+}
+
+// globScopeDir collects all the files in the given subdir across all scopes that match the given glob, e.g. '*.jar' or 'api/*.txt'.
+// <subdir>/<scope>/<glob> for all scope.
+func globScopeDir(mctx android.LoadHookContext, subdir string, subdir_glob string) []string {
 	var files []string
 	dir := mctx.ModuleDir() + "/" + subdir
 	for _, scope := range []string{"public", "system", "test", "core", "module-lib", "system-server"} {
-		glob := fmt.Sprintf("%s/%s/%s", dir, scope, name)
+		glob := fmt.Sprintf("%s/%s/%s", dir, scope, subdir_glob)
 		vfiles, err := mctx.GlobWithDeps(glob, nil)
 		if err != nil {
-			mctx.ModuleErrorf("failed to glob %s files under %q: %s", name, dir+"/"+scope, err)
+			mctx.ModuleErrorf("failed to glob %s files under %q: %s", subdir_glob, dir+"/"+scope, err)
 		}
 		files = append(files, vfiles...)
 	}
+	for i, f := range files {
+		files[i] = strings.TrimPrefix(f, mctx.ModuleDir()+"/")
+	}
 	return files
 }
 
 func prebuiltSdkStubs(mctx android.LoadHookContext, p *prebuiltApis) {
-	mydir := mctx.ModuleDir() + "/"
 	// <apiver>/<scope>/<module>.jar
-	files := getPrebuiltFiles(mctx, p, "*.jar")
+	files := globApiDirs(mctx, p, "*.jar")
 
 	sdkVersion := proptools.StringDefault(p.properties.Imports_sdk_version, "current")
 	compileDex := proptools.BoolDefault(p.properties.Imports_compile_dex, false)
 
 	for _, f := range files {
 		// create a Import module for each jar file
-		localPath := strings.TrimPrefix(f, mydir)
+		module, version, scope := parsePrebuiltPath(mctx, f)
-		module, apiver, scope := parseJarPath(localPath)
+		createImport(mctx, module, scope, version, f, sdkVersion, compileDex)
-		createImport(mctx, module, scope, apiver, localPath, sdkVersion, compileDex)
 
 		if module == "core-for-system-modules" {
-			createSystemModules(mctx, apiver, scope)
+			createSystemModules(mctx, version, scope)
 		}
 	}
 }
 
-func createSystemModules(mctx android.LoadHookContext, apiver string, scope string) {
+func createSystemModules(mctx android.LoadHookContext, version, scope string) {
 	props := struct {
 		Name *string
 		Libs []string
 	}{}
-	props.Name = proptools.StringPtr(prebuiltApiModuleName(mctx, "system_modules", scope, apiver))
+	props.Name = proptools.StringPtr(prebuiltApiModuleName(mctx, "system_modules", scope, version))
-	props.Libs = append(props.Libs, prebuiltApiModuleName(mctx, "core-for-system-modules", scope, apiver))
+	props.Libs = append(props.Libs, prebuiltApiModuleName(mctx, "core-for-system-modules", scope, version))
 
 	mctx.CreateModule(systemModulesImportFactory, &props)
 }
 
 func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) {
-	mydir := mctx.ModuleDir() + "/"
 	// <apiver>/<scope>/api/<module>.txt
-	files := getPrebuiltFiles(mctx, p, "api/*.txt")
+	apiLevelFiles := globApiDirs(mctx, p, "api/*.txt")
-
+	if len(apiLevelFiles) == 0 {
-	if len(files) == 0 {
+		mctx.ModuleErrorf("no api file found under %q", mctx.ModuleDir())
-		mctx.ModuleErrorf("no api file found under %q", mydir)
-	}
-
-	// construct a map to find out the latest api file path
-	// for each (<module>, <scope>) pair.
-	type latestApiInfo struct {
-		module  string
-		scope   string
-		version int
-		path    string
 	}
 
 	// Create modules for all (<module>, <scope, <version>) triplets,
-	// and a "latest" module variant for each (<module>, <scope>) pair
 	apiModuleName := func(module, scope, version string) string {
 		return module + ".api." + scope + "." + version
 	}
-	m := make(map[string]latestApiInfo)
+	for _, f := range apiLevelFiles {
-	for _, f := range files {
+		module, version, scope := parseFinalizedPrebuiltPath(mctx, f)
-		localPath := strings.TrimPrefix(f, mydir)
+		createApiModule(mctx, apiModuleName(module, scope, strconv.Itoa(version)), f)
-		module, apiver, scope := parseApiFilePath(mctx, localPath)
-		createApiModule(mctx, apiModuleName(module, scope, apiver), localPath)
-
-		version, err := strconv.Atoi(apiver)
-		if err != nil {
-			mctx.ModuleErrorf("Found finalized API files in non-numeric dir %v", apiver)
-			return
 	}
 
-		// Track latest version of each module/scope, except for incompatibilities
+	// Figure out the latest version of each module/scope
-		if !strings.HasSuffix(module, "incompatibilities") {
+	type latestApiInfo struct {
+		module, scope, path string
+		version             int
+	}
+
+	getLatest := func(files []string) map[string]latestApiInfo {
+		m := make(map[string]latestApiInfo)
+		for _, f := range files {
+			module, version, scope := parseFinalizedPrebuiltPath(mctx, f)
+			if strings.HasSuffix(module, "incompatibilities") {
+				continue
+			}
 			key := module + "." + scope
-			info, ok := m[key]
+			info, exists := m[key]
-			if !ok {
+			if !exists || version > info.version {
-				m[key] = latestApiInfo{module, scope, version, localPath}
+				m[key] = latestApiInfo{module, scope, f, version}
-			} else if version > info.version {
+			}
-				info.version = version
+		}
-				info.path = localPath
+		return m
-				m[key] = info
+	}
+
+	latest := getLatest(apiLevelFiles)
+	if p.properties.Extensions_dir != nil {
+		extensionApiFiles := globExtensionDirs(mctx, p, "api/*.txt")
+		for k, v := range getLatest(extensionApiFiles) {
+			if v.version > mctx.Config().PlatformBaseSdkExtensionVersion() {
+				if _, exists := latest[k]; !exists {
+					mctx.ModuleErrorf("Module %v finalized for extension %d but never during an API level; likely error", v.module, v.version)
+				}
+				latest[k] = v
 			}
 		}
 	}
 
 	// Sort the keys in order to make build.ninja stable
-	for _, k := range android.SortedStringKeys(m) {
+	for _, k := range android.SortedStringKeys(latest) {
-		info := m[k]
+		info := latest[k]
 		name := apiModuleName(info.module, info.scope, "latest")
 		createApiModule(mctx, name, info.path)
 	}
@@ -244,21 +273,20 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) {
 	// Create incompatibilities tracking files for all modules, if we have a "next" api.
 	incompatibilities := make(map[string]bool)
 	if nextApiDir := String(p.properties.Next_api_dir); nextApiDir != "" {
-		files := getPrebuiltFilesInSubdir(mctx, nextApiDir, "api/*incompatibilities.txt")
+		files := globScopeDir(mctx, nextApiDir, "api/*incompatibilities.txt")
 		for _, f := range files {
-			localPath := strings.TrimPrefix(f, mydir)
+			filename, _, scope := parsePrebuiltPath(mctx, f)
-			filename, _, scope := parseApiFilePath(mctx, localPath)
 			referencedModule := strings.TrimSuffix(filename, "-incompatibilities")
 
-			createApiModule(mctx, apiModuleName(referencedModule+"-incompatibilities", scope, "latest"), localPath)
+			createApiModule(mctx, apiModuleName(referencedModule+"-incompatibilities", scope, "latest"), f)
 
 			incompatibilities[referencedModule+"."+scope] = true
 		}
 	}
 	// Create empty incompatibilities files for remaining modules
-	for _, k := range android.SortedStringKeys(m) {
+	for _, k := range android.SortedStringKeys(latest) {
 		if _, ok := incompatibilities[k]; !ok {
-			createEmptyFile(mctx, apiModuleName(m[k].module+"-incompatibilities", m[k].scope, "latest"))
+			createEmptyFile(mctx, apiModuleName(latest[k].module+"-incompatibilities", latest[k].scope, "latest"))
 		}
 	}
 }

java/prebuilt_apis_test.go

@@ -20,9 +20,14 @@ import (
 	"testing"
 
 	"android/soong/android"
+
 	"github.com/google/blueprint"
 )
 
+func intPtr(v int) *int {
+	return &v
+}
+
 func TestPrebuiltApis_SystemModulesCreation(t *testing.T) {
 	result := android.GroupFixturePreparers(
 		prepareForJavaTest,
@@ -54,3 +59,34 @@ func TestPrebuiltApis_SystemModulesCreation(t *testing.T) {
 	sort.Strings(expected)
 	android.AssertArrayString(t, "sdk system modules", expected, sdkSystemModules)
 }
+
+func TestPrebuiltApis_WithExtensions(t *testing.T) {
+	runTestWithBaseExtensionLevel := func(v int) (foo_input string, bar_input string) {
+		result := android.GroupFixturePreparers(
+			prepareForJavaTest,
+			android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) {
+				variables.Platform_base_sdk_extension_version = intPtr(v)
+			}),
+			FixtureWithPrebuiltApisAndExtensions(map[string][]string{
+				"31":      {"foo"},
+				"32":      {"foo", "bar"},
+				"current": {"foo", "bar"},
+			}, map[string][]string{
+				"1": {"foo"},
+				"2": {"foo", "bar"},
+			}),
+		).RunTest(t)
+		foo_input = result.ModuleForTests("foo.api.public.latest", "").Rule("generator").Implicits[0].String()
+		bar_input = result.ModuleForTests("bar.api.public.latest", "").Rule("generator").Implicits[0].String()
+		return
+	}
+	// Here, the base extension level is 1, so extension level 2 is the latest
+	foo_input, bar_input := runTestWithBaseExtensionLevel(1)
+	android.AssertStringEquals(t, "Expected latest = extension level 2", "prebuilts/sdk/extensions/2/public/api/foo.txt", foo_input)
+	android.AssertStringEquals(t, "Expected latest = extension level 2", "prebuilts/sdk/extensions/2/public/api/bar.txt", bar_input)
+
+	// Here, the base extension level is 2, so 2 is not later than 32.
+	foo_input, bar_input = runTestWithBaseExtensionLevel(2)
+	android.AssertStringEquals(t, "Expected latest = api level 32", "prebuilts/sdk/32/public/api/foo.txt", foo_input)
+	android.AssertStringEquals(t, "Expected latest = api level 32", "prebuilts/sdk/32/public/api/bar.txt", bar_input)
+}

java/testing.go

@@ -146,6 +146,10 @@ var PrepareForTestWithPrebuiltsOfCurrentApi = FixtureWithPrebuiltApis(map[string
 // This defines a file in the mock file system in a predefined location (prebuilts/sdk/Android.bp)
 // and so only one instance of this can be used in each fixture.
 func FixtureWithPrebuiltApis(release2Modules map[string][]string) android.FixturePreparer {
+	return FixtureWithPrebuiltApisAndExtensions(release2Modules, nil)
+}
+
+func FixtureWithPrebuiltApisAndExtensions(apiLevel2Modules map[string][]string, extensionLevel2Modules map[string][]string) android.FixturePreparer {
 	mockFS := android.MockFS{}
 	path := "prebuilts/sdk/Android.bp"
 
@@ -153,14 +157,20 @@ func FixtureWithPrebuiltApis(release2Modules map[string][]string) android.Fixtur
 			prebuilt_apis {
 				name: "sdk",
 				api_dirs: ["%s"],
+				extensions_dir: "extensions",
 				imports_sdk_version: "none",
 				imports_compile_dex: true,
 			}
-		`, strings.Join(android.SortedStringKeys(release2Modules), `", "`))
+		`, strings.Join(android.SortedStringKeys(apiLevel2Modules), `", "`))
 
-	for release, modules := range release2Modules {
+	for release, modules := range apiLevel2Modules {
 		mockFS.Merge(prebuiltApisFilesForModules([]string{release}, modules))
 	}
+	if extensionLevel2Modules != nil {
+		for release, modules := range extensionLevel2Modules {
+			mockFS.Merge(prebuiltExtensionApiFiles([]string{release}, modules))
+		}
+	}
 	return android.GroupFixturePreparers(
 		android.FixtureAddTextFile(path, bp),
 		android.FixtureMergeMockFs(mockFS),
@@ -198,6 +208,19 @@ func prebuiltApisFilesForModules(apiLevels []string, modules []string) map[strin
 	return fs
 }
 
+func prebuiltExtensionApiFiles(extensionLevels []string, modules []string) map[string][]byte {
+	fs := make(map[string][]byte)
+	for _, level := range extensionLevels {
+		for _, sdkKind := range []android.SdkKind{android.SdkPublic, android.SdkSystem, android.SdkModule, android.SdkSystemServer} {
+			for _, lib := range modules {
+				fs[fmt.Sprintf("prebuilts/sdk/extensions/%s/%s/api/%s.txt", level, sdkKind, lib)] = nil
+				fs[fmt.Sprintf("prebuilts/sdk/extensions/%s/%s/api/%s-removed.txt", level, sdkKind, lib)] = nil
+			}
+		}
+	}
+	return fs
+}
+
 // FixtureConfigureBootJars configures the boot jars in both the dexpreopt.GlobalConfig and
 // Config.productVariables structs. As a side effect that enables dexpreopt.
 func FixtureConfigureBootJars(bootJars ...string) android.FixturePreparer {

scripts/build-ndk-prebuilts.sh

@@ -23,6 +23,7 @@ TOP=$(pwd)
 
 source build/envsetup.sh
 PLATFORM_SDK_VERSION=$(get_build_var PLATFORM_SDK_VERSION)
+PLATFORM_BASE_SDK_EXTENSION_VERSION=$(get_build_var PLATFORM_BASE_SDK_EXTENSION_VERSION)
 PLATFORM_VERSION_ALL_CODENAMES=$(get_build_var PLATFORM_VERSION_ALL_CODENAMES)
 
 # PLATFORM_VERSION_ALL_CODENAMES is a comma separated list like O,P. We need to
@@ -46,6 +47,7 @@ mkdir -p ${SOONG_OUT}
 cat > ${SOONG_OUT}/soong.variables << EOF
 {
     "Platform_sdk_version": ${PLATFORM_SDK_VERSION},
+    "Platform_base_sdk_extension_version": ${PLATFORM_BASE_SDK_EXTENSION_VERSION},
     "Platform_version_active_codenames": ${PLATFORM_VERSION_ALL_CODENAMES},
 
     "DeviceName": "generic_arm64",