Merge "Create scripts to update and freeze a module SDK"

Android.bp

@@ -491,6 +491,7 @@ bootstrap_go_package {
     ],
     srcs: [
         "sdk/sdk.go",
+        "sdk/update.go",
     ],
     testSrcs: [
         "sdk/sdk_test.go",

android/sdk.go

@@ -39,25 +39,17 @@ type SdkRef struct {
 	Version string
 }
 
-const (
+// Unversioned determines if the SdkRef is referencing to the unversioned SDK module
-	// currentVersion refers to the in-development version of an SDK
+func (s SdkRef) Unversioned() bool {
-	currentVersion = "current"
+	return s.Version == ""
-)
-
-// IsCurrentVersion determines if the SdkRef is referencing to an in-development version of an SDK
-func (s SdkRef) IsCurrentVersion() bool {
-	return s.Version == currentVersion
 }
 
-// IsCurrentVersionOf determines if the SdkRef is referencing to an in-development version of the
+// SdkVersionSeparator is a character used to separate an sdk name and its version
-// specified SDK
+const SdkVersionSeparator = '@'
-func (s SdkRef) IsCurrentVersionOf(name string) bool {
-	return s.Name == name && s.IsCurrentVersion()
-}
 
-// ParseSdkRef parses a `name#version` style string into a corresponding SdkRef struct
+// ParseSdkRef parses a `name@version` style string into a corresponding SdkRef struct
 func ParseSdkRef(ctx BaseModuleContext, str string, property string) SdkRef {
-	tokens := strings.Split(str, "#")
+	tokens := strings.Split(str, string(SdkVersionSeparator))
 	if len(tokens) < 1 || len(tokens) > 2 {
 		ctx.PropertyErrorf(property, "%q does not follow name#version syntax", str)
 		return SdkRef{Name: "invalid sdk name", Version: "invalid sdk version"}
@@ -65,7 +57,7 @@ func ParseSdkRef(ctx BaseModuleContext, str string, property string) SdkRef {
 
 	name := tokens[0]
 
-	version := currentVersion // If version is omitted, defaults to "current"
+	var version string
 	if len(tokens) == 2 {
 		version = tokens[1]
 	}
@@ -75,6 +67,7 @@ func ParseSdkRef(ctx BaseModuleContext, str string, property string) SdkRef {
 
 type SdkRefs []SdkRef
 
+// Contains tells if the given SdkRef is in this list of SdkRef's
 func (refs SdkRefs) Contains(s SdkRef) bool {
 	for _, r := range refs {
 		if r == s {
@@ -105,7 +98,7 @@ func (s *SdkBase) sdkBase() *SdkBase {
 	return s
 }
 
-// MakeMemberof sets this module to be a member of a specific SDK
+// MakeMemberOf sets this module to be a member of a specific SDK
 func (s *SdkBase) MakeMemberOf(sdk SdkRef) {
 	s.properties.ContainingSdk = &sdk
 }
@@ -120,10 +113,10 @@ func (s *SdkBase) ContainingSdk() SdkRef {
 	if s.properties.ContainingSdk != nil {
 		return *s.properties.ContainingSdk
 	}
-	return SdkRef{Name: "", Version: currentVersion}
+	return SdkRef{Name: "", Version: ""}
 }
 
-// Membername returns the name of the module that this SDK member is overriding
+// MemberName returns the name of the module that this SDK member is overriding
 func (s *SdkBase) MemberName() string {
 	return proptools.String(s.properties.Sdk_member_name)
 }

java/androidmk.go

@@ -145,7 +145,7 @@ func (j *TestHelperLibrary) AndroidMkEntries() android.AndroidMkEntries {
 }
 
 func (prebuilt *Import) AndroidMkEntries() android.AndroidMkEntries {
-	if !prebuilt.IsForPlatform() || !prebuilt.ContainingSdk().IsCurrentVersion() {
+	if !prebuilt.IsForPlatform() || !prebuilt.ContainingSdk().Unversioned() {
 		return android.AndroidMkEntries{
 			Disabled: true,
 		}

sdk/sdk.go

@@ -15,6 +15,9 @@
 package sdk
 
 import (
+	"fmt"
+	"strconv"
+
 	"github.com/google/blueprint"
 
 	"android/soong/android"
@@ -25,6 +28,7 @@ import (
 
 func init() {
 	android.RegisterModuleType("sdk", ModuleFactory)
+	android.RegisterModuleType("sdk_snapshot", SnapshotModuleFactory)
 	android.PreDepsMutators(RegisterPreDepsMutators)
 	android.PostDepsMutators(RegisterPostDepsMutators)
 }
@@ -34,12 +38,18 @@ type sdk struct {
 	android.DefaultableModuleBase
 
 	properties sdkProperties
+
+	updateScript android.OutputPath
+	freezeScript android.OutputPath
 }
 
 type sdkProperties struct {
-	// The list of java_import modules that provide Java stubs for this SDK
+	// The list of java libraries in this SDK
-	Java_libs          []string
+	Java_libs []string
+	// The list of native libraries in this SDK
 	Native_shared_libs []string
+
+	Snapshot bool `blueprint:"mutated"`
 }
 
 // sdk defines an SDK which is a logical group of modules (e.g. native libs, headers, java libs, etc.)
@@ -52,8 +62,44 @@ func ModuleFactory() android.Module {
 	return s
 }
 
+// sdk_snapshot is a versioned snapshot of an SDK. This is an auto-generated module.
+func SnapshotModuleFactory() android.Module {
+	s := ModuleFactory()
+	s.(*sdk).properties.Snapshot = true
+	return s
+}
+
+func (s *sdk) snapshot() bool {
+	return s.properties.Snapshot
+}
+
+func (s *sdk) frozenVersions(ctx android.BaseModuleContext) []string {
+	if s.snapshot() {
+		panic(fmt.Errorf("frozenVersions() called for sdk_snapshot %q", ctx.ModuleName()))
+	}
+	versions := []string{}
+	ctx.WalkDeps(func(child android.Module, parent android.Module) bool {
+		depTag := ctx.OtherModuleDependencyTag(child)
+		if depTag == sdkMemberDepTag {
+			return true
+		}
+		if versionedDepTag, ok := depTag.(sdkMemberVesionedDepTag); ok {
+			v := versionedDepTag.version
+			if v != "current" && !android.InList(v, versions) {
+				versions = append(versions, versionedDepTag.version)
+			}
+		}
+		return false
+	})
+	return android.SortedUniqueStrings(versions)
+}
+
 func (s *sdk) GenerateAndroidBuildActions(ctx android.ModuleContext) {
-	// TODO(jiyong): add build rules for creating stubs from members of this SDK
+	s.buildSnapshotGenerationScripts(ctx)
+}
+
+func (s *sdk) AndroidMkEntries() android.AndroidMkEntries {
+	return s.androidMkEntriesForScript()
 }
 
 // RegisterPreDepsMutators registers pre-deps mutators to support modules implementing SdkAware
@@ -112,8 +158,21 @@ func memberMutator(mctx android.BottomUpMutatorContext) {
 
 // Step 2: record that dependencies of SDK modules are members of the SDK modules
 func memberDepsMutator(mctx android.TopDownMutatorContext) {
-	if _, ok := mctx.Module().(*sdk); ok {
+	if s, ok := mctx.Module().(*sdk); ok {
 		mySdkRef := android.ParseSdkRef(mctx, mctx.ModuleName(), "name")
+		if s.snapshot() && mySdkRef.Unversioned() {
+			mctx.PropertyErrorf("name", "sdk_snapshot should be named as <name>@<version>. "+
+				"Did you manually modify Android.bp?")
+		}
+		if !s.snapshot() && !mySdkRef.Unversioned() {
+			mctx.PropertyErrorf("name", "sdk shouldn't be named as <name>@<version>.")
+		}
+		if mySdkRef.Version != "" && mySdkRef.Version != "current" {
+			if _, err := strconv.Atoi(mySdkRef.Version); err != nil {
+				mctx.PropertyErrorf("name", "version %q is neither a number nor \"current\"", mySdkRef.Version)
+			}
+		}
+
 		mctx.VisitDirectDeps(func(child android.Module) {
 			if member, ok := child.(android.SdkAware); ok {
 				member.MakeMemberOf(mySdkRef)
@@ -122,7 +181,7 @@ func memberDepsMutator(mctx android.TopDownMutatorContext) {
 	}
 }
 
-// Step 3: create dependencies from the in-development version of an SDK member to frozen versions
+// Step 3: create dependencies from the unversioned SDK member to snapshot versions
 // of the same member. By having these dependencies, they are mutated for multiple Mainline modules
 // (apex and apk), each of which might want different sdks to be built with. For example, if both
 // apex A and B are referencing libfoo which is a member of sdk 'mysdk', the two APEXes can be
@@ -130,7 +189,7 @@ func memberDepsMutator(mctx android.TopDownMutatorContext) {
 // using.
 func memberInterVersionMutator(mctx android.BottomUpMutatorContext) {
 	if m, ok := mctx.Module().(android.SdkAware); ok && m.IsInAnySdk() {
-		if !m.ContainingSdk().IsCurrentVersion() {
+		if !m.ContainingSdk().Unversioned() {
 			memberName := m.MemberName()
 			tag := sdkMemberVesionedDepTag{member: memberName, version: m.ContainingSdk().Version}
 			mctx.AddReverseDependency(mctx.Module(), tag, memberName)
@@ -159,7 +218,7 @@ func sdkDepsMutator(mctx android.TopDownMutatorContext) {
 // versioned module is used instead of the un-versioned (in-development) module libfoo
 func sdkDepsReplaceMutator(mctx android.BottomUpMutatorContext) {
 	if m, ok := mctx.Module().(android.SdkAware); ok && m.IsInAnySdk() {
-		if sdk := m.ContainingSdk(); !sdk.IsCurrentVersion() {
+		if sdk := m.ContainingSdk(); !sdk.Unversioned() {
 			if m.RequiredSdks().Contains(sdk) {
 				// Note that this replacement is done only for the modules that have the same
 				// variations as the current module. Since current module is already mutated for

sdk/sdk_test.go

@@ -69,6 +69,7 @@ func testSdkContext(t *testing.T, bp string) (*android.TestContext, android.Conf
 
 	// from this package
 	ctx.RegisterModuleType("sdk", android.ModuleFactoryAdaptor(ModuleFactory))
+	ctx.RegisterModuleType("sdk_snapshot", android.ModuleFactoryAdaptor(SnapshotModuleFactory))
 	ctx.PreDepsMutators(RegisterPreDepsMutators)
 	ctx.PostDepsMutators(RegisterPostDepsMutators)
 
@@ -155,12 +156,17 @@ func pathsToStrings(paths android.Paths) []string {
 func TestBasicSdkWithJava(t *testing.T) {
 	ctx, _ := testSdk(t, `
 		sdk {
-			name: "mysdk#1",
+			name: "mysdk",
+			java_libs: ["sdkmember"],
+		}
+
+		sdk_snapshot {
+			name: "mysdk@1",
 			java_libs: ["sdkmember_mysdk_1"],
 		}
 
-		sdk {
+		sdk_snapshot {
-			name: "mysdk#2",
+			name: "mysdk@2",
 			java_libs: ["sdkmember_mysdk_2"],
 		}
 
@@ -195,7 +201,7 @@ func TestBasicSdkWithJava(t *testing.T) {
 		apex {
 			name: "myapex",
 			java_libs: ["myjavalib"],
-			uses_sdks: ["mysdk#1"],
+			uses_sdks: ["mysdk@1"],
 			key: "myapex.key",
 			certificate: ":myapex.cert",
 		}
@@ -203,7 +209,7 @@ func TestBasicSdkWithJava(t *testing.T) {
 		apex {
 			name: "myapex2",
 			java_libs: ["myjavalib"],
-			uses_sdks: ["mysdk#2"],
+			uses_sdks: ["mysdk@2"],
 			key: "myapex.key",
 			certificate: ":myapex.cert",
 		}
@@ -223,12 +229,17 @@ func TestBasicSdkWithJava(t *testing.T) {
 func TestBasicSdkWithCc(t *testing.T) {
 	ctx, _ := testSdk(t, `
 		sdk {
-			name: "mysdk#1",
+			name: "mysdk",
+			native_shared_libs: ["sdkmember"],
+		}
+
+		sdk_snapshot {
+			name: "mysdk@1",
 			native_shared_libs: ["sdkmember_mysdk_1"],
 		}
 
-		sdk {
+		sdk_snapshot {
-			name: "mysdk#2",
+			name: "mysdk@2",
 			native_shared_libs: ["sdkmember_mysdk_2"],
 		}
 
@@ -267,7 +278,7 @@ func TestBasicSdkWithCc(t *testing.T) {
 		apex {
 			name: "myapex",
 			native_shared_libs: ["mycpplib"],
-			uses_sdks: ["mysdk#1"],
+			uses_sdks: ["mysdk@1"],
 			key: "myapex.key",
 			certificate: ":myapex.cert",
 		}
@@ -275,7 +286,7 @@ func TestBasicSdkWithCc(t *testing.T) {
 		apex {
 			name: "myapex2",
 			native_shared_libs: ["mycpplib"],
-			uses_sdks: ["mysdk#2"],
+			uses_sdks: ["mysdk@2"],
 			key: "myapex.key",
 			certificate: ":myapex.cert",
 		}

sdk/update.go

@@ -0,0 +1,228 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sdk
+
+import (
+	"fmt"
+	"io"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/google/blueprint/proptools"
+
+	"android/soong/android"
+	"android/soong/java"
+)
+
+var pctx = android.NewPackageContext("android/soong/sdk")
+
+// generatedFile abstracts operations for writing contents into a file and emit a build rule
+// for the file.
+type generatedFile struct {
+	path    android.OutputPath
+	content strings.Builder
+}
+
+func newGeneratedFile(ctx android.ModuleContext, name string) *generatedFile {
+	return &generatedFile{
+		path: android.PathForModuleOut(ctx, name).OutputPath,
+	}
+}
+
+func (gf *generatedFile) printfln(format string, args ...interface{}) {
+	// ninja consumes newline characters in rspfile_content. Prevent it by
+	// escaping the backslash in the newline character. The extra backshash
+	// is removed when the rspfile is written to the actual script file
+	fmt.Fprintf(&(gf.content), format+"\\n", args...)
+}
+
+func (gf *generatedFile) build(pctx android.PackageContext, ctx android.BuilderContext, implicits android.Paths) {
+	rb := android.NewRuleBuilder()
+	// convert \\n to \n
+	rb.Command().
+		Implicits(implicits).
+		Text("echo").Text(proptools.ShellEscape(gf.content.String())).
+		Text("| sed 's/\\\\n/\\n/g' >").Output(gf.path)
+	rb.Command().
+		Text("chmod a+x").Output(gf.path)
+	rb.Build(pctx, ctx, gf.path.Base(), "Build "+gf.path.Base())
+}
+
+func (s *sdk) javaMemberNames(ctx android.ModuleContext) []string {
+	result := []string{}
+	ctx.VisitDirectDeps(func(m android.Module) {
+		if _, ok := m.(*java.Library); ok {
+			result = append(result, m.Name())
+		}
+	})
+	return result
+}
+
+// buildAndroidBp creates the blueprint file that defines prebuilt modules for each of
+// the SDK members, and the sdk_snapshot module for the specified version
+func (s *sdk) buildAndroidBp(ctx android.ModuleContext, version string) android.OutputPath {
+	bp := newGeneratedFile(ctx, "blueprint-"+version+".sh")
+
+	makePrebuiltName := func(name string) string {
+		return ctx.ModuleName() + "_" + name + string(android.SdkVersionSeparator) + version
+	}
+
+	javaLibs := s.javaMemberNames(ctx)
+	for _, name := range javaLibs {
+		prebuiltName := makePrebuiltName(name)
+		jar := filepath.Join("java", name, "stub.jar")
+
+		bp.printfln("java_import {")
+		bp.printfln("    name: %q,", prebuiltName)
+		bp.printfln("    jars: [%q],", jar)
+		bp.printfln("    sdk_member_name: %q,", name)
+		bp.printfln("}")
+		bp.printfln("")
+
+		// This module is for the case when the source tree for the unversioned module
+		// doesn't exist (i.e. building in an unbundled tree). "prefer:" is set to false
+		// so that this module does not eclipse the unversioned module if it exists.
+		bp.printfln("java_import {")
+		bp.printfln("    name: %q,", name)
+		bp.printfln("    jars: [%q],", jar)
+		bp.printfln("    prefer: false,")
+		bp.printfln("}")
+		bp.printfln("")
+
+	}
+
+	// TODO(jiyong): emit cc_prebuilt_library_shared for the native libs
+
+	bp.printfln("sdk_snapshot {")
+	bp.printfln("    name: %q,", ctx.ModuleName()+string(android.SdkVersionSeparator)+version)
+	bp.printfln("    java_libs: [")
+	for _, n := range javaLibs {
+		bp.printfln("        %q,", makePrebuiltName(n))
+	}
+	bp.printfln("    ],")
+	// TODO(jiyong): emit native_shared_libs
+	bp.printfln("}")
+	bp.printfln("")
+
+	bp.build(pctx, ctx, nil)
+	return bp.path
+}
+
+func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.OutputPath {
+	sh := newGeneratedFile(ctx, "update_prebuilt-"+version+".sh")
+
+	snapshotRoot := filepath.Join(ctx.ModuleDir(), version)
+	aidlIncludeDir := filepath.Join(snapshotRoot, "aidl")
+	javaStubsDir := filepath.Join(snapshotRoot, "java")
+
+	sh.printfln("#!/bin/bash")
+	sh.printfln("echo Updating snapshot of %s in %s", ctx.ModuleName(), snapshotRoot)
+	sh.printfln("pushd $ANDROID_BUILD_TOP > /dev/null")
+	sh.printfln("rm -rf %s", snapshotRoot)
+	sh.printfln("mkdir -p %s", aidlIncludeDir)
+	sh.printfln("mkdir -p %s", javaStubsDir)
+	// TODO(jiyong): mkdir the 'native' dir
+
+	var implicits android.Paths
+	ctx.VisitDirectDeps(func(m android.Module) {
+		if javaLib, ok := m.(*java.Library); ok {
+			headerJars := javaLib.HeaderJars()
+			if len(headerJars) != 1 {
+				panic(fmt.Errorf("there must be only one header jar from %q", m.Name()))
+			}
+			implicits = append(implicits, headerJars...)
+
+			exportedAidlIncludeDirs := javaLib.AidlIncludeDirs()
+			for _, dir := range exportedAidlIncludeDirs {
+				// Using tar to copy with the directory structure
+				// TODO(jiyong): copy parcelable declarations only
+				sh.printfln("find %s -name \"*.aidl\" | tar cf - -T - | (cd %s; tar xf -)",
+					dir.String(), aidlIncludeDir)
+			}
+
+			copiedHeaderJar := filepath.Join(javaStubsDir, m.Name(), "stub.jar")
+			sh.printfln("mkdir -p $(dirname %s) && cp %s %s",
+				copiedHeaderJar, headerJars[0].String(), copiedHeaderJar)
+		}
+		// TODO(jiyong): emit the commands for copying the headers and stub libraries for native libs
+	})
+
+	bp := s.buildAndroidBp(ctx, version)
+	implicits = append(implicits, bp)
+	sh.printfln("cp %s %s", bp.String(), filepath.Join(snapshotRoot, "Android.bp"))
+
+	sh.printfln("popd > /dev/null")
+	sh.printfln("rm -- \"$0\"") // self deleting so that stale script is not used
+	sh.printfln("echo Done")
+
+	sh.build(pctx, ctx, implicits)
+	return sh.path
+}
+
+func (s *sdk) buildSnapshotGenerationScripts(ctx android.ModuleContext) {
+	if s.snapshot() {
+		// we don't need a script for sdk_snapshot.. as they are frozen
+		return
+	}
+
+	// script to update the 'current' snapshot
+	s.updateScript = s.buildScript(ctx, "current")
+
+	versions := s.frozenVersions(ctx)
+	newVersion := "1"
+	if len(versions) >= 1 {
+		lastVersion := versions[len(versions)-1]
+		lastVersionNum, err := strconv.Atoi(lastVersion)
+		if err != nil {
+			panic(err)
+			return
+		}
+		newVersion = strconv.Itoa(lastVersionNum + 1)
+	}
+	// script to create a new frozen version of snapshot
+	s.freezeScript = s.buildScript(ctx, newVersion)
+}
+
+func (s *sdk) androidMkEntriesForScript() android.AndroidMkEntries {
+	if s.snapshot() {
+		// we don't need a script for sdk_snapshot.. as they are frozen
+		return android.AndroidMkEntries{}
+	}
+
+	entries := android.AndroidMkEntries{
+		Class: "FAKE",
+		// TODO(jiyong): remove this? but androidmk.go expects OutputFile to be specified anyway
+		OutputFile: android.OptionalPathForPath(s.updateScript),
+		Include:    "$(BUILD_SYSTEM)/base_rules.mk",
+		ExtraEntries: []android.AndroidMkExtraEntriesFunc{
+			func(entries *android.AndroidMkEntries) {
+				entries.AddStrings("LOCAL_ADDITIONAL_DEPENDENCIES",
+					s.updateScript.String(), s.freezeScript.String())
+			},
+		},
+		ExtraFooters: []android.AndroidMkExtraFootersFunc{
+			func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) {
+				fmt.Fprintln(w, "$(LOCAL_BUILT_MODULE): $(LOCAL_ADDITIONAL_DEPENDENCIES)")
+				fmt.Fprintln(w, "	touch $@")
+				fmt.Fprintln(w, "	echo ##################################################")
+				fmt.Fprintln(w, "	echo To update current SDK: execute", s.updateScript.String())
+				fmt.Fprintln(w, "	echo To freeze current SDK: execute", s.freezeScript.String())
+				fmt.Fprintln(w, "	echo ##################################################")
+			},
+		},
+	}
+	return entries
+}