SDK snapshot is dist'ed

`m module_sdk dist` produces snapshots of all SDKs in the source tree.
A snapshot is a zip file consists of Android.bp, exported headers,
exported AIDL files, stubs for native libs and jars. The zip file is
expected to be downloaded from the build server and extracted to a
directory (which probably will be
/prebuilts/module_sdks/<module_name>/current).

Bug: 138182343
Test: m (sdk_test.go updated)

Change-Id: Idbe4bc24795fe08f26fc1cf7497028f9d162053a

cc/cc.go

@@ -858,21 +858,28 @@ func (c *Module) ExportedIncludeDirs() android.Paths {
 	if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok {
 		return flagsProducer.exportedDirs()
 	}
-	return []android.Path{}
+	return nil
 }
 
 func (c *Module) ExportedSystemIncludeDirs() android.Paths {
 	if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok {
 		return flagsProducer.exportedSystemDirs()
 	}
-	return []android.Path{}
+	return nil
 }
 
 func (c *Module) ExportedFlags() []string {
 	if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok {
 		return flagsProducer.exportedFlags()
 	}
-	return []string{}
+	return nil
+}
+
+func (c *Module) ExportedDeps() android.Paths {
+	if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok {
+		return flagsProducer.exportedDeps()
+	}
+	return nil
 }
 
 func isBionic(name string) bool {

sdk/sdk.go

@@ -29,6 +29,7 @@ import (
 )
 
 func init() {
+	pctx.Import("android/soong/android")
 	android.RegisterModuleType("sdk", ModuleFactory)
 	android.RegisterModuleType("sdk_snapshot", SnapshotModuleFactory)
 	android.PreDepsMutators(RegisterPreDepsMutators)
@@ -41,8 +42,7 @@ type sdk struct {
 
 	properties sdkProperties
 
-	updateScript android.OutputPath
+	snapshotFile android.OptionalPath
-	freezeScript android.OutputPath
 }
 
 type sdkProperties struct {
@@ -104,11 +104,24 @@ func (s *sdk) frozenVersions(ctx android.BaseModuleContext) []string {
 }
 
 func (s *sdk) GenerateAndroidBuildActions(ctx android.ModuleContext) {
-	s.buildSnapshotGenerationScripts(ctx)
+	if !s.snapshot() {
+		// We don't need to create a snapshot out of sdk_snapshot.
+		// That doesn't make sense. We need a snapshot to create sdk_snapshot.
+		s.snapshotFile = android.OptionalPathForPath(s.buildSnapshot(ctx))
+	}
 }
 
 func (s *sdk) AndroidMkEntries() android.AndroidMkEntries {
-	return s.androidMkEntriesForScript()
+	if !s.snapshotFile.Valid() {
+		return android.AndroidMkEntries{}
+	}
+
+	return android.AndroidMkEntries{
+		Class:      "FAKE",
+		OutputFile: s.snapshotFile,
+		DistFile:   s.snapshotFile,
+		Include:    "$(BUILD_PHONY_PACKAGE)",
+	}
 }
 
 // RegisterPreDepsMutators registers pre-deps mutators to support modules implementing SdkAware

sdk/sdk_test.go

@@ -17,6 +17,7 @@ package sdk
 import (
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -100,6 +101,8 @@ func testSdkContext(t *testing.T, bp string) (*android.TestContext, android.Conf
 		"myapex.pk8":                                 nil,
 		"Test.java":                                  nil,
 		"Test.cpp":                                   nil,
+		"include/Test.h":                             nil,
+		"aidl/foo/bar/Test.aidl":                     nil,
 		"libfoo.so":                                  nil,
 	})
 
@@ -398,7 +401,9 @@ func TestSdkIsCompileMultilibBoth(t *testing.T) {
 	var inputs []string
 	buildParams := ctx.ModuleForTests("mysdk", "android_common").Module().BuildParamsForTests()
 	for _, bp := range buildParams {
-		inputs = append(inputs, bp.Implicits.Strings()...)
+		if bp.Input != nil {
+			inputs = append(inputs, bp.Input.String())
+		}
 	}
 
 	// ensure that both 32/64 outputs are inputs of the sdk snapshot
@@ -406,6 +411,69 @@ func TestSdkIsCompileMultilibBoth(t *testing.T) {
 	ensureListContains(t, inputs, arm64Output.String())
 }
 
+func TestSnapshot(t *testing.T) {
+	ctx, config := testSdk(t, `
+		sdk {
+			name: "mysdk",
+			java_libs: ["myjavalib"],
+			native_shared_libs: ["mynativelib"],
+		}
+
+		java_library {
+			name: "myjavalib",
+			srcs: ["Test.java"],
+			aidl: {
+				export_include_dirs: ["aidl"],
+			},
+			system_modules: "none",
+			sdk_version: "none",
+			compile_dex: true,
+			host_supported: true,
+		}
+
+		cc_library_shared {
+			name: "mynativelib",
+			srcs: [
+				"Test.cpp",
+				"aidl/foo/bar/Test.aidl",
+			],
+			export_include_dirs: ["include"],
+			aidl: {
+				export_aidl_headers: true,
+			},
+			system_shared_libs: [],
+			stl: "none",
+		}
+	`)
+
+	var copySrcs []string
+	var copyDests []string
+	buildParams := ctx.ModuleForTests("mysdk", "android_common").Module().BuildParamsForTests()
+	for _, bp := range buildParams {
+		if bp.Rule.String() == "android/soong/android.Cp" {
+			copySrcs = append(copySrcs, bp.Input.String())
+			copyDests = append(copyDests, bp.Output.Rel()) // rooted at the snapshot root
+		}
+	}
+
+	buildDir := config.BuildDir()
+	ensureListContains(t, copySrcs, "aidl/foo/bar/Test.aidl")
+	ensureListContains(t, copySrcs, "include/Test.h")
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/gen/aidl/aidl/foo/bar/BnTest.h"))
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/gen/aidl/aidl/foo/bar/BpTest.h"))
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/gen/aidl/aidl/foo/bar/Test.h"))
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar"))
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/mynativelib.so"))
+
+	ensureListContains(t, copyDests, "aidl/aidl/foo/bar/Test.aidl")
+	ensureListContains(t, copyDests, "arm64/include/include/Test.h")
+	ensureListContains(t, copyDests, "arm64/include_gen/mynativelib/aidl/foo/bar/BnTest.h")
+	ensureListContains(t, copyDests, "arm64/include_gen/mynativelib/aidl/foo/bar/BpTest.h")
+	ensureListContains(t, copyDests, "arm64/include_gen/mynativelib/aidl/foo/bar/Test.h")
+	ensureListContains(t, copyDests, "java/myjavalib.jar")
+	ensureListContains(t, copyDests, "arm64/lib/mynativelib.so")
+}
+
 var buildDir string
 
 func setUp() {

sdk/update.go

@@ -16,9 +16,7 @@ package sdk
 
 import (
 	"fmt"
-	"io"
 	"path/filepath"
-	"strconv"
 	"strings"
 
 	"github.com/google/blueprint/proptools"
@@ -38,9 +36,9 @@ type generatedFile struct {
 	indentLevel int
 }
 
-func newGeneratedFile(ctx android.ModuleContext, name string) *generatedFile {
+func newGeneratedFile(ctx android.ModuleContext, path ...string) *generatedFile {
 	return &generatedFile{
-		path:        android.PathForModuleOut(ctx, name).OutputPath,
+		path:        android.PathForModuleOut(ctx, path...).OutputPath,
 		indentLevel: 0,
 	}
 }
@@ -89,6 +87,7 @@ type archSpecificNativeLibInfo struct {
 	exportedIncludeDirs       android.Paths
 	exportedSystemIncludeDirs android.Paths
 	exportedFlags             []string
+	exportedDeps              android.Paths
 	outputFile                android.Path
 }
 
@@ -132,6 +131,7 @@ func (s *sdk) nativeMemberInfos(ctx android.ModuleContext) []*nativeLibInfo {
 			exportedIncludeDirs:       ccModule.ExportedIncludeDirs(),
 			exportedSystemIncludeDirs: ccModule.ExportedSystemIncludeDirs(),
 			exportedFlags:             ccModule.ExportedFlags(),
+			exportedDeps:              ccModule.ExportedDeps(),
 			outputFile:                ccModule.OutputFile().Path(),
 		})
 	})
@@ -169,11 +169,11 @@ func (s *sdk) nativeMemberInfos(ctx android.ModuleContext) []*nativeLibInfo {
 //         aidl/
 //            frameworks/base/core/..../IFoo.aidl   : an exported AIDL file
 //         java/
-//            java/<module_name>/stub.jar    : a stub jar for a java library 'module_name'
+//            <module_name>.jar    : the stub jar for a java library 'module_name'
 //         include/
 //            bionic/libc/include/stdlib.h   : an exported header file
 //         include_gen/
-//            com/android/.../IFoo.h : a generated header file
+//            <module_name>/com/android/.../IFoo.h : a generated header file
 //         <arch>/include/   : arch-specific exported headers
 //         <arch>/include_gen/   : arch-specific generated headers
 //         <arch>/lib/
@@ -182,7 +182,7 @@ func (s *sdk) nativeMemberInfos(ctx android.ModuleContext) []*nativeLibInfo {
 const (
 	aidlIncludeDir            = "aidl"
 	javaStubDir               = "java"
-	javaStubFile              = "stub.jar"
+	javaStubFileSuffix        = ".jar"
 	nativeIncludeDir          = "include"
 	nativeGeneratedIncludeDir = "include_gen"
 	nativeStubDir             = "lib"
@@ -191,7 +191,7 @@ const (
 
 // path to the stub file of a java library. Relative to <sdk_root>/<api_dir>
 func javaStubFilePathFor(javaLib *java.Library) string {
-	return filepath.Join(javaStubDir, javaLib.Name(), javaStubFile)
+	return filepath.Join(javaStubDir, javaLib.Name()+javaStubFileSuffix)
 }
 
 // path to the stub file of a native shared library. Relative to <sdk_root>/<api_dir>
@@ -204,7 +204,6 @@ func nativeStubFilePathFor(lib archSpecificNativeLibInfo) string {
 func nativeIncludeDirPathsFor(ctx android.ModuleContext, lib archSpecificNativeLibInfo,
 	systemInclude bool, archSpecific bool) []string {
 	var result []string
-	buildDir := ctx.Config().BuildDir()
 	var includeDirs []android.Path
 	if !systemInclude {
 		includeDirs = lib.exportedIncludeDirs
@@ -213,8 +212,8 @@ func nativeIncludeDirPathsFor(ctx android.ModuleContext, lib archSpecificNativeL
 	}
 	for _, dir := range includeDirs {
 		var path string
-		if gen := strings.HasPrefix(dir.String(), buildDir); gen {
+		if _, gen := dir.(android.WritablePath); gen {
-			path = filepath.Join(nativeGeneratedIncludeDir, dir.Rel())
+			path = filepath.Join(nativeGeneratedIncludeDir, lib.name)
 		} else {
 			path = filepath.Join(nativeIncludeDir, dir.String())
 		}
@@ -226,21 +225,19 @@ func nativeIncludeDirPathsFor(ctx android.ModuleContext, lib archSpecificNativeL
 	return result
 }
 
-// A name that uniquely identifies an prebuilt SDK member for a version of SDK snapshot
+// A name that uniquely identifies a prebuilt SDK member for a version of SDK snapshot
 // This isn't visible to users, so could be changed in future.
 func versionedSdkMemberName(ctx android.ModuleContext, memberName string, version string) string {
 	return ctx.ModuleName() + "_" + memberName + string(android.SdkVersionSeparator) + version
 }
 
-// arm64, arm, x86, x86_64, etc.
-func archTypeOf(module android.Module) string {
-	return module.Target().Arch.ArchType.String()
-}
-
 // buildAndroidBp creates the blueprint file that defines prebuilt modules for each of
 // the SDK members, and the entire sdk_snapshot module for the specified version
+// TODO(jiyong): create a meta info file (e.g. json, protobuf, etc.) instead, and convert it to
+// Android.bp in the (presumably old) branch where the snapshots will be used. This will give us
+// some flexibility to introduce backwards incompatible changes in soong.
 func (s *sdk) buildAndroidBp(ctx android.ModuleContext, version string) android.OutputPath {
-	bp := newGeneratedFile(ctx, "blueprint-"+version+".bp")
+	bp := newGeneratedFile(ctx, "snapshot", "Android.bp")
 	bp.printfln("// This is auto-generated. DO NOT EDIT.")
 	bp.printfln("")
 
@@ -352,52 +349,42 @@ func (s *sdk) buildAndroidBp(ctx android.ModuleContext, version string) android.
 	return bp.path
 }
 
-func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.OutputPath {
+// buildSnapshot is the main function in this source file. It creates rules to copy
-	sh := newGeneratedFile(ctx, "update_prebuilt-"+version+".sh")
+// the contents (header files, stub libraries, etc) into the zip file.
-	buildDir := ctx.Config().BuildDir()
+func (s *sdk) buildSnapshot(ctx android.ModuleContext) android.OutputPath {
-
+	snapshotPath := func(paths ...string) android.OutputPath {
-	snapshotPath := func(paths ...string) string {
+		return android.PathForModuleOut(ctx, "snapshot").Join(ctx, paths...)
-		return filepath.Join(ctx.ModuleDir(), version, filepath.Join(paths...))
 	}
 
-	// TODO(jiyong) instead of creating script, create a zip file having the Android.bp, the headers,
+	var filesToZip android.Paths
-	// and the stubs and put it to the dist directory. The dist'ed zip file then would be downloaded,
+	// copy src to dest and add the dest to the zip
-	// unzipped and then uploaded to gerrit again.
+	copy := func(src android.Path, dest android.OutputPath) {
-	sh.printfln("#!/bin/bash")
+		ctx.Build(pctx, android.BuildParams{
-	sh.printfln("echo Updating snapshot of %s in %s", ctx.ModuleName(), snapshotPath())
+			Rule:   android.Cp,
-	sh.printfln("pushd $ANDROID_BUILD_TOP > /dev/null")
+			Input:  src,
-	sh.printfln("mkdir -p %s", snapshotPath(aidlIncludeDir))
+			Output: dest,
-	sh.printfln("mkdir -p %s", snapshotPath(javaStubDir))
+		})
-	sh.printfln("mkdir -p %s", snapshotPath(nativeIncludeDir))
+		filesToZip = append(filesToZip, dest)
-	sh.printfln("mkdir -p %s", snapshotPath(nativeGeneratedIncludeDir))
-	for _, target := range ctx.MultiTargets() {
-		arch := target.Arch.ArchType.String()
-		sh.printfln("mkdir -p %s", snapshotPath(arch, nativeStubDir))
-		sh.printfln("mkdir -p %s", snapshotPath(arch, nativeIncludeDir))
-		sh.printfln("mkdir -p %s", snapshotPath(arch, nativeGeneratedIncludeDir))
 	}
 
-	var implicits android.Paths
+	// copy exported AIDL files and stub jar files
 	for _, m := range s.javaLibs(ctx) {
 		headerJars := m.HeaderJars()
 		if len(headerJars) != 1 {
 			panic(fmt.Errorf("there must be only one header jar from %q", m.Name()))
 		}
-		implicits = append(implicits, headerJars...)
+		copy(headerJars[0], snapshotPath(javaStubFilePathFor(m)))
 
-		exportedAidlIncludeDirs := m.AidlIncludeDirs()
+		for _, dir := range m.AidlIncludeDirs() {
-		for _, dir := range exportedAidlIncludeDirs {
-			// Using tar to copy with the directory structure
 			// TODO(jiyong): copy parcelable declarations only
-			sh.printfln("find %s -name \"*.aidl\" | tar cf - -T - | (cd %s; tar xf -)",
+			aidlFiles, _ := ctx.GlobWithDeps(dir.String()+"/**/*.aidl", nil)
-				dir.String(), snapshotPath(aidlIncludeDir))
+			for _, file := range aidlFiles {
+				copy(android.PathForSource(ctx, file), snapshotPath(aidlIncludeDir, file))
+			}
 		}
-
-		copyTarget := snapshotPath(javaStubFilePathFor(m))
-		sh.printfln("mkdir -p %s && cp %s %s",
-			filepath.Dir(copyTarget), headerJars[0].String(), copyTarget)
 	}
 
+	// copy exported header files and stub *.so files
 	nativeLibInfos := s.nativeMemberInfos(ctx)
 	for _, info := range nativeLibInfos {
 
@@ -409,26 +396,32 @@ func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.Out
 				return
 			}
 			for _, dir := range includeDirs {
-				gen := strings.HasPrefix(dir.String(), buildDir)
+				if _, gen := dir.(android.WritablePath); gen {
-				targetDir := nativeIncludeDir
+					// generated headers are copied via exportedDeps. See below.
-				if gen {
+					continue
-					targetDir = nativeGeneratedIncludeDir
 				}
+				targetDir := nativeIncludeDir
 				if info.hasArchSpecificFlags {
 					targetDir = filepath.Join(lib.archType, targetDir)
 				}
-				targetDir = snapshotPath(targetDir)
 
-				sourceDirRoot := "."
-				sourceDirRel := dir.String()
-				if gen {
-					// ex) out/soong/.intermediate/foo/bar/gen/aidl
-					sourceDirRoot = strings.TrimSuffix(dir.String(), dir.Rel())
-					sourceDirRel = dir.Rel()
-				}
 				// TODO(jiyong) copy headers having other suffixes
-				sh.printfln("(cd %s; find %s -name \"*.h\" | tar cf - -T - ) | (cd %s; tar xf -)",
+				headers, _ := ctx.GlobWithDeps(dir.String()+"/**/*.h", nil)
-					sourceDirRoot, sourceDirRel, targetDir)
+				for _, file := range headers {
+					src := android.PathForSource(ctx, file)
+					dest := snapshotPath(targetDir, file)
+					copy(src, dest)
+				}
+			}
+
+			genHeaders := lib.exportedDeps
+			for _, file := range genHeaders {
+				targetDir := nativeGeneratedIncludeDir
+				if info.hasArchSpecificFlags {
+					targetDir = filepath.Join(lib.archType, targetDir)
+				}
+				dest := snapshotPath(targetDir, lib.name, file.Rel())
+				copy(file, dest)
 			}
 		}
 
@@ -438,10 +431,7 @@ func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.Out
 
 		// for each architecture
 		for _, av := range info.archVariants {
-			stub := av.outputFile
+			copy(av.outputFile, snapshotPath(nativeStubFilePathFor(av)))
-			implicits = append(implicits, stub)
-			copiedStub := snapshotPath(nativeStubFilePathFor(av))
-			sh.printfln("cp %s %s", stub.String(), copiedStub)
 
 			if info.hasArchSpecificFlags {
 				printExportedDirCopyCommandsForNativeLibs(av)
@@ -449,69 +439,19 @@ func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.Out
 		}
 	}
 
-	bp := s.buildAndroidBp(ctx, version)
+	// generate Android.bp
-	implicits = append(implicits, bp)
+	bp := s.buildAndroidBp(ctx, "current")
-	sh.printfln("cp %s %s", bp.String(), snapshotPath("Android.bp"))
+	filesToZip = append(filesToZip, bp)
 
-	sh.printfln("popd > /dev/null")
+	// zip them all
-	sh.printfln("rm -- \"$0\"") // self deleting so that stale script is not used
+	zipFile := android.PathForModuleOut(ctx, ctx.ModuleName()+"-current.zip").OutputPath
-	sh.printfln("echo Done")
+	rb := android.NewRuleBuilder()
+	rb.Command().
+		BuiltTool(ctx, "soong_zip").
+		FlagWithArg("-C ", snapshotPath().String()).
+		FlagWithRspFileInputList("-l ", filesToZip).
+		FlagWithOutput("-o ", zipFile)
+	rb.Build(pctx, ctx, "snapshot", "Building snapshot for "+ctx.ModuleName())
 
-	sh.build(pctx, ctx, implicits)
+	return zipFile
-	return sh.path
-}
-
-func (s *sdk) buildSnapshotGenerationScripts(ctx android.ModuleContext) {
-	if s.snapshot() {
-		// we don't need a script for sdk_snapshot.. as they are frozen
-		return
-	}
-
-	// script to update the 'current' snapshot
-	s.updateScript = s.buildScript(ctx, "current")
-
-	versions := s.frozenVersions(ctx)
-	newVersion := "1"
-	if len(versions) >= 1 {
-		lastVersion := versions[len(versions)-1]
-		lastVersionNum, err := strconv.Atoi(lastVersion)
-		if err != nil {
-			panic(err)
-			return
-		}
-		newVersion = strconv.Itoa(lastVersionNum + 1)
-	}
-	// script to create a new frozen version of snapshot
-	s.freezeScript = s.buildScript(ctx, newVersion)
-}
-
-func (s *sdk) androidMkEntriesForScript() android.AndroidMkEntries {
-	if s.snapshot() {
-		// we don't need a script for sdk_snapshot.. as they are frozen
-		return android.AndroidMkEntries{}
-	}
-
-	entries := android.AndroidMkEntries{
-		Class: "FAKE",
-		// TODO(jiyong): remove this? but androidmk.go expects OutputFile to be specified anyway
-		OutputFile: android.OptionalPathForPath(s.updateScript),
-		Include:    "$(BUILD_SYSTEM)/base_rules.mk",
-		ExtraEntries: []android.AndroidMkExtraEntriesFunc{
-			func(entries *android.AndroidMkEntries) {
-				entries.AddStrings("LOCAL_ADDITIONAL_DEPENDENCIES",
-					s.updateScript.String(), s.freezeScript.String())
-			},
-		},
-		ExtraFooters: []android.AndroidMkExtraFootersFunc{
-			func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) {
-				fmt.Fprintln(w, "$(LOCAL_BUILT_MODULE): $(LOCAL_ADDITIONAL_DEPENDENCIES)")
-				fmt.Fprintln(w, "	touch $@")
-				fmt.Fprintln(w, "	echo ##################################################")
-				fmt.Fprintln(w, "	echo To update current SDK: execute", filepath.Join("\\$$ANDROID_BUILD_TOP", s.updateScript.String()))
-				fmt.Fprintln(w, "	echo To freeze current SDK: execute", filepath.Join("\\$$ANDROID_BUILD_TOP", s.freezeScript.String()))
-				fmt.Fprintln(w, "	echo ##################################################")
-			},
-		},
-	}
-	return entries
 }