Merge "SDK snapshot is dist'ed"

cc/cc.go

@@ -858,21 +858,28 @@ func (c *Module) ExportedIncludeDirs() android.Paths {
 	if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok {
 		return flagsProducer.exportedDirs()
 	}
-	return []android.Path{}
+	return nil
 }
 
 func (c *Module) ExportedSystemIncludeDirs() android.Paths {
 	if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok {
 		return flagsProducer.exportedSystemDirs()
 	}
-	return []android.Path{}
+	return nil
 }
 
 func (c *Module) ExportedFlags() []string {
 	if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok {
 		return flagsProducer.exportedFlags()
 	}
-	return []string{}
+	return nil
+}
+
+func (c *Module) ExportedDeps() android.Paths {
+	if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok {
+		return flagsProducer.exportedDeps()
+	}
+	return nil
 }
 
 func isBionic(name string) bool {

sdk/sdk.go

@@ -29,6 +29,7 @@ import (
 )
 
 func init() {
+	pctx.Import("android/soong/android")
 	android.RegisterModuleType("sdk", ModuleFactory)
 	android.RegisterModuleType("sdk_snapshot", SnapshotModuleFactory)
 	android.PreDepsMutators(RegisterPreDepsMutators)
@@ -41,8 +42,7 @@ type sdk struct {
 
 	properties sdkProperties
 
-	updateScript android.OutputPath
+	snapshotFile android.OptionalPath
-	freezeScript android.OutputPath
 }
 
 type sdkProperties struct {
@@ -104,11 +104,24 @@ func (s *sdk) frozenVersions(ctx android.BaseModuleContext) []string {
 }
 
 func (s *sdk) GenerateAndroidBuildActions(ctx android.ModuleContext) {
-	s.buildSnapshotGenerationScripts(ctx)
+	if !s.snapshot() {
+		// We don't need to create a snapshot out of sdk_snapshot.
+		// That doesn't make sense. We need a snapshot to create sdk_snapshot.
+		s.snapshotFile = android.OptionalPathForPath(s.buildSnapshot(ctx))
+	}
 }
 
 func (s *sdk) AndroidMkEntries() android.AndroidMkEntries {
-	return s.androidMkEntriesForScript()
+	if !s.snapshotFile.Valid() {
+		return android.AndroidMkEntries{}
+	}
+
+	return android.AndroidMkEntries{
+		Class:      "FAKE",
+		OutputFile: s.snapshotFile,
+		DistFile:   s.snapshotFile,
+		Include:    "$(BUILD_PHONY_PACKAGE)",
+	}
 }
 
 // RegisterPreDepsMutators registers pre-deps mutators to support modules implementing SdkAware

sdk/sdk_test.go

@@ -17,6 +17,7 @@ package sdk
 import (
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -100,6 +101,8 @@ func testSdkContext(t *testing.T, bp string) (*android.TestContext, android.Conf
 		"myapex.pk8":                                 nil,
 		"Test.java":                                  nil,
 		"Test.cpp":                                   nil,
+		"include/Test.h":                             nil,
+		"aidl/foo/bar/Test.aidl":                     nil,
 		"libfoo.so":                                  nil,
 	})
 
@@ -398,7 +401,9 @@ func TestSdkIsCompileMultilibBoth(t *testing.T) {
 	var inputs []string
 	buildParams := ctx.ModuleForTests("mysdk", "android_common").Module().BuildParamsForTests()
 	for _, bp := range buildParams {
-		inputs = append(inputs, bp.Implicits.Strings()...)
+		if bp.Input != nil {
+			inputs = append(inputs, bp.Input.String())
+		}
 	}
 
 	// ensure that both 32/64 outputs are inputs of the sdk snapshot
@@ -406,6 +411,69 @@ func TestSdkIsCompileMultilibBoth(t *testing.T) {
 	ensureListContains(t, inputs, arm64Output.String())
 }
 
+func TestSnapshot(t *testing.T) {
+	ctx, config := testSdk(t, `
+		sdk {
+			name: "mysdk",
+			java_libs: ["myjavalib"],
+			native_shared_libs: ["mynativelib"],
+		}
+
+		java_library {
+			name: "myjavalib",
+			srcs: ["Test.java"],
+			aidl: {
+				export_include_dirs: ["aidl"],
+			},
+			system_modules: "none",
+			sdk_version: "none",
+			compile_dex: true,
+			host_supported: true,
+		}
+
+		cc_library_shared {
+			name: "mynativelib",
+			srcs: [
+				"Test.cpp",
+				"aidl/foo/bar/Test.aidl",
+			],
+			export_include_dirs: ["include"],
+			aidl: {
+				export_aidl_headers: true,
+			},
+			system_shared_libs: [],
+			stl: "none",
+		}
+	`)
+
+	var copySrcs []string
+	var copyDests []string
+	buildParams := ctx.ModuleForTests("mysdk", "android_common").Module().BuildParamsForTests()
+	for _, bp := range buildParams {
+		if bp.Rule.String() == "android/soong/android.Cp" {
+			copySrcs = append(copySrcs, bp.Input.String())
+			copyDests = append(copyDests, bp.Output.Rel()) // rooted at the snapshot root
+		}
+	}
+
+	buildDir := config.BuildDir()
+	ensureListContains(t, copySrcs, "aidl/foo/bar/Test.aidl")
+	ensureListContains(t, copySrcs, "include/Test.h")
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/gen/aidl/aidl/foo/bar/BnTest.h"))
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/gen/aidl/aidl/foo/bar/BpTest.h"))
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/gen/aidl/aidl/foo/bar/Test.h"))
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar"))
+	ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/mynativelib.so"))
+
+	ensureListContains(t, copyDests, "aidl/aidl/foo/bar/Test.aidl")
+	ensureListContains(t, copyDests, "arm64/include/include/Test.h")
+	ensureListContains(t, copyDests, "arm64/include_gen/mynativelib/aidl/foo/bar/BnTest.h")
+	ensureListContains(t, copyDests, "arm64/include_gen/mynativelib/aidl/foo/bar/BpTest.h")
+	ensureListContains(t, copyDests, "arm64/include_gen/mynativelib/aidl/foo/bar/Test.h")
+	ensureListContains(t, copyDests, "java/myjavalib.jar")
+	ensureListContains(t, copyDests, "arm64/lib/mynativelib.so")
+}
+
 var buildDir string
 
 func setUp() {

sdk/update.go

@@ -16,9 +16,7 @@ package sdk
 
 import (
 	"fmt"
-	"io"
 	"path/filepath"
-	"strconv"
 	"strings"
 
 	"github.com/google/blueprint/proptools"
@@ -38,9 +36,9 @@ type generatedFile struct {
 	indentLevel int
 }
 
-func newGeneratedFile(ctx android.ModuleContext, name string) *generatedFile {
+func newGeneratedFile(ctx android.ModuleContext, path ...string) *generatedFile {
 	return &generatedFile{
-		path:        android.PathForModuleOut(ctx, name).OutputPath,
+		path:        android.PathForModuleOut(ctx, path...).OutputPath,
 		indentLevel: 0,
 	}
 }
@@ -89,6 +87,7 @@ type archSpecificNativeLibInfo struct {
 	exportedIncludeDirs       android.Paths
 	exportedSystemIncludeDirs android.Paths
 	exportedFlags             []string
+	exportedDeps              android.Paths
 	outputFile                android.Path
 }
 
@@ -132,6 +131,7 @@ func (s *sdk) nativeMemberInfos(ctx android.ModuleContext) []*nativeLibInfo {
 			exportedIncludeDirs:       ccModule.ExportedIncludeDirs(),
 			exportedSystemIncludeDirs: ccModule.ExportedSystemIncludeDirs(),
 			exportedFlags:             ccModule.ExportedFlags(),
+			exportedDeps:              ccModule.ExportedDeps(),
 			outputFile:                ccModule.OutputFile().Path(),
 		})
 	})
@@ -169,11 +169,11 @@ func (s *sdk) nativeMemberInfos(ctx android.ModuleContext) []*nativeLibInfo {
 //         aidl/
 //            frameworks/base/core/..../IFoo.aidl   : an exported AIDL file
 //         java/
-//            java/<module_name>/stub.jar    : a stub jar for a java library 'module_name'
+//            <module_name>.jar    : the stub jar for a java library 'module_name'
 //         include/
 //            bionic/libc/include/stdlib.h   : an exported header file
 //         include_gen/
-//            com/android/.../IFoo.h : a generated header file
+//            <module_name>/com/android/.../IFoo.h : a generated header file
 //         <arch>/include/   : arch-specific exported headers
 //         <arch>/include_gen/   : arch-specific generated headers
 //         <arch>/lib/
@@ -182,7 +182,7 @@ func (s *sdk) nativeMemberInfos(ctx android.ModuleContext) []*nativeLibInfo {
 const (
 	aidlIncludeDir            = "aidl"
 	javaStubDir               = "java"
-	javaStubFile              = "stub.jar"
+	javaStubFileSuffix        = ".jar"
 	nativeIncludeDir          = "include"
 	nativeGeneratedIncludeDir = "include_gen"
 	nativeStubDir             = "lib"
@@ -191,7 +191,7 @@ const (
 
 // path to the stub file of a java library. Relative to <sdk_root>/<api_dir>
 func javaStubFilePathFor(javaLib *java.Library) string {
-	return filepath.Join(javaStubDir, javaLib.Name(), javaStubFile)
+	return filepath.Join(javaStubDir, javaLib.Name()+javaStubFileSuffix)
 }
 
 // path to the stub file of a native shared library. Relative to <sdk_root>/<api_dir>
@@ -204,7 +204,6 @@ func nativeStubFilePathFor(lib archSpecificNativeLibInfo) string {
 func nativeIncludeDirPathsFor(ctx android.ModuleContext, lib archSpecificNativeLibInfo,
 	systemInclude bool, archSpecific bool) []string {
 	var result []string
-	buildDir := ctx.Config().BuildDir()
 	var includeDirs []android.Path
 	if !systemInclude {
 		includeDirs = lib.exportedIncludeDirs
@@ -213,8 +212,8 @@ func nativeIncludeDirPathsFor(ctx android.ModuleContext, lib archSpecificNativeL
 	}
 	for _, dir := range includeDirs {
 		var path string
-		if gen := strings.HasPrefix(dir.String(), buildDir); gen {
+		if _, gen := dir.(android.WritablePath); gen {
-			path = filepath.Join(nativeGeneratedIncludeDir, dir.Rel())
+			path = filepath.Join(nativeGeneratedIncludeDir, lib.name)
 		} else {
 			path = filepath.Join(nativeIncludeDir, dir.String())
 		}
@@ -226,21 +225,19 @@ func nativeIncludeDirPathsFor(ctx android.ModuleContext, lib archSpecificNativeL
 	return result
 }
 
-// A name that uniquely identifies an prebuilt SDK member for a version of SDK snapshot
+// A name that uniquely identifies a prebuilt SDK member for a version of SDK snapshot
 // This isn't visible to users, so could be changed in future.
 func versionedSdkMemberName(ctx android.ModuleContext, memberName string, version string) string {
 	return ctx.ModuleName() + "_" + memberName + string(android.SdkVersionSeparator) + version
 }
 
-// arm64, arm, x86, x86_64, etc.
-func archTypeOf(module android.Module) string {
-	return module.Target().Arch.ArchType.String()
-}
-
 // buildAndroidBp creates the blueprint file that defines prebuilt modules for each of
 // the SDK members, and the entire sdk_snapshot module for the specified version
+// TODO(jiyong): create a meta info file (e.g. json, protobuf, etc.) instead, and convert it to
+// Android.bp in the (presumably old) branch where the snapshots will be used. This will give us
+// some flexibility to introduce backwards incompatible changes in soong.
 func (s *sdk) buildAndroidBp(ctx android.ModuleContext, version string) android.OutputPath {
-	bp := newGeneratedFile(ctx, "blueprint-"+version+".bp")
+	bp := newGeneratedFile(ctx, "snapshot", "Android.bp")
 	bp.printfln("// This is auto-generated. DO NOT EDIT.")
 	bp.printfln("")
 
@@ -352,52 +349,42 @@ func (s *sdk) buildAndroidBp(ctx android.ModuleContext, version string) android.
 	return bp.path
 }
 
-func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.OutputPath {
+// buildSnapshot is the main function in this source file. It creates rules to copy
-	sh := newGeneratedFile(ctx, "update_prebuilt-"+version+".sh")
+// the contents (header files, stub libraries, etc) into the zip file.
-	buildDir := ctx.Config().BuildDir()
+func (s *sdk) buildSnapshot(ctx android.ModuleContext) android.OutputPath {
-
+	snapshotPath := func(paths ...string) android.OutputPath {
-	snapshotPath := func(paths ...string) string {
+		return android.PathForModuleOut(ctx, "snapshot").Join(ctx, paths...)
-		return filepath.Join(ctx.ModuleDir(), version, filepath.Join(paths...))
 	}
 
-	// TODO(jiyong) instead of creating script, create a zip file having the Android.bp, the headers,
+	var filesToZip android.Paths
-	// and the stubs and put it to the dist directory. The dist'ed zip file then would be downloaded,
+	// copy src to dest and add the dest to the zip
-	// unzipped and then uploaded to gerrit again.
+	copy := func(src android.Path, dest android.OutputPath) {
-	sh.printfln("#!/bin/bash")
+		ctx.Build(pctx, android.BuildParams{
-	sh.printfln("echo Updating snapshot of %s in %s", ctx.ModuleName(), snapshotPath())
+			Rule:   android.Cp,
-	sh.printfln("pushd $ANDROID_BUILD_TOP > /dev/null")
+			Input:  src,
-	sh.printfln("mkdir -p %s", snapshotPath(aidlIncludeDir))
+			Output: dest,
-	sh.printfln("mkdir -p %s", snapshotPath(javaStubDir))
+		})
-	sh.printfln("mkdir -p %s", snapshotPath(nativeIncludeDir))
+		filesToZip = append(filesToZip, dest)
-	sh.printfln("mkdir -p %s", snapshotPath(nativeGeneratedIncludeDir))
-	for _, target := range ctx.MultiTargets() {
-		arch := target.Arch.ArchType.String()
-		sh.printfln("mkdir -p %s", snapshotPath(arch, nativeStubDir))
-		sh.printfln("mkdir -p %s", snapshotPath(arch, nativeIncludeDir))
-		sh.printfln("mkdir -p %s", snapshotPath(arch, nativeGeneratedIncludeDir))
 	}
 
-	var implicits android.Paths
+	// copy exported AIDL files and stub jar files
 	for _, m := range s.javaLibs(ctx) {
 		headerJars := m.HeaderJars()
 		if len(headerJars) != 1 {
 			panic(fmt.Errorf("there must be only one header jar from %q", m.Name()))
 		}
-		implicits = append(implicits, headerJars...)
+		copy(headerJars[0], snapshotPath(javaStubFilePathFor(m)))
 
-		exportedAidlIncludeDirs := m.AidlIncludeDirs()
+		for _, dir := range m.AidlIncludeDirs() {
-		for _, dir := range exportedAidlIncludeDirs {
-			// Using tar to copy with the directory structure
 			// TODO(jiyong): copy parcelable declarations only
-			sh.printfln("find %s -name \"*.aidl\" | tar cf - -T - | (cd %s; tar xf -)",
+			aidlFiles, _ := ctx.GlobWithDeps(dir.String()+"/**/*.aidl", nil)
-				dir.String(), snapshotPath(aidlIncludeDir))
+			for _, file := range aidlFiles {
+				copy(android.PathForSource(ctx, file), snapshotPath(aidlIncludeDir, file))
+			}
 		}
-
-		copyTarget := snapshotPath(javaStubFilePathFor(m))
-		sh.printfln("mkdir -p %s && cp %s %s",
-			filepath.Dir(copyTarget), headerJars[0].String(), copyTarget)
 	}
 
+	// copy exported header files and stub *.so files
 	nativeLibInfos := s.nativeMemberInfos(ctx)
 	for _, info := range nativeLibInfos {
 
@@ -409,26 +396,32 @@ func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.Out
 				return
 			}
 			for _, dir := range includeDirs {
-				gen := strings.HasPrefix(dir.String(), buildDir)
+				if _, gen := dir.(android.WritablePath); gen {
-				targetDir := nativeIncludeDir
+					// generated headers are copied via exportedDeps. See below.
-				if gen {
+					continue
-					targetDir = nativeGeneratedIncludeDir
 				}
+				targetDir := nativeIncludeDir
 				if info.hasArchSpecificFlags {
 					targetDir = filepath.Join(lib.archType, targetDir)
 				}
-				targetDir = snapshotPath(targetDir)
 
-				sourceDirRoot := "."
-				sourceDirRel := dir.String()
-				if gen {
-					// ex) out/soong/.intermediate/foo/bar/gen/aidl
-					sourceDirRoot = strings.TrimSuffix(dir.String(), dir.Rel())
-					sourceDirRel = dir.Rel()
-				}
 				// TODO(jiyong) copy headers having other suffixes
-				sh.printfln("(cd %s; find %s -name \"*.h\" | tar cf - -T - ) | (cd %s; tar xf -)",
+				headers, _ := ctx.GlobWithDeps(dir.String()+"/**/*.h", nil)
-					sourceDirRoot, sourceDirRel, targetDir)
+				for _, file := range headers {
+					src := android.PathForSource(ctx, file)
+					dest := snapshotPath(targetDir, file)
+					copy(src, dest)
+				}
+			}
+
+			genHeaders := lib.exportedDeps
+			for _, file := range genHeaders {
+				targetDir := nativeGeneratedIncludeDir
+				if info.hasArchSpecificFlags {
+					targetDir = filepath.Join(lib.archType, targetDir)
+				}
+				dest := snapshotPath(targetDir, lib.name, file.Rel())
+				copy(file, dest)
 			}
 		}
 
@@ -438,10 +431,7 @@ func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.Out
 
 		// for each architecture
 		for _, av := range info.archVariants {
-			stub := av.outputFile
+			copy(av.outputFile, snapshotPath(nativeStubFilePathFor(av)))
-			implicits = append(implicits, stub)
-			copiedStub := snapshotPath(nativeStubFilePathFor(av))
-			sh.printfln("cp %s %s", stub.String(), copiedStub)
 
 			if info.hasArchSpecificFlags {
 				printExportedDirCopyCommandsForNativeLibs(av)
@@ -449,69 +439,19 @@ func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.Out
 		}
 	}
 
-	bp := s.buildAndroidBp(ctx, version)
+	// generate Android.bp
-	implicits = append(implicits, bp)
+	bp := s.buildAndroidBp(ctx, "current")
-	sh.printfln("cp %s %s", bp.String(), snapshotPath("Android.bp"))
+	filesToZip = append(filesToZip, bp)
 
-	sh.printfln("popd > /dev/null")
+	// zip them all
-	sh.printfln("rm -- \"$0\"") // self deleting so that stale script is not used
+	zipFile := android.PathForModuleOut(ctx, ctx.ModuleName()+"-current.zip").OutputPath
-	sh.printfln("echo Done")
+	rb := android.NewRuleBuilder()
+	rb.Command().
+		BuiltTool(ctx, "soong_zip").
+		FlagWithArg("-C ", snapshotPath().String()).
+		FlagWithRspFileInputList("-l ", filesToZip).
+		FlagWithOutput("-o ", zipFile)
+	rb.Build(pctx, ctx, "snapshot", "Building snapshot for "+ctx.ModuleName())
 
-	sh.build(pctx, ctx, implicits)
+	return zipFile
-	return sh.path
-}
-
-func (s *sdk) buildSnapshotGenerationScripts(ctx android.ModuleContext) {
-	if s.snapshot() {
-		// we don't need a script for sdk_snapshot.. as they are frozen
-		return
-	}
-
-	// script to update the 'current' snapshot
-	s.updateScript = s.buildScript(ctx, "current")
-
-	versions := s.frozenVersions(ctx)
-	newVersion := "1"
-	if len(versions) >= 1 {
-		lastVersion := versions[len(versions)-1]
-		lastVersionNum, err := strconv.Atoi(lastVersion)
-		if err != nil {
-			panic(err)
-			return
-		}
-		newVersion = strconv.Itoa(lastVersionNum + 1)
-	}
-	// script to create a new frozen version of snapshot
-	s.freezeScript = s.buildScript(ctx, newVersion)
-}
-
-func (s *sdk) androidMkEntriesForScript() android.AndroidMkEntries {
-	if s.snapshot() {
-		// we don't need a script for sdk_snapshot.. as they are frozen
-		return android.AndroidMkEntries{}
-	}
-
-	entries := android.AndroidMkEntries{
-		Class: "FAKE",
-		// TODO(jiyong): remove this? but androidmk.go expects OutputFile to be specified anyway
-		OutputFile: android.OptionalPathForPath(s.updateScript),
-		Include:    "$(BUILD_SYSTEM)/base_rules.mk",
-		ExtraEntries: []android.AndroidMkExtraEntriesFunc{
-			func(entries *android.AndroidMkEntries) {
-				entries.AddStrings("LOCAL_ADDITIONAL_DEPENDENCIES",
-					s.updateScript.String(), s.freezeScript.String())
-			},
-		},
-		ExtraFooters: []android.AndroidMkExtraFootersFunc{
-			func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) {
-				fmt.Fprintln(w, "$(LOCAL_BUILT_MODULE): $(LOCAL_ADDITIONAL_DEPENDENCIES)")
-				fmt.Fprintln(w, "	touch $@")
-				fmt.Fprintln(w, "	echo ##################################################")
-				fmt.Fprintln(w, "	echo To update current SDK: execute", filepath.Join("\\$$ANDROID_BUILD_TOP", s.updateScript.String()))
-				fmt.Fprintln(w, "	echo To freeze current SDK: execute", filepath.Join("\\$$ANDROID_BUILD_TOP", s.freezeScript.String()))
-				fmt.Fprintln(w, "	echo ##################################################")
-			},
-		},
-	}
-	return entries
 }