From 232e785b9894a2df107b557378b84a900bbb68e5 Mon Sep 17 00:00:00 2001 From: Jiyong Park Date: Mon, 4 Nov 2019 12:23:40 +0900 Subject: [PATCH] SDK snapshot is dist'ed `m module_sdk dist` produces snapshots of all SDKs in the source tree. A snapshot is a zip file consists of Android.bp, exported headers, exported AIDL files, stubs for native libs and jars. The zip file is expected to be downloaded from the build server and extracted to a directory (which probably will be /prebuilts/module_sdks//current). Bug: 138182343 Test: m (sdk_test.go updated) Change-Id: Idbe4bc24795fe08f26fc1cf7497028f9d162053a --- cc/cc.go | 13 +++- sdk/sdk.go | 21 ++++- sdk/sdk_test.go | 70 ++++++++++++++++- sdk/update.go | 202 +++++++++++++++++------------------------------- 4 files changed, 167 insertions(+), 139 deletions(-) diff --git a/cc/cc.go b/cc/cc.go index 5dfc56386..abe8c1a66 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -858,21 +858,28 @@ func (c *Module) ExportedIncludeDirs() android.Paths { if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { return flagsProducer.exportedDirs() } - return []android.Path{} + return nil } func (c *Module) ExportedSystemIncludeDirs() android.Paths { if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { return flagsProducer.exportedSystemDirs() } - return []android.Path{} + return nil } func (c *Module) ExportedFlags() []string { if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { return flagsProducer.exportedFlags() } - return []string{} + return nil +} + +func (c *Module) ExportedDeps() android.Paths { + if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { + return flagsProducer.exportedDeps() + } + return nil } func isBionic(name string) bool { diff --git a/sdk/sdk.go b/sdk/sdk.go index cb81a1466..4eb3665fb 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -29,6 +29,7 @@ import ( ) func init() { + pctx.Import("android/soong/android") android.RegisterModuleType("sdk", ModuleFactory) android.RegisterModuleType("sdk_snapshot", SnapshotModuleFactory) android.PreDepsMutators(RegisterPreDepsMutators) @@ -41,8 +42,7 @@ type sdk struct { properties sdkProperties - updateScript android.OutputPath - freezeScript android.OutputPath + snapshotFile android.OptionalPath } type sdkProperties struct { @@ -104,11 +104,24 @@ func (s *sdk) frozenVersions(ctx android.BaseModuleContext) []string { } func (s *sdk) GenerateAndroidBuildActions(ctx android.ModuleContext) { - s.buildSnapshotGenerationScripts(ctx) + if !s.snapshot() { + // We don't need to create a snapshot out of sdk_snapshot. + // That doesn't make sense. We need a snapshot to create sdk_snapshot. + s.snapshotFile = android.OptionalPathForPath(s.buildSnapshot(ctx)) + } } func (s *sdk) AndroidMkEntries() android.AndroidMkEntries { - return s.androidMkEntriesForScript() + if !s.snapshotFile.Valid() { + return android.AndroidMkEntries{} + } + + return android.AndroidMkEntries{ + Class: "FAKE", + OutputFile: s.snapshotFile, + DistFile: s.snapshotFile, + Include: "$(BUILD_PHONY_PACKAGE)", + } } // RegisterPreDepsMutators registers pre-deps mutators to support modules implementing SdkAware diff --git a/sdk/sdk_test.go b/sdk/sdk_test.go index 96129b838..3471bc9da 100644 --- a/sdk/sdk_test.go +++ b/sdk/sdk_test.go @@ -17,6 +17,7 @@ package sdk import ( "io/ioutil" "os" + "path/filepath" "strings" "testing" @@ -100,6 +101,8 @@ func testSdkContext(t *testing.T, bp string) (*android.TestContext, android.Conf "myapex.pk8": nil, "Test.java": nil, "Test.cpp": nil, + "include/Test.h": nil, + "aidl/foo/bar/Test.aidl": nil, "libfoo.so": nil, }) @@ -398,7 +401,9 @@ func TestSdkIsCompileMultilibBoth(t *testing.T) { var inputs []string buildParams := ctx.ModuleForTests("mysdk", "android_common").Module().BuildParamsForTests() for _, bp := range buildParams { - inputs = append(inputs, bp.Implicits.Strings()...) + if bp.Input != nil { + inputs = append(inputs, bp.Input.String()) + } } // ensure that both 32/64 outputs are inputs of the sdk snapshot @@ -406,6 +411,69 @@ func TestSdkIsCompileMultilibBoth(t *testing.T) { ensureListContains(t, inputs, arm64Output.String()) } +func TestSnapshot(t *testing.T) { + ctx, config := testSdk(t, ` + sdk { + name: "mysdk", + java_libs: ["myjavalib"], + native_shared_libs: ["mynativelib"], + } + + java_library { + name: "myjavalib", + srcs: ["Test.java"], + aidl: { + export_include_dirs: ["aidl"], + }, + system_modules: "none", + sdk_version: "none", + compile_dex: true, + host_supported: true, + } + + cc_library_shared { + name: "mynativelib", + srcs: [ + "Test.cpp", + "aidl/foo/bar/Test.aidl", + ], + export_include_dirs: ["include"], + aidl: { + export_aidl_headers: true, + }, + system_shared_libs: [], + stl: "none", + } + `) + + var copySrcs []string + var copyDests []string + buildParams := ctx.ModuleForTests("mysdk", "android_common").Module().BuildParamsForTests() + for _, bp := range buildParams { + if bp.Rule.String() == "android/soong/android.Cp" { + copySrcs = append(copySrcs, bp.Input.String()) + copyDests = append(copyDests, bp.Output.Rel()) // rooted at the snapshot root + } + } + + buildDir := config.BuildDir() + ensureListContains(t, copySrcs, "aidl/foo/bar/Test.aidl") + ensureListContains(t, copySrcs, "include/Test.h") + ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/gen/aidl/aidl/foo/bar/BnTest.h")) + ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/gen/aidl/aidl/foo/bar/BpTest.h")) + ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/gen/aidl/aidl/foo/bar/Test.h")) + ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar")) + ensureListContains(t, copySrcs, filepath.Join(buildDir, ".intermediates/mynativelib/android_arm64_armv8-a_core_shared/mynativelib.so")) + + ensureListContains(t, copyDests, "aidl/aidl/foo/bar/Test.aidl") + ensureListContains(t, copyDests, "arm64/include/include/Test.h") + ensureListContains(t, copyDests, "arm64/include_gen/mynativelib/aidl/foo/bar/BnTest.h") + ensureListContains(t, copyDests, "arm64/include_gen/mynativelib/aidl/foo/bar/BpTest.h") + ensureListContains(t, copyDests, "arm64/include_gen/mynativelib/aidl/foo/bar/Test.h") + ensureListContains(t, copyDests, "java/myjavalib.jar") + ensureListContains(t, copyDests, "arm64/lib/mynativelib.so") +} + var buildDir string func setUp() { diff --git a/sdk/update.go b/sdk/update.go index ce6082799..171bb3f8c 100644 --- a/sdk/update.go +++ b/sdk/update.go @@ -16,9 +16,7 @@ package sdk import ( "fmt" - "io" "path/filepath" - "strconv" "strings" "github.com/google/blueprint/proptools" @@ -38,9 +36,9 @@ type generatedFile struct { indentLevel int } -func newGeneratedFile(ctx android.ModuleContext, name string) *generatedFile { +func newGeneratedFile(ctx android.ModuleContext, path ...string) *generatedFile { return &generatedFile{ - path: android.PathForModuleOut(ctx, name).OutputPath, + path: android.PathForModuleOut(ctx, path...).OutputPath, indentLevel: 0, } } @@ -89,6 +87,7 @@ type archSpecificNativeLibInfo struct { exportedIncludeDirs android.Paths exportedSystemIncludeDirs android.Paths exportedFlags []string + exportedDeps android.Paths outputFile android.Path } @@ -132,6 +131,7 @@ func (s *sdk) nativeMemberInfos(ctx android.ModuleContext) []*nativeLibInfo { exportedIncludeDirs: ccModule.ExportedIncludeDirs(), exportedSystemIncludeDirs: ccModule.ExportedSystemIncludeDirs(), exportedFlags: ccModule.ExportedFlags(), + exportedDeps: ccModule.ExportedDeps(), outputFile: ccModule.OutputFile().Path(), }) }) @@ -169,11 +169,11 @@ func (s *sdk) nativeMemberInfos(ctx android.ModuleContext) []*nativeLibInfo { // aidl/ // frameworks/base/core/..../IFoo.aidl : an exported AIDL file // java/ -// java//stub.jar : a stub jar for a java library 'module_name' +// .jar : the stub jar for a java library 'module_name' // include/ // bionic/libc/include/stdlib.h : an exported header file // include_gen/ -// com/android/.../IFoo.h : a generated header file +// /com/android/.../IFoo.h : a generated header file // /include/ : arch-specific exported headers // /include_gen/ : arch-specific generated headers // /lib/ @@ -182,7 +182,7 @@ func (s *sdk) nativeMemberInfos(ctx android.ModuleContext) []*nativeLibInfo { const ( aidlIncludeDir = "aidl" javaStubDir = "java" - javaStubFile = "stub.jar" + javaStubFileSuffix = ".jar" nativeIncludeDir = "include" nativeGeneratedIncludeDir = "include_gen" nativeStubDir = "lib" @@ -191,7 +191,7 @@ const ( // path to the stub file of a java library. Relative to / func javaStubFilePathFor(javaLib *java.Library) string { - return filepath.Join(javaStubDir, javaLib.Name(), javaStubFile) + return filepath.Join(javaStubDir, javaLib.Name()+javaStubFileSuffix) } // path to the stub file of a native shared library. Relative to / @@ -204,7 +204,6 @@ func nativeStubFilePathFor(lib archSpecificNativeLibInfo) string { func nativeIncludeDirPathsFor(ctx android.ModuleContext, lib archSpecificNativeLibInfo, systemInclude bool, archSpecific bool) []string { var result []string - buildDir := ctx.Config().BuildDir() var includeDirs []android.Path if !systemInclude { includeDirs = lib.exportedIncludeDirs @@ -213,8 +212,8 @@ func nativeIncludeDirPathsFor(ctx android.ModuleContext, lib archSpecificNativeL } for _, dir := range includeDirs { var path string - if gen := strings.HasPrefix(dir.String(), buildDir); gen { - path = filepath.Join(nativeGeneratedIncludeDir, dir.Rel()) + if _, gen := dir.(android.WritablePath); gen { + path = filepath.Join(nativeGeneratedIncludeDir, lib.name) } else { path = filepath.Join(nativeIncludeDir, dir.String()) } @@ -226,21 +225,19 @@ func nativeIncludeDirPathsFor(ctx android.ModuleContext, lib archSpecificNativeL return result } -// A name that uniquely identifies an prebuilt SDK member for a version of SDK snapshot +// A name that uniquely identifies a prebuilt SDK member for a version of SDK snapshot // This isn't visible to users, so could be changed in future. func versionedSdkMemberName(ctx android.ModuleContext, memberName string, version string) string { return ctx.ModuleName() + "_" + memberName + string(android.SdkVersionSeparator) + version } -// arm64, arm, x86, x86_64, etc. -func archTypeOf(module android.Module) string { - return module.Target().Arch.ArchType.String() -} - // buildAndroidBp creates the blueprint file that defines prebuilt modules for each of // the SDK members, and the entire sdk_snapshot module for the specified version +// TODO(jiyong): create a meta info file (e.g. json, protobuf, etc.) instead, and convert it to +// Android.bp in the (presumably old) branch where the snapshots will be used. This will give us +// some flexibility to introduce backwards incompatible changes in soong. func (s *sdk) buildAndroidBp(ctx android.ModuleContext, version string) android.OutputPath { - bp := newGeneratedFile(ctx, "blueprint-"+version+".bp") + bp := newGeneratedFile(ctx, "snapshot", "Android.bp") bp.printfln("// This is auto-generated. DO NOT EDIT.") bp.printfln("") @@ -352,52 +349,42 @@ func (s *sdk) buildAndroidBp(ctx android.ModuleContext, version string) android. return bp.path } -func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.OutputPath { - sh := newGeneratedFile(ctx, "update_prebuilt-"+version+".sh") - buildDir := ctx.Config().BuildDir() - - snapshotPath := func(paths ...string) string { - return filepath.Join(ctx.ModuleDir(), version, filepath.Join(paths...)) +// buildSnapshot is the main function in this source file. It creates rules to copy +// the contents (header files, stub libraries, etc) into the zip file. +func (s *sdk) buildSnapshot(ctx android.ModuleContext) android.OutputPath { + snapshotPath := func(paths ...string) android.OutputPath { + return android.PathForModuleOut(ctx, "snapshot").Join(ctx, paths...) } - // TODO(jiyong) instead of creating script, create a zip file having the Android.bp, the headers, - // and the stubs and put it to the dist directory. The dist'ed zip file then would be downloaded, - // unzipped and then uploaded to gerrit again. - sh.printfln("#!/bin/bash") - sh.printfln("echo Updating snapshot of %s in %s", ctx.ModuleName(), snapshotPath()) - sh.printfln("pushd $ANDROID_BUILD_TOP > /dev/null") - sh.printfln("mkdir -p %s", snapshotPath(aidlIncludeDir)) - sh.printfln("mkdir -p %s", snapshotPath(javaStubDir)) - sh.printfln("mkdir -p %s", snapshotPath(nativeIncludeDir)) - sh.printfln("mkdir -p %s", snapshotPath(nativeGeneratedIncludeDir)) - for _, target := range ctx.MultiTargets() { - arch := target.Arch.ArchType.String() - sh.printfln("mkdir -p %s", snapshotPath(arch, nativeStubDir)) - sh.printfln("mkdir -p %s", snapshotPath(arch, nativeIncludeDir)) - sh.printfln("mkdir -p %s", snapshotPath(arch, nativeGeneratedIncludeDir)) + var filesToZip android.Paths + // copy src to dest and add the dest to the zip + copy := func(src android.Path, dest android.OutputPath) { + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: src, + Output: dest, + }) + filesToZip = append(filesToZip, dest) } - var implicits android.Paths + // copy exported AIDL files and stub jar files for _, m := range s.javaLibs(ctx) { headerJars := m.HeaderJars() if len(headerJars) != 1 { panic(fmt.Errorf("there must be only one header jar from %q", m.Name())) } - implicits = append(implicits, headerJars...) + copy(headerJars[0], snapshotPath(javaStubFilePathFor(m))) - exportedAidlIncludeDirs := m.AidlIncludeDirs() - for _, dir := range exportedAidlIncludeDirs { - // Using tar to copy with the directory structure + for _, dir := range m.AidlIncludeDirs() { // TODO(jiyong): copy parcelable declarations only - sh.printfln("find %s -name \"*.aidl\" | tar cf - -T - | (cd %s; tar xf -)", - dir.String(), snapshotPath(aidlIncludeDir)) + aidlFiles, _ := ctx.GlobWithDeps(dir.String()+"/**/*.aidl", nil) + for _, file := range aidlFiles { + copy(android.PathForSource(ctx, file), snapshotPath(aidlIncludeDir, file)) + } } - - copyTarget := snapshotPath(javaStubFilePathFor(m)) - sh.printfln("mkdir -p %s && cp %s %s", - filepath.Dir(copyTarget), headerJars[0].String(), copyTarget) } + // copy exported header files and stub *.so files nativeLibInfos := s.nativeMemberInfos(ctx) for _, info := range nativeLibInfos { @@ -409,26 +396,32 @@ func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.Out return } for _, dir := range includeDirs { - gen := strings.HasPrefix(dir.String(), buildDir) - targetDir := nativeIncludeDir - if gen { - targetDir = nativeGeneratedIncludeDir + if _, gen := dir.(android.WritablePath); gen { + // generated headers are copied via exportedDeps. See below. + continue } + targetDir := nativeIncludeDir if info.hasArchSpecificFlags { targetDir = filepath.Join(lib.archType, targetDir) } - targetDir = snapshotPath(targetDir) - sourceDirRoot := "." - sourceDirRel := dir.String() - if gen { - // ex) out/soong/.intermediate/foo/bar/gen/aidl - sourceDirRoot = strings.TrimSuffix(dir.String(), dir.Rel()) - sourceDirRel = dir.Rel() - } // TODO(jiyong) copy headers having other suffixes - sh.printfln("(cd %s; find %s -name \"*.h\" | tar cf - -T - ) | (cd %s; tar xf -)", - sourceDirRoot, sourceDirRel, targetDir) + headers, _ := ctx.GlobWithDeps(dir.String()+"/**/*.h", nil) + for _, file := range headers { + src := android.PathForSource(ctx, file) + dest := snapshotPath(targetDir, file) + copy(src, dest) + } + } + + genHeaders := lib.exportedDeps + for _, file := range genHeaders { + targetDir := nativeGeneratedIncludeDir + if info.hasArchSpecificFlags { + targetDir = filepath.Join(lib.archType, targetDir) + } + dest := snapshotPath(targetDir, lib.name, file.Rel()) + copy(file, dest) } } @@ -438,10 +431,7 @@ func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.Out // for each architecture for _, av := range info.archVariants { - stub := av.outputFile - implicits = append(implicits, stub) - copiedStub := snapshotPath(nativeStubFilePathFor(av)) - sh.printfln("cp %s %s", stub.String(), copiedStub) + copy(av.outputFile, snapshotPath(nativeStubFilePathFor(av))) if info.hasArchSpecificFlags { printExportedDirCopyCommandsForNativeLibs(av) @@ -449,69 +439,19 @@ func (s *sdk) buildScript(ctx android.ModuleContext, version string) android.Out } } - bp := s.buildAndroidBp(ctx, version) - implicits = append(implicits, bp) - sh.printfln("cp %s %s", bp.String(), snapshotPath("Android.bp")) + // generate Android.bp + bp := s.buildAndroidBp(ctx, "current") + filesToZip = append(filesToZip, bp) - sh.printfln("popd > /dev/null") - sh.printfln("rm -- \"$0\"") // self deleting so that stale script is not used - sh.printfln("echo Done") + // zip them all + zipFile := android.PathForModuleOut(ctx, ctx.ModuleName()+"-current.zip").OutputPath + rb := android.NewRuleBuilder() + rb.Command(). + BuiltTool(ctx, "soong_zip"). + FlagWithArg("-C ", snapshotPath().String()). + FlagWithRspFileInputList("-l ", filesToZip). + FlagWithOutput("-o ", zipFile) + rb.Build(pctx, ctx, "snapshot", "Building snapshot for "+ctx.ModuleName()) - sh.build(pctx, ctx, implicits) - return sh.path -} - -func (s *sdk) buildSnapshotGenerationScripts(ctx android.ModuleContext) { - if s.snapshot() { - // we don't need a script for sdk_snapshot.. as they are frozen - return - } - - // script to update the 'current' snapshot - s.updateScript = s.buildScript(ctx, "current") - - versions := s.frozenVersions(ctx) - newVersion := "1" - if len(versions) >= 1 { - lastVersion := versions[len(versions)-1] - lastVersionNum, err := strconv.Atoi(lastVersion) - if err != nil { - panic(err) - return - } - newVersion = strconv.Itoa(lastVersionNum + 1) - } - // script to create a new frozen version of snapshot - s.freezeScript = s.buildScript(ctx, newVersion) -} - -func (s *sdk) androidMkEntriesForScript() android.AndroidMkEntries { - if s.snapshot() { - // we don't need a script for sdk_snapshot.. as they are frozen - return android.AndroidMkEntries{} - } - - entries := android.AndroidMkEntries{ - Class: "FAKE", - // TODO(jiyong): remove this? but androidmk.go expects OutputFile to be specified anyway - OutputFile: android.OptionalPathForPath(s.updateScript), - Include: "$(BUILD_SYSTEM)/base_rules.mk", - ExtraEntries: []android.AndroidMkExtraEntriesFunc{ - func(entries *android.AndroidMkEntries) { - entries.AddStrings("LOCAL_ADDITIONAL_DEPENDENCIES", - s.updateScript.String(), s.freezeScript.String()) - }, - }, - ExtraFooters: []android.AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { - fmt.Fprintln(w, "$(LOCAL_BUILT_MODULE): $(LOCAL_ADDITIONAL_DEPENDENCIES)") - fmt.Fprintln(w, " touch $@") - fmt.Fprintln(w, " echo ##################################################") - fmt.Fprintln(w, " echo To update current SDK: execute", filepath.Join("\\$$ANDROID_BUILD_TOP", s.updateScript.String())) - fmt.Fprintln(w, " echo To freeze current SDK: execute", filepath.Join("\\$$ANDROID_BUILD_TOP", s.freezeScript.String())) - fmt.Fprintln(w, " echo ##################################################") - }, - }, - } - return entries + return zipFile }