diff --git a/android/module.go b/android/module.go index 11f63bd49..b40310b85 100644 --- a/android/module.go +++ b/android/module.go @@ -413,6 +413,7 @@ type ModuleContext interface { InstallInDebugRamdisk() bool InstallInRecovery() bool InstallInRoot() bool + InstallInVendor() bool InstallBypassMake() bool InstallForceOS() (*OsType, *ArchType) @@ -471,6 +472,7 @@ type Module interface { InstallInDebugRamdisk() bool InstallInRecovery() bool InstallInRoot() bool + InstallInVendor() bool InstallBypassMake() bool InstallForceOS() (*OsType, *ArchType) HideFromMake() @@ -1579,6 +1581,10 @@ func (m *ModuleBase) InstallInRecovery() bool { return Bool(m.commonProperties.Recovery) } +func (m *ModuleBase) InstallInVendor() bool { + return Bool(m.commonProperties.Vendor) +} + func (m *ModuleBase) InstallInRoot() bool { return false } @@ -2645,6 +2651,10 @@ func (m *moduleContext) InstallForceOS() (*OsType, *ArchType) { return m.module.InstallForceOS() } +func (m *moduleContext) InstallInVendor() bool { + return m.module.InstallInVendor() +} + func (m *moduleContext) skipInstall() bool { if m.module.base().commonProperties.SkipInstall { return true diff --git a/cc/util.go b/cc/util.go index 1220d8432..9bba87676 100644 --- a/cc/util.go +++ b/cc/util.go @@ -21,6 +21,7 @@ import ( "strings" "android/soong/android" + "android/soong/snapshot" ) // Efficiently converts a list of include directories to a single string @@ -126,20 +127,6 @@ func makeSymlinkCmd(linkDirOnDevice string, linkName string, target string) stri "ln -sf " + target + " " + filepath.Join(dir, linkName) } -func copyFileRule(ctx android.SingletonContext, path android.Path, out string) android.OutputPath { - outPath := android.PathForOutput(ctx, out) - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cp, - Input: path, - Output: outPath, - Description: "copy " + path.String() + " -> " + out, - Args: map[string]string{ - "cpFlags": "-f -L", - }, - }) - return outPath -} - func combineNoticesRule(ctx android.SingletonContext, paths android.Paths, out string) android.OutputPath { outPath := android.PathForOutput(ctx, out) ctx.Build(pctx, android.BuildParams{ @@ -151,12 +138,6 @@ func combineNoticesRule(ctx android.SingletonContext, paths android.Paths, out s return outPath } -func writeStringToFileRule(ctx android.SingletonContext, content, out string) android.OutputPath { - outPath := android.PathForOutput(ctx, out) - android.WriteFileRule(ctx, outPath, content) - return outPath -} - // Dump a map to a list file as: // // {key1} {value1} @@ -172,5 +153,5 @@ func installMapListFileRule(ctx android.SingletonContext, m map[string]string, p txtBuilder.WriteString(" ") txtBuilder.WriteString(m[k]) } - return writeStringToFileRule(ctx, txtBuilder.String(), path) + return snapshot.WriteStringToFileRule(ctx, txtBuilder.String(), path) } diff --git a/cc/vendor_snapshot.go b/cc/vendor_snapshot.go index 6705d557b..ba4d79fcf 100644 --- a/cc/vendor_snapshot.go +++ b/cc/vendor_snapshot.go @@ -210,9 +210,9 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS if fake { // All prebuilt binaries and headers are installed by copyFile function. This makes a fake // snapshot just touch prebuilts and headers, rather than installing real files. - return writeStringToFileRule(ctx, "", out) + return snapshot.WriteStringToFileRule(ctx, "", out) } else { - return copyFileRule(ctx, path, out) + return snapshot.CopyFileRule(pctx, ctx, path, out) } } @@ -350,7 +350,7 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS ctx.Errorf("json marshal to %q failed: %#v", propOut, err) return nil } - ret = append(ret, writeStringToFileRule(ctx, string(j), propOut)) + ret = append(ret, snapshot.WriteStringToFileRule(ctx, string(j), propOut)) return ret } diff --git a/cc/vndk.go b/cc/vndk.go index 499d4282c..1ae79de05 100644 --- a/cc/vndk.go +++ b/cc/vndk.go @@ -25,6 +25,7 @@ import ( "android/soong/android" "android/soong/cc/config" "android/soong/etc" + "android/soong/snapshot" "github.com/google/blueprint" ) @@ -698,7 +699,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex libPath := m.outputFile.Path() snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, "shared", vndkType, libPath.Base()) - ret = append(ret, copyFileRule(ctx, libPath, snapshotLibOut)) + ret = append(ret, snapshot.CopyFileRule(pctx, ctx, libPath, snapshotLibOut)) if ctx.Config().VndkSnapshotBuildArtifacts() { prop := struct { @@ -720,7 +721,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex ctx.Errorf("json marshal to %q failed: %#v", propOut, err) return nil, false } - ret = append(ret, writeStringToFileRule(ctx, string(j), propOut)) + ret = append(ret, snapshot.WriteStringToFileRule(ctx, string(j), propOut)) } return ret, true } @@ -778,8 +779,8 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex // install all headers after removing duplicates for _, header := range android.FirstUniquePaths(headers) { - snapshotOutputs = append(snapshotOutputs, copyFileRule( - ctx, header, filepath.Join(includeDir, header.String()))) + snapshotOutputs = append(snapshotOutputs, snapshot.CopyFileRule( + pctx, ctx, header, filepath.Join(includeDir, header.String()))) } // install *.libraries.txt except vndkcorevariant.libraries.txt @@ -788,8 +789,8 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex if !ok || !m.Enabled() || m.Name() == vndkUsingCoreVariantLibrariesTxt { return } - snapshotOutputs = append(snapshotOutputs, copyFileRule( - ctx, m.OutputFile(), filepath.Join(configsDir, m.Name()))) + snapshotOutputs = append(snapshotOutputs, snapshot.CopyFileRule( + pctx, ctx, m.OutputFile(), filepath.Join(configsDir, m.Name()))) }) /* diff --git a/etc/Android.bp b/etc/Android.bp index cab7389b6..06a2fa15d 100644 --- a/etc/Android.bp +++ b/etc/Android.bp @@ -9,6 +9,7 @@ bootstrap_go_package { "blueprint", "soong", "soong-android", + "soong-snapshot", ], srcs: [ "prebuilt_etc.go", diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index 4dd383d77..4107916a7 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -28,12 +28,15 @@ package etc // various `prebuilt_*` mutators. import ( + "encoding/json" "fmt" + "path/filepath" "strings" "github.com/google/blueprint/proptools" "android/soong/android" + "android/soong/snapshot" ) var pctx = android.NewPackageContext("android/soong/etc") @@ -43,6 +46,7 @@ var pctx = android.NewPackageContext("android/soong/etc") func init() { pctx.Import("android/soong/android") RegisterPrebuiltEtcBuildComponents(android.InitRegistrationContext) + snapshot.RegisterSnapshotAction(generatePrebuiltSnapshot) } func RegisterPrebuiltEtcBuildComponents(ctx android.RegistrationContext) { @@ -128,6 +132,9 @@ type PrebuiltEtc struct { android.ModuleBase android.DefaultableModuleBase + snapshot.VendorSnapshotModuleInterface + snapshot.RecoverySnapshotModuleInterface + properties prebuiltEtcProperties subdirProperties prebuiltSubdirProperties @@ -183,7 +190,7 @@ func (p *PrebuiltEtc) InstallInDebugRamdisk() bool { return p.inDebugRamdisk() } -func (p *PrebuiltEtc) inRecovery() bool { +func (p *PrebuiltEtc) InRecovery() bool { return p.ModuleBase.InRecovery() || p.ModuleBase.InstallInRecovery() } @@ -192,7 +199,7 @@ func (p *PrebuiltEtc) onlyInRecovery() bool { } func (p *PrebuiltEtc) InstallInRecovery() bool { - return p.inRecovery() + return p.InRecovery() } var _ android.ImageInterface = (*PrebuiltEtc)(nil) @@ -271,6 +278,18 @@ func (p *PrebuiltEtc) Installable() bool { return p.properties.Installable == nil || proptools.Bool(p.properties.Installable) } +func (p *PrebuiltEtc) InVendor() bool { + return p.ModuleBase.InstallInVendor() +} + +func (p *PrebuiltEtc) ExcludeFromVendorSnapshot() bool { + return false +} + +func (p *PrebuiltEtc) ExcludeFromRecoverySnapshot() bool { + return false +} + func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { if p.properties.Src == nil { ctx.PropertyErrorf("src", "missing prebuilt source file") @@ -344,7 +363,7 @@ func (p *PrebuiltEtc) AndroidMkEntries() []android.AndroidMkEntries { if p.inDebugRamdisk() && !p.onlyInDebugRamdisk() { nameSuffix = ".debug_ramdisk" } - if p.inRecovery() && !p.onlyInRecovery() { + if p.InRecovery() && !p.onlyInRecovery() { nameSuffix = ".recovery" } return []android.AndroidMkEntries{android.AndroidMkEntries{ @@ -494,3 +513,137 @@ func PrebuiltRFSAFactory() android.Module { android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) return module } + +// Flags to be included in the snapshot +type snapshotJsonFlags struct { + ModuleName string `json:",omitempty"` + Filename string `json:",omitempty"` + RelativeInstallPath string `json:",omitempty"` +} + +// Copy file into the snapshot +func copyFile(ctx android.SingletonContext, path android.Path, out string, fake bool) android.OutputPath { + if fake { + // Create empty file instead for the fake snapshot + return snapshot.WriteStringToFileRule(ctx, "", out) + } else { + return snapshot.CopyFileRule(pctx, ctx, path, out) + } +} + +// Check if the module is target of the snapshot +func isSnapshotAware(ctx android.SingletonContext, m *PrebuiltEtc, image snapshot.SnapshotImage) bool { + if !m.Enabled() { + return false + } + + // Skip if the module is not included in the image + if !image.InImage(m)() { + return false + } + + // When android/prebuilt.go selects between source and prebuilt, it sets + // HideFromMake on the other one to avoid duplicate install rules in make. + if m.IsHideFromMake() { + return false + } + + // There are some prebuilt_etc module with multiple definition of same name. + // Check if the target would be included from the build + if !m.ExportedToMake() { + return false + } + + // Skip if the module is in the predefined path list to skip + if image.IsProprietaryPath(ctx.ModuleDir(m), ctx.DeviceConfig()) { + return false + } + + // Skip if the module should be excluded + if image.ExcludeFromSnapshot(m) || image.ExcludeFromDirectedSnapshot(ctx.DeviceConfig(), m.BaseModuleName()) { + return false + } + + // Skip from other exceptional cases + if m.Target().Os.Class != android.Device { + return false + } + if m.Target().NativeBridge == android.NativeBridgeEnabled { + return false + } + + return true +} + +func generatePrebuiltSnapshot(s snapshot.SnapshotSingleton, ctx android.SingletonContext, snapshotArchDir string) android.Paths { + /* + Snapshot zipped artifacts directory structure for etc modules: + {SNAPSHOT_ARCH}/ + arch-{TARGET_ARCH}-{TARGET_ARCH_VARIANT}/ + etc/ + (prebuilt etc files) + arch-{TARGET_2ND_ARCH}-{TARGET_2ND_ARCH_VARIANT}/ + etc/ + (prebuilt etc files) + NOTICE_FILES/ + (notice files) + */ + var snapshotOutputs android.Paths + noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES") + installedNotices := make(map[string]bool) + + ctx.VisitAllModules(func(module android.Module) { + m, ok := module.(*PrebuiltEtc) + if !ok { + return + } + + if !isSnapshotAware(ctx, m, s.Image) { + return + } + + targetArch := "arch-" + m.Target().Arch.ArchType.String() + + snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, "etc", m.BaseModuleName()) + snapshotOutputs = append(snapshotOutputs, copyFile(ctx, m.OutputFile(), snapshotLibOut, s.Fake)) + + prop := snapshotJsonFlags{} + propOut := snapshotLibOut + ".json" + prop.ModuleName = m.BaseModuleName() + if m.subdirProperties.Relative_install_path != nil { + prop.RelativeInstallPath = *m.subdirProperties.Relative_install_path + } + + if m.properties.Filename != nil { + prop.Filename = *m.properties.Filename + } + + j, err := json.Marshal(prop) + if err != nil { + ctx.Errorf("json marshal to %q failed: %#v", propOut, err) + return + } + snapshotOutputs = append(snapshotOutputs, snapshot.WriteStringToFileRule(ctx, string(j), propOut)) + + if len(m.EffectiveLicenseFiles()) > 0 { + noticeName := ctx.ModuleName(m) + ".txt" + noticeOut := filepath.Join(noticeDir, noticeName) + // skip already copied notice file + if !installedNotices[noticeOut] { + installedNotices[noticeOut] = true + + noticeOutPath := android.PathForOutput(ctx, noticeOut) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cat, + Inputs: m.EffectiveLicenseFiles(), + Output: noticeOutPath, + Description: "combine notices for " + noticeOut, + }) + snapshotOutputs = append(snapshotOutputs, noticeOutPath) + } + } + + }) + + return snapshotOutputs +} diff --git a/etc/prebuilt_etc_test.go b/etc/prebuilt_etc_test.go index 354f6bb6e..0c5cfe4f4 100644 --- a/etc/prebuilt_etc_test.go +++ b/etc/prebuilt_etc_test.go @@ -15,11 +15,15 @@ package etc import ( + "fmt" "os" "path/filepath" "testing" + "github.com/google/blueprint/proptools" + "android/soong/android" + "android/soong/snapshot" ) func TestMain(m *testing.M) { @@ -36,6 +40,18 @@ var prepareForPrebuiltEtcTest = android.GroupFixturePreparers( }), ) +var prepareForPrebuiltEtcSnapshotTest = android.GroupFixturePreparers( + prepareForPrebuiltEtcTest, + android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + snapshot.VendorSnapshotImageSingleton.Init(ctx) + snapshot.RecoverySnapshotImageSingleton.Init(ctx) + }), + android.FixtureModifyConfig(func(config android.Config) { + config.TestProductVariables.DeviceVndkVersion = proptools.StringPtr("current") + config.TestProductVariables.RecoverySnapshotVersion = proptools.StringPtr("current") + }), +) + func TestPrebuiltEtcVariants(t *testing.T) { result := prepareForPrebuiltEtcTest.RunTestWithBp(t, ` prebuilt_etc { @@ -346,3 +362,110 @@ func TestPrebuiltRFSADirPath(t *testing.T) { }) } } + +func checkIfSnapshotTaken(t *testing.T, result *android.TestResult, image string, moduleName string) { + checkIfSnapshotExistAsExpected(t, result, image, moduleName, true) +} + +func checkIfSnapshotNotTaken(t *testing.T, result *android.TestResult, image string, moduleName string) { + checkIfSnapshotExistAsExpected(t, result, image, moduleName, false) +} + +func checkIfSnapshotExistAsExpected(t *testing.T, result *android.TestResult, image string, moduleName string, expectToExist bool) { + snapshotSingleton := result.SingletonForTests(image + "-snapshot") + archType := "arm64" + archVariant := "armv8-a" + archDir := fmt.Sprintf("arch-%s", archType) + + snapshotDir := fmt.Sprintf("%s-snapshot", image) + snapshotVariantPath := filepath.Join(snapshotDir, archType) + outputDir := filepath.Join(snapshotVariantPath, archDir, "etc") + imageVariant := "" + if image == "recovery" { + imageVariant = "recovery_" + } + mod := result.ModuleForTests(moduleName, fmt.Sprintf("android_%s%s_%s", imageVariant, archType, archVariant)) + outputFiles := mod.OutputFiles(t, "") + if len(outputFiles) != 1 { + t.Errorf("%q must have single output\n", moduleName) + return + } + snapshotPath := filepath.Join(outputDir, moduleName) + + if expectToExist { + out := snapshotSingleton.Output(snapshotPath) + + if out.Input.String() != outputFiles[0].String() { + t.Errorf("The input of snapshot %q must be %q, but %q", "prebuilt_vendor", out.Input.String(), outputFiles[0]) + } + + snapshotJsonPath := snapshotPath + ".json" + + if snapshotSingleton.MaybeOutput(snapshotJsonPath).Rule == nil { + t.Errorf("%q expected but not found", snapshotJsonPath) + } + } else { + out := snapshotSingleton.MaybeOutput(snapshotPath) + if out.Rule != nil { + t.Errorf("There must be no rule for module %q output file %q", moduleName, outputFiles[0]) + } + } +} + +func TestPrebuiltTakeSnapshot(t *testing.T) { + var testBp = ` + prebuilt_etc { + name: "prebuilt_vendor", + src: "foo.conf", + vendor: true, + } + + prebuilt_etc { + name: "prebuilt_vendor_indirect", + src: "foo.conf", + vendor: true, + } + + prebuilt_etc { + name: "prebuilt_recovery", + src: "bar.conf", + recovery: true, + } + + prebuilt_etc { + name: "prebuilt_recovery_indirect", + src: "bar.conf", + recovery: true, + } + ` + + t.Run("prebuilt: vendor and recovery snapshot", func(t *testing.T) { + result := prepareForPrebuiltEtcSnapshotTest.RunTestWithBp(t, testBp) + + checkIfSnapshotTaken(t, result, "vendor", "prebuilt_vendor") + checkIfSnapshotTaken(t, result, "vendor", "prebuilt_vendor_indirect") + checkIfSnapshotTaken(t, result, "recovery", "prebuilt_recovery") + checkIfSnapshotTaken(t, result, "recovery", "prebuilt_recovery_indirect") + }) + + t.Run("prebuilt: directed snapshot", func(t *testing.T) { + prepareForPrebuiltEtcDirectedSnapshotTest := android.GroupFixturePreparers( + prepareForPrebuiltEtcSnapshotTest, + android.FixtureModifyConfig(func(config android.Config) { + config.TestProductVariables.DirectedVendorSnapshot = true + config.TestProductVariables.VendorSnapshotModules = make(map[string]bool) + config.TestProductVariables.VendorSnapshotModules["prebuilt_vendor"] = true + config.TestProductVariables.DirectedRecoverySnapshot = true + config.TestProductVariables.RecoverySnapshotModules = make(map[string]bool) + config.TestProductVariables.RecoverySnapshotModules["prebuilt_recovery"] = true + }), + ) + + result := prepareForPrebuiltEtcDirectedSnapshotTest.RunTestWithBp(t, testBp) + + checkIfSnapshotTaken(t, result, "vendor", "prebuilt_vendor") + checkIfSnapshotNotTaken(t, result, "vendor", "prebuilt_vendor_indirect") + checkIfSnapshotTaken(t, result, "recovery", "prebuilt_recovery") + checkIfSnapshotNotTaken(t, result, "recovery", "prebuilt_recovery_indirect") + }) +} diff --git a/snapshot/Android.bp b/snapshot/Android.bp index c7e9d7e10..f17ac532a 100644 --- a/snapshot/Android.bp +++ b/snapshot/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "recovery_snapshot.go", "snapshot.go", "snapshot_base.go", + "util.go", "vendor_snapshot.go", ], pluginFor: ["soong_build"], diff --git a/snapshot/util.go b/snapshot/util.go new file mode 100644 index 000000000..2297dfc2b --- /dev/null +++ b/snapshot/util.go @@ -0,0 +1,36 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package snapshot + +import "android/soong/android" + +func WriteStringToFileRule(ctx android.SingletonContext, content, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + android.WriteFileRule(ctx, outPath, content) + return outPath +} + +func CopyFileRule(pctx android.PackageContext, ctx android.SingletonContext, path android.Path, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: path, + Output: outPath, + Description: "copy " + path.String() + " -> " + out, + Args: map[string]string{ + "cpFlags": "-f -L", + }, + }) + return outPath +}