Separate snapshot definition

Current snapshot definition is located in the CC module, so it is
difficult to capture non-CC module (such as prebuilt_etc) to the
snapshot. Separate general snapshot definition from cc so other modules
can also define its own snapshot.

Bug: 192430376
Test: m nothing passed
Change-Id: Ifb69fb3d2ec555b629aa31ec03e7ce5831fd3063

snapshot/Android.bp

@@ -0,0 +1,21 @@
+package {
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+bootstrap_go_package {
+    name: "soong-snapshot",
+    pkgPath: "android/soong/snapshot",
+    deps: [
+        "blueprint",
+        "blueprint-pathtools",
+        "soong",
+        "soong-android",
+    ],
+    srcs: [
+        "recovery_snapshot.go",
+        "snapshot.go",
+        "snapshot_base.go",
+        "vendor_snapshot.go",
+    ],
+    pluginFor: ["soong_build"],
+}

snapshot/recovery_snapshot.go

@@ -0,0 +1,130 @@
+// Copyright 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package snapshot
+
+import "android/soong/android"
+
+// Interface for modules which can be captured in the recovery snapshot.
+type RecoverySnapshotModuleInterface interface {
+	SnapshotModuleInterfaceBase
+	InRecovery() bool
+	ExcludeFromRecoverySnapshot() bool
+}
+
+var recoverySnapshotSingleton = SnapshotSingleton{
+	"recovery",                     // name
+	"SOONG_RECOVERY_SNAPSHOT_ZIP",  // makeVar
+	android.OptionalPath{},         // snapshotZipFile
+	RecoverySnapshotImageSingleton, // Image
+	false,                          // Fake
+}
+
+func RecoverySnapshotSingleton() android.Singleton {
+	return &recoverySnapshotSingleton
+}
+
+// Determine if a dir under source tree is an SoC-owned proprietary directory based
+// on recovery snapshot configuration
+// Examples: device/, vendor/
+func isRecoveryProprietaryPath(dir string, deviceConfig android.DeviceConfig) bool {
+	return RecoverySnapshotSingleton().(*SnapshotSingleton).Image.IsProprietaryPath(dir, deviceConfig)
+}
+
+func IsRecoveryProprietaryModule(ctx android.BaseModuleContext) bool {
+
+	// Any module in a recovery proprietary path is a recovery proprietary
+	// module.
+	if isRecoveryProprietaryPath(ctx.ModuleDir(), ctx.DeviceConfig()) {
+		return true
+	}
+
+	// However if the module is not in a recovery proprietary path, it may
+	// still be a recovery proprietary module. This happens for cc modules
+	// that are excluded from the recovery snapshot, and it means that the
+	// vendor has assumed control of the framework-provided module.
+
+	if c, ok := ctx.Module().(RecoverySnapshotModuleInterface); ok {
+		if c.ExcludeFromRecoverySnapshot() {
+			return true
+		}
+	}
+
+	return false
+}
+
+var RecoverySnapshotImageName = "recovery"
+
+type RecoverySnapshotImage struct{}
+
+func (RecoverySnapshotImage) Init(ctx android.RegistrationContext) {
+	ctx.RegisterSingletonType("recovery-snapshot", RecoverySnapshotSingleton)
+}
+
+func (RecoverySnapshotImage) shouldGenerateSnapshot(ctx android.SingletonContext) bool {
+	// RECOVERY_SNAPSHOT_VERSION must be set to 'current' in order to generate a
+	// snapshot.
+	return ctx.DeviceConfig().RecoverySnapshotVersion() == "current"
+}
+
+func (RecoverySnapshotImage) InImage(m SnapshotModuleInterfaceBase) func() bool {
+	r, ok := m.(RecoverySnapshotModuleInterface)
+
+	if !ok {
+		// This module does not support recovery snapshot
+		return func() bool { return false }
+	}
+	return r.InRecovery
+}
+
+func (RecoverySnapshotImage) IsProprietaryPath(dir string, deviceConfig android.DeviceConfig) bool {
+	return isDirectoryExcluded(dir, deviceConfig.RecoverySnapshotDirsExcludedMap(), deviceConfig.RecoverySnapshotDirsIncludedMap())
+}
+
+func (RecoverySnapshotImage) ExcludeFromSnapshot(m SnapshotModuleInterfaceBase) bool {
+	r, ok := m.(RecoverySnapshotModuleInterface)
+
+	if !ok {
+		// This module does not support recovery snapshot
+		return true
+	}
+	return r.ExcludeFromRecoverySnapshot()
+}
+
+func (RecoverySnapshotImage) IsUsingSnapshot(cfg android.DeviceConfig) bool {
+	recoverySnapshotVersion := cfg.RecoverySnapshotVersion()
+	return recoverySnapshotVersion != "current" && recoverySnapshotVersion != ""
+}
+
+func (RecoverySnapshotImage) TargetSnapshotVersion(cfg android.DeviceConfig) string {
+	return cfg.RecoverySnapshotVersion()
+}
+
+func (RecoverySnapshotImage) ExcludeFromDirectedSnapshot(cfg android.DeviceConfig, name string) bool {
+	// If we're using full snapshot, not directed snapshot, capture every module
+	if !cfg.DirectedRecoverySnapshot() {
+		return false
+	}
+	// Else, checks if name is in RECOVERY_SNAPSHOT_MODULES.
+	return !cfg.RecoverySnapshotModules()[name]
+}
+
+func (RecoverySnapshotImage) ImageName() string {
+	return RecoverySnapshotImageName
+}
+
+var RecoverySnapshotImageSingleton RecoverySnapshotImage
+
+func init() {
+	RecoverySnapshotImageSingleton.Init(android.InitRegistrationContext)
+}

snapshot/snapshot.go

@@ -0,0 +1,122 @@
+// Copyright 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package snapshot
+
+import (
+	"path/filepath"
+	"sort"
+
+	"android/soong/android"
+)
+
+// This file contains singletons to capture snapshots. This singleton will generate snapshot of each target
+// image, and capturing snapshot module will be delegated to each module which implements GenerateSnapshotAction
+// function and register with RegisterSnapshotAction.
+
+var pctx = android.NewPackageContext("android/soong/snapshot")
+
+type SnapshotSingleton struct {
+	// Name, e.g., "vendor", "recovery", "ramdisk".
+	name string
+
+	// Make variable that points to the snapshot file, e.g.,
+	// "SOONG_RECOVERY_SNAPSHOT_ZIP".
+	makeVar string
+
+	// Path to the snapshot zip file.
+	snapshotZipFile android.OptionalPath
+
+	// Implementation of the image interface specific to the image
+	// associated with this snapshot (e.g., specific to the vendor image,
+	// recovery image, etc.).
+	Image SnapshotImage
+
+	// Whether this singleton is for fake snapshot or not.
+	// Fake snapshot is a snapshot whose prebuilt binaries and headers are empty.
+	// It is much faster to generate, and can be used to inspect dependencies.
+	Fake bool
+}
+
+// Interface of function to capture snapshot from each module
+type GenerateSnapshotAction func(snapshot SnapshotSingleton, ctx android.SingletonContext, snapshotArchDir string) android.Paths
+
+var snapshotActionList []GenerateSnapshotAction
+
+// Register GenerateSnapshotAction function so it can be called while generating snapshot
+func RegisterSnapshotAction(x GenerateSnapshotAction) {
+	snapshotActionList = append(snapshotActionList, x)
+}
+
+func (c *SnapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+	if !c.Image.shouldGenerateSnapshot(ctx) {
+		return
+	}
+
+	var snapshotOutputs android.Paths
+
+	// Snapshot zipped artifacts will be captured under {SNAPSHOT_ARCH} directory
+
+	snapshotDir := c.name + "-snapshot"
+	if c.Fake {
+		// If this is a fake snapshot singleton, place all files under fake/ subdirectory to avoid
+		// collision with real snapshot files
+		snapshotDir = filepath.Join("fake", snapshotDir)
+	}
+	snapshotArchDir := filepath.Join(snapshotDir, ctx.DeviceConfig().DeviceArch())
+
+	for _, f := range snapshotActionList {
+		snapshotOutputs = append(snapshotOutputs, f(*c, ctx, snapshotArchDir)...)
+	}
+
+	// All artifacts are ready. Sort them to normalize ninja and then zip.
+	sort.Slice(snapshotOutputs, func(i, j int) bool {
+		return snapshotOutputs[i].String() < snapshotOutputs[j].String()
+	})
+
+	zipPath := android.PathForOutput(
+		ctx,
+		snapshotDir,
+		c.name+"-"+ctx.Config().DeviceName()+".zip")
+	zipRule := android.NewRuleBuilder(pctx, ctx)
+
+	// filenames in rspfile from FlagWithRspFileInputList might be single-quoted. Remove it with tr
+	snapshotOutputList := android.PathForOutput(
+		ctx,
+		snapshotDir,
+		c.name+"-"+ctx.Config().DeviceName()+"_list")
+	rspFile := snapshotOutputList.ReplaceExtension(ctx, "rsp")
+	zipRule.Command().
+		Text("tr").
+		FlagWithArg("-d ", "\\'").
+		FlagWithRspFileInputList("< ", rspFile, snapshotOutputs).
+		FlagWithOutput("> ", snapshotOutputList)
+
+	zipRule.Temporary(snapshotOutputList)
+
+	zipRule.Command().
+		BuiltTool("soong_zip").
+		FlagWithOutput("-o ", zipPath).
+		FlagWithArg("-C ", android.PathForOutput(ctx, snapshotDir).String()).
+		FlagWithInput("-l ", snapshotOutputList)
+
+	zipRule.Build(zipPath.String(), c.name+" snapshot "+zipPath.String())
+	zipRule.DeleteTemporaryFiles()
+	c.snapshotZipFile = android.OptionalPathForPath(zipPath)
+}
+
+func (c *SnapshotSingleton) MakeVars(ctx android.MakeVarsContext) {
+	ctx.Strict(
+		c.makeVar,
+		c.snapshotZipFile.String())
+}

snapshot/snapshot_base.go

@@ -0,0 +1,104 @@
+// Copyright 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package snapshot
+
+import (
+	"android/soong/android"
+	"path/filepath"
+)
+
+// Interface for modules which can be captured in the snapshot.
+type SnapshotModuleInterfaceBase interface{}
+
+// Defines the specifics of different images to which the snapshot process is applicable, e.g.,
+// vendor, recovery, ramdisk.
+type SnapshotImage interface {
+	// Returns true if a snapshot should be generated for this image.
+	shouldGenerateSnapshot(ctx android.SingletonContext) bool
+
+	// Function that returns true if the module is included in this image.
+	// Using a function return instead of a value to prevent early
+	// evalution of a function that may be not be defined.
+	InImage(m SnapshotModuleInterfaceBase) func() bool
+
+	// Returns true if a dir under source tree is an SoC-owned proprietary
+	// directory, such as device/, vendor/, etc.
+	//
+	// For a given snapshot (e.g., vendor, recovery, etc.) if
+	// isProprietaryPath(dir, deviceConfig) returns true, then the module in dir
+	// will be built from sources.
+	IsProprietaryPath(dir string, deviceConfig android.DeviceConfig) bool
+
+	// Whether a given module has been explicitly excluded from the
+	// snapshot, e.g., using the exclude_from_vendor_snapshot or
+	// exclude_from_recovery_snapshot properties.
+	ExcludeFromSnapshot(m SnapshotModuleInterfaceBase) bool
+
+	// Returns true if the build is using a snapshot for this image.
+	IsUsingSnapshot(cfg android.DeviceConfig) bool
+
+	// Returns a version of which the snapshot should be used in this target.
+	// This will only be meaningful when isUsingSnapshot is true.
+	TargetSnapshotVersion(cfg android.DeviceConfig) string
+
+	// Whether to exclude a given module from the directed snapshot or not.
+	// If the makefile variable DIRECTED_{IMAGE}_SNAPSHOT is true, directed snapshot is turned on,
+	// and only modules listed in {IMAGE}_SNAPSHOT_MODULES will be captured.
+	ExcludeFromDirectedSnapshot(cfg android.DeviceConfig, name string) bool
+
+	// Returns target image name
+	ImageName() string
+}
+
+type directoryMap map[string]bool
+
+var (
+	// Modules under following directories are ignored. They are OEM's and vendor's
+	// proprietary modules(device/, kernel/, vendor/, and hardware/).
+	defaultDirectoryExcludedMap = directoryMap{
+		"device":   true,
+		"hardware": true,
+		"kernel":   true,
+		"vendor":   true,
+	}
+
+	// Modules under following directories are included as they are in AOSP,
+	// although hardware/ and kernel/ are normally for vendor's own.
+	defaultDirectoryIncludedMap = directoryMap{
+		"kernel/configs":              true,
+		"kernel/prebuilts":            true,
+		"kernel/tests":                true,
+		"hardware/interfaces":         true,
+		"hardware/libhardware":        true,
+		"hardware/libhardware_legacy": true,
+		"hardware/ril":                true,
+	}
+)
+
+func isDirectoryExcluded(dir string, excludedMap directoryMap, includedMap directoryMap) bool {
+	if dir == "." || dir == "/" {
+		return false
+	}
+	if includedMap[dir] {
+		return false
+	} else if excludedMap[dir] {
+		return true
+	} else if defaultDirectoryIncludedMap[dir] {
+		return false
+	} else if defaultDirectoryExcludedMap[dir] {
+		return true
+	} else {
+		return isDirectoryExcluded(filepath.Dir(dir), excludedMap, includedMap)
+	}
+}

snapshot/vendor_snapshot.go

@@ -0,0 +1,147 @@
+// Copyright 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package snapshot
+
+import "android/soong/android"
+
+// Interface for modules which can be captured in the vendor snapshot.
+type VendorSnapshotModuleInterface interface {
+	SnapshotModuleInterfaceBase
+	InVendor() bool
+	ExcludeFromVendorSnapshot() bool
+}
+
+var vendorSnapshotSingleton = SnapshotSingleton{
+	"vendor",                     // name
+	"SOONG_VENDOR_SNAPSHOT_ZIP",  // makeVar
+	android.OptionalPath{},       // snapshotZipFile
+	VendorSnapshotImageSingleton, // Image
+	false,                        // Fake
+}
+
+var vendorFakeSnapshotSingleton = SnapshotSingleton{
+	"vendor",                         // name
+	"SOONG_VENDOR_FAKE_SNAPSHOT_ZIP", // makeVar
+	android.OptionalPath{},           // snapshotZipFile
+	VendorSnapshotImageSingleton,     // Image
+	true,                             // Fake
+}
+
+func VendorSnapshotSingleton() android.Singleton {
+	return &vendorSnapshotSingleton
+}
+
+func VendorFakeSnapshotSingleton() android.Singleton {
+	return &vendorFakeSnapshotSingleton
+}
+
+// Determine if a dir under source tree is an SoC-owned proprietary directory based
+// on vendor snapshot configuration
+// Examples: device/, vendor/
+func isVendorProprietaryPath(dir string, deviceConfig android.DeviceConfig) bool {
+	return VendorSnapshotSingleton().(*SnapshotSingleton).Image.IsProprietaryPath(dir, deviceConfig)
+}
+
+func IsVendorProprietaryModule(ctx android.BaseModuleContext) bool {
+	// Any module in a vendor proprietary path is a vendor proprietary
+	// module.
+	if isVendorProprietaryPath(ctx.ModuleDir(), ctx.DeviceConfig()) {
+		return true
+	}
+
+	// However if the module is not in a vendor proprietary path, it may
+	// still be a vendor proprietary module. This happens for cc modules
+	// that are excluded from the vendor snapshot, and it means that the
+	// vendor has assumed control of the framework-provided module.
+	if c, ok := ctx.Module().(VendorSnapshotModuleInterface); ok {
+		if c.ExcludeFromVendorSnapshot() {
+			return true
+		}
+	}
+
+	return false
+}
+
+var VendorSnapshotImageName = "vendor"
+
+type VendorSnapshotImage struct{}
+
+func (VendorSnapshotImage) Init(ctx android.RegistrationContext) {
+	ctx.RegisterSingletonType("vendor-snapshot", VendorSnapshotSingleton)
+	ctx.RegisterSingletonType("vendor-fake-snapshot", VendorFakeSnapshotSingleton)
+}
+
+func (VendorSnapshotImage) RegisterAdditionalModule(ctx android.RegistrationContext, name string, factory android.ModuleFactory) {
+	ctx.RegisterModuleType(name, factory)
+}
+
+func (VendorSnapshotImage) shouldGenerateSnapshot(ctx android.SingletonContext) bool {
+	// BOARD_VNDK_VERSION must be set to 'current' in order to generate a snapshot.
+	return ctx.DeviceConfig().VndkVersion() == "current"
+}
+
+func (VendorSnapshotImage) InImage(m SnapshotModuleInterfaceBase) func() bool {
+	v, ok := m.(VendorSnapshotModuleInterface)
+
+	if !ok {
+		// This module does not support Vendor snapshot
+		return func() bool { return false }
+	}
+
+	return v.InVendor
+}
+
+func (VendorSnapshotImage) IsProprietaryPath(dir string, deviceConfig android.DeviceConfig) bool {
+	return isDirectoryExcluded(dir, deviceConfig.VendorSnapshotDirsExcludedMap(), deviceConfig.VendorSnapshotDirsIncludedMap())
+}
+
+func (VendorSnapshotImage) ExcludeFromSnapshot(m SnapshotModuleInterfaceBase) bool {
+	v, ok := m.(VendorSnapshotModuleInterface)
+
+	if !ok {
+		// This module does not support Vendor snapshot
+		return true
+	}
+
+	return v.ExcludeFromVendorSnapshot()
+}
+
+func (VendorSnapshotImage) IsUsingSnapshot(cfg android.DeviceConfig) bool {
+	vndkVersion := cfg.VndkVersion()
+	return vndkVersion != "current" && vndkVersion != ""
+}
+
+func (VendorSnapshotImage) TargetSnapshotVersion(cfg android.DeviceConfig) string {
+	return cfg.VndkVersion()
+}
+
+// returns true iff a given module SHOULD BE EXCLUDED, false if included
+func (VendorSnapshotImage) ExcludeFromDirectedSnapshot(cfg android.DeviceConfig, name string) bool {
+	// If we're using full snapshot, not directed snapshot, capture every module
+	if !cfg.DirectedVendorSnapshot() {
+		return false
+	}
+	// Else, checks if name is in VENDOR_SNAPSHOT_MODULES.
+	return !cfg.VendorSnapshotModules()[name]
+}
+
+func (VendorSnapshotImage) ImageName() string {
+	return VendorSnapshotImageName
+}
+
+var VendorSnapshotImageSingleton VendorSnapshotImage
+
+func init() {
+	VendorSnapshotImageSingleton.Init(android.InitRegistrationContext)
+}