Update prebuilt_etc available for snapshot

Make prebuilt_etc module available for the snapshot. This change
includes implementing snapshot interface for the prebuilt_etc module so
it can be added to the snapshot from the fake snapshot, or when the
module is specified in the list.

Bug: 192430376
Test: m nothing passed
Test: Checked if the module is included in the snapshot properly
Change-Id: I3574d2a1b8f8f4e5f083f3913e8768f5088d0c46

android/module.go

@@ -413,6 +413,7 @@ type ModuleContext interface {
 	InstallInDebugRamdisk() bool
 	InstallInRecovery() bool
 	InstallInRoot() bool
+	InstallInVendor() bool
 	InstallBypassMake() bool
 	InstallForceOS() (*OsType, *ArchType)
 
@@ -471,6 +472,7 @@ type Module interface {
 	InstallInDebugRamdisk() bool
 	InstallInRecovery() bool
 	InstallInRoot() bool
+	InstallInVendor() bool
 	InstallBypassMake() bool
 	InstallForceOS() (*OsType, *ArchType)
 	HideFromMake()
@@ -1579,6 +1581,10 @@ func (m *ModuleBase) InstallInRecovery() bool {
 	return Bool(m.commonProperties.Recovery)
 }
 
+func (m *ModuleBase) InstallInVendor() bool {
+	return Bool(m.commonProperties.Vendor)
+}
+
 func (m *ModuleBase) InstallInRoot() bool {
 	return false
 }
@@ -2645,6 +2651,10 @@ func (m *moduleContext) InstallForceOS() (*OsType, *ArchType) {
 	return m.module.InstallForceOS()
 }
 
+func (m *moduleContext) InstallInVendor() bool {
+	return m.module.InstallInVendor()
+}
+
 func (m *moduleContext) skipInstall() bool {
 	if m.module.base().commonProperties.SkipInstall {
 		return true

cc/util.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"android/soong/android"
+	"android/soong/snapshot"
 )
 
 // Efficiently converts a list of include directories to a single string
@@ -126,20 +127,6 @@ func makeSymlinkCmd(linkDirOnDevice string, linkName string, target string) stri
 		"ln -sf " + target + " " + filepath.Join(dir, linkName)
 }
 
-func copyFileRule(ctx android.SingletonContext, path android.Path, out string) android.OutputPath {
-	outPath := android.PathForOutput(ctx, out)
-	ctx.Build(pctx, android.BuildParams{
-		Rule:        android.Cp,
-		Input:       path,
-		Output:      outPath,
-		Description: "copy " + path.String() + " -> " + out,
-		Args: map[string]string{
-			"cpFlags": "-f -L",
-		},
-	})
-	return outPath
-}
-
 func combineNoticesRule(ctx android.SingletonContext, paths android.Paths, out string) android.OutputPath {
 	outPath := android.PathForOutput(ctx, out)
 	ctx.Build(pctx, android.BuildParams{
@@ -151,12 +138,6 @@ func combineNoticesRule(ctx android.SingletonContext, paths android.Paths, out s
 	return outPath
 }
 
-func writeStringToFileRule(ctx android.SingletonContext, content, out string) android.OutputPath {
-	outPath := android.PathForOutput(ctx, out)
-	android.WriteFileRule(ctx, outPath, content)
-	return outPath
-}
-
 // Dump a map to a list file as:
 //
 // {key1} {value1}
@@ -172,5 +153,5 @@ func installMapListFileRule(ctx android.SingletonContext, m map[string]string, p
 		txtBuilder.WriteString(" ")
 		txtBuilder.WriteString(m[k])
 	}
-	return writeStringToFileRule(ctx, txtBuilder.String(), path)
+	return snapshot.WriteStringToFileRule(ctx, txtBuilder.String(), path)
 }

cc/vendor_snapshot.go

@@ -210,9 +210,9 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS
 		if fake {
 			// All prebuilt binaries and headers are installed by copyFile function. This makes a fake
 			// snapshot just touch prebuilts and headers, rather than installing real files.
-			return writeStringToFileRule(ctx, "", out)
+			return snapshot.WriteStringToFileRule(ctx, "", out)
 		} else {
-			return copyFileRule(ctx, path, out)
+			return snapshot.CopyFileRule(pctx, ctx, path, out)
 		}
 	}
 
@@ -350,7 +350,7 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS
 			ctx.Errorf("json marshal to %q failed: %#v", propOut, err)
 			return nil
 		}
-		ret = append(ret, writeStringToFileRule(ctx, string(j), propOut))
+		ret = append(ret, snapshot.WriteStringToFileRule(ctx, string(j), propOut))
 
 		return ret
 	}

cc/vndk.go

@@ -25,6 +25,7 @@ import (
 	"android/soong/android"
 	"android/soong/cc/config"
 	"android/soong/etc"
+	"android/soong/snapshot"
 
 	"github.com/google/blueprint"
 )
@@ -698,7 +699,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex
 
 		libPath := m.outputFile.Path()
 		snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, "shared", vndkType, libPath.Base())
-		ret = append(ret, copyFileRule(ctx, libPath, snapshotLibOut))
+		ret = append(ret, snapshot.CopyFileRule(pctx, ctx, libPath, snapshotLibOut))
 
 		if ctx.Config().VndkSnapshotBuildArtifacts() {
 			prop := struct {
@@ -720,7 +721,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex
 				ctx.Errorf("json marshal to %q failed: %#v", propOut, err)
 				return nil, false
 			}
-			ret = append(ret, writeStringToFileRule(ctx, string(j), propOut))
+			ret = append(ret, snapshot.WriteStringToFileRule(ctx, string(j), propOut))
 		}
 		return ret, true
 	}
@@ -778,8 +779,8 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex
 
 	// install all headers after removing duplicates
 	for _, header := range android.FirstUniquePaths(headers) {
-		snapshotOutputs = append(snapshotOutputs, copyFileRule(
+		snapshotOutputs = append(snapshotOutputs, snapshot.CopyFileRule(
-			ctx, header, filepath.Join(includeDir, header.String())))
+			pctx, ctx, header, filepath.Join(includeDir, header.String())))
 	}
 
 	// install *.libraries.txt except vndkcorevariant.libraries.txt
@@ -788,8 +789,8 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex
 		if !ok || !m.Enabled() || m.Name() == vndkUsingCoreVariantLibrariesTxt {
 			return
 		}
-		snapshotOutputs = append(snapshotOutputs, copyFileRule(
+		snapshotOutputs = append(snapshotOutputs, snapshot.CopyFileRule(
-			ctx, m.OutputFile(), filepath.Join(configsDir, m.Name())))
+			pctx, ctx, m.OutputFile(), filepath.Join(configsDir, m.Name())))
 	})
 
 	/*

etc/Android.bp

@@ -9,6 +9,7 @@ bootstrap_go_package {
         "blueprint",
         "soong",
         "soong-android",
+        "soong-snapshot",
     ],
     srcs: [
         "prebuilt_etc.go",

etc/prebuilt_etc.go

@@ -28,12 +28,15 @@ package etc
 // various `prebuilt_*` mutators.
 
 import (
+	"encoding/json"
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	"github.com/google/blueprint/proptools"
 
 	"android/soong/android"
+	"android/soong/snapshot"
 )
 
 var pctx = android.NewPackageContext("android/soong/etc")
@@ -43,6 +46,7 @@ var pctx = android.NewPackageContext("android/soong/etc")
 func init() {
 	pctx.Import("android/soong/android")
 	RegisterPrebuiltEtcBuildComponents(android.InitRegistrationContext)
+	snapshot.RegisterSnapshotAction(generatePrebuiltSnapshot)
 }
 
 func RegisterPrebuiltEtcBuildComponents(ctx android.RegistrationContext) {
@@ -128,6 +132,9 @@ type PrebuiltEtc struct {
 	android.ModuleBase
 	android.DefaultableModuleBase
 
+	snapshot.VendorSnapshotModuleInterface
+	snapshot.RecoverySnapshotModuleInterface
+
 	properties       prebuiltEtcProperties
 	subdirProperties prebuiltSubdirProperties
 
@@ -183,7 +190,7 @@ func (p *PrebuiltEtc) InstallInDebugRamdisk() bool {
 	return p.inDebugRamdisk()
 }
 
-func (p *PrebuiltEtc) inRecovery() bool {
+func (p *PrebuiltEtc) InRecovery() bool {
 	return p.ModuleBase.InRecovery() || p.ModuleBase.InstallInRecovery()
 }
 
@@ -192,7 +199,7 @@ func (p *PrebuiltEtc) onlyInRecovery() bool {
 }
 
 func (p *PrebuiltEtc) InstallInRecovery() bool {
-	return p.inRecovery()
+	return p.InRecovery()
 }
 
 var _ android.ImageInterface = (*PrebuiltEtc)(nil)
@@ -271,6 +278,18 @@ func (p *PrebuiltEtc) Installable() bool {
 	return p.properties.Installable == nil || proptools.Bool(p.properties.Installable)
 }
 
+func (p *PrebuiltEtc) InVendor() bool {
+	return p.ModuleBase.InstallInVendor()
+}
+
+func (p *PrebuiltEtc) ExcludeFromVendorSnapshot() bool {
+	return false
+}
+
+func (p *PrebuiltEtc) ExcludeFromRecoverySnapshot() bool {
+	return false
+}
+
 func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) {
 	if p.properties.Src == nil {
 		ctx.PropertyErrorf("src", "missing prebuilt source file")
@@ -344,7 +363,7 @@ func (p *PrebuiltEtc) AndroidMkEntries() []android.AndroidMkEntries {
 	if p.inDebugRamdisk() && !p.onlyInDebugRamdisk() {
 		nameSuffix = ".debug_ramdisk"
 	}
-	if p.inRecovery() && !p.onlyInRecovery() {
+	if p.InRecovery() && !p.onlyInRecovery() {
 		nameSuffix = ".recovery"
 	}
 	return []android.AndroidMkEntries{android.AndroidMkEntries{
@@ -494,3 +513,137 @@ func PrebuiltRFSAFactory() android.Module {
 	android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst)
 	return module
 }
+
+// Flags to be included in the snapshot
+type snapshotJsonFlags struct {
+	ModuleName          string `json:",omitempty"`
+	Filename            string `json:",omitempty"`
+	RelativeInstallPath string `json:",omitempty"`
+}
+
+// Copy file into the snapshot
+func copyFile(ctx android.SingletonContext, path android.Path, out string, fake bool) android.OutputPath {
+	if fake {
+		// Create empty file instead for the fake snapshot
+		return snapshot.WriteStringToFileRule(ctx, "", out)
+	} else {
+		return snapshot.CopyFileRule(pctx, ctx, path, out)
+	}
+}
+
+// Check if the module is target of the snapshot
+func isSnapshotAware(ctx android.SingletonContext, m *PrebuiltEtc, image snapshot.SnapshotImage) bool {
+	if !m.Enabled() {
+		return false
+	}
+
+	// Skip if the module is not included in the image
+	if !image.InImage(m)() {
+		return false
+	}
+
+	// When android/prebuilt.go selects between source and prebuilt, it sets
+	// HideFromMake on the other one to avoid duplicate install rules in make.
+	if m.IsHideFromMake() {
+		return false
+	}
+
+	// There are some prebuilt_etc module with multiple definition of same name.
+	// Check if the target would be included from the build
+	if !m.ExportedToMake() {
+		return false
+	}
+
+	// Skip if the module is in the predefined path list to skip
+	if image.IsProprietaryPath(ctx.ModuleDir(m), ctx.DeviceConfig()) {
+		return false
+	}
+
+	// Skip if the module should be excluded
+	if image.ExcludeFromSnapshot(m) || image.ExcludeFromDirectedSnapshot(ctx.DeviceConfig(), m.BaseModuleName()) {
+		return false
+	}
+
+	// Skip from other exceptional cases
+	if m.Target().Os.Class != android.Device {
+		return false
+	}
+	if m.Target().NativeBridge == android.NativeBridgeEnabled {
+		return false
+	}
+
+	return true
+}
+
+func generatePrebuiltSnapshot(s snapshot.SnapshotSingleton, ctx android.SingletonContext, snapshotArchDir string) android.Paths {
+	/*
+		Snapshot zipped artifacts directory structure for etc modules:
+		{SNAPSHOT_ARCH}/
+			arch-{TARGET_ARCH}-{TARGET_ARCH_VARIANT}/
+				etc/
+					(prebuilt etc files)
+			arch-{TARGET_2ND_ARCH}-{TARGET_2ND_ARCH_VARIANT}/
+				etc/
+					(prebuilt etc files)
+			NOTICE_FILES/
+				(notice files)
+	*/
+	var snapshotOutputs android.Paths
+	noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES")
+	installedNotices := make(map[string]bool)
+
+	ctx.VisitAllModules(func(module android.Module) {
+		m, ok := module.(*PrebuiltEtc)
+		if !ok {
+			return
+		}
+
+		if !isSnapshotAware(ctx, m, s.Image) {
+			return
+		}
+
+		targetArch := "arch-" + m.Target().Arch.ArchType.String()
+
+		snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, "etc", m.BaseModuleName())
+		snapshotOutputs = append(snapshotOutputs, copyFile(ctx, m.OutputFile(), snapshotLibOut, s.Fake))
+
+		prop := snapshotJsonFlags{}
+		propOut := snapshotLibOut + ".json"
+		prop.ModuleName = m.BaseModuleName()
+		if m.subdirProperties.Relative_install_path != nil {
+			prop.RelativeInstallPath = *m.subdirProperties.Relative_install_path
+		}
+
+		if m.properties.Filename != nil {
+			prop.Filename = *m.properties.Filename
+		}
+
+		j, err := json.Marshal(prop)
+		if err != nil {
+			ctx.Errorf("json marshal to %q failed: %#v", propOut, err)
+			return
+		}
+		snapshotOutputs = append(snapshotOutputs, snapshot.WriteStringToFileRule(ctx, string(j), propOut))
+
+		if len(m.EffectiveLicenseFiles()) > 0 {
+			noticeName := ctx.ModuleName(m) + ".txt"
+			noticeOut := filepath.Join(noticeDir, noticeName)
+			// skip already copied notice file
+			if !installedNotices[noticeOut] {
+				installedNotices[noticeOut] = true
+
+				noticeOutPath := android.PathForOutput(ctx, noticeOut)
+				ctx.Build(pctx, android.BuildParams{
+					Rule:        android.Cat,
+					Inputs:      m.EffectiveLicenseFiles(),
+					Output:      noticeOutPath,
+					Description: "combine notices for " + noticeOut,
+				})
+				snapshotOutputs = append(snapshotOutputs, noticeOutPath)
+			}
+		}
+
+	})
+
+	return snapshotOutputs
+}

etc/prebuilt_etc_test.go

@@ -15,11 +15,15 @@
 package etc
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/google/blueprint/proptools"
+
 	"android/soong/android"
+	"android/soong/snapshot"
 )
 
 func TestMain(m *testing.M) {
@@ -36,6 +40,18 @@ var prepareForPrebuiltEtcTest = android.GroupFixturePreparers(
 	}),
 )
 
+var prepareForPrebuiltEtcSnapshotTest = android.GroupFixturePreparers(
+	prepareForPrebuiltEtcTest,
+	android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) {
+		snapshot.VendorSnapshotImageSingleton.Init(ctx)
+		snapshot.RecoverySnapshotImageSingleton.Init(ctx)
+	}),
+	android.FixtureModifyConfig(func(config android.Config) {
+		config.TestProductVariables.DeviceVndkVersion = proptools.StringPtr("current")
+		config.TestProductVariables.RecoverySnapshotVersion = proptools.StringPtr("current")
+	}),
+)
+
 func TestPrebuiltEtcVariants(t *testing.T) {
 	result := prepareForPrebuiltEtcTest.RunTestWithBp(t, `
 		prebuilt_etc {
@@ -346,3 +362,110 @@ func TestPrebuiltRFSADirPath(t *testing.T) {
 		})
 	}
 }
+
+func checkIfSnapshotTaken(t *testing.T, result *android.TestResult, image string, moduleName string) {
+	checkIfSnapshotExistAsExpected(t, result, image, moduleName, true)
+}
+
+func checkIfSnapshotNotTaken(t *testing.T, result *android.TestResult, image string, moduleName string) {
+	checkIfSnapshotExistAsExpected(t, result, image, moduleName, false)
+}
+
+func checkIfSnapshotExistAsExpected(t *testing.T, result *android.TestResult, image string, moduleName string, expectToExist bool) {
+	snapshotSingleton := result.SingletonForTests(image + "-snapshot")
+	archType := "arm64"
+	archVariant := "armv8-a"
+	archDir := fmt.Sprintf("arch-%s", archType)
+
+	snapshotDir := fmt.Sprintf("%s-snapshot", image)
+	snapshotVariantPath := filepath.Join(snapshotDir, archType)
+	outputDir := filepath.Join(snapshotVariantPath, archDir, "etc")
+	imageVariant := ""
+	if image == "recovery" {
+		imageVariant = "recovery_"
+	}
+	mod := result.ModuleForTests(moduleName, fmt.Sprintf("android_%s%s_%s", imageVariant, archType, archVariant))
+	outputFiles := mod.OutputFiles(t, "")
+	if len(outputFiles) != 1 {
+		t.Errorf("%q must have single output\n", moduleName)
+		return
+	}
+	snapshotPath := filepath.Join(outputDir, moduleName)
+
+	if expectToExist {
+		out := snapshotSingleton.Output(snapshotPath)
+
+		if out.Input.String() != outputFiles[0].String() {
+			t.Errorf("The input of snapshot %q must be %q, but %q", "prebuilt_vendor", out.Input.String(), outputFiles[0])
+		}
+
+		snapshotJsonPath := snapshotPath + ".json"
+
+		if snapshotSingleton.MaybeOutput(snapshotJsonPath).Rule == nil {
+			t.Errorf("%q expected but not found", snapshotJsonPath)
+		}
+	} else {
+		out := snapshotSingleton.MaybeOutput(snapshotPath)
+		if out.Rule != nil {
+			t.Errorf("There must be no rule for module %q output file %q", moduleName, outputFiles[0])
+		}
+	}
+}
+
+func TestPrebuiltTakeSnapshot(t *testing.T) {
+	var testBp = `
+	prebuilt_etc {
+		name: "prebuilt_vendor",
+		src: "foo.conf",
+		vendor: true,
+	}
+
+	prebuilt_etc {
+		name: "prebuilt_vendor_indirect",
+		src: "foo.conf",
+		vendor: true,
+	}
+
+	prebuilt_etc {
+		name: "prebuilt_recovery",
+		src: "bar.conf",
+		recovery: true,
+	}
+
+	prebuilt_etc {
+		name: "prebuilt_recovery_indirect",
+		src: "bar.conf",
+		recovery: true,
+	}
+	`
+
+	t.Run("prebuilt: vendor and recovery snapshot", func(t *testing.T) {
+		result := prepareForPrebuiltEtcSnapshotTest.RunTestWithBp(t, testBp)
+
+		checkIfSnapshotTaken(t, result, "vendor", "prebuilt_vendor")
+		checkIfSnapshotTaken(t, result, "vendor", "prebuilt_vendor_indirect")
+		checkIfSnapshotTaken(t, result, "recovery", "prebuilt_recovery")
+		checkIfSnapshotTaken(t, result, "recovery", "prebuilt_recovery_indirect")
+	})
+
+	t.Run("prebuilt: directed snapshot", func(t *testing.T) {
+		prepareForPrebuiltEtcDirectedSnapshotTest := android.GroupFixturePreparers(
+			prepareForPrebuiltEtcSnapshotTest,
+			android.FixtureModifyConfig(func(config android.Config) {
+				config.TestProductVariables.DirectedVendorSnapshot = true
+				config.TestProductVariables.VendorSnapshotModules = make(map[string]bool)
+				config.TestProductVariables.VendorSnapshotModules["prebuilt_vendor"] = true
+				config.TestProductVariables.DirectedRecoverySnapshot = true
+				config.TestProductVariables.RecoverySnapshotModules = make(map[string]bool)
+				config.TestProductVariables.RecoverySnapshotModules["prebuilt_recovery"] = true
+			}),
+		)
+
+		result := prepareForPrebuiltEtcDirectedSnapshotTest.RunTestWithBp(t, testBp)
+
+		checkIfSnapshotTaken(t, result, "vendor", "prebuilt_vendor")
+		checkIfSnapshotNotTaken(t, result, "vendor", "prebuilt_vendor_indirect")
+		checkIfSnapshotTaken(t, result, "recovery", "prebuilt_recovery")
+		checkIfSnapshotNotTaken(t, result, "recovery", "prebuilt_recovery_indirect")
+	})
+}

snapshot/Android.bp

@@ -15,6 +15,7 @@ bootstrap_go_package {
         "recovery_snapshot.go",
         "snapshot.go",
         "snapshot_base.go",
+        "util.go",
         "vendor_snapshot.go",
     ],
     pluginFor: ["soong_build"],

snapshot/util.go

@@ -0,0 +1,36 @@
+// Copyright 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package snapshot
+
+import "android/soong/android"
+
+func WriteStringToFileRule(ctx android.SingletonContext, content, out string) android.OutputPath {
+	outPath := android.PathForOutput(ctx, out)
+	android.WriteFileRule(ctx, outPath, content)
+	return outPath
+}
+
+func CopyFileRule(pctx android.PackageContext, ctx android.SingletonContext, path android.Path, out string) android.OutputPath {
+	outPath := android.PathForOutput(ctx, out)
+	ctx.Build(pctx, android.BuildParams{
+		Rule:        android.Cp,
+		Input:       path,
+		Output:      outPath,
+		Description: "copy " + path.String() + " -> " + out,
+		Args: map[string]string{
+			"cpFlags": "-f -L",
+		},
+	})
+	return outPath
+}