Move fsverity metadata generation to Makefile

build_image.py has been handling fsverity metadata generation in the
packing step, but it can cause issues because the metadata files are
missing in the $OUT directory, and they only exist in result system.img.
This change moves the generation logic into Makefile, and makes the
metadata tracked by ninja graph.

Bug: 206326351
Test: PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA := true and build
Change-Id: I1f910d8ac6e2cc3c54f35916871733c632f18e44

core/Makefile

@@ -526,16 +526,6 @@ $(foreach kmd,$(BOARD_KERNEL_MODULE_DIRS), \
     $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-recovery-as-boot-load,$(kmd))),\
     $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-image-kernel-modules-dir,GENERIC_RAMDISK,$(TARGET_RAMDISK_OUT),,modules.load,,$(kmd)))))
 
-# -----------------------------------------------------------------
-# FSVerity metadata generation
-ifeq ($(PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA),true)
-
-FSVERITY_APK_KEY_PATH := $(DEFAULT_SYSTEM_DEV_CERTIFICATE)
-FSVERITY_APK_OUT := system/etc/security/fsverity/BuildManifest.apk
-FSVERITY_APK_MANIFEST_PATH := system/security/fsverity/AndroidManifest.xml
-
-endif # PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA
-
 # -----------------------------------------------------------------
 # Cert-to-package mapping.  Used by the post-build signing tools.
 # Use a macro to add newline to each echo command
@@ -1744,11 +1734,6 @@ define generate-image-prop-dictionary
 $(if $(filter $(2),system),\
     $(if $(INTERNAL_SYSTEM_OTHER_PARTITION_SIZE),$(hide) echo "system_other_size=$(INTERNAL_SYSTEM_OTHER_PARTITION_SIZE)" >> $(1))
     $(if $(PRODUCT_SYSTEM_HEADROOM),$(hide) echo "system_headroom=$(PRODUCT_SYSTEM_HEADROOM)" >> $(1))
-    $(if $(filter true,$(PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA)),$(hide) echo "fsverity=$(HOST_OUT_EXECUTABLES)/fsverity" >> $(1))
-    $(if $(filter true,$(PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA)),$(hide) echo "fsverity_generate_metadata=true" >> $(1))
-    $(if $(filter true,$(PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA)),$(hide) echo "fsverity_apk_key=$(FSVERITY_APK_KEY_PATH)" >> $(1))
-    $(if $(filter true,$(PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA)),$(hide) echo "fsverity_apk_manifest=$(FSVERITY_APK_MANIFEST_PATH)" >> $(1))
-    $(if $(filter true,$(PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA)),$(hide) echo "fsverity_apk_out=$(FSVERITY_APK_OUT)" >> $(1))
     $(call add-common-ro-flags-to-image-props,system,$(1))
 )
 $(if $(filter $(2),system_other),\
@@ -2765,6 +2750,55 @@ endef
 # -----------------------------------------------------------------
 # system image
 
+# FSVerity metadata generation
+# Generate fsverity metadata files (.fsv_meta) and build manifest
+# (system/etc/security/fsverity/BuildManifest.apk) BEFORE filtering systemimage files below
+ifeq ($(PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA),true)
+
+# Generate fsv_meta
+fsverity-metadata-targets := $(sort $(filter \
+  $(TARGET_OUT)/framework/%.jar \
+  $(foreach arch,$(TARGET_ARCH) $(TARGET_2ND_ARCH),$(foreach ext,oat vdex art, \
+    $(TARGET_OUT)/framework/oat/$(arch)/%.$(ext))) \
+  $(TARGET_OUT)/etc/boot-image.prof \
+  $(TARGET_OUT)/etc/dirty-image-objects \
+  $(TARGET_OUT)/etc/updatable-bcp-packages.txt, \
+  $(ALL_GENERATED_SOURCES) $(ALL_DEFAULT_INSTALLED_MODULES)))
+
+define fsverity-generate-metadata
+$(1).fsv_meta: PRIVATE_SRC := $(1)
+$(1).fsv_meta: PRIVATE_FSVERITY := $(HOST_OUT_EXECUTABLES)/fsverity
+$(1).fsv_meta: $(HOST_OUT_EXECUTABLES)/fsverity_metadata_generator $(HOST_OUT_EXECUTABLES)/fsverity $(1)
+	$$< --fsverity-path $$(PRIVATE_FSVERITY) --signature none \
+	    --hash-alg sha256 --output $$@ $$(PRIVATE_SRC)
+endef
+
+$(foreach f,$(fsverity-metadata-targets),$(eval $(call fsverity-generate-metadata,$(f))))
+ALL_DEFAULT_INSTALLED_MODULES += $(addsuffix .fsv_meta,$(fsverity-metadata-targets))
+
+# Generate BuildManifest.apk
+FSVERITY_APK_KEY_PATH := $(DEFAULT_SYSTEM_DEV_CERTIFICATE)
+FSVERITY_APK_OUT := $(TARGET_OUT)/etc/security/fsverity/BuildManifest.apk
+FSVERITY_APK_MANIFEST_PATH := system/security/fsverity/AndroidManifest.xml
+$(FSVERITY_APK_OUT): PRIVATE_FSVERITY := $(HOST_OUT_EXECUTABLES)/fsverity
+$(FSVERITY_APK_OUT): PRIVATE_AAPT2 := $(HOST_OUT_EXECUTABLES)/aapt2
+$(FSVERITY_APK_OUT): PRIVATE_APKSIGNER := $(HOST_OUT_EXECUTABLES)/apksigner
+$(FSVERITY_APK_OUT): PRIVATE_MANIFEST := $(FSVERITY_APK_MANIFEST_PATH)
+$(FSVERITY_APK_OUT): PRIVATE_KEY := $(FSVERITY_APK_KEY_PATH)
+$(FSVERITY_APK_OUT): PRIVATE_INPUTS := $(fsverity-metadata-targets)
+$(FSVERITY_APK_OUT): $(HOST_OUT_EXECUTABLES)/fsverity_manifest_generator \
+    $(HOST_OUT_EXECUTABLES)/fsverity $(HOST_OUT_EXECUTABLES)/aapt2 \
+    $(HOST_OUT_EXECUTABLES)/apksigner $(FSVERITY_APK_MANIFEST_PATH) \
+    $(FSVERITY_APK_KEY_PATH).x509.pem $(FSVERITY_APK_KEY_PATH).pk8
+	$< --fsverity-path $(PRIVATE_FSVERITY) --aapt2-path $(PRIVATE_AAPT2) \
+	    --apksigner-path $(PRIVATE_APKSIGNER) --apk-key-path $(PRIVATE_KEY) \
+	    --apk-manifest-path $(PRIVATE_MANIFEST) --output $@ \
+	    --base-dir $(PRODUCT_OUT) $(PRIVATE_INPUTS)
+
+ALL_DEFAULT_INSTALLED_MODULES += $(FSVERITY_APK_OUT)
+
+endif  # PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA
+
 INTERNAL_SYSTEMIMAGE_FILES := $(sort $(filter $(TARGET_OUT)/%, \
     $(ALL_GENERATED_SOURCES) \
     $(ALL_DEFAULT_INSTALLED_MODULES)))
@@ -2864,10 +2898,6 @@ endef
 ifeq ($(BOARD_AVB_ENABLE),true)
 $(BUILT_SYSTEMIMAGE): $(BOARD_AVB_SYSTEM_KEY_PATH)
 endif
-ifeq ($(PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA),true)
-$(BUILT_SYSTEMIMAGE): $(HOST_OUT_EXECUTABLES)/fsverity $(HOST_OUT_EXECUTABLES)/aapt2 $(HOST_OUT_EXECUTABLES)/apksigner \
-    $(FSVERITY_APK_MANIFEST_PATH) $(FSVERITY_APK_KEY_PATH).x509.pem $(FSVERITY_APK_KEY_PATH).pk8
-endif
 $(BUILT_SYSTEMIMAGE): $(FULL_SYSTEMIMAGE_DEPS) $(INSTALLED_FILES_FILE)
 	$(call build-systemimage-target,$@)
 

tools/releasetools/Android.bp

@@ -554,12 +554,25 @@ python_binary_host {
 }
 
 python_binary_host {
-    name: "fsverity_metadata_generator",
+    name: "fsverity_manifest_generator",
     srcs: [
-        "fsverity_metadata_generator.py",
+        "fsverity_manifest_generator.py",
     ],
     libs: [
         "fsverity_digests_proto_python",
+        "releasetools_common",
+    ],
+    required: [
+        "aapt2",
+        "apksigner",
+        "fsverity",
+    ],
+}
+
+python_binary_host {
+    name: "fsverity_metadata_generator",
+    srcs: [
+        "fsverity_metadata_generator.py",
     ],
     required: [
         "fsverity",

tools/releasetools/build_image.py

@@ -35,9 +35,6 @@ import sys
 import common
 import verity_utils
 
-from fsverity_digests_pb2 import FSVerityDigests
-from fsverity_metadata_generator import FSVerityMetadataGenerator
-
 logger = logging.getLogger(__name__)
 
 OPTIONS = common.OPTIONS
@@ -451,69 +448,6 @@ def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config):
 
   return mkfs_output
 
-def GenerateFSVerityMetadata(in_dir, fsverity_path, apk_key_path, apk_manifest_path, apk_out_path):
-  """Generates fsverity metadata files.
-
-  By setting PRODUCT_SYSTEM_FSVERITY_GENERATE_METADATA := true, fsverity
-  metadata files will be generated. For the input files, see `patterns` below.
-
-  One metadata file per one input file will be generated with the suffix
-  .fsv_meta. e.g. system/framework/foo.jar -> system/framework/foo.jar.fsv_meta
-  Also a mapping file containing fsverity digests will be generated to
-  system/etc/security/fsverity/BuildManifest.apk.
-
-  Args:
-    in_dir: temporary working directory (same as BuildImage)
-    fsverity_path: path to host tool fsverity
-    apk_key_path: path to key (e.g. build/make/target/product/security/platform)
-    apk_manifest_path: path to AndroidManifest.xml for APK
-    apk_out_path: path to the output APK
-
-  Returns:
-    None. The files are generated directly under in_dir.
-  """
-
-  patterns = [
-    "system/framework/*.jar",
-    "system/framework/oat/*/*.oat",
-    "system/framework/oat/*/*.vdex",
-    "system/framework/oat/*/*.art",
-    "system/etc/boot-image.prof",
-    "system/etc/dirty-image-objects",
-  ]
-  files = []
-  for pattern in patterns:
-    files += glob.glob(os.path.join(in_dir, pattern))
-  files = sorted(set(files))
-
-  generator = FSVerityMetadataGenerator(fsverity_path)
-  generator.set_hash_alg("sha256")
-
-  digests = FSVerityDigests()
-  for f in files:
-    generator.generate(f)
-    # f is a full path for now; make it relative so it starts with {mount_point}/
-    digest = digests.digests[os.path.relpath(f, in_dir)]
-    digest.digest = generator.digest(f)
-    digest.hash_alg = "sha256"
-
-  temp_dir = common.MakeTempDir()
-
-  os.mkdir(os.path.join(temp_dir, "assets"))
-  metadata_path = os.path.join(temp_dir, "assets", "build_manifest")
-  with open(metadata_path, "wb") as f:
-    f.write(digests.SerializeToString())
-
-  apk_path = os.path.join(in_dir, apk_out_path)
-
-  common.RunAndCheckOutput(["aapt2", "link",
-      "-A", os.path.join(temp_dir, "assets"),
-      "-o", apk_path,
-      "--manifest", apk_manifest_path])
-  common.RunAndCheckOutput(["apksigner", "sign", "--in", apk_path,
-      "--cert", apk_key_path + ".x509.pem",
-      "--key", apk_key_path + ".pk8"])
-
 def BuildImage(in_dir, prop_dict, out_file, target_out=None):
   """Builds an image for the files under in_dir and writes it to out_file.
 
@@ -541,13 +475,6 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None):
   elif fs_type.startswith("f2fs") and prop_dict.get("f2fs_compress") == "true":
     fs_spans_partition = False
 
-  if "fsverity_generate_metadata" in prop_dict:
-    GenerateFSVerityMetadata(in_dir,
-        fsverity_path=prop_dict["fsverity"],
-        apk_key_path=prop_dict["fsverity_apk_key"],
-        apk_manifest_path=prop_dict["fsverity_apk_manifest"],
-        apk_out_path=prop_dict["fsverity_apk_out"])
-
   # Get a builder for creating an image that's to be verified by Verified Boot,
   # or None if not applicable.
   verity_image_builder = verity_utils.CreateVerityImageBuilder(prop_dict)
@@ -801,11 +728,6 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
     copy_prop("system_root_image", "system_root_image")
     copy_prop("root_dir", "root_dir")
     copy_prop("root_fs_config", "root_fs_config")
-    copy_prop("fsverity", "fsverity")
-    copy_prop("fsverity_generate_metadata", "fsverity_generate_metadata")
-    copy_prop("fsverity_apk_key","fsverity_apk_key")
-    copy_prop("fsverity_apk_manifest","fsverity_apk_manifest")
-    copy_prop("fsverity_apk_out","fsverity_apk_out")
   elif mount_point == "data":
     # Copy the generic fs type first, override with specific one if available.
     copy_prop("flash_logical_block_size", "flash_logical_block_size")

tools/releasetools/fsverity_manifest_generator.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+#
+# Copyright 2022 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+`fsverity_manifest_generator` generates build manifest APK file containing
+digests of target files. The APK file is signed so the manifest inside the APK
+can be trusted.
+"""
+
+import argparse
+import common
+import os
+import subprocess
+import sys
+from fsverity_digests_pb2 import FSVerityDigests
+
+HASH_ALGORITHM = 'sha256'
+
+def _digest(fsverity_path, input_file):
+  cmd = [fsverity_path, 'digest', input_file]
+  cmd.extend(['--compact'])
+  cmd.extend(['--hash-alg', HASH_ALGORITHM])
+  out = subprocess.check_output(cmd, universal_newlines=True).strip()
+  return bytes(bytearray.fromhex(out))
+
+if __name__ == '__main__':
+  p = argparse.ArgumentParser()
+  p.add_argument(
+      '--output',
+      help='Path to the output manifest APK',
+      required=True)
+  p.add_argument(
+      '--fsverity-path',
+      help='path to the fsverity program',
+      required=True)
+  p.add_argument(
+      '--aapt2-path',
+      help='path to the aapt2 program',
+      required=True)
+  p.add_argument(
+      '--apksigner-path',
+      help='path to the apksigner program',
+      required=True)
+  p.add_argument(
+      '--apk-key-path',
+      help='path to the apk key',
+      required=True)
+  p.add_argument(
+      '--apk-manifest-path',
+      help='path to AndroidManifest.xml',
+      required=True)
+  p.add_argument(
+      '--base-dir',
+      help='directory to use as a relative root for the inputs',
+      required=True)
+  p.add_argument(
+      'inputs',
+      nargs='+',
+      help='input file for the build manifest')
+  args = p.parse_args(sys.argv[1:])
+
+  digests = FSVerityDigests()
+  for f in sorted(args.inputs):
+    # f is a full path for now; make it relative so it starts with {mount_point}/
+    digest = digests.digests[os.path.relpath(f, args.base_dir)]
+    digest.digest = _digest(args.fsverity_path, f)
+    digest.hash_alg = HASH_ALGORITHM
+
+  temp_dir = common.MakeTempDir()
+
+  os.mkdir(os.path.join(temp_dir, "assets"))
+  metadata_path = os.path.join(temp_dir, "assets", "build_manifest.pb")
+  with open(metadata_path, "wb") as f:
+    f.write(digests.SerializeToString())
+
+  common.RunAndCheckOutput([args.aapt2_path, "link",
+      "-A", os.path.join(temp_dir, "assets"),
+      "-o", args.output,
+      "--manifest", args.apk_manifest_path])
+  common.RunAndCheckOutput([args.apksigner_path, "sign", "--in", args.output,
+      "--cert", args.apk_key_path + ".x509.pem",
+      "--key", args.apk_key_path + ".pk8"])