From 9afb221c9e6b21f53ea022436ac8d4b9756e1688 Mon Sep 17 00:00:00 2001 From: Tianjie Xu Date: Sun, 10 May 2020 21:48:15 +0000 Subject: [PATCH] Reland "Calculate the runtime fingerprint prefixes from build prop" This reverts commit b21e48b499850bfb5ac9e6e155ed96e8e724c5c0. In practice, some partners use the 'import' statement to override the device fingerprint at runtime. The runtime fingerprint will later add to the metadata of OTA package, so that the OTA server can deliver the package to corresponding devices correctly. This CL supports parsing a subset of import statement that the init process recognizes. And we loose the restriction based on how the dynamic fingerprint is used in practice. Right now, we only searches for the override of brand, name and device. And the placeholder format should be ${placeholder}, with its value supplied by the script caller. As part of the implementation, we generate all the possible combinations of the input boot variables. And recalculate the fingerprint for each of the combination. Though we load the build.prop multiple times, the logic is easier to follow. Also, it's more convenient to enhance the logic if we only want to allow some of the boot variables combination later. Bug: 152167826 Change-Id: I4a9fa35c7ac037ff1cf4f9a4bdff602beac3894b Test: unittests pass --- tools/releasetools/common.py | 77 ++++++- tools/releasetools/ota_from_target_files.py | 34 +++ tools/releasetools/test_common.py | 216 +++++++++++++++++- .../test_ota_from_target_files.py | 125 +++++++++- 4 files changed, 429 insertions(+), 23 deletions(-) diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index 71ae21fa8a..7805e30ce8 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -738,18 +738,22 @@ class PartitionBuildProps(object): partition: name of the partition. props_allow_override: a list of build properties to search for the alternative values during runtime. - build_props: a dictionary of build properties for the given partition. - prop_overrides: a dict of list. And each list holds the overridden values - for props_allow_override. + build_props: a dict of build properties for the given partition. + prop_overrides: a set of props that are overridden by import. + placeholder_values: A dict of runtime variables' values to replace the + placeholders in the build.prop file. We expect exactly one value for + each of the variables. """ - - def __init__(self, input_file, name): + def __init__(self, input_file, name, placeholder_values=None): self.input_file = input_file self.partition = name self.props_allow_override = [props.format(name) for props in [ - 'ro.product.{}.name', 'ro.product.{}.device']] + 'ro.product.{}.brand', 'ro.product.{}.name', 'ro.product.{}.device']] self.build_props = {} - self.prop_overrides = {} + self.prop_overrides = set() + self.placeholder_values = {} + if placeholder_values: + self.placeholder_values = copy.deepcopy(placeholder_values) @staticmethod def FromDictionary(name, build_props): @@ -760,9 +764,8 @@ class PartitionBuildProps(object): return props @staticmethod - def FromInputFile(input_file, name): + def FromInputFile(input_file, name, placeholder_values=None): """Loads the build.prop file and builds the attributes.""" - data = '' for prop_file in ['{}/etc/build.prop'.format(name.upper()), '{}/build.prop'.format(name.upper())]: @@ -772,10 +775,62 @@ class PartitionBuildProps(object): except KeyError: logger.warning('Failed to read %s', prop_file) - props = PartitionBuildProps(input_file, name) - props.build_props = LoadDictionaryFromLines(data.split('\n')) + props = PartitionBuildProps(input_file, name, placeholder_values) + props._LoadBuildProp(data) return props + def _LoadBuildProp(self, data): + for line in data.split('\n'): + line = line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("import"): + overrides = self._ImportParser(line) + duplicates = self.prop_overrides.intersection(overrides.keys()) + if duplicates: + raise ValueError('prop {} is overridden multiple times'.format( + ','.join(duplicates))) + self.prop_overrides = self.prop_overrides.union(overrides.keys()) + self.build_props.update(overrides) + elif "=" in line: + name, value = line.split("=", 1) + if name in self.prop_overrides: + raise ValueError('prop {} is set again after overridden by import ' + 'statement'.format(name)) + self.build_props[name] = value + + def _ImportParser(self, line): + """Parses the build prop in a given import statement.""" + + tokens = line.split() + if len(tokens) != 2 or tokens[0] != 'import': + raise ValueError('Unrecognized import statement {}'.format(line)) + import_path = tokens[1] + if not re.match(r'^/{}/.*\.prop$'.format(self.partition), import_path): + raise ValueError('Unrecognized import path {}'.format(line)) + + # We only recognize a subset of import statement that the init process + # supports. And we can loose the restriction based on how the dynamic + # fingerprint is used in practice. The placeholder format should be + # ${placeholder}, and its value should be provided by the caller through + # the placeholder_values. + for prop, value in self.placeholder_values.items(): + prop_place_holder = '${{{}}}'.format(prop) + if prop_place_holder in import_path: + import_path = import_path.replace(prop_place_holder, value) + if '$' in import_path: + logger.info('Unresolved place holder in import path %s', import_path) + return {} + + import_path = import_path.replace('/{}'.format(self.partition), + self.partition.upper()) + logger.info('Parsing build props override from %s', import_path) + + lines = ReadFromInputFile(self.input_file, import_path).split('\n') + d = LoadDictionaryFromLines(lines) + return {key: val for key, val in d.items() + if key in self.props_allow_override} + def GetProp(self, prop): return self.build_props.get(prop) diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py index 92a46a20cd..47ad3d8a14 100755 --- a/tools/releasetools/ota_from_target_files.py +++ b/tools/releasetools/ota_from_target_files.py @@ -193,6 +193,8 @@ A/B OTA specific options from __future__ import print_function import collections +import copy +import itertools import logging import multiprocessing import os.path @@ -229,6 +231,7 @@ OPTIONS.include_secondary = False OPTIONS.no_signing = False OPTIONS.block_based = True OPTIONS.updater_binary = None +OPTIONS.oem_dicts = None OPTIONS.oem_source = None OPTIONS.oem_no_mount = False OPTIONS.full_radio = False @@ -247,6 +250,7 @@ OPTIONS.retrofit_dynamic_partitions = False OPTIONS.skip_compatibility_check = False OPTIONS.output_metadata_path = None OPTIONS.disable_fec_computation = False +OPTIONS.boot_variable_values = None METADATA_NAME = 'META-INF/com/android/metadata' @@ -1959,6 +1963,36 @@ def GenerateNonAbOtaPackage(target_file, output_file, source_file=None): output_file) +def CalculateRuntimeFingerprints(): + """Returns a set of runtime fingerprints based on the boot variables.""" + + build_info = common.BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts) + fingerprints = {build_info.fingerprint} + + if not OPTIONS.boot_variable_values: + return fingerprints + + # Calculate all possible combinations of the values for the boot variables. + keys = OPTIONS.boot_variable_values.keys() + value_list = OPTIONS.boot_variable_values.values() + combinations = [dict(zip(keys, values)) + for values in itertools.product(*value_list)] + for placeholder_values in combinations: + # Reload the info_dict as some build properties may change their values + # based on the value of ro.boot* properties. + info_dict = copy.deepcopy(OPTIONS.info_dict) + for partition in common.PARTITIONS_WITH_CARE_MAP: + partition_prop_key = "{}.build.prop".format(partition) + old_props = info_dict[partition_prop_key] + info_dict[partition_prop_key] = common.PartitionBuildProps.FromInputFile( + old_props.input_file, partition, placeholder_values) + info_dict["build.prop"] = info_dict["system.build.prop"] + + build_info = common.BuildInfo(info_dict, OPTIONS.oem_dicts) + fingerprints.add(build_info.fingerprint) + return fingerprints + + def main(argv): def option_handler(o, a): diff --git a/tools/releasetools/test_common.py b/tools/releasetools/test_common.py index c0c2d3f32a..787e6757ec 100644 --- a/tools/releasetools/test_common.py +++ b/tools/releasetools/test_common.py @@ -1899,7 +1899,7 @@ super_group_foo_group_size={group_foo_size} class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase): def setUp(self): - self.build_prop = [ + self.odm_build_prop = [ 'ro.odm.build.date.utc=1578430045', 'ro.odm.build.fingerprint=' 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', @@ -1918,13 +1918,81 @@ class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase): def test_parseBuildProps_noImportStatement(self): build_prop = [ - 'ro.odm.build.date.utc=1578430045', - 'ro.odm.build.fingerprint=' - 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', - 'ro.product.odm.device=coral', + 'ro.odm.build.date.utc=1578430045', + 'ro.odm.build.fingerprint=' + 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', + 'ro.product.odm.device=coral', ] input_file = self._BuildZipFile({ - 'ODM/etc/build.prop': '\n'.join(build_prop), + 'ODM/etc/build.prop': '\n'.join(build_prop), + }) + + with zipfile.ZipFile(input_file, 'r') as input_zip: + placeholder_values = { + 'ro.boot.product.device_name': ['std', 'pro'] + } + partition_props = common.PartitionBuildProps.FromInputFile( + input_zip, 'odm', placeholder_values) + + self.assertEqual({ + 'ro.odm.build.date.utc': '1578430045', + 'ro.odm.build.fingerprint': + 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', + 'ro.product.odm.device': 'coral', + }, partition_props.build_props) + + self.assertEqual(set(), partition_props.prop_overrides) + + def test_parseBuildProps_singleImportStatement(self): + build_std_prop = [ + 'ro.product.odm.device=coral', + 'ro.product.odm.name=product1', + ] + build_pro_prop = [ + 'ro.product.odm.device=coralpro', + 'ro.product.odm.name=product2', + ] + + input_file = self._BuildZipFile({ + 'ODM/etc/build.prop': '\n'.join(self.odm_build_prop), + 'ODM/etc/build_std.prop': '\n'.join(build_std_prop), + 'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop), + }) + + with zipfile.ZipFile(input_file, 'r') as input_zip: + placeholder_values = { + 'ro.boot.product.device_name': 'std' + } + partition_props = common.PartitionBuildProps.FromInputFile( + input_zip, 'odm', placeholder_values) + + self.assertEqual({ + 'ro.odm.build.date.utc': '1578430045', + 'ro.odm.build.fingerprint': + 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', + 'ro.product.odm.device': 'coral', + 'ro.product.odm.name': 'product1', + }, partition_props.build_props) + + with zipfile.ZipFile(input_file, 'r') as input_zip: + placeholder_values = { + 'ro.boot.product.device_name': 'pro' + } + partition_props = common.PartitionBuildProps.FromInputFile( + input_zip, 'odm', placeholder_values) + + self.assertEqual({ + 'ro.odm.build.date.utc': '1578430045', + 'ro.odm.build.fingerprint': + 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', + 'ro.product.odm.device': 'coralpro', + 'ro.product.odm.name': 'product2', + }, partition_props.build_props) + + def test_parseBuildProps_noPlaceHolders(self): + build_prop = copy.copy(self.odm_build_prop) + input_file = self._BuildZipFile({ + 'ODM/etc/build.prop': '\n'.join(build_prop), }) with zipfile.ZipFile(input_file, 'r') as input_zip: @@ -1932,10 +2000,136 @@ class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase): input_zip, 'odm') self.assertEqual({ - 'ro.odm.build.date.utc': '1578430045', - 'ro.odm.build.fingerprint': - 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', - 'ro.product.odm.device': 'coral', + 'ro.odm.build.date.utc': '1578430045', + 'ro.odm.build.fingerprint': + 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', + 'ro.product.odm.device': 'coral', }, partition_props.build_props) - self.assertEqual({}, partition_props.prop_overrides) + self.assertEqual(set(), partition_props.prop_overrides) + + def test_parseBuildProps_multipleImportStatements(self): + build_prop = copy.deepcopy(self.odm_build_prop) + build_prop.append( + 'import /odm/etc/build_${ro.boot.product.product_name}.prop') + + build_std_prop = [ + 'ro.product.odm.device=coral', + ] + build_pro_prop = [ + 'ro.product.odm.device=coralpro', + ] + + product1_prop = [ + 'ro.product.odm.name=product1', + 'ro.product.not_care=not_care', + ] + + product2_prop = [ + 'ro.product.odm.name=product2', + 'ro.product.not_care=not_care', + ] + + input_file = self._BuildZipFile({ + 'ODM/etc/build.prop': '\n'.join(build_prop), + 'ODM/etc/build_std.prop': '\n'.join(build_std_prop), + 'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop), + 'ODM/etc/build_product1.prop': '\n'.join(product1_prop), + 'ODM/etc/build_product2.prop': '\n'.join(product2_prop), + }) + + with zipfile.ZipFile(input_file, 'r') as input_zip: + placeholder_values = { + 'ro.boot.product.device_name': 'std', + 'ro.boot.product.product_name': 'product1', + 'ro.boot.product.not_care': 'not_care', + } + partition_props = common.PartitionBuildProps.FromInputFile( + input_zip, 'odm', placeholder_values) + + self.assertEqual({ + 'ro.odm.build.date.utc': '1578430045', + 'ro.odm.build.fingerprint': + 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', + 'ro.product.odm.device': 'coral', + 'ro.product.odm.name': 'product1' + }, partition_props.build_props) + + with zipfile.ZipFile(input_file, 'r') as input_zip: + placeholder_values = { + 'ro.boot.product.device_name': 'pro', + 'ro.boot.product.product_name': 'product2', + 'ro.boot.product.not_care': 'not_care', + } + partition_props = common.PartitionBuildProps.FromInputFile( + input_zip, 'odm', placeholder_values) + + self.assertEqual({ + 'ro.odm.build.date.utc': '1578430045', + 'ro.odm.build.fingerprint': + 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', + 'ro.product.odm.device': 'coralpro', + 'ro.product.odm.name': 'product2' + }, partition_props.build_props) + + def test_parseBuildProps_defineAfterOverride(self): + build_prop = copy.deepcopy(self.odm_build_prop) + build_prop.append('ro.product.odm.device=coral') + + build_std_prop = [ + 'ro.product.odm.device=coral', + ] + build_pro_prop = [ + 'ro.product.odm.device=coralpro', + ] + + input_file = self._BuildZipFile({ + 'ODM/etc/build.prop': '\n'.join(build_prop), + 'ODM/etc/build_std.prop': '\n'.join(build_std_prop), + 'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop), + }) + + with zipfile.ZipFile(input_file, 'r') as input_zip: + placeholder_values = { + 'ro.boot.product.device_name': 'std', + } + + self.assertRaises(ValueError, common.PartitionBuildProps.FromInputFile, + input_zip, 'odm', placeholder_values) + + def test_parseBuildProps_duplicateOverride(self): + build_prop = copy.deepcopy(self.odm_build_prop) + build_prop.append( + 'import /odm/etc/build_${ro.boot.product.product_name}.prop') + + build_std_prop = [ + 'ro.product.odm.device=coral', + 'ro.product.odm.name=product1', + ] + build_pro_prop = [ + 'ro.product.odm.device=coralpro', + ] + + product1_prop = [ + 'ro.product.odm.name=product1', + ] + + product2_prop = [ + 'ro.product.odm.name=product2', + ] + + input_file = self._BuildZipFile({ + 'ODM/etc/build.prop': '\n'.join(build_prop), + 'ODM/etc/build_std.prop': '\n'.join(build_std_prop), + 'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop), + 'ODM/etc/build_product1.prop': '\n'.join(product1_prop), + 'ODM/etc/build_product2.prop': '\n'.join(product2_prop), + }) + + with zipfile.ZipFile(input_file, 'r') as input_zip: + placeholder_values = { + 'ro.boot.product.device_name': 'std', + 'ro.boot.product.product_name': 'product1', + } + self.assertRaises(ValueError, common.PartitionBuildProps.FromInputFile, + input_zip, 'odm', placeholder_values) diff --git a/tools/releasetools/test_ota_from_target_files.py b/tools/releasetools/test_ota_from_target_files.py index e0078632c7..4077d06e69 100644 --- a/tools/releasetools/test_ota_from_target_files.py +++ b/tools/releasetools/test_ota_from_target_files.py @@ -26,7 +26,8 @@ from ota_from_target_files import ( GetPackageMetadata, GetTargetFilesZipForSecondaryImages, GetTargetFilesZipWithoutPostinstallConfig, NonAbOtaPropertyFiles, Payload, PayloadSigner, POSTINSTALL_CONFIG, PropertyFiles, - StreamingPropertyFiles, WriteFingerprintAssertion) + StreamingPropertyFiles, WriteFingerprintAssertion, + CalculateRuntimeFingerprints) def construct_target_files(secondary=False): @@ -1318,3 +1319,125 @@ class PayloadTest(test_utils.ReleaseToolsTestCase): Payload.SECONDARY_PAYLOAD_PROPERTIES_TXT): continue self.assertEqual(zipfile.ZIP_STORED, entry_info.compress_type) + + +class RuntimeFingerprintTest(test_utils.ReleaseToolsTestCase): + MISC_INFO = [ + 'recovery_api_version=3', + 'fstab_version=2', + 'recovery_as_boot=true', + ] + + BUILD_PROP = [ + 'ro.build.version.release=version-release', + 'ro.build.id=build-id', + 'ro.build.version.incremental=version-incremental', + 'ro.build.type=build-type', + 'ro.build.tags=build-tags', + ] + + VENDOR_BUILD_PROP = [ + 'ro.product.vendor.brand=vendor-product-brand', + 'ro.product.vendor.name=vendor-product-name', + 'ro.product.vendor.device=vendor-product-device' + ] + + def setUp(self): + common.OPTIONS.oem_dicts = None + self.test_dir = common.MakeTempDir() + self.writeFiles({'META/misc_info.txt': '\n'.join(self.MISC_INFO)}) + + def writeFiles(self, contents_dict): + for path, content in contents_dict.items(): + abs_path = os.path.join(self.test_dir, path) + dir_name = os.path.dirname(abs_path) + if not os.path.exists(dir_name): + os.makedirs(dir_name) + with open(abs_path, 'w') as f: + f.write(content) + + @staticmethod + def constructFingerprint(prefix): + return '{}:version-release/build-id/version-incremental:' \ + 'build-type/build-tags'.format(prefix) + + def test_CalculatePossibleFingerprints_no_dynamic_fingerprint(self): + build_prop = copy.deepcopy(self.BUILD_PROP) + build_prop.extend([ + 'ro.product.brand=product-brand', + 'ro.product.name=product-name', + 'ro.product.device=product-device', + ]) + self.writeFiles({ + 'SYSTEM/build.prop': '\n'.join(build_prop), + 'VENDOR/build.prop': '\n'.join(self.VENDOR_BUILD_PROP), + }) + common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir) + + self.assertEqual({ + self.constructFingerprint('product-brand/product-name/product-device') + }, CalculateRuntimeFingerprints()) + + def test_CalculatePossibleFingerprints_single_override(self): + vendor_build_prop = copy.deepcopy(self.VENDOR_BUILD_PROP) + vendor_build_prop.extend([ + 'import /vendor/etc/build_${ro.boot.sku_name}.prop', + ]) + self.writeFiles({ + 'SYSTEM/build.prop': '\n'.join(self.BUILD_PROP), + 'VENDOR/build.prop': '\n'.join(vendor_build_prop), + 'VENDOR/etc/build_std.prop': + 'ro.product.vendor.name=vendor-product-std', + 'VENDOR/etc/build_pro.prop': + 'ro.product.vendor.name=vendor-product-pro', + }) + common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir) + common.OPTIONS.boot_variable_values = { + 'ro.boot.sku_name': ['std', 'pro'] + } + + self.assertEqual({ + self.constructFingerprint( + 'vendor-product-brand/vendor-product-name/vendor-product-device'), + self.constructFingerprint( + 'vendor-product-brand/vendor-product-std/vendor-product-device'), + self.constructFingerprint( + 'vendor-product-brand/vendor-product-pro/vendor-product-device'), + }, CalculateRuntimeFingerprints()) + + def test_CalculatePossibleFingerprints_multiple_overrides(self): + vendor_build_prop = copy.deepcopy(self.VENDOR_BUILD_PROP) + vendor_build_prop.extend([ + 'import /vendor/etc/build_${ro.boot.sku_name}.prop', + 'import /vendor/etc/build_${ro.boot.device_name}.prop', + ]) + self.writeFiles({ + 'SYSTEM/build.prop': '\n'.join(self.BUILD_PROP), + 'VENDOR/build.prop': '\n'.join(vendor_build_prop), + 'VENDOR/etc/build_std.prop': + 'ro.product.vendor.name=vendor-product-std', + 'VENDOR/etc/build_product1.prop': + 'ro.product.vendor.device=vendor-device-product1', + 'VENDOR/etc/build_pro.prop': + 'ro.product.vendor.name=vendor-product-pro', + 'VENDOR/etc/build_product2.prop': + 'ro.product.vendor.device=vendor-device-product2', + }) + common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir) + common.OPTIONS.boot_variable_values = { + 'ro.boot.sku_name': ['std', 'pro'], + 'ro.boot.device_name': ['product1', 'product2'], + } + + self.assertEqual({ + self.constructFingerprint( + 'vendor-product-brand/vendor-product-name/vendor-product-device'), + self.constructFingerprint( + 'vendor-product-brand/vendor-product-std/vendor-device-product1'), + self.constructFingerprint( + 'vendor-product-brand/vendor-product-pro/vendor-device-product1'), + self.constructFingerprint( + 'vendor-product-brand/vendor-product-std/vendor-device-product2'), + self.constructFingerprint( + 'vendor-product-brand/vendor-product-pro/vendor-device-product2'), + }, CalculateRuntimeFingerprints())