Calculate the runtime fingerprint prefixes from build prop

In practice, some partners use the 'import' statement to override
the device fingerprint at runtime. The runtime fingerprint will
later add to the metadata of OTA package, so that the OTA server
can deliver the package to corresponding devices correctly.

This CL supports parsing a subset of import statement that the init
process recognizes. And we loose the restriction based on how the
dynamic fingerprint is used in practice. Right now, we only searches
for the override of brand, name and device. And the placeholder
format should be ${placeholder}, with its value supplied by the
script caller.

As part of the implementation, we generate all the possible
combinations of the input boot variables. And recalculate the
fingerprint for each of the combination. Though we load the
build.prop multiple times, the logic is easier to follow. Also,
it's more convenient to enhance the logic if we only want to
allow some of the boot variables combination later.

Bug: 152167826
Test: unittests pass
Change-Id: Iac093a40dc6f873c5e1858efa44cb2bd6082508a

tools/releasetools/common.py

@@ -738,18 +738,22 @@ class PartitionBuildProps(object):
     partition: name of the partition.
     props_allow_override: a list of build properties to search for the
         alternative values during runtime.
-    build_props: a dictionary of build properties for the given partition.
+    build_props: a dict of build properties for the given partition.
-    prop_overrides: a dict of list. And each list holds the overridden values
+    prop_overrides: a set of props that are overridden by import.
-        for props_allow_override.
+    placeholder_values: A dict of runtime variables' values to replace the
+        placeholders in the build.prop file. We expect exactly one value for
+        each of the variables.
   """
-
+  def __init__(self, input_file, name, placeholder_values=None):
-  def __init__(self, input_file, name):
     self.input_file = input_file
     self.partition = name
     self.props_allow_override = [props.format(name) for props in [
-        'ro.product.{}.name', 'ro.product.{}.device']]
+        'ro.product.{}.brand', 'ro.product.{}.name', 'ro.product.{}.device']]
     self.build_props = {}
-    self.prop_overrides = {}
+    self.prop_overrides = set()
+    self.placeholder_values = {}
+    if placeholder_values:
+      self.placeholder_values = copy.deepcopy(placeholder_values)
 
   @staticmethod
   def FromDictionary(name, build_props):
@@ -760,9 +764,8 @@ class PartitionBuildProps(object):
     return props
 
   @staticmethod
-  def FromInputFile(input_file, name):
+  def FromInputFile(input_file, name, placeholder_values=None):
     """Loads the build.prop file and builds the attributes."""
-
     data = ''
     for prop_file in ['{}/etc/build.prop'.format(name.upper()),
                       '{}/build.prop'.format(name.upper())]:
@@ -772,10 +775,62 @@ class PartitionBuildProps(object):
       except KeyError:
         logger.warning('Failed to read %s', prop_file)
 
-    props = PartitionBuildProps(input_file, name)
+    props = PartitionBuildProps(input_file, name, placeholder_values)
-    props.build_props = LoadDictionaryFromLines(data.split('\n'))
+    props._LoadBuildProp(data)
     return props
 
+  def _LoadBuildProp(self, data):
+    for line in data.split('\n'):
+      line = line.strip()
+      if not line or line.startswith("#"):
+        continue
+      if line.startswith("import"):
+        overrides = self._ImportParser(line)
+        duplicates = self.prop_overrides.intersection(overrides.keys())
+        if duplicates:
+          raise ValueError('prop {} is overridden multiple times'.format(
+              ','.join(duplicates)))
+        self.prop_overrides = self.prop_overrides.union(overrides.keys())
+        self.build_props.update(overrides)
+      elif "=" in line:
+        name, value = line.split("=", 1)
+        if name in self.prop_overrides:
+          raise ValueError('prop {} is set again after overridden by import '
+                           'statement'.format(name))
+        self.build_props[name] = value
+
+  def _ImportParser(self, line):
+    """Parses the build prop in a given import statement."""
+
+    tokens = line.split()
+    if len(tokens) != 2 or tokens[0] != 'import':
+      raise ValueError('Unrecognized import statement {}'.format(line))
+    import_path = tokens[1]
+    if not re.match(r'^/{}/.*\.prop$'.format(self.partition), import_path):
+      raise ValueError('Unrecognized import path {}'.format(line))
+
+    # We only recognize a subset of import statement that the init process
+    # supports. And we can loose the restriction based on how the dynamic
+    # fingerprint is used in practice. The placeholder format should be
+    # ${placeholder}, and its value should be provided by the caller through
+    # the placeholder_values.
+    for prop, value in self.placeholder_values.items():
+      prop_place_holder = '${{{}}}'.format(prop)
+      if prop_place_holder in import_path:
+        import_path = import_path.replace(prop_place_holder, value)
+    if '$' in import_path:
+      logger.info('Unresolved place holder in import path %s', import_path)
+      return {}
+
+    import_path = import_path.replace('/{}'.format(self.partition),
+                                      self.partition.upper())
+    logger.info('Parsing build props override from %s', import_path)
+
+    lines = ReadFromInputFile(self.input_file, import_path).split('\n')
+    d = LoadDictionaryFromLines(lines)
+    return {key: val for key, val in d.items()
+            if key in self.props_allow_override}
+
   def GetProp(self, prop):
     return self.build_props.get(prop)
 

tools/releasetools/ota_from_target_files.py

@@ -193,6 +193,8 @@ A/B OTA specific options
 from __future__ import print_function
 
 import collections
+import copy
+import itertools
 import logging
 import multiprocessing
 import os.path
@@ -229,6 +231,7 @@ OPTIONS.include_secondary = False
 OPTIONS.no_signing = False
 OPTIONS.block_based = True
 OPTIONS.updater_binary = None
+OPTIONS.oem_dicts = None
 OPTIONS.oem_source = None
 OPTIONS.oem_no_mount = False
 OPTIONS.full_radio = False
@@ -247,6 +250,7 @@ OPTIONS.retrofit_dynamic_partitions = False
 OPTIONS.skip_compatibility_check = False
 OPTIONS.output_metadata_path = None
 OPTIONS.disable_fec_computation = False
+OPTIONS.boot_variable_values = None
 
 
 METADATA_NAME = 'META-INF/com/android/metadata'
@@ -1959,6 +1963,36 @@ def GenerateNonAbOtaPackage(target_file, output_file, source_file=None):
           output_file)
 
 
+def CalculateRuntimeFingerprints():
+  """Returns a set of runtime fingerprints based on the boot variables."""
+
+  build_info = common.BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts)
+  fingerprints = {build_info.fingerprint}
+
+  if not OPTIONS.boot_variable_values:
+    return fingerprints
+
+  # Calculate all possible combinations of the values for the boot variables.
+  keys = OPTIONS.boot_variable_values.keys()
+  value_list = OPTIONS.boot_variable_values.values()
+  combinations = [dict(zip(keys, values))
+                  for values in itertools.product(*value_list)]
+  for placeholder_values in combinations:
+    # Reload the info_dict as some build properties may change their values
+    # based on the value of ro.boot* properties.
+    info_dict = copy.deepcopy(OPTIONS.info_dict)
+    for partition in common.PARTITIONS_WITH_CARE_MAP:
+      partition_prop_key = "{}.build.prop".format(partition)
+      old_props = info_dict[partition_prop_key]
+      info_dict[partition_prop_key] = common.PartitionBuildProps.FromInputFile(
+          old_props.input_file, partition, placeholder_values)
+    info_dict["build.prop"] = info_dict["system.build.prop"]
+
+    build_info = common.BuildInfo(info_dict, OPTIONS.oem_dicts)
+    fingerprints.add(build_info.fingerprint)
+  return fingerprints
+
+
 def main(argv):
 
   def option_handler(o, a):

tools/releasetools/test_common.py

@@ -1899,7 +1899,7 @@ super_group_foo_group_size={group_foo_size}
 
 class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase):
   def setUp(self):
-    self.build_prop = [
+    self.odm_build_prop = [
         'ro.odm.build.date.utc=1578430045',
         'ro.odm.build.fingerprint='
         'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
@@ -1918,13 +1918,81 @@ class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase):
 
   def test_parseBuildProps_noImportStatement(self):
     build_prop = [
-      'ro.odm.build.date.utc=1578430045',
+        'ro.odm.build.date.utc=1578430045',
-      'ro.odm.build.fingerprint='
+        'ro.odm.build.fingerprint='
-      'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
+        'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
-      'ro.product.odm.device=coral',
+        'ro.product.odm.device=coral',
     ]
     input_file = self._BuildZipFile({
-      'ODM/etc/build.prop': '\n'.join(build_prop),
+        'ODM/etc/build.prop': '\n'.join(build_prop),
+    })
+
+    with zipfile.ZipFile(input_file, 'r') as input_zip:
+      placeholder_values = {
+          'ro.boot.product.device_name': ['std', 'pro']
+      }
+      partition_props = common.PartitionBuildProps.FromInputFile(
+          input_zip, 'odm', placeholder_values)
+
+    self.assertEqual({
+        'ro.odm.build.date.utc': '1578430045',
+        'ro.odm.build.fingerprint':
+        'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
+        'ro.product.odm.device': 'coral',
+    }, partition_props.build_props)
+
+    self.assertEqual(set(), partition_props.prop_overrides)
+
+  def test_parseBuildProps_singleImportStatement(self):
+    build_std_prop = [
+        'ro.product.odm.device=coral',
+        'ro.product.odm.name=product1',
+    ]
+    build_pro_prop = [
+        'ro.product.odm.device=coralpro',
+        'ro.product.odm.name=product2',
+    ]
+
+    input_file = self._BuildZipFile({
+        'ODM/etc/build.prop': '\n'.join(self.odm_build_prop),
+        'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
+        'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
+    })
+
+    with zipfile.ZipFile(input_file, 'r') as input_zip:
+      placeholder_values = {
+          'ro.boot.product.device_name': 'std'
+      }
+      partition_props = common.PartitionBuildProps.FromInputFile(
+          input_zip, 'odm', placeholder_values)
+
+    self.assertEqual({
+      'ro.odm.build.date.utc': '1578430045',
+      'ro.odm.build.fingerprint':
+      'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
+      'ro.product.odm.device': 'coral',
+      'ro.product.odm.name': 'product1',
+    }, partition_props.build_props)
+
+    with zipfile.ZipFile(input_file, 'r') as input_zip:
+      placeholder_values = {
+          'ro.boot.product.device_name': 'pro'
+      }
+      partition_props = common.PartitionBuildProps.FromInputFile(
+          input_zip, 'odm', placeholder_values)
+
+    self.assertEqual({
+        'ro.odm.build.date.utc': '1578430045',
+        'ro.odm.build.fingerprint':
+        'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
+        'ro.product.odm.device': 'coralpro',
+        'ro.product.odm.name': 'product2',
+    }, partition_props.build_props)
+
+  def test_parseBuildProps_noPlaceHolders(self):
+    build_prop = copy.copy(self.odm_build_prop)
+    input_file = self._BuildZipFile({
+        'ODM/etc/build.prop': '\n'.join(build_prop),
     })
 
     with zipfile.ZipFile(input_file, 'r') as input_zip:
@@ -1932,10 +2000,136 @@ class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase):
           input_zip, 'odm')
 
     self.assertEqual({
-      'ro.odm.build.date.utc': '1578430045',
+        'ro.odm.build.date.utc': '1578430045',
-      'ro.odm.build.fingerprint':
+        'ro.odm.build.fingerprint':
-      'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
+        'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
-      'ro.product.odm.device': 'coral',
+        'ro.product.odm.device': 'coral',
     }, partition_props.build_props)
 
-    self.assertEqual({}, partition_props.prop_overrides)
+    self.assertEqual(set(), partition_props.prop_overrides)
+
+  def test_parseBuildProps_multipleImportStatements(self):
+    build_prop = copy.deepcopy(self.odm_build_prop)
+    build_prop.append(
+        'import /odm/etc/build_${ro.boot.product.product_name}.prop')
+
+    build_std_prop = [
+        'ro.product.odm.device=coral',
+    ]
+    build_pro_prop = [
+        'ro.product.odm.device=coralpro',
+    ]
+
+    product1_prop = [
+        'ro.product.odm.name=product1',
+        'ro.product.not_care=not_care',
+    ]
+
+    product2_prop = [
+        'ro.product.odm.name=product2',
+        'ro.product.not_care=not_care',
+    ]
+
+    input_file = self._BuildZipFile({
+        'ODM/etc/build.prop': '\n'.join(build_prop),
+        'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
+        'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
+        'ODM/etc/build_product1.prop': '\n'.join(product1_prop),
+        'ODM/etc/build_product2.prop': '\n'.join(product2_prop),
+    })
+
+    with zipfile.ZipFile(input_file, 'r') as input_zip:
+      placeholder_values = {
+          'ro.boot.product.device_name': 'std',
+          'ro.boot.product.product_name': 'product1',
+          'ro.boot.product.not_care': 'not_care',
+      }
+      partition_props = common.PartitionBuildProps.FromInputFile(
+          input_zip, 'odm', placeholder_values)
+
+    self.assertEqual({
+        'ro.odm.build.date.utc': '1578430045',
+        'ro.odm.build.fingerprint':
+        'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
+        'ro.product.odm.device': 'coral',
+        'ro.product.odm.name': 'product1'
+    }, partition_props.build_props)
+
+    with zipfile.ZipFile(input_file, 'r') as input_zip:
+      placeholder_values = {
+          'ro.boot.product.device_name': 'pro',
+          'ro.boot.product.product_name': 'product2',
+          'ro.boot.product.not_care': 'not_care',
+      }
+      partition_props = common.PartitionBuildProps.FromInputFile(
+          input_zip, 'odm', placeholder_values)
+
+    self.assertEqual({
+        'ro.odm.build.date.utc': '1578430045',
+        'ro.odm.build.fingerprint':
+        'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
+        'ro.product.odm.device': 'coralpro',
+        'ro.product.odm.name': 'product2'
+    }, partition_props.build_props)
+
+  def test_parseBuildProps_defineAfterOverride(self):
+    build_prop = copy.deepcopy(self.odm_build_prop)
+    build_prop.append('ro.product.odm.device=coral')
+
+    build_std_prop = [
+        'ro.product.odm.device=coral',
+    ]
+    build_pro_prop = [
+        'ro.product.odm.device=coralpro',
+    ]
+
+    input_file = self._BuildZipFile({
+        'ODM/etc/build.prop': '\n'.join(build_prop),
+        'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
+        'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
+    })
+
+    with zipfile.ZipFile(input_file, 'r') as input_zip:
+      placeholder_values = {
+          'ro.boot.product.device_name': 'std',
+      }
+
+      self.assertRaises(ValueError, common.PartitionBuildProps.FromInputFile,
+                        input_zip, 'odm', placeholder_values)
+
+  def test_parseBuildProps_duplicateOverride(self):
+    build_prop = copy.deepcopy(self.odm_build_prop)
+    build_prop.append(
+        'import /odm/etc/build_${ro.boot.product.product_name}.prop')
+
+    build_std_prop = [
+        'ro.product.odm.device=coral',
+        'ro.product.odm.name=product1',
+    ]
+    build_pro_prop = [
+        'ro.product.odm.device=coralpro',
+    ]
+
+    product1_prop = [
+        'ro.product.odm.name=product1',
+    ]
+
+    product2_prop = [
+        'ro.product.odm.name=product2',
+    ]
+
+    input_file = self._BuildZipFile({
+        'ODM/etc/build.prop': '\n'.join(build_prop),
+        'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
+        'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
+        'ODM/etc/build_product1.prop': '\n'.join(product1_prop),
+        'ODM/etc/build_product2.prop': '\n'.join(product2_prop),
+    })
+
+    with zipfile.ZipFile(input_file, 'r') as input_zip:
+      placeholder_values = {
+          'ro.boot.product.device_name': 'std',
+          'ro.boot.product.product_name': 'product1',
+      }
+      self.assertRaises(ValueError, common.PartitionBuildProps.FromInputFile,
+                        input_zip, 'odm', placeholder_values)

tools/releasetools/test_ota_from_target_files.py

@@ -26,7 +26,8 @@ from ota_from_target_files import (
     GetPackageMetadata, GetTargetFilesZipForSecondaryImages,
     GetTargetFilesZipWithoutPostinstallConfig, NonAbOtaPropertyFiles,
     Payload, PayloadSigner, POSTINSTALL_CONFIG, PropertyFiles,
-    StreamingPropertyFiles, WriteFingerprintAssertion)
+    StreamingPropertyFiles, WriteFingerprintAssertion,
+    CalculateRuntimeFingerprints)
 
 
 def construct_target_files(secondary=False):
@@ -1318,3 +1319,125 @@ class PayloadTest(test_utils.ReleaseToolsTestCase):
             Payload.SECONDARY_PAYLOAD_PROPERTIES_TXT):
           continue
         self.assertEqual(zipfile.ZIP_STORED, entry_info.compress_type)
+
+
+class RuntimeFingerprintTest(test_utils.ReleaseToolsTestCase):
+  MISC_INFO = [
+      'recovery_api_version=3',
+      'fstab_version=2',
+      'recovery_as_boot=true',
+  ]
+
+  BUILD_PROP = [
+      'ro.build.version.release=version-release',
+      'ro.build.id=build-id',
+      'ro.build.version.incremental=version-incremental',
+      'ro.build.type=build-type',
+      'ro.build.tags=build-tags',
+  ]
+
+  VENDOR_BUILD_PROP = [
+      'ro.product.vendor.brand=vendor-product-brand',
+      'ro.product.vendor.name=vendor-product-name',
+      'ro.product.vendor.device=vendor-product-device'
+  ]
+
+  def setUp(self):
+    common.OPTIONS.oem_dicts = None
+    self.test_dir = common.MakeTempDir()
+    self.writeFiles({'META/misc_info.txt': '\n'.join(self.MISC_INFO)})
+
+  def writeFiles(self, contents_dict):
+    for path, content in contents_dict.items():
+      abs_path = os.path.join(self.test_dir, path)
+      dir_name = os.path.dirname(abs_path)
+      if not os.path.exists(dir_name):
+        os.makedirs(dir_name)
+      with open(abs_path, 'w') as f:
+        f.write(content)
+
+  @staticmethod
+  def constructFingerprint(prefix):
+    return '{}:version-release/build-id/version-incremental:' \
+           'build-type/build-tags'.format(prefix)
+
+  def test_CalculatePossibleFingerprints_no_dynamic_fingerprint(self):
+    build_prop = copy.deepcopy(self.BUILD_PROP)
+    build_prop.extend([
+        'ro.product.brand=product-brand',
+        'ro.product.name=product-name',
+        'ro.product.device=product-device',
+    ])
+    self.writeFiles({
+        'SYSTEM/build.prop': '\n'.join(build_prop),
+        'VENDOR/build.prop': '\n'.join(self.VENDOR_BUILD_PROP),
+    })
+    common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir)
+
+    self.assertEqual({
+        self.constructFingerprint('product-brand/product-name/product-device')
+    }, CalculateRuntimeFingerprints())
+
+  def test_CalculatePossibleFingerprints_single_override(self):
+    vendor_build_prop = copy.deepcopy(self.VENDOR_BUILD_PROP)
+    vendor_build_prop.extend([
+        'import /vendor/etc/build_${ro.boot.sku_name}.prop',
+    ])
+    self.writeFiles({
+        'SYSTEM/build.prop': '\n'.join(self.BUILD_PROP),
+        'VENDOR/build.prop': '\n'.join(vendor_build_prop),
+        'VENDOR/etc/build_std.prop':
+        'ro.product.vendor.name=vendor-product-std',
+        'VENDOR/etc/build_pro.prop':
+        'ro.product.vendor.name=vendor-product-pro',
+    })
+    common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir)
+    common.OPTIONS.boot_variable_values = {
+        'ro.boot.sku_name': ['std', 'pro']
+    }
+
+    self.assertEqual({
+        self.constructFingerprint(
+            'vendor-product-brand/vendor-product-name/vendor-product-device'),
+        self.constructFingerprint(
+            'vendor-product-brand/vendor-product-std/vendor-product-device'),
+        self.constructFingerprint(
+            'vendor-product-brand/vendor-product-pro/vendor-product-device'),
+    }, CalculateRuntimeFingerprints())
+
+  def test_CalculatePossibleFingerprints_multiple_overrides(self):
+    vendor_build_prop = copy.deepcopy(self.VENDOR_BUILD_PROP)
+    vendor_build_prop.extend([
+        'import /vendor/etc/build_${ro.boot.sku_name}.prop',
+        'import /vendor/etc/build_${ro.boot.device_name}.prop',
+    ])
+    self.writeFiles({
+        'SYSTEM/build.prop': '\n'.join(self.BUILD_PROP),
+        'VENDOR/build.prop': '\n'.join(vendor_build_prop),
+        'VENDOR/etc/build_std.prop':
+        'ro.product.vendor.name=vendor-product-std',
+        'VENDOR/etc/build_product1.prop':
+        'ro.product.vendor.device=vendor-device-product1',
+        'VENDOR/etc/build_pro.prop':
+        'ro.product.vendor.name=vendor-product-pro',
+        'VENDOR/etc/build_product2.prop':
+        'ro.product.vendor.device=vendor-device-product2',
+    })
+    common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir)
+    common.OPTIONS.boot_variable_values = {
+        'ro.boot.sku_name': ['std', 'pro'],
+        'ro.boot.device_name': ['product1', 'product2'],
+    }
+
+    self.assertEqual({
+        self.constructFingerprint(
+            'vendor-product-brand/vendor-product-name/vendor-product-device'),
+        self.constructFingerprint(
+            'vendor-product-brand/vendor-product-std/vendor-device-product1'),
+        self.constructFingerprint(
+            'vendor-product-brand/vendor-product-pro/vendor-device-product1'),
+        self.constructFingerprint(
+            'vendor-product-brand/vendor-product-std/vendor-device-product2'),
+        self.constructFingerprint(
+            'vendor-product-brand/vendor-product-pro/vendor-device-product2'),
+    }, CalculateRuntimeFingerprints())