diff --git a/scripts/hiddenapi/generate_hiddenapi_lists.py b/scripts/hiddenapi/generate_hiddenapi_lists.py index 5ab93d185..35e0948f0 100755 --- a/scripts/hiddenapi/generate_hiddenapi_lists.py +++ b/scripts/hiddenapi/generate_hiddenapi_lists.py @@ -16,8 +16,6 @@ """Generate API lists for non-SDK API enforcement.""" import argparse from collections import defaultdict, namedtuple -import functools -import os import re import sys @@ -60,15 +58,15 @@ ALL_FLAGS_SET = set(ALL_FLAGS) # For example, the max-target-P list is checked in as it was in P, # but signatures have changes since then. The flag instructs this # script to skip any entries which do not exist any more. -FLAG_IGNORE_CONFLICTS = "ignore-conflicts" +FLAG_IGNORE_CONFLICTS = 'ignore-conflicts' # Option specified after one of FLAGS_API_LIST to express that all # apis within a given set of packages should be assign the given flag. -FLAG_PACKAGES = "packages" +FLAG_PACKAGES = 'packages' # Option specified after one of FLAGS_API_LIST to indicate an extra # tag that should be added to the matching APIs. -FLAG_TAG = "tag" +FLAG_TAG = 'tag' # Regex patterns of fields/methods used in serialization. These are # considered public API despite being hidden. @@ -84,24 +82,30 @@ SERIALIZATION_PATTERNS = [ # Single regex used to match serialization API. It combines all the # SERIALIZATION_PATTERNS into a single regular expression. -SERIALIZATION_REGEX = re.compile(r'.*->(' + '|'.join(SERIALIZATION_PATTERNS) + r')$') +SERIALIZATION_REGEX = re.compile(r'.*->(' + '|'.join(SERIALIZATION_PATTERNS) + + r')$') # Predicates to be used with filter_apis. -HAS_NO_API_LIST_ASSIGNED = lambda api, flags: not FLAGS_API_LIST_SET.intersection(flags) +HAS_NO_API_LIST_ASSIGNED = \ + lambda api,flags: not FLAGS_API_LIST_SET.intersection(flags) + IS_SERIALIZATION = lambda api, flags: SERIALIZATION_REGEX.match(api) class StoreOrderedOptions(argparse.Action): - """An argparse action that stores a number of option arguments in the order that - they were specified. + """An argparse action that stores a number of option arguments in the order + + that they were specified. """ - def __call__(self, parser, args, values, option_string = None): + + def __call__(self, parser, args, values, option_string=None): items = getattr(args, self.dest, None) if items is None: items = [] items.append([option_string.lstrip('-'), values]) setattr(args, self.dest, items) + def get_args(): """Parses command line arguments. @@ -110,22 +114,43 @@ def get_args(): """ parser = argparse.ArgumentParser() parser.add_argument('--output', required=True) - parser.add_argument('--csv', nargs='*', default=[], metavar='CSV_FILE', + parser.add_argument( + '--csv', + nargs='*', + default=[], + metavar='CSV_FILE', help='CSV files to be merged into output') for flag in ALL_FLAGS: - parser.add_argument('--' + flag, dest='ordered_flags', metavar='TXT_FILE', - action=StoreOrderedOptions, help='lists of entries with flag "' + flag + '"') - parser.add_argument('--' + FLAG_IGNORE_CONFLICTS, dest='ordered_flags', nargs=0, - action=StoreOrderedOptions, help='Indicates that only known and otherwise unassigned ' - 'entries should be assign the given flag. Must follow a list of entries and applies ' - 'to the preceding such list.') - parser.add_argument('--' + FLAG_PACKAGES, dest='ordered_flags', nargs=0, - action=StoreOrderedOptions, help='Indicates that the previous list of entries ' - 'is a list of packages. All members in those packages will be given the flag. ' - 'Must follow a list of entries and applies to the preceding such list.') - parser.add_argument('--' + FLAG_TAG, dest='ordered_flags', nargs=1, - action=StoreOrderedOptions, help='Adds an extra tag to the previous list of entries. ' + parser.add_argument( + '--' + flag, + dest='ordered_flags', + metavar='TXT_FILE', + action=StoreOrderedOptions, + help='lists of entries with flag "' + flag + '"') + parser.add_argument( + '--' + FLAG_IGNORE_CONFLICTS, + dest='ordered_flags', + nargs=0, + action=StoreOrderedOptions, + help='Indicates that only known and otherwise unassigned ' + 'entries should be assign the given flag. Must follow a list of ' + 'entries and applies to the preceding such list.') + parser.add_argument( + '--' + FLAG_PACKAGES, + dest='ordered_flags', + nargs=0, + action=StoreOrderedOptions, + help='Indicates that the previous list of entries ' + 'is a list of packages. All members in those packages will be given ' + 'the flag. Must follow a list of entries and applies to the preceding ' + 'such list.') + parser.add_argument( + '--' + FLAG_TAG, + dest='ordered_flags', + nargs=1, + action=StoreOrderedOptions, + help='Adds an extra tag to the previous list of entries. ' 'Must follow a list of entries and applies to the preceding such list.') return parser.parse_args() @@ -143,9 +168,9 @@ def read_lines(filename): Lines of the file as a list of string. """ with open(filename, 'r') as f: - lines = f.readlines(); - lines = filter(lambda line: not line.startswith('#'), lines) - lines = map(lambda line: line.strip(), lines) + lines = f.readlines() + lines = [line for line in lines if not line.startswith('#')] + lines = [line.strip() for line in lines] return set(lines) @@ -156,7 +181,7 @@ def write_lines(filename, lines): filename (string): Path to the file to be writing into. lines (list): List of strings to write into the file. """ - lines = map(lambda line: line + '\n', lines) + lines = [line + '\n' for line in lines] with open(filename, 'w') as f: f.writelines(lines) @@ -170,17 +195,19 @@ def extract_package(signature): Returns: The package name of the class containing the field/method. """ - full_class_name = signature.split(";->")[0] + full_class_name = signature.split(';->')[0] # Example: Landroid/hardware/radio/V1_2/IRadio$Proxy - if (full_class_name[0] != "L"): - raise ValueError("Expected to start with 'L': %s" % full_class_name) + if full_class_name[0] != 'L': + raise ValueError("Expected to start with 'L': %s" + % full_class_name) full_class_name = full_class_name[1:] # If full_class_name doesn't contain '/', then package_name will be ''. - package_name = full_class_name.rpartition("/")[0] + package_name = full_class_name.rpartition('/')[0] return package_name.replace('/', '.') class FlagsDict: + def __init__(self): self._dict_keyset = set() self._dict = defaultdict(set) @@ -188,37 +215,43 @@ class FlagsDict: def _check_entries_set(self, keys_subset, source): assert isinstance(keys_subset, set) assert keys_subset.issubset(self._dict_keyset), ( - "Error: {} specifies signatures not present in code:\n" - "{}" - "Please visit go/hiddenapi for more information.").format( - source, "".join(map(lambda x: " " + str(x) + "\n", keys_subset - self._dict_keyset))) + 'Error: {} specifies signatures not present in code:\n' + '{}' + 'Please visit go/hiddenapi for more information.').format( + source, ''.join( + [' ' + str(x) + '\n' for x in + keys_subset - self._dict_keyset])) def _check_flags_set(self, flags_subset, source): assert isinstance(flags_subset, set) assert flags_subset.issubset(ALL_FLAGS_SET), ( - "Error processing: {}\n" - "The following flags were not recognized: \n" - "{}\n" - "Please visit go/hiddenapi for more information.").format( - source, "\n".join(flags_subset - ALL_FLAGS_SET)) + 'Error processing: {}\n' + 'The following flags were not recognized: \n' + '{}\n' + 'Please visit go/hiddenapi for more information.').format( + source, '\n'.join(flags_subset - ALL_FLAGS_SET)) def filter_apis(self, filter_fn): """Returns APIs which match a given predicate. - This is a helper function which allows to filter on both signatures (keys) and - flags (values). The built-in filter() invokes the lambda only with dict's keys. + This is a helper function which allows to filter on both signatures + (keys) and + flags (values). The built-in filter() invokes the lambda only with + dict's keys. Args: - filter_fn : Function which takes two arguments (signature/flags) and returns a boolean. + filter_fn : Function which takes two arguments (signature/flags) and + returns a boolean. Returns: A set of APIs which match the predicate. """ - return set(filter(lambda x: filter_fn(x, self._dict[x]), self._dict_keyset)) + return {x for x in self._dict_keyset if filter_fn(x, self._dict[x])} def get_valid_subset_of_unassigned_apis(self, api_subset): - """Sanitizes a key set input to only include keys which exist in the dictionary - and have not been assigned any API list flags. + """Sanitizes a key set input to only include keys which exist in the + + dictionary and have not been assigned any API list flags. Args: entries_subset (set/list): Key set to be sanitized. @@ -227,7 +260,8 @@ class FlagsDict: Sanitized key set. """ assert isinstance(api_subset, set) - return api_subset.intersection(self.filter_apis(HAS_NO_API_LIST_ASSIGNED)) + return api_subset.intersection( + self.filter_apis(HAS_NO_API_LIST_ASSIGNED)) def generate_csv(self): """Constructs CSV entries from a dictionary. @@ -235,15 +269,16 @@ class FlagsDict: Old versions of flags are used to generate the file. Returns: - List of lines comprising a CSV file. See "parse_and_merge_csv" for format description. + List of lines comprising a CSV file. See "parse_and_merge_csv" for + format description. """ lines = [] for api in self._dict: - flags = sorted(self._dict[api]) - lines.append(",".join([api] + flags)) + flags = sorted(self._dict[api]) + lines.append(','.join([api] + flags)) return sorted(lines) - def parse_and_merge_csv(self, csv_lines, source = ""): + def parse_and_merge_csv(self, csv_lines, source=''): """Parses CSV entries and merges them into a given dictionary. The expected CSV format is: @@ -251,21 +286,20 @@ class FlagsDict: Args: csv_lines (list of strings): Lines read from a CSV file. - source (string): Origin of `csv_lines`. Will be printed in error messages. - - Throws: - AssertionError if parsed flags are invalid. + source (string): Origin of `csv_lines`. Will be printed in error + messages. + Throws: AssertionError if parsed flags are invalid. """ # Split CSV lines into arrays of values. - csv_values = [ line.split(',') for line in csv_lines ] + csv_values = [line.split(',') for line in csv_lines] # Update the full set of API signatures. - self._dict_keyset.update([ csv[0] for csv in csv_values ]) + self._dict_keyset.update([csv[0] for csv in csv_values]) # Check that all flags are known. csv_flags = set() for csv in csv_values: - csv_flags.update(csv[1:]) + csv_flags.update(csv[1:]) self._check_flags_set(csv_flags, source) # Iterate over all CSV lines, find entry in dict and append flags to it. @@ -275,47 +309,53 @@ class FlagsDict: flags.append(FLAG_SDK) self._dict[csv[0]].update(flags) - def assign_flag(self, flag, apis, source="", tag = None): + def assign_flag(self, flag, apis, source='', tag=None): """Assigns a flag to given subset of entries. Args: flag (string): One of ALL_FLAGS. apis (set): Subset of APIs to receive the flag. - source (string): Origin of `entries_subset`. Will be printed in error messages. - - Throws: - AssertionError if parsed API signatures of flags are invalid. + source (string): Origin of `entries_subset`. Will be printed in + error messages. + Throws: AssertionError if parsed API signatures of flags are invalid. """ # Check that all APIs exist in the dict. self._check_entries_set(apis, source) # Check that the flag is known. - self._check_flags_set(set([ flag ]), source) + self._check_flags_set(set([flag]), source) - # Iterate over the API subset, find each entry in dict and assign the flag to it. + # Iterate over the API subset, find each entry in dict and assign the + # flag to it. for api in apis: self._dict[api].add(flag) if tag: self._dict[api].add(tag) -FlagFile = namedtuple('FlagFile', ('flag', 'file', 'ignore_conflicts', 'packages', 'tag')) +FlagFile = namedtuple('FlagFile', + ('flag', 'file', 'ignore_conflicts', 'packages', 'tag')) + def parse_ordered_flags(ordered_flags): r = [] - currentflag, file, ignore_conflicts, packages, tag = None, None, False, False, None + currentflag, file, ignore_conflicts, packages, tag = None, None, False, \ + False, None for flag_value in ordered_flags: flag, value = flag_value[0], flag_value[1] if flag in ALL_FLAGS_SET: if currentflag: - r.append(FlagFile(currentflag, file, ignore_conflicts, packages, tag)) + r.append( + FlagFile(currentflag, file, ignore_conflicts, packages, + tag)) ignore_conflicts, packages, tag = False, False, None currentflag = flag file = value else: if currentflag is None: - raise argparse.ArgumentError('--%s is only allowed after one of %s' % ( - flag, ' '.join(['--%s' % f for f in ALL_FLAGS_SET]))) + raise argparse.ArgumentError( #pylint: disable=no-value-for-parameter + '--%s is only allowed after one of %s' % + (flag, ' '.join(['--%s' % f for f in ALL_FLAGS_SET]))) if flag == FLAG_IGNORE_CONFLICTS: ignore_conflicts = True elif flag == FLAG_PACKAGES: @@ -323,13 +363,12 @@ def parse_ordered_flags(ordered_flags): elif flag == FLAG_TAG: tag = value[0] - if currentflag: r.append(FlagFile(currentflag, file, ignore_conflicts, packages, tag)) return r -def main(argv): +def main(argv): #pylint: disable=unused-argument # Parse arguments. args = vars(get_args()) flagfiles = parse_ordered_flags(args['ordered_flags'] or []) @@ -342,7 +381,7 @@ def main(argv): # contain the full set of APIs. Subsequent additions from text files # will be able to detect invalid entries, and/or filter all as-yet # unassigned entries. - for filename in args["csv"]: + for filename in args['csv']: flags.parse_and_merge_csv(read_lines(filename), filename) # Combine inputs which do not require any particular order. @@ -352,24 +391,28 @@ def main(argv): # (2) Merge text files with a known flag into the dictionary. for info in flagfiles: if (not info.ignore_conflicts) and (not info.packages): - flags.assign_flag(info.flag, read_lines(info.file), info.file, info.tag) + flags.assign_flag(info.flag, read_lines(info.file), info.file, + info.tag) # Merge text files where conflicts should be ignored. # This will only assign the given flag if: # (a) the entry exists, and # (b) it has not been assigned any other flag. - # Because of (b), this must run after all strict assignments have been performed. + # Because of (b), this must run after all strict assignments have been + # performed. for info in flagfiles: if info.ignore_conflicts: - valid_entries = flags.get_valid_subset_of_unassigned_apis(read_lines(info.file)) - flags.assign_flag(info.flag, valid_entries, filename, info.tag) + valid_entries = flags.get_valid_subset_of_unassigned_apis( + read_lines(info.file)) + flags.assign_flag(info.flag, valid_entries, filename, info.tag) #pylint: disable=undefined-loop-variable - # All members in the specified packages will be assigned the appropriate flag. + # All members in the specified packages will be assigned the appropriate + # flag. for info in flagfiles: if info.packages: packages_needing_list = set(read_lines(info.file)) - should_add_signature_to_list = lambda sig,lists: extract_package( - sig) in packages_needing_list and not lists + should_add_signature_to_list = lambda sig, lists: extract_package( + sig) in packages_needing_list and not lists #pylint: disable=cell-var-from-loop valid_entries = flags.filter_apis(should_add_signature_to_list) flags.assign_flag(info.flag, valid_entries, info.file, info.tag) @@ -377,7 +420,8 @@ def main(argv): flags.assign_flag(FLAG_BLOCKED, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED)) # Write output. - write_lines(args["output"], flags.generate_csv()) + write_lines(args['output'], flags.generate_csv()) -if __name__ == "__main__": + +if __name__ == '__main__': main(sys.argv) diff --git a/scripts/hiddenapi/generate_hiddenapi_lists_test.py b/scripts/hiddenapi/generate_hiddenapi_lists_test.py index b81424b7a..204de9723 100755 --- a/scripts/hiddenapi/generate_hiddenapi_lists_test.py +++ b/scripts/hiddenapi/generate_hiddenapi_lists_test.py @@ -15,34 +15,39 @@ # limitations under the License. """Unit tests for Hidden API list generation.""" import unittest -from generate_hiddenapi_lists import * +from generate_hiddenapi_lists import * # pylint: disable=wildcard-import,unused-wildcard-import + class TestHiddenapiListGeneration(unittest.TestCase): - def test_filter_apis(self): # Initialize flags so that A and B are put on the allow list and # C, D, E are left unassigned. Try filtering for the unassigned ones. flags = FlagsDict() - flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B,' + FLAG_SDK, - 'C', 'D', 'E']) + flags.parse_and_merge_csv( + ['A,' + FLAG_SDK, 'B,' + FLAG_SDK, 'C', 'D', 'E'] + ) filter_set = flags.filter_apis(lambda api, flags: not flags) self.assertTrue(isinstance(filter_set, set)) - self.assertEqual(filter_set, set([ 'C', 'D', 'E' ])) + self.assertEqual(filter_set, set(['C', 'D', 'E'])) def test_get_valid_subset_of_unassigned_keys(self): # Create flags where only A is unassigned. flags = FlagsDict() flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B', 'C']) flags.assign_flag(FLAG_UNSUPPORTED, set(['C'])) - self.assertEqual(flags.generate_csv(), - [ 'A,' + FLAG_SDK, 'B', 'C,' + FLAG_UNSUPPORTED ]) + self.assertEqual( + flags.generate_csv(), + ['A,' + FLAG_SDK, 'B', 'C,' + FLAG_UNSUPPORTED], + ) # Check three things: # (1) B is selected as valid unassigned # (2) A is not selected because it is assigned to the allow list # (3) D is not selected because it is not a valid key self.assertEqual( - flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), set([ 'B' ])) + flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), + set(['B']), + ) def test_parse_and_merge_csv(self): flags = FlagsDict() @@ -51,41 +56,48 @@ class TestHiddenapiListGeneration(unittest.TestCase): self.assertEqual(flags.generate_csv(), []) # Test new additions. - flags.parse_and_merge_csv([ - 'A,' + FLAG_UNSUPPORTED, - 'B,' + FLAG_BLOCKED + ',' + FLAG_MAX_TARGET_O, - 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, - 'D,' + FLAG_UNSUPPORTED + ',' + FLAG_TEST_API, - 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, - ]) - self.assertEqual(flags.generate_csv(), [ - 'A,' + FLAG_UNSUPPORTED, - 'B,' + FLAG_BLOCKED + "," + FLAG_MAX_TARGET_O, - 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, - 'D,' + FLAG_TEST_API + ',' + FLAG_UNSUPPORTED, - 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, - ]) + flags.parse_and_merge_csv( + [ + 'A,' + FLAG_UNSUPPORTED, + 'B,' + FLAG_BLOCKED + ',' + FLAG_MAX_TARGET_O, + 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, + 'D,' + FLAG_UNSUPPORTED + ',' + FLAG_TEST_API, + 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, + ] + ) + self.assertEqual( + flags.generate_csv(), + [ + 'A,' + FLAG_UNSUPPORTED, + 'B,' + FLAG_BLOCKED + "," + FLAG_MAX_TARGET_O, + 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, + 'D,' + FLAG_TEST_API + ',' + FLAG_UNSUPPORTED, + 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, + ], + ) # Test unknown flag. with self.assertRaises(AssertionError): - flags.parse_and_merge_csv([ 'Z,foo' ]) + flags.parse_and_merge_csv(['Z,foo']) def test_assign_flag(self): flags = FlagsDict() flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B']) # Test new additions. - flags.assign_flag(FLAG_UNSUPPORTED, set([ 'A', 'B' ])) - self.assertEqual(flags.generate_csv(), - [ 'A,' + FLAG_SDK + "," + FLAG_UNSUPPORTED, 'B,' + FLAG_UNSUPPORTED ]) + flags.assign_flag(FLAG_UNSUPPORTED, set(['A', 'B'])) + self.assertEqual( + flags.generate_csv(), + ['A,' + FLAG_SDK + "," + FLAG_UNSUPPORTED, 'B,' + FLAG_UNSUPPORTED], + ) # Test invalid API signature. with self.assertRaises(AssertionError): - flags.assign_flag(FLAG_SDK, set([ 'C' ])) + flags.assign_flag(FLAG_SDK, set(['C'])) # Test invalid flag. with self.assertRaises(AssertionError): - flags.assign_flag('foo', set([ 'A' ])) + flags.assign_flag('foo', set(['A'])) def test_extract_package(self): signature = 'Lcom/foo/bar/Baz;->method1()Lcom/bar/Baz;' @@ -100,5 +112,6 @@ class TestHiddenapiListGeneration(unittest.TestCase): expected_package = 'com.foo_bar.baz' self.assertEqual(extract_package(signature), expected_package) + if __name__ == '__main__': unittest.main(verbosity=2)