diff --git a/scripts/hiddenapi/verify_overlaps.py b/scripts/hiddenapi/verify_overlaps.py index e5214dfc8..0d4f66945 100755 --- a/scripts/hiddenapi/verify_overlaps.py +++ b/scripts/hiddenapi/verify_overlaps.py @@ -23,13 +23,13 @@ from itertools import chain from signature_trie import signature_trie -def dict_reader(csvfile): +def dict_reader(csv_file): return csv.DictReader( - csvfile, delimiter=",", quotechar="|", fieldnames=["signature"]) + csv_file, delimiter=",", quotechar="|", fieldnames=["signature"]) def read_flag_trie_from_file(file): - with open(file, "r") as stream: + with open(file, "r", encoding="utf8") as stream: return read_flag_trie_from_stream(stream) @@ -43,24 +43,24 @@ def read_flag_trie_from_stream(stream): def extract_subset_from_monolithic_flags_as_dict_from_file( - monolithicTrie, patternsFile): + monolithic_trie, patterns_file): """Extract a subset of flags from the dict of monolithic flags. - :param monolithicFlagsDict: the dict containing all the monolithic flags. - :param patternsFile: a file containing a list of signature patterns that + :param monolithic_trie: the trie containing all the monolithic flags. + :param patterns_file: a file containing a list of signature patterns that define the subset. :return: the dict from signature to row. """ - with open(patternsFile, "r") as stream: + with open(patterns_file, "r", encoding="utf8") as stream: return extract_subset_from_monolithic_flags_as_dict_from_stream( - monolithicTrie, stream) + monolithic_trie, stream) def extract_subset_from_monolithic_flags_as_dict_from_stream( - monolithicTrie, stream): + monolithic_trie, stream): """Extract a subset of flags from the trie of monolithic flags. - :param monolithicTrie: the trie containing all the monolithic flags. + :param monolithic_trie: the trie containing all the monolithic flags. :param stream: a stream containing a list of signature patterns that define the subset. :return: the dict from signature to row. @@ -68,7 +68,7 @@ def extract_subset_from_monolithic_flags_as_dict_from_stream( dict_signature_to_row = {} for pattern in stream: pattern = pattern.rstrip() - rows = monolithicTrie.get_matching_rows(pattern) + rows = monolithic_trie.get_matching_rows(pattern) for row in rows: signature = row["signature"] dict_signature_to_row[signature] = row @@ -93,48 +93,48 @@ def read_signature_csv_from_stream_as_dict(stream): return dict_signature_to_row -def read_signature_csv_from_file_as_dict(csvFile): +def read_signature_csv_from_file_as_dict(csv_file): """Read the csvFile into a dict. The first column is assumed to be the signature and used as the key. The whole row is stored as the value. - :param csvFile: the csv file to read + :param csv_file: the csv file to read :return: the dict from signature to row. """ - with open(csvFile, "r") as f: + with open(csv_file, "r", encoding="utf8") as f: return read_signature_csv_from_stream_as_dict(f) -def compare_signature_flags(monolithicFlagsDict, modularFlagsDict): +def compare_signature_flags(monolithic_flags_dict, modular_flags_dict): """Compare the signature flags between the two dicts. - :param monolithicFlagsDict: the dict containing the subset of the monolithic - flags that should be equal to the modular flags. - :param modularFlagsDict:the dict containing the flags produced by a single + :param monolithic_flags_dict: the dict containing the subset of the + monolithic flags that should be equal to the modular flags. + :param modular_flags_dict:the dict containing the flags produced by a single bootclasspath_fragment module. :return: list of mismatches., each mismatch is a tuple where the first item is the signature, and the second and third items are lists of the flags from modular dict, and monolithic dict respectively. """ - mismatchingSignatures = [] + mismatching_signatures = [] # Create a sorted set of all the signatures from both the monolithic and # modular dicts. - allSignatures = sorted( - set(chain(monolithicFlagsDict.keys(), modularFlagsDict.keys()))) - for signature in allSignatures: - monolithicRow = monolithicFlagsDict.get(signature, {}) - monolithicFlags = monolithicRow.get(None, []) - if signature in modularFlagsDict: - modularRow = modularFlagsDict.get(signature, {}) - modularFlags = modularRow.get(None, []) + all_signatures = sorted( + set(chain(monolithic_flags_dict.keys(), modular_flags_dict.keys()))) + for signature in all_signatures: + monolithic_row = monolithic_flags_dict.get(signature, {}) + monolithic_flags = monolithic_row.get(None, []) + if signature in modular_flags_dict: + modular_row = modular_flags_dict.get(signature, {}) + modular_flags = modular_row.get(None, []) else: - modularFlags = ["blocked"] - if monolithicFlags != modularFlags: - mismatchingSignatures.append( - (signature, modularFlags, monolithicFlags)) - return mismatchingSignatures + modular_flags = ["blocked"] + if monolithic_flags != modular_flags: + mismatching_signatures.append( + (signature, modular_flags, monolithic_flags)) + return mismatching_signatures def main(argv): @@ -149,30 +149,30 @@ def main(argv): args = args_parser.parse_args(argv[1:]) # Read in all the flags into the trie - monolithicFlagsPath = args.monolithicFlags - monolithicTrie = read_flag_trie_from_file(monolithicFlagsPath) + monolithic_flags_path = args.monolithicFlags + monolithic_trie = read_flag_trie_from_file(monolithic_flags_path) # For each subset specified on the command line, create dicts for the flags # provided by the subset and the corresponding flags from the complete set # of flags and compare them. failed = False - for modularPair in args.modularFlags: - parts = modularPair.split(":") - modularFlagsPath = parts[0] - modularPatternsPath = parts[1] - modularFlagsDict = read_signature_csv_from_file_as_dict( - modularFlagsPath) - monolithicFlagsSubsetDict = \ + for modular_pair in args.modularFlags: + parts = modular_pair.split(":") + modular_flags_path = parts[0] + modular_patterns_path = parts[1] + modular_flags_dict = read_signature_csv_from_file_as_dict( + modular_flags_path) + monolithic_flags_subset_dict = \ extract_subset_from_monolithic_flags_as_dict_from_file( - monolithicTrie, modularPatternsPath) - mismatchingSignatures = compare_signature_flags( - monolithicFlagsSubsetDict, modularFlagsDict) - if mismatchingSignatures: + monolithic_trie, modular_patterns_path) + mismatching_signatures = compare_signature_flags( + monolithic_flags_subset_dict, modular_flags_dict) + if mismatching_signatures: failed = True print("ERROR: Hidden API flags are inconsistent:") - print("< " + modularFlagsPath) - print("> " + monolithicFlagsPath) - for mismatch in mismatchingSignatures: + print("< " + modular_flags_path) + print("> " + monolithic_flags_path) + for mismatch in mismatching_signatures: signature = mismatch[0] print() print("< " + ",".join([signature] + mismatch[1])) diff --git a/scripts/hiddenapi/verify_overlaps_test.py b/scripts/hiddenapi/verify_overlaps_test.py index 8cf295962..ead8a4e96 100755 --- a/scripts/hiddenapi/verify_overlaps_test.py +++ b/scripts/hiddenapi/verify_overlaps_test.py @@ -17,24 +17,26 @@ import io import unittest -from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import +import verify_overlaps as vo -#pylint: disable=line-too-long class TestDetectOverlaps(unittest.TestCase): - def read_flag_trie_from_string(self, csvdata): + @staticmethod + def read_flag_trie_from_string(csvdata): with io.StringIO(csvdata) as f: - return read_flag_trie_from_stream(f) + return vo.read_flag_trie_from_stream(f) - def read_signature_csv_from_string_as_dict(self, csvdata): + @staticmethod + def read_signature_csv_from_string_as_dict(csvdata): with io.StringIO(csvdata) as f: - return read_signature_csv_from_stream_as_dict(f) + return vo.read_signature_csv_from_stream_as_dict(f) + @staticmethod def extract_subset_from_monolithic_flags_as_dict_from_string( - self, monolithic, patterns): + monolithic, patterns): with io.StringIO(patterns) as f: - return extract_subset_from_monolithic_flags_as_dict_from_stream( + return vo.extract_subset_from_monolithic_flags_as_dict_from_stream( monolithic, f) extractInput = """ @@ -50,14 +52,14 @@ Ljava/lang/ProcessBuilder$Redirect$1;->()V,blocked monolithic = self.read_flag_trie_from_string( TestDetectOverlaps.extractInput) - patterns = 'Ljava/lang/Object;->hashCode()I' + patterns = "Ljava/lang/Object;->hashCode()I" subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( monolithic, patterns) expected = { - 'Ljava/lang/Object;->hashCode()I': { - None: ['public-api', 'system-api', 'test-api'], - 'signature': 'Ljava/lang/Object;->hashCode()I', + "Ljava/lang/Object;->hashCode()I": { + None: ["public-api", "system-api", "test-api"], + "signature": "Ljava/lang/Object;->hashCode()I", }, } self.assertEqual(expected, subset) @@ -66,18 +68,18 @@ Ljava/lang/ProcessBuilder$Redirect$1;->()V,blocked monolithic = self.read_flag_trie_from_string( TestDetectOverlaps.extractInput) - patterns = 'java/lang/Object' + patterns = "java/lang/Object" subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( monolithic, patterns) expected = { - 'Ljava/lang/Object;->hashCode()I': { - None: ['public-api', 'system-api', 'test-api'], - 'signature': 'Ljava/lang/Object;->hashCode()I', + "Ljava/lang/Object;->hashCode()I": { + None: ["public-api", "system-api", "test-api"], + "signature": "Ljava/lang/Object;->hashCode()I", }, - 'Ljava/lang/Object;->toString()Ljava/lang/String;': { - None: ['blocked'], - 'signature': 'Ljava/lang/Object;->toString()Ljava/lang/String;', + "Ljava/lang/Object;->toString()Ljava/lang/String;": { + None: ["blocked"], + "signature": "Ljava/lang/Object;->toString()Ljava/lang/String;", }, } self.assertEqual(expected, subset) @@ -86,20 +88,20 @@ Ljava/lang/ProcessBuilder$Redirect$1;->()V,blocked monolithic = self.read_flag_trie_from_string( TestDetectOverlaps.extractInput) - patterns = 'java/lang/Character' + patterns = "java/lang/Character" subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( monolithic, patterns) expected = { - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': - { - None: ['blocked'], - 'signature': - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', - }, - 'Ljava/lang/Character;->serialVersionUID:J': { - None: ['sdk'], - 'signature': 'Ljava/lang/Character;->serialVersionUID:J', + "Ljava/lang/Character$UnicodeScript;" + "->of(I)Ljava/lang/Character$UnicodeScript;": { + None: ["blocked"], + "signature": "Ljava/lang/Character$UnicodeScript;" + "->of(I)Ljava/lang/Character$UnicodeScript;", + }, + "Ljava/lang/Character;->serialVersionUID:J": { + None: ["sdk"], + "signature": "Ljava/lang/Character;->serialVersionUID:J", }, } self.assertEqual(expected, subset) @@ -108,17 +110,17 @@ Ljava/lang/ProcessBuilder$Redirect$1;->()V,blocked monolithic = self.read_flag_trie_from_string( TestDetectOverlaps.extractInput) - patterns = 'java/lang/Character$UnicodeScript' + patterns = "java/lang/Character$UnicodeScript" subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( monolithic, patterns) expected = { - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': - { - None: ['blocked'], - 'signature': - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', - }, + "Ljava/lang/Character$UnicodeScript;" + "->of(I)Ljava/lang/Character$UnicodeScript;": { + None: ["blocked"], + "signature": "Ljava/lang/Character$UnicodeScript;" + "->of(I)Ljava/lang/Character$UnicodeScript;", + }, } self.assertEqual(expected, subset) @@ -126,32 +128,32 @@ Ljava/lang/ProcessBuilder$Redirect$1;->()V,blocked monolithic = self.read_flag_trie_from_string( TestDetectOverlaps.extractInput) - patterns = 'java/lang/*' + patterns = "java/lang/*" subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( monolithic, patterns) expected = { - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': - { - None: ['blocked'], - 'signature': - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', - }, - 'Ljava/lang/Character;->serialVersionUID:J': { - None: ['sdk'], - 'signature': 'Ljava/lang/Character;->serialVersionUID:J', + "Ljava/lang/Character$UnicodeScript;" + "->of(I)Ljava/lang/Character$UnicodeScript;": { + None: ["blocked"], + "signature": "Ljava/lang/Character$UnicodeScript;" + "->of(I)Ljava/lang/Character$UnicodeScript;", }, - 'Ljava/lang/Object;->hashCode()I': { - None: ['public-api', 'system-api', 'test-api'], - 'signature': 'Ljava/lang/Object;->hashCode()I', + "Ljava/lang/Character;->serialVersionUID:J": { + None: ["sdk"], + "signature": "Ljava/lang/Character;->serialVersionUID:J", }, - 'Ljava/lang/Object;->toString()Ljava/lang/String;': { - None: ['blocked'], - 'signature': 'Ljava/lang/Object;->toString()Ljava/lang/String;', + "Ljava/lang/Object;->hashCode()I": { + None: ["public-api", "system-api", "test-api"], + "signature": "Ljava/lang/Object;->hashCode()I", }, - 'Ljava/lang/ProcessBuilder$Redirect$1;->()V': { - None: ['blocked'], - 'signature': 'Ljava/lang/ProcessBuilder$Redirect$1;->()V', + "Ljava/lang/Object;->toString()Ljava/lang/String;": { + None: ["blocked"], + "signature": "Ljava/lang/Object;->toString()Ljava/lang/String;", + }, + "Ljava/lang/ProcessBuilder$Redirect$1;->()V": { + None: ["blocked"], + "signature": "Ljava/lang/ProcessBuilder$Redirect$1;->()V", }, } self.assertEqual(expected, subset) @@ -160,36 +162,36 @@ Ljava/lang/ProcessBuilder$Redirect$1;->()V,blocked monolithic = self.read_flag_trie_from_string( TestDetectOverlaps.extractInput) - patterns = 'java/**' + patterns = "java/**" subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( monolithic, patterns) expected = { - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': - { - None: ['blocked'], - 'signature': - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', - }, - 'Ljava/lang/Character;->serialVersionUID:J': { - None: ['sdk'], - 'signature': 'Ljava/lang/Character;->serialVersionUID:J', + "Ljava/lang/Character$UnicodeScript;" + "->of(I)Ljava/lang/Character$UnicodeScript;": { + None: ["blocked"], + "signature": "Ljava/lang/Character$UnicodeScript;" + "->of(I)Ljava/lang/Character$UnicodeScript;", }, - 'Ljava/lang/Object;->hashCode()I': { - None: ['public-api', 'system-api', 'test-api'], - 'signature': 'Ljava/lang/Object;->hashCode()I', + "Ljava/lang/Character;->serialVersionUID:J": { + None: ["sdk"], + "signature": "Ljava/lang/Character;->serialVersionUID:J", }, - 'Ljava/lang/Object;->toString()Ljava/lang/String;': { - None: ['blocked'], - 'signature': 'Ljava/lang/Object;->toString()Ljava/lang/String;', + "Ljava/lang/Object;->hashCode()I": { + None: ["public-api", "system-api", "test-api"], + "signature": "Ljava/lang/Object;->hashCode()I", }, - 'Ljava/lang/ProcessBuilder$Redirect$1;->()V': { - None: ['blocked'], - 'signature': 'Ljava/lang/ProcessBuilder$Redirect$1;->()V', + "Ljava/lang/Object;->toString()Ljava/lang/String;": { + None: ["blocked"], + "signature": "Ljava/lang/Object;->toString()Ljava/lang/String;", }, - 'Ljava/util/zip/ZipFile;->()V': { - None: ['blocked'], - 'signature': 'Ljava/util/zip/ZipFile;->()V', + "Ljava/lang/ProcessBuilder$Redirect$1;->()V": { + None: ["blocked"], + "signature": "Ljava/lang/ProcessBuilder$Redirect$1;->()V", + }, + "Ljava/util/zip/ZipFile;->()V": { + None: ["blocked"], + "signature": "Ljava/util/zip/ZipFile;->()V", }, } self.assertEqual(expected, subset) @@ -200,7 +202,7 @@ Ljava/lang/ProcessBuilder$Redirect$1;->()V,blocked Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api Ljava/lang/Object;->hashCode()I,blocked """) - self.assertTrue('Duplicate signature: Ljava/lang/Object;->hashCode()I' + self.assertTrue("Duplicate signature: Ljava/lang/Object;->hashCode()I" in str(context.exception)) def test_read_trie_missing_member(self): @@ -209,8 +211,8 @@ Ljava/lang/Object;->hashCode()I,blocked Ljava/lang/Object,public-api,system-api,test-api """) self.assertTrue( - 'Invalid signature: Ljava/lang/Object, does not identify a specific member' - in str(context.exception)) + "Invalid signature: Ljava/lang/Object, " + "does not identify a specific member" in str(context.exception)) def test_match(self): monolithic = self.read_signature_csv_from_string_as_dict(""" @@ -219,7 +221,7 @@ Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api """) - mismatches = compare_signature_flags(monolithic, modular) + mismatches = vo.compare_signature_flags(monolithic, modular) expected = [] self.assertEqual(expected, mismatches) @@ -230,12 +232,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api """) - mismatches = compare_signature_flags(monolithic, modular) + mismatches = vo.compare_signature_flags(monolithic, modular) expected = [ ( - 'Ljava/lang/Object;->toString()Ljava/lang/String;', - ['public-api', 'system-api', 'test-api'], - ['public-api'], + "Ljava/lang/Object;->toString()Ljava/lang/String;", + ["public-api", "system-api", "test-api"], + ["public-api"], ), ] self.assertEqual(expected, mismatches) @@ -247,12 +249,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,blocked modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api """) - mismatches = compare_signature_flags(monolithic, modular) + mismatches = vo.compare_signature_flags(monolithic, modular) expected = [ ( - 'Ljava/lang/Object;->toString()Ljava/lang/String;', - ['public-api', 'system-api', 'test-api'], - ['blocked'], + "Ljava/lang/Object;->toString()Ljava/lang/String;", + ["public-api", "system-api", "test-api"], + ["blocked"], ), ] self.assertEqual(expected, mismatches) @@ -264,26 +266,26 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,blocked """) - mismatches = compare_signature_flags(monolithic, modular) + mismatches = vo.compare_signature_flags(monolithic, modular) expected = [ ( - 'Ljava/lang/Object;->toString()Ljava/lang/String;', - ['blocked'], - ['public-api', 'system-api', 'test-api'], + "Ljava/lang/Object;->toString()Ljava/lang/String;", + ["blocked"], + ["public-api", "system-api", "test-api"], ), ] self.assertEqual(expected, mismatches) def test_match_treat_missing_from_modular_as_blocked(self): - monolithic = self.read_signature_csv_from_string_as_dict('') + monolithic = self.read_signature_csv_from_string_as_dict("") modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api """) - mismatches = compare_signature_flags(monolithic, modular) + mismatches = vo.compare_signature_flags(monolithic, modular) expected = [ ( - 'Ljava/lang/Object;->toString()Ljava/lang/String;', - ['public-api', 'system-api', 'test-api'], + "Ljava/lang/Object;->toString()Ljava/lang/String;", + ["public-api", "system-api", "test-api"], [], ), ] @@ -294,12 +296,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api """) modular = {} - mismatches = compare_signature_flags(monolithic, modular) + mismatches = vo.compare_signature_flags(monolithic, modular) expected = [ ( - 'Ljava/lang/Object;->hashCode()I', - ['blocked'], - ['public-api', 'system-api', 'test-api'], + "Ljava/lang/Object;->hashCode()I", + ["blocked"], + ["public-api", "system-api", "test-api"], ), ] self.assertEqual(expected, mismatches) @@ -309,12 +311,10 @@ Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api Ljava/lang/Object;->hashCode()I,blocked """) modular = {} - mismatches = compare_signature_flags(monolithic, modular) + mismatches = vo.compare_signature_flags(monolithic, modular) expected = [] self.assertEqual(expected, mismatches) -#pylint: enable=line-too-long - -if __name__ == '__main__': +if __name__ == "__main__": unittest.main(verbosity=2)