StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup verify_overlaps*.py

Browse Source 
Fix issues reported by pylint, Intellij Python checks and also try and
adhere to the Google Python Style Guide.

Bug: 194063708
Test: atest --host verify_overlaps_test
      m out/soong/hiddenapi/hiddenapi-flags.csv
      /usr/bin/pylint --rcfile $ANDROID_BUILD_TOP/tools/repohooks/tools/pylintrc scripts/hiddenapi/verify_overlaps*.py
      pyformat -s 4 --force_quote_type single -i scripts/hiddenapi/verify_overlaps*.py
Change-Id: I6273456abc3a4da4af780ae1a0473ca63521dff5
This commit is contained in:
Paul Duffin
2022-04-08 00:03:32 +01:00
parent 70b9a7e493
commit 181b56cf4d
2 changed files with 153 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
scripts/hiddenapi/verify_overlaps.py
View File

@@ -23,13 +23,13 @@ from itertools import chain
from signature_trie import signature_trie
def dict_reader(csvfile):
def dict_reader(csv_file):
return csv.DictReader(
csvfile, delimiter=",", quotechar="|", fieldnames=["signature"])
csv_file, delimiter=",", quotechar="|", fieldnames=["signature"])
def read_flag_trie_from_file(file):
with open(file, "r") as stream:
with open(file, "r", encoding="utf8") as stream:
return read_flag_trie_from_stream(stream)
@@ -43,24 +43,24 @@ def read_flag_trie_from_stream(stream):
def extract_subset_from_monolithic_flags_as_dict_from_file(
monolithicTrie, patternsFile):
monolithic_trie, patterns_file):
"""Extract a subset of flags from the dict of monolithic flags.
:param monolithicFlagsDict: the dict containing all the monolithic flags.
:param patternsFile: a file containing a list of signature patterns that
:param monolithic_trie: the trie containing all the monolithic flags.
:param patterns_file: a file containing a list of signature patterns that
define the subset.
:return: the dict from signature to row.
"""
with open(patternsFile, "r") as stream:
with open(patterns_file, "r", encoding="utf8") as stream:
return extract_subset_from_monolithic_flags_as_dict_from_stream(
monolithicTrie, stream)
monolithic_trie, stream)
def extract_subset_from_monolithic_flags_as_dict_from_stream(
monolithicTrie, stream):
monolithic_trie, stream):
"""Extract a subset of flags from the trie of monolithic flags.
:param monolithicTrie: the trie containing all the monolithic flags.
:param monolithic_trie: the trie containing all the monolithic flags.
:param stream: a stream containing a list of signature patterns that define
the subset.
:return: the dict from signature to row.
@@ -68,7 +68,7 @@ def extract_subset_from_monolithic_flags_as_dict_from_stream(
dict_signature_to_row = {}
for pattern in stream:
pattern = pattern.rstrip()
rows = monolithicTrie.get_matching_rows(pattern)
rows = monolithic_trie.get_matching_rows(pattern)
for row in rows:
signature = row["signature"]
dict_signature_to_row[signature] = row
@@ -93,48 +93,48 @@ def read_signature_csv_from_stream_as_dict(stream):
return dict_signature_to_row
def read_signature_csv_from_file_as_dict(csvFile):
def read_signature_csv_from_file_as_dict(csv_file):
"""Read the csvFile into a dict.
The first column is assumed to be the signature and used as the
key.
The whole row is stored as the value.
:param csvFile: the csv file to read
:param csv_file: the csv file to read
:return: the dict from signature to row.
"""
with open(csvFile, "r") as f:
with open(csv_file, "r", encoding="utf8") as f:
return read_signature_csv_from_stream_as_dict(f)
def compare_signature_flags(monolithicFlagsDict, modularFlagsDict):
def compare_signature_flags(monolithic_flags_dict, modular_flags_dict):
"""Compare the signature flags between the two dicts.
:param monolithicFlagsDict: the dict containing the subset of the monolithic
flags that should be equal to the modular flags.
:param modularFlagsDict:the dict containing the flags produced by a single
:param monolithic_flags_dict: the dict containing the subset of the
monolithic flags that should be equal to the modular flags.
:param modular_flags_dict:the dict containing the flags produced by a single
bootclasspath_fragment module.
:return: list of mismatches., each mismatch is a tuple where the first item
is the signature, and the second and third items are lists of the flags from
modular dict, and monolithic dict respectively.
"""
mismatchingSignatures = []
mismatching_signatures = []
# Create a sorted set of all the signatures from both the monolithic and
# modular dicts.
allSignatures = sorted(
set(chain(monolithicFlagsDict.keys(), modularFlagsDict.keys())))
for signature in allSignatures:
monolithicRow = monolithicFlagsDict.get(signature, {})
monolithicFlags = monolithicRow.get(None, [])
if signature in modularFlagsDict:
modularRow = modularFlagsDict.get(signature, {})
modularFlags = modularRow.get(None, [])
all_signatures = sorted(
set(chain(monolithic_flags_dict.keys(), modular_flags_dict.keys())))
for signature in all_signatures:
monolithic_row = monolithic_flags_dict.get(signature, {})
monolithic_flags = monolithic_row.get(None, [])
if signature in modular_flags_dict:
modular_row = modular_flags_dict.get(signature, {})
modular_flags = modular_row.get(None, [])
else:
modularFlags = ["blocked"]
if monolithicFlags != modularFlags:
mismatchingSignatures.append(
(signature, modularFlags, monolithicFlags))
return mismatchingSignatures
modular_flags = ["blocked"]
if monolithic_flags != modular_flags:
mismatching_signatures.append(
(signature, modular_flags, monolithic_flags))
return mismatching_signatures
def main(argv):
@@ -149,30 +149,30 @@ def main(argv):
args = args_parser.parse_args(argv[1:])
# Read in all the flags into the trie
monolithicFlagsPath = args.monolithicFlags
monolithicTrie = read_flag_trie_from_file(monolithicFlagsPath)
monolithic_flags_path = args.monolithicFlags
monolithic_trie = read_flag_trie_from_file(monolithic_flags_path)
# For each subset specified on the command line, create dicts for the flags
# provided by the subset and the corresponding flags from the complete set
# of flags and compare them.
failed = False
for modularPair in args.modularFlags:
parts = modularPair.split(":")
modularFlagsPath = parts[0]
modularPatternsPath = parts[1]
modularFlagsDict = read_signature_csv_from_file_as_dict(
modularFlagsPath)
monolithicFlagsSubsetDict = \
for modular_pair in args.modularFlags:
parts = modular_pair.split(":")
modular_flags_path = parts[0]
modular_patterns_path = parts[1]
modular_flags_dict = read_signature_csv_from_file_as_dict(
modular_flags_path)
monolithic_flags_subset_dict = \
extract_subset_from_monolithic_flags_as_dict_from_file(
monolithicTrie, modularPatternsPath)
mismatchingSignatures = compare_signature_flags(
monolithicFlagsSubsetDict, modularFlagsDict)
if mismatchingSignatures:
monolithic_trie, modular_patterns_path)
mismatching_signatures = compare_signature_flags(
monolithic_flags_subset_dict, modular_flags_dict)
if mismatching_signatures:
failed = True
print("ERROR: Hidden API flags are inconsistent:")
print("< " + modularFlagsPath)
print("> " + monolithicFlagsPath)
for mismatch in mismatchingSignatures:
print("< " + modular_flags_path)
print("> " + monolithic_flags_path)
for mismatch in mismatching_signatures:
signature = mismatch[0]
print()
print("< " + ",".join([signature] + mismatch[1]))

202
scripts/hiddenapi/verify_overlaps_test.py
View File

@@ -17,24 +17,26 @@
import io
import unittest
from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import
import verify_overlaps as vo
#pylint: disable=line-too-long
class TestDetectOverlaps(unittest.TestCase):
def read_flag_trie_from_string(self, csvdata):
@staticmethod
def read_flag_trie_from_string(csvdata):
with io.StringIO(csvdata) as f:
return read_flag_trie_from_stream(f)
return vo.read_flag_trie_from_stream(f)
def read_signature_csv_from_string_as_dict(self, csvdata):
@staticmethod
def read_signature_csv_from_string_as_dict(csvdata):
with io.StringIO(csvdata) as f:
return read_signature_csv_from_stream_as_dict(f)
return vo.read_signature_csv_from_stream_as_dict(f)
@staticmethod
def extract_subset_from_monolithic_flags_as_dict_from_string(
self, monolithic, patterns):
monolithic, patterns):
with io.StringIO(patterns) as f:
return extract_subset_from_monolithic_flags_as_dict_from_stream(
return vo.extract_subset_from_monolithic_flags_as_dict_from_stream(
monolithic, f)
extractInput = """
@@ -50,14 +52,14 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
monolithic = self.read_flag_trie_from_string(
TestDetectOverlaps.extractInput)
patterns = 'Ljava/lang/Object;->hashCode()I'
patterns = "Ljava/lang/Object;->hashCode()I"
subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
monolithic, patterns)
expected = {
'Ljava/lang/Object;->hashCode()I': {
None: ['public-api', 'system-api', 'test-api'],
'signature': 'Ljava/lang/Object;->hashCode()I',
"Ljava/lang/Object;->hashCode()I": {
None: ["public-api", "system-api", "test-api"],
"signature": "Ljava/lang/Object;->hashCode()I",
},
}
self.assertEqual(expected, subset)
@@ -66,18 +68,18 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
monolithic = self.read_flag_trie_from_string(
TestDetectOverlaps.extractInput)
patterns = 'java/lang/Object'
patterns = "java/lang/Object"
subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
monolithic, patterns)
expected = {
'Ljava/lang/Object;->hashCode()I': {
None: ['public-api', 'system-api', 'test-api'],
'signature': 'Ljava/lang/Object;->hashCode()I',
"Ljava/lang/Object;->hashCode()I": {
None: ["public-api", "system-api", "test-api"],
"signature": "Ljava/lang/Object;->hashCode()I",
},
'Ljava/lang/Object;->toString()Ljava/lang/String;': {
None: ['blocked'],
'signature': 'Ljava/lang/Object;->toString()Ljava/lang/String;',
"Ljava/lang/Object;->toString()Ljava/lang/String;": {
None: ["blocked"],
"signature": "Ljava/lang/Object;->toString()Ljava/lang/String;",
},
}
self.assertEqual(expected, subset)
@@ -86,20 +88,20 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
monolithic = self.read_flag_trie_from_string(
TestDetectOverlaps.extractInput)
patterns = 'java/lang/Character'
patterns = "java/lang/Character"
subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
monolithic, patterns)
expected = {
'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;':
{
None: ['blocked'],
'signature':
'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;',
"Ljava/lang/Character$UnicodeScript;"
"->of(I)Ljava/lang/Character$UnicodeScript;": {
None: ["blocked"],
"signature": "Ljava/lang/Character$UnicodeScript;"
"->of(I)Ljava/lang/Character$UnicodeScript;",
},
'Ljava/lang/Character;->serialVersionUID:J': {
None: ['sdk'],
'signature': 'Ljava/lang/Character;->serialVersionUID:J',
"Ljava/lang/Character;->serialVersionUID:J": {
None: ["sdk"],
"signature": "Ljava/lang/Character;->serialVersionUID:J",
},
}
self.assertEqual(expected, subset)
@@ -108,16 +110,16 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
monolithic = self.read_flag_trie_from_string(
TestDetectOverlaps.extractInput)
patterns = 'java/lang/Character$UnicodeScript'
patterns = "java/lang/Character$UnicodeScript"
subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
monolithic, patterns)
expected = {
'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;':
{
None: ['blocked'],
'signature':
'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;',
"Ljava/lang/Character$UnicodeScript;"
"->of(I)Ljava/lang/Character$UnicodeScript;": {
None: ["blocked"],
"signature": "Ljava/lang/Character$UnicodeScript;"
"->of(I)Ljava/lang/Character$UnicodeScript;",
},
}
self.assertEqual(expected, subset)
@@ -126,32 +128,32 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
monolithic = self.read_flag_trie_from_string(
TestDetectOverlaps.extractInput)
patterns = 'java/lang/*'
patterns = "java/lang/*"
subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
monolithic, patterns)
expected = {
'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;':
{
None: ['blocked'],
'signature':
'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;',
"Ljava/lang/Character$UnicodeScript;"
"->of(I)Ljava/lang/Character$UnicodeScript;": {
None: ["blocked"],
"signature": "Ljava/lang/Character$UnicodeScript;"
"->of(I)Ljava/lang/Character$UnicodeScript;",
},
'Ljava/lang/Character;->serialVersionUID:J': {
None: ['sdk'],
'signature': 'Ljava/lang/Character;->serialVersionUID:J',
"Ljava/lang/Character;->serialVersionUID:J": {
None: ["sdk"],
"signature": "Ljava/lang/Character;->serialVersionUID:J",
},
'Ljava/lang/Object;->hashCode()I': {
None: ['public-api', 'system-api', 'test-api'],
'signature': 'Ljava/lang/Object;->hashCode()I',
"Ljava/lang/Object;->hashCode()I": {
None: ["public-api", "system-api", "test-api"],
"signature": "Ljava/lang/Object;->hashCode()I",
},
'Ljava/lang/Object;->toString()Ljava/lang/String;': {
None: ['blocked'],
'signature': 'Ljava/lang/Object;->toString()Ljava/lang/String;',
"Ljava/lang/Object;->toString()Ljava/lang/String;": {
None: ["blocked"],
"signature": "Ljava/lang/Object;->toString()Ljava/lang/String;",
},
'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V': {
None: ['blocked'],
'signature': 'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V',
"Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V": {
None: ["blocked"],
"signature": "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V",
},
}
self.assertEqual(expected, subset)
@@ -160,36 +162,36 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
monolithic = self.read_flag_trie_from_string(
TestDetectOverlaps.extractInput)
patterns = 'java/**'
patterns = "java/**"
subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
monolithic, patterns)
expected = {
'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;':
{
None: ['blocked'],
'signature':
'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;',
"Ljava/lang/Character$UnicodeScript;"
"->of(I)Ljava/lang/Character$UnicodeScript;": {
None: ["blocked"],
"signature": "Ljava/lang/Character$UnicodeScript;"
"->of(I)Ljava/lang/Character$UnicodeScript;",
},
'Ljava/lang/Character;->serialVersionUID:J': {
None: ['sdk'],
'signature': 'Ljava/lang/Character;->serialVersionUID:J',
"Ljava/lang/Character;->serialVersionUID:J": {
None: ["sdk"],
"signature": "Ljava/lang/Character;->serialVersionUID:J",
},
'Ljava/lang/Object;->hashCode()I': {
None: ['public-api', 'system-api', 'test-api'],
'signature': 'Ljava/lang/Object;->hashCode()I',
"Ljava/lang/Object;->hashCode()I": {
None: ["public-api", "system-api", "test-api"],
"signature": "Ljava/lang/Object;->hashCode()I",
},
'Ljava/lang/Object;->toString()Ljava/lang/String;': {
None: ['blocked'],
'signature': 'Ljava/lang/Object;->toString()Ljava/lang/String;',
"Ljava/lang/Object;->toString()Ljava/lang/String;": {
None: ["blocked"],
"signature": "Ljava/lang/Object;->toString()Ljava/lang/String;",
},
'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V': {
None: ['blocked'],
'signature': 'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V',
"Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V": {
None: ["blocked"],
"signature": "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V",
},
'Ljava/util/zip/ZipFile;-><clinit>()V': {
None: ['blocked'],
'signature': 'Ljava/util/zip/ZipFile;-><clinit>()V',
"Ljava/util/zip/ZipFile;-><clinit>()V": {
None: ["blocked"],
"signature": "Ljava/util/zip/ZipFile;-><clinit>()V",
},
}
self.assertEqual(expected, subset)
@@ -200,7 +202,7 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
Ljava/lang/Object;->hashCode()I,blocked
""")
self.assertTrue('Duplicate signature: Ljava/lang/Object;->hashCode()I'
self.assertTrue("Duplicate signature: Ljava/lang/Object;->hashCode()I"
in str(context.exception))
def test_read_trie_missing_member(self):
@@ -209,8 +211,8 @@ Ljava/lang/Object;->hashCode()I,blocked
Ljava/lang/Object,public-api,system-api,test-api
""")
self.assertTrue(
'Invalid signature: Ljava/lang/Object, does not identify a specific member'
in str(context.exception))
"Invalid signature: Ljava/lang/Object, "
"does not identify a specific member" in str(context.exception))
def test_match(self):
monolithic = self.read_signature_csv_from_string_as_dict("""
@@ -219,7 +221,7 @@ Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
modular = self.read_signature_csv_from_string_as_dict("""
Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
""")
mismatches = compare_signature_flags(monolithic, modular)
mismatches = vo.compare_signature_flags(monolithic, modular)
expected = []
self.assertEqual(expected, mismatches)
@@ -230,12 +232,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api
modular = self.read_signature_csv_from_string_as_dict("""
Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
""")
mismatches = compare_signature_flags(monolithic, modular)
mismatches = vo.compare_signature_flags(monolithic, modular)
expected = [
(
'Ljava/lang/Object;->toString()Ljava/lang/String;',
['public-api', 'system-api', 'test-api'],
['public-api'],
"Ljava/lang/Object;->toString()Ljava/lang/String;",
["public-api", "system-api", "test-api"],
["public-api"],
),
]
self.assertEqual(expected, mismatches)
@@ -247,12 +249,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,blocked
modular = self.read_signature_csv_from_string_as_dict("""
Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
""")
mismatches = compare_signature_flags(monolithic, modular)
mismatches = vo.compare_signature_flags(monolithic, modular)
expected = [
(
'Ljava/lang/Object;->toString()Ljava/lang/String;',
['public-api', 'system-api', 'test-api'],
['blocked'],
"Ljava/lang/Object;->toString()Ljava/lang/String;",
["public-api", "system-api", "test-api"],
["blocked"],
),
]
self.assertEqual(expected, mismatches)
@@ -264,26 +266,26 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
modular = self.read_signature_csv_from_string_as_dict("""
Ljava/lang/Object;->toString()Ljava/lang/String;,blocked
""")
mismatches = compare_signature_flags(monolithic, modular)
mismatches = vo.compare_signature_flags(monolithic, modular)
expected = [
(
'Ljava/lang/Object;->toString()Ljava/lang/String;',
['blocked'],
['public-api', 'system-api', 'test-api'],
"Ljava/lang/Object;->toString()Ljava/lang/String;",
["blocked"],
["public-api", "system-api", "test-api"],
),
]
self.assertEqual(expected, mismatches)
def test_match_treat_missing_from_modular_as_blocked(self):
monolithic = self.read_signature_csv_from_string_as_dict('')
monolithic = self.read_signature_csv_from_string_as_dict("")
modular = self.read_signature_csv_from_string_as_dict("""
Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
""")
mismatches = compare_signature_flags(monolithic, modular)
mismatches = vo.compare_signature_flags(monolithic, modular)
expected = [
(
'Ljava/lang/Object;->toString()Ljava/lang/String;',
['public-api', 'system-api', 'test-api'],
"Ljava/lang/Object;->toString()Ljava/lang/String;",
["public-api", "system-api", "test-api"],
[],
),
]
@@ -294,12 +296,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
""")
modular = {}
mismatches = compare_signature_flags(monolithic, modular)
mismatches = vo.compare_signature_flags(monolithic, modular)
expected = [
(
'Ljava/lang/Object;->hashCode()I',
['blocked'],
['public-api', 'system-api', 'test-api'],
"Ljava/lang/Object;->hashCode()I",
["blocked"],
["public-api", "system-api", "test-api"],
),
]
self.assertEqual(expected, mismatches)
@@ -309,12 +311,10 @@ Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
Ljava/lang/Object;->hashCode()I,blocked
""")
modular = {}
mismatches = compare_signature_flags(monolithic, modular)
mismatches = vo.compare_signature_flags(monolithic, modular)
expected = []
self.assertEqual(expected, mismatches)
#pylint: enable=line-too-long
if __name__ == '__main__':
if __name__ == "__main__":
unittest.main(verbosity=2)