Extract signature_trie.py from verify_overlaps.py
Makes the efficient pattern matching of hidden API flags that is used by verify_overlaps.py available for use in other scripts. As part of the move this cleans up the python to use consistent quotes, and fix pylint issues. Bug: 202154151 Test: m out/soong/hiddenapi/hiddenapi-flags.csv atest --host signature_trie_test verify_overlaps_test pyformat -s 4 --force_quote_type double -i scripts/hiddenapi/signature_trie* /usr/bin/pylint --rcfile $ANDROID_BUILD_TOP/tools/repohooks/tools/pylintrc scripts/hiddenapi/signature_trie* Change-Id: I758ca70bb5b7e6806f14b72fd04f821a069f188f Change-Id: I73fdb7e02127a8c0171a285221d9e6024310953d
This commit is contained in:
@@ -69,10 +69,37 @@ python_test_host {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
python_library_host {
|
||||||
|
name: "signature_trie",
|
||||||
|
srcs: ["signature_trie.py"],
|
||||||
|
}
|
||||||
|
|
||||||
|
python_test_host {
|
||||||
|
name: "signature_trie_test",
|
||||||
|
main: "signature_trie_test.py",
|
||||||
|
srcs: ["signature_trie_test.py"],
|
||||||
|
libs: ["signature_trie"],
|
||||||
|
version: {
|
||||||
|
py2: {
|
||||||
|
enabled: false,
|
||||||
|
},
|
||||||
|
py3: {
|
||||||
|
enabled: true,
|
||||||
|
embedded_launcher: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
test_options: {
|
||||||
|
unit_test: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
python_binary_host {
|
python_binary_host {
|
||||||
name: "verify_overlaps",
|
name: "verify_overlaps",
|
||||||
main: "verify_overlaps.py",
|
main: "verify_overlaps.py",
|
||||||
srcs: ["verify_overlaps.py"],
|
srcs: ["verify_overlaps.py"],
|
||||||
|
libs: [
|
||||||
|
"signature_trie",
|
||||||
|
],
|
||||||
version: {
|
version: {
|
||||||
py2: {
|
py2: {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
@@ -91,6 +118,9 @@ python_test_host {
|
|||||||
"verify_overlaps.py",
|
"verify_overlaps.py",
|
||||||
"verify_overlaps_test.py",
|
"verify_overlaps_test.py",
|
||||||
],
|
],
|
||||||
|
libs: [
|
||||||
|
"signature_trie",
|
||||||
|
],
|
||||||
version: {
|
version: {
|
||||||
py2: {
|
py2: {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
|
252
scripts/hiddenapi/signature_trie.py
Normal file
252
scripts/hiddenapi/signature_trie.py
Normal file
@@ -0,0 +1,252 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
#
|
||||||
|
# Copyright (C) 2022 The Android Open Source Project
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""Verify that one set of hidden API flags is a subset of another."""
|
||||||
|
|
||||||
|
from itertools import chain
|
||||||
|
|
||||||
|
|
||||||
|
# pylint: disable=line-too-long
|
||||||
|
class InteriorNode:
|
||||||
|
"""An interior node in a trie.
|
||||||
|
|
||||||
|
Each interior node has a dict that maps from an element of a signature to
|
||||||
|
either another interior node or a leaf. Each interior node represents either
|
||||||
|
a package, class or nested class. Class members are represented by a Leaf.
|
||||||
|
|
||||||
|
Associating the set of flags [public-api] with the signature
|
||||||
|
"Ljava/lang/Object;->String()Ljava/lang/String;" will cause the following
|
||||||
|
nodes to be created:
|
||||||
|
Node()
|
||||||
|
^- package:java -> Node()
|
||||||
|
^- package:lang -> Node()
|
||||||
|
^- class:Object -> Node()
|
||||||
|
^- member:String()Ljava/lang/String; -> Leaf([public-api])
|
||||||
|
|
||||||
|
Associating the set of flags [blocked,core-platform-api] with the signature
|
||||||
|
"Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;"
|
||||||
|
will cause the following nodes to be created:
|
||||||
|
Node()
|
||||||
|
^- package:java -> Node()
|
||||||
|
^- package:lang -> Node()
|
||||||
|
^- class:Character -> Node()
|
||||||
|
^- class:UnicodeScript -> Node()
|
||||||
|
^- member:of(I)Ljava/lang/Character$UnicodeScript;
|
||||||
|
-> Leaf([blocked,core-platform-api])
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
nodes: a dict from an element of the signature to the Node/Leaf
|
||||||
|
containing the next element/value.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# pylint: enable=line-too-long
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.nodes = {}
|
||||||
|
|
||||||
|
# pylint: disable=line-too-long
|
||||||
|
@staticmethod
|
||||||
|
def signature_to_elements(signature):
|
||||||
|
"""Split a signature or a prefix into a number of elements:
|
||||||
|
|
||||||
|
1. The packages (excluding the leading L preceding the first package).
|
||||||
|
2. The class names, from outermost to innermost.
|
||||||
|
3. The member signature.
|
||||||
|
e.g.
|
||||||
|
Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;
|
||||||
|
will be broken down into these elements:
|
||||||
|
1. package:java
|
||||||
|
2. package:lang
|
||||||
|
3. class:Character
|
||||||
|
4. class:UnicodeScript
|
||||||
|
5. member:of(I)Ljava/lang/Character$UnicodeScript;
|
||||||
|
"""
|
||||||
|
# Remove the leading L.
|
||||||
|
# - java/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;
|
||||||
|
text = signature.removeprefix("L")
|
||||||
|
# Split the signature between qualified class name and the class member
|
||||||
|
# signature.
|
||||||
|
# 0 - java/lang/Character$UnicodeScript
|
||||||
|
# 1 - of(I)Ljava/lang/Character$UnicodeScript;
|
||||||
|
parts = text.split(";->")
|
||||||
|
member = parts[1:]
|
||||||
|
# Split the qualified class name into packages, and class name.
|
||||||
|
# 0 - java
|
||||||
|
# 1 - lang
|
||||||
|
# 2 - Character$UnicodeScript
|
||||||
|
elements = parts[0].split("/")
|
||||||
|
packages = elements[0:-1]
|
||||||
|
class_name = elements[-1]
|
||||||
|
if class_name in ("*", "**"): # pylint: disable=no-else-return
|
||||||
|
# Cannot specify a wildcard and target a specific member
|
||||||
|
if len(member) != 0:
|
||||||
|
raise Exception(f"Invalid signature {signature}: contains "
|
||||||
|
f"wildcard {class_name} and "
|
||||||
|
f"member signature {member[0]}")
|
||||||
|
wildcard = [class_name]
|
||||||
|
# Assemble the parts into a single list, adding prefixes to identify
|
||||||
|
# the different parts.
|
||||||
|
# 0 - package:java
|
||||||
|
# 1 - package:lang
|
||||||
|
# 2 - *
|
||||||
|
return list(chain(["package:" + x for x in packages], wildcard))
|
||||||
|
else:
|
||||||
|
# Split the class name into outer / inner classes
|
||||||
|
# 0 - Character
|
||||||
|
# 1 - UnicodeScript
|
||||||
|
classes = class_name.split("$")
|
||||||
|
# Assemble the parts into a single list, adding prefixes to identify
|
||||||
|
# the different parts.
|
||||||
|
# 0 - package:java
|
||||||
|
# 1 - package:lang
|
||||||
|
# 2 - class:Character
|
||||||
|
# 3 - class:UnicodeScript
|
||||||
|
# 4 - member:of(I)Ljava/lang/Character$UnicodeScript;
|
||||||
|
return list(
|
||||||
|
chain(["package:" + x for x in packages],
|
||||||
|
["class:" + x for x in classes],
|
||||||
|
["member:" + x for x in member]))
|
||||||
|
|
||||||
|
# pylint: enable=line-too-long
|
||||||
|
|
||||||
|
def add(self, signature, value):
|
||||||
|
"""Associate the value with the specific signature.
|
||||||
|
|
||||||
|
:param signature: the member signature
|
||||||
|
:param value: the value to associated with the signature
|
||||||
|
:return: n/a
|
||||||
|
"""
|
||||||
|
# Split the signature into elements.
|
||||||
|
elements = self.signature_to_elements(signature)
|
||||||
|
# Find the Node associated with the deepest class.
|
||||||
|
node = self
|
||||||
|
for element in elements[:-1]:
|
||||||
|
if element in node.nodes:
|
||||||
|
node = node.nodes[element]
|
||||||
|
else:
|
||||||
|
next_node = InteriorNode()
|
||||||
|
node.nodes[element] = next_node
|
||||||
|
node = next_node
|
||||||
|
# Add a Leaf containing the value and associate it with the member
|
||||||
|
# signature within the class.
|
||||||
|
last_element = elements[-1]
|
||||||
|
if not last_element.startswith("member:"):
|
||||||
|
raise Exception(
|
||||||
|
f"Invalid signature: {signature}, does not identify a "
|
||||||
|
"specific member")
|
||||||
|
if last_element in node.nodes:
|
||||||
|
raise Exception(f"Duplicate signature: {signature}")
|
||||||
|
node.nodes[last_element] = Leaf(value)
|
||||||
|
|
||||||
|
def get_matching_rows(self, pattern):
|
||||||
|
"""Get the values (plural) associated with the pattern.
|
||||||
|
|
||||||
|
e.g. If the pattern is a full signature then this will return a list
|
||||||
|
containing the value associated with that signature.
|
||||||
|
|
||||||
|
If the pattern is a class then this will return a list containing the
|
||||||
|
values associated with all members of that class.
|
||||||
|
|
||||||
|
If the pattern is a package then this will return a list containing the
|
||||||
|
values associated with all the members of all the classes in that
|
||||||
|
package and sub-packages.
|
||||||
|
|
||||||
|
If the pattern ends with "*" then the preceding part is treated as a
|
||||||
|
package and this will return a list containing the values associated
|
||||||
|
with all the members of all the classes in that package.
|
||||||
|
|
||||||
|
If the pattern ends with "**" then the preceding part is treated
|
||||||
|
as a package and this will return a list containing the values
|
||||||
|
associated with all the members of all the classes in that package and
|
||||||
|
all sub-packages.
|
||||||
|
|
||||||
|
:param pattern: the pattern which could be a complete signature or a
|
||||||
|
class, or package wildcard.
|
||||||
|
:return: an iterable containing all the values associated with the
|
||||||
|
pattern.
|
||||||
|
"""
|
||||||
|
elements = self.signature_to_elements(pattern)
|
||||||
|
node = self
|
||||||
|
|
||||||
|
# Include all values from this node and all its children.
|
||||||
|
selector = lambda x: True
|
||||||
|
|
||||||
|
last_element = elements[-1]
|
||||||
|
if last_element in ("*", "**"):
|
||||||
|
elements = elements[:-1]
|
||||||
|
if last_element == "*":
|
||||||
|
# Do not include values from sub-packages.
|
||||||
|
selector = lambda x: not x.startswith("package:")
|
||||||
|
|
||||||
|
for element in elements:
|
||||||
|
if element in node.nodes:
|
||||||
|
node = node.nodes[element]
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
return chain.from_iterable(node.values(selector))
|
||||||
|
|
||||||
|
def values(self, selector):
|
||||||
|
""":param selector: a function that can be applied to a key in the nodes
|
||||||
|
|
||||||
|
attribute to determine whether to return its values.
|
||||||
|
|
||||||
|
:return: A list of iterables of all the values associated with
|
||||||
|
this node and its children.
|
||||||
|
"""
|
||||||
|
values = []
|
||||||
|
self.append_values(values, selector)
|
||||||
|
return values
|
||||||
|
|
||||||
|
def append_values(self, values, selector):
|
||||||
|
"""Append the values associated with this node and its children.
|
||||||
|
|
||||||
|
For each item (key, child) in nodes the child node's values are returned
|
||||||
|
if and only if the selector returns True when called on its key. A child
|
||||||
|
node's values are all the values associated with it and all its
|
||||||
|
descendant nodes.
|
||||||
|
|
||||||
|
:param selector: a function that can be applied to a key in the nodes
|
||||||
|
attribute to determine whether to return its values.
|
||||||
|
:param values: a list of a iterables of values.
|
||||||
|
"""
|
||||||
|
for key, node in self.nodes.items():
|
||||||
|
if selector(key):
|
||||||
|
node.append_values(values, lambda x: True)
|
||||||
|
|
||||||
|
|
||||||
|
class Leaf:
|
||||||
|
"""A leaf of the trie
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
value: the value associated with this leaf.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, value):
|
||||||
|
self.value = value
|
||||||
|
|
||||||
|
def values(self, selector): # pylint: disable=unused-argument
|
||||||
|
""":return: A list of a list of the value associated with this node."""
|
||||||
|
return [[self.value]]
|
||||||
|
|
||||||
|
def append_values(self, values, selector): # pylint: disable=unused-argument
|
||||||
|
"""Appends a list of the value associated with this node to the list.
|
||||||
|
|
||||||
|
:param values: a list of a iterables of values.
|
||||||
|
"""
|
||||||
|
values.append([self.value])
|
||||||
|
|
||||||
|
|
||||||
|
def signature_trie():
|
||||||
|
return InteriorNode()
|
192
scripts/hiddenapi/signature_trie_test.py
Executable file
192
scripts/hiddenapi/signature_trie_test.py
Executable file
@@ -0,0 +1,192 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
#
|
||||||
|
# Copyright (C) 2022 The Android Open Source Project
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the 'License');
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an 'AS IS' BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""Unit tests for verify_overlaps_test.py."""
|
||||||
|
import io
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from signature_trie import InteriorNode
|
||||||
|
from signature_trie import signature_trie
|
||||||
|
|
||||||
|
|
||||||
|
class TestSignatureToElements(unittest.TestCase):
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def signature_to_elements(signature):
|
||||||
|
return InteriorNode.signature_to_elements(signature)
|
||||||
|
|
||||||
|
def test_nested_inner_classes(self):
|
||||||
|
elements = [
|
||||||
|
"package:java",
|
||||||
|
"package:lang",
|
||||||
|
"class:ProcessBuilder",
|
||||||
|
"class:Redirect",
|
||||||
|
"class:1",
|
||||||
|
"member:<init>()V",
|
||||||
|
]
|
||||||
|
signature = "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V"
|
||||||
|
self.assertEqual(elements, self.signature_to_elements(signature))
|
||||||
|
|
||||||
|
def test_basic_member(self):
|
||||||
|
elements = [
|
||||||
|
"package:java",
|
||||||
|
"package:lang",
|
||||||
|
"class:Object",
|
||||||
|
"member:hashCode()I",
|
||||||
|
]
|
||||||
|
signature = "Ljava/lang/Object;->hashCode()I"
|
||||||
|
self.assertEqual(elements, self.signature_to_elements(signature))
|
||||||
|
|
||||||
|
def test_double_dollar_class(self):
|
||||||
|
elements = [
|
||||||
|
"package:java",
|
||||||
|
"package:lang",
|
||||||
|
"class:CharSequence",
|
||||||
|
"class:",
|
||||||
|
"class:ExternalSyntheticLambda0",
|
||||||
|
"member:<init>(Ljava/lang/CharSequence;)V",
|
||||||
|
]
|
||||||
|
signature = "Ljava/lang/CharSequence$$ExternalSyntheticLambda0;" \
|
||||||
|
"-><init>(Ljava/lang/CharSequence;)V"
|
||||||
|
self.assertEqual(elements, self.signature_to_elements(signature))
|
||||||
|
|
||||||
|
def test_no_member(self):
|
||||||
|
elements = [
|
||||||
|
"package:java",
|
||||||
|
"package:lang",
|
||||||
|
"class:CharSequence",
|
||||||
|
"class:",
|
||||||
|
"class:ExternalSyntheticLambda0",
|
||||||
|
]
|
||||||
|
signature = "Ljava/lang/CharSequence$$ExternalSyntheticLambda0"
|
||||||
|
self.assertEqual(elements, self.signature_to_elements(signature))
|
||||||
|
|
||||||
|
def test_wildcard(self):
|
||||||
|
elements = [
|
||||||
|
"package:java",
|
||||||
|
"package:lang",
|
||||||
|
"*",
|
||||||
|
]
|
||||||
|
signature = "java/lang/*"
|
||||||
|
self.assertEqual(elements, self.signature_to_elements(signature))
|
||||||
|
|
||||||
|
def test_recursive_wildcard(self):
|
||||||
|
elements = [
|
||||||
|
"package:java",
|
||||||
|
"package:lang",
|
||||||
|
"**",
|
||||||
|
]
|
||||||
|
signature = "java/lang/**"
|
||||||
|
self.assertEqual(elements, self.signature_to_elements(signature))
|
||||||
|
|
||||||
|
def test_no_packages_wildcard(self):
|
||||||
|
elements = [
|
||||||
|
"*",
|
||||||
|
]
|
||||||
|
signature = "*"
|
||||||
|
self.assertEqual(elements, self.signature_to_elements(signature))
|
||||||
|
|
||||||
|
def test_no_packages_recursive_wildcard(self):
|
||||||
|
elements = [
|
||||||
|
"**",
|
||||||
|
]
|
||||||
|
signature = "**"
|
||||||
|
self.assertEqual(elements, self.signature_to_elements(signature))
|
||||||
|
|
||||||
|
def test_non_standard_class_name(self):
|
||||||
|
elements = [
|
||||||
|
"package:javax",
|
||||||
|
"package:crypto",
|
||||||
|
"class:extObjectInputStream",
|
||||||
|
]
|
||||||
|
signature = "Ljavax/crypto/extObjectInputStream"
|
||||||
|
self.assertEqual(elements, self.signature_to_elements(signature))
|
||||||
|
|
||||||
|
def test_invalid_pattern_wildcard_and_member(self):
|
||||||
|
pattern = "Ljava/lang/*;->hashCode()I"
|
||||||
|
with self.assertRaises(Exception) as context:
|
||||||
|
self.signature_to_elements(pattern)
|
||||||
|
self.assertIn("contains wildcard * and member signature hashCode()I",
|
||||||
|
str(context.exception))
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetMatchingRows(unittest.TestCase):
|
||||||
|
extractInput = """
|
||||||
|
Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;
|
||||||
|
Ljava/lang/Character;->serialVersionUID:J
|
||||||
|
Ljava/lang/Object;->hashCode()I
|
||||||
|
Ljava/lang/Object;->toString()Ljava/lang/String;
|
||||||
|
Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V
|
||||||
|
Ljava/util/zip/ZipFile;-><clinit>()V
|
||||||
|
"""
|
||||||
|
|
||||||
|
def read_trie(self):
|
||||||
|
trie = signature_trie()
|
||||||
|
with io.StringIO(self.extractInput.strip()) as f:
|
||||||
|
for line in iter(f.readline, ""):
|
||||||
|
line = line.rstrip()
|
||||||
|
trie.add(line, line)
|
||||||
|
return trie
|
||||||
|
|
||||||
|
def check_patterns(self, pattern, expected):
|
||||||
|
trie = self.read_trie()
|
||||||
|
self.check_node_patterns(trie, pattern, expected)
|
||||||
|
|
||||||
|
def check_node_patterns(self, node, pattern, expected):
|
||||||
|
actual = list(node.get_matching_rows(pattern))
|
||||||
|
actual.sort()
|
||||||
|
self.assertEqual(expected, actual)
|
||||||
|
|
||||||
|
def test_member_pattern(self):
|
||||||
|
self.check_patterns("java/util/zip/ZipFile;-><clinit>()V",
|
||||||
|
["Ljava/util/zip/ZipFile;-><clinit>()V"])
|
||||||
|
|
||||||
|
def test_class_pattern(self):
|
||||||
|
self.check_patterns("java/lang/Object", [
|
||||||
|
"Ljava/lang/Object;->hashCode()I",
|
||||||
|
"Ljava/lang/Object;->toString()Ljava/lang/String;",
|
||||||
|
])
|
||||||
|
|
||||||
|
# pylint: disable=line-too-long
|
||||||
|
def test_nested_class_pattern(self):
|
||||||
|
self.check_patterns("java/lang/Character", [
|
||||||
|
"Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;",
|
||||||
|
"Ljava/lang/Character;->serialVersionUID:J",
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_wildcard(self):
|
||||||
|
self.check_patterns("java/lang/*", [
|
||||||
|
"Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;",
|
||||||
|
"Ljava/lang/Character;->serialVersionUID:J",
|
||||||
|
"Ljava/lang/Object;->hashCode()I",
|
||||||
|
"Ljava/lang/Object;->toString()Ljava/lang/String;",
|
||||||
|
"Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V",
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_recursive_wildcard(self):
|
||||||
|
self.check_patterns("java/**", [
|
||||||
|
"Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;",
|
||||||
|
"Ljava/lang/Character;->serialVersionUID:J",
|
||||||
|
"Ljava/lang/Object;->hashCode()I",
|
||||||
|
"Ljava/lang/Object;->toString()Ljava/lang/String;",
|
||||||
|
"Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V",
|
||||||
|
"Ljava/util/zip/ZipFile;-><clinit>()V",
|
||||||
|
])
|
||||||
|
|
||||||
|
# pylint: enable=line-too-long
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
@@ -13,239 +13,14 @@
|
|||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
"""Verify that one set of hidden API flags is a subset of another.
|
"""Verify that one set of hidden API flags is a subset of another."""
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import csv
|
import csv
|
||||||
import sys
|
import sys
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
|
|
||||||
#pylint: disable=line-too-long
|
from signature_trie import signature_trie
|
||||||
class InteriorNode:
|
|
||||||
"""An interior node in a trie.
|
|
||||||
|
|
||||||
Each interior node has a dict that maps from an element of a signature to
|
|
||||||
either another interior node or a leaf. Each interior node represents either
|
|
||||||
a package, class or nested class. Class members are represented by a Leaf.
|
|
||||||
|
|
||||||
Associating the set of flags [public-api] with the signature
|
|
||||||
"Ljava/lang/Object;->String()Ljava/lang/String;" will cause the following
|
|
||||||
nodes to be created:
|
|
||||||
Node()
|
|
||||||
^- package:java -> Node()
|
|
||||||
^- package:lang -> Node()
|
|
||||||
^- class:Object -> Node()
|
|
||||||
^- member:String()Ljava/lang/String; -> Leaf([public-api])
|
|
||||||
|
|
||||||
Associating the set of flags [blocked,core-platform-api] with the signature
|
|
||||||
"Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;"
|
|
||||||
will cause the following nodes to be created:
|
|
||||||
Node()
|
|
||||||
^- package:java -> Node()
|
|
||||||
^- package:lang -> Node()
|
|
||||||
^- class:Character -> Node()
|
|
||||||
^- class:UnicodeScript -> Node()
|
|
||||||
^- member:of(I)Ljava/lang/Character$UnicodeScript;
|
|
||||||
-> Leaf([blocked,core-platform-api])
|
|
||||||
|
|
||||||
Attributes:
|
|
||||||
nodes: a dict from an element of the signature to the Node/Leaf
|
|
||||||
containing the next element/value.
|
|
||||||
"""
|
|
||||||
#pylint: enable=line-too-long
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.nodes = {}
|
|
||||||
|
|
||||||
#pylint: disable=line-too-long
|
|
||||||
def signatureToElements(self, signature):
|
|
||||||
"""Split a signature or a prefix into a number of elements:
|
|
||||||
1. The packages (excluding the leading L preceding the first package).
|
|
||||||
2. The class names, from outermost to innermost.
|
|
||||||
3. The member signature.
|
|
||||||
e.g.
|
|
||||||
Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;
|
|
||||||
will be broken down into these elements:
|
|
||||||
1. package:java
|
|
||||||
2. package:lang
|
|
||||||
3. class:Character
|
|
||||||
4. class:UnicodeScript
|
|
||||||
5. member:of(I)Ljava/lang/Character$UnicodeScript;
|
|
||||||
"""
|
|
||||||
# Remove the leading L.
|
|
||||||
# - java/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;
|
|
||||||
text = signature.removeprefix("L")
|
|
||||||
# Split the signature between qualified class name and the class member
|
|
||||||
# signature.
|
|
||||||
# 0 - java/lang/Character$UnicodeScript
|
|
||||||
# 1 - of(I)Ljava/lang/Character$UnicodeScript;
|
|
||||||
parts = text.split(";->")
|
|
||||||
member = parts[1:]
|
|
||||||
# Split the qualified class name into packages, and class name.
|
|
||||||
# 0 - java
|
|
||||||
# 1 - lang
|
|
||||||
# 2 - Character$UnicodeScript
|
|
||||||
elements = parts[0].split("/")
|
|
||||||
packages = elements[0:-1]
|
|
||||||
className = elements[-1]
|
|
||||||
if className in ("*" , "**"): #pylint: disable=no-else-return
|
|
||||||
# Cannot specify a wildcard and target a specific member
|
|
||||||
if len(member) != 0:
|
|
||||||
raise Exception(
|
|
||||||
"Invalid signature %s: contains wildcard %s and member " \
|
|
||||||
"signature %s"
|
|
||||||
% (signature, className, member[0]))
|
|
||||||
wildcard = [className]
|
|
||||||
# Assemble the parts into a single list, adding prefixes to identify
|
|
||||||
# the different parts.
|
|
||||||
# 0 - package:java
|
|
||||||
# 1 - package:lang
|
|
||||||
# 2 - *
|
|
||||||
return list(
|
|
||||||
chain(["package:" + x for x in packages], wildcard))
|
|
||||||
else:
|
|
||||||
# Split the class name into outer / inner classes
|
|
||||||
# 0 - Character
|
|
||||||
# 1 - UnicodeScript
|
|
||||||
classes = className.split("$")
|
|
||||||
# Assemble the parts into a single list, adding prefixes to identify
|
|
||||||
# the different parts.
|
|
||||||
# 0 - package:java
|
|
||||||
# 1 - package:lang
|
|
||||||
# 2 - class:Character
|
|
||||||
# 3 - class:UnicodeScript
|
|
||||||
# 4 - member:of(I)Ljava/lang/Character$UnicodeScript;
|
|
||||||
return list(
|
|
||||||
chain(
|
|
||||||
["package:" + x for x in packages],
|
|
||||||
["class:" + x for x in classes],
|
|
||||||
["member:" + x for x in member]))
|
|
||||||
#pylint: enable=line-too-long
|
|
||||||
|
|
||||||
def add(self, signature, value):
|
|
||||||
"""Associate the value with the specific signature.
|
|
||||||
|
|
||||||
:param signature: the member signature
|
|
||||||
:param value: the value to associated with the signature
|
|
||||||
:return: n/a
|
|
||||||
"""
|
|
||||||
# Split the signature into elements.
|
|
||||||
elements = self.signatureToElements(signature)
|
|
||||||
# Find the Node associated with the deepest class.
|
|
||||||
node = self
|
|
||||||
for element in elements[:-1]:
|
|
||||||
if element in node.nodes:
|
|
||||||
node = node.nodes[element]
|
|
||||||
else:
|
|
||||||
next_node = InteriorNode()
|
|
||||||
node.nodes[element] = next_node
|
|
||||||
node = next_node
|
|
||||||
# Add a Leaf containing the value and associate it with the member
|
|
||||||
# signature within the class.
|
|
||||||
lastElement = elements[-1]
|
|
||||||
if not lastElement.startswith("member:"):
|
|
||||||
raise Exception(
|
|
||||||
"Invalid signature: %s, does not identify a specific member" %
|
|
||||||
signature)
|
|
||||||
if lastElement in node.nodes:
|
|
||||||
raise Exception("Duplicate signature: %s" % signature)
|
|
||||||
node.nodes[lastElement] = Leaf(value)
|
|
||||||
|
|
||||||
def getMatchingRows(self, pattern):
|
|
||||||
"""Get the values (plural) associated with the pattern.
|
|
||||||
|
|
||||||
e.g. If the pattern is a full signature then this will return a list
|
|
||||||
containing the value associated with that signature.
|
|
||||||
|
|
||||||
If the pattern is a class then this will return a list containing the
|
|
||||||
values associated with all members of that class.
|
|
||||||
|
|
||||||
If the pattern is a package then this will return a list containing the
|
|
||||||
values associated with all the members of all the classes in that
|
|
||||||
package and sub-packages.
|
|
||||||
|
|
||||||
If the pattern ends with "*" then the preceding part is treated as a
|
|
||||||
package and this will return a list containing the values associated
|
|
||||||
with all the members of all the classes in that package.
|
|
||||||
|
|
||||||
If the pattern ends with "**" then the preceding part is treated
|
|
||||||
as a package and this will return a list containing the values
|
|
||||||
associated with all the members of all the classes in that package and
|
|
||||||
all sub-packages.
|
|
||||||
|
|
||||||
:param pattern: the pattern which could be a complete signature or a
|
|
||||||
class, or package wildcard.
|
|
||||||
:return: an iterable containing all the values associated with the
|
|
||||||
pattern.
|
|
||||||
"""
|
|
||||||
elements = self.signatureToElements(pattern)
|
|
||||||
node = self
|
|
||||||
# Include all values from this node and all its children.
|
|
||||||
selector = lambda x: True
|
|
||||||
lastElement = elements[-1]
|
|
||||||
if lastElement in ("*", "**"):
|
|
||||||
elements = elements[:-1]
|
|
||||||
if lastElement == "*":
|
|
||||||
# Do not include values from sub-packages.
|
|
||||||
selector = lambda x: not x.startswith("package:")
|
|
||||||
for element in elements:
|
|
||||||
if element in node.nodes:
|
|
||||||
node = node.nodes[element]
|
|
||||||
else:
|
|
||||||
return []
|
|
||||||
return chain.from_iterable(node.values(selector))
|
|
||||||
|
|
||||||
def values(self, selector):
|
|
||||||
""":param selector: a function that can be applied to a key in the nodes
|
|
||||||
attribute to determine whether to return its values.
|
|
||||||
|
|
||||||
:return: A list of iterables of all the values associated with
|
|
||||||
this node and its children.
|
|
||||||
"""
|
|
||||||
values = []
|
|
||||||
self.appendValues(values, selector)
|
|
||||||
return values
|
|
||||||
|
|
||||||
def appendValues(self, values, selector):
|
|
||||||
"""Append the values associated with this node and its children to the
|
|
||||||
list.
|
|
||||||
|
|
||||||
For each item (key, child) in nodes the child node's values are returned
|
|
||||||
if and only if the selector returns True when called on its key. A child
|
|
||||||
node's values are all the values associated with it and all its
|
|
||||||
descendant nodes.
|
|
||||||
|
|
||||||
:param selector: a function that can be applied to a key in the nodes
|
|
||||||
attribute to determine whether to return its values.
|
|
||||||
:param values: a list of a iterables of values.
|
|
||||||
"""
|
|
||||||
for key, node in self.nodes.items():
|
|
||||||
if selector(key):
|
|
||||||
node.appendValues(values, lambda x: True)
|
|
||||||
|
|
||||||
|
|
||||||
class Leaf:
|
|
||||||
"""A leaf of the trie
|
|
||||||
|
|
||||||
Attributes:
|
|
||||||
value: the value associated with this leaf.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, value):
|
|
||||||
self.value = value
|
|
||||||
|
|
||||||
def values(self, selector): #pylint: disable=unused-argument
|
|
||||||
""":return: A list of a list of the value associated with this node.
|
|
||||||
"""
|
|
||||||
return [[self.value]]
|
|
||||||
|
|
||||||
def appendValues(self, values, selector): #pylint: disable=unused-argument
|
|
||||||
"""Appends a list of the value associated with this node to the list.
|
|
||||||
|
|
||||||
:param values: a list of a iterables of values.
|
|
||||||
"""
|
|
||||||
values.append([self.value])
|
|
||||||
|
|
||||||
|
|
||||||
def dict_reader(csvfile):
|
def dict_reader(csvfile):
|
||||||
@@ -259,7 +34,7 @@ def read_flag_trie_from_file(file):
|
|||||||
|
|
||||||
|
|
||||||
def read_flag_trie_from_stream(stream):
|
def read_flag_trie_from_stream(stream):
|
||||||
trie = InteriorNode()
|
trie = signature_trie()
|
||||||
reader = dict_reader(stream)
|
reader = dict_reader(stream)
|
||||||
for row in reader:
|
for row in reader:
|
||||||
signature = row["signature"]
|
signature = row["signature"]
|
||||||
@@ -269,8 +44,7 @@ def read_flag_trie_from_stream(stream):
|
|||||||
|
|
||||||
def extract_subset_from_monolithic_flags_as_dict_from_file(
|
def extract_subset_from_monolithic_flags_as_dict_from_file(
|
||||||
monolithicTrie, patternsFile):
|
monolithicTrie, patternsFile):
|
||||||
"""Extract a subset of flags from the dict containing all the monolithic
|
"""Extract a subset of flags from the dict of monolithic flags.
|
||||||
flags.
|
|
||||||
|
|
||||||
:param monolithicFlagsDict: the dict containing all the monolithic flags.
|
:param monolithicFlagsDict: the dict containing all the monolithic flags.
|
||||||
:param patternsFile: a file containing a list of signature patterns that
|
:param patternsFile: a file containing a list of signature patterns that
|
||||||
@@ -284,8 +58,7 @@ def extract_subset_from_monolithic_flags_as_dict_from_file(
|
|||||||
|
|
||||||
def extract_subset_from_monolithic_flags_as_dict_from_stream(
|
def extract_subset_from_monolithic_flags_as_dict_from_stream(
|
||||||
monolithicTrie, stream):
|
monolithicTrie, stream):
|
||||||
"""Extract a subset of flags from the trie containing all the monolithic
|
"""Extract a subset of flags from the trie of monolithic flags.
|
||||||
flags.
|
|
||||||
|
|
||||||
:param monolithicTrie: the trie containing all the monolithic flags.
|
:param monolithicTrie: the trie containing all the monolithic flags.
|
||||||
:param stream: a stream containing a list of signature patterns that define
|
:param stream: a stream containing a list of signature patterns that define
|
||||||
@@ -295,7 +68,7 @@ def extract_subset_from_monolithic_flags_as_dict_from_stream(
|
|||||||
dict_signature_to_row = {}
|
dict_signature_to_row = {}
|
||||||
for pattern in stream:
|
for pattern in stream:
|
||||||
pattern = pattern.rstrip()
|
pattern = pattern.rstrip()
|
||||||
rows = monolithicTrie.getMatchingRows(pattern)
|
rows = monolithicTrie.get_matching_rows(pattern)
|
||||||
for row in rows:
|
for row in rows:
|
||||||
signature = row["signature"]
|
signature = row["signature"]
|
||||||
dict_signature_to_row[signature] = row
|
dict_signature_to_row[signature] = row
|
||||||
@@ -303,8 +76,10 @@ def extract_subset_from_monolithic_flags_as_dict_from_stream(
|
|||||||
|
|
||||||
|
|
||||||
def read_signature_csv_from_stream_as_dict(stream):
|
def read_signature_csv_from_stream_as_dict(stream):
|
||||||
"""Read the csv contents from the stream into a dict. The first column is
|
"""Read the csv contents from the stream into a dict.
|
||||||
assumed to be the signature and used as the key.
|
|
||||||
|
The first column is assumed to be the signature and used as the
|
||||||
|
key.
|
||||||
|
|
||||||
The whole row is stored as the value.
|
The whole row is stored as the value.
|
||||||
:param stream: the csv contents to read
|
:param stream: the csv contents to read
|
||||||
@@ -319,8 +94,10 @@ def read_signature_csv_from_stream_as_dict(stream):
|
|||||||
|
|
||||||
|
|
||||||
def read_signature_csv_from_file_as_dict(csvFile):
|
def read_signature_csv_from_file_as_dict(csvFile):
|
||||||
"""Read the csvFile into a dict. The first column is assumed to be the
|
"""Read the csvFile into a dict.
|
||||||
signature and used as the key.
|
|
||||||
|
The first column is assumed to be the signature and used as the
|
||||||
|
key.
|
||||||
|
|
||||||
The whole row is stored as the value.
|
The whole row is stored as the value.
|
||||||
:param csvFile: the csv file to read
|
:param csvFile: the csv file to read
|
||||||
@@ -363,8 +140,7 @@ def compare_signature_flags(monolithicFlagsDict, modularFlagsDict):
|
|||||||
def main(argv):
|
def main(argv):
|
||||||
args_parser = argparse.ArgumentParser(
|
args_parser = argparse.ArgumentParser(
|
||||||
description="Verify that sets of hidden API flags are each a subset of "
|
description="Verify that sets of hidden API flags are each a subset of "
|
||||||
"the monolithic flag file."
|
"the monolithic flag file.")
|
||||||
)
|
|
||||||
args_parser.add_argument("monolithicFlags", help="The monolithic flag file")
|
args_parser.add_argument("monolithicFlags", help="The monolithic flag file")
|
||||||
args_parser.add_argument(
|
args_parser.add_argument(
|
||||||
"modularFlags",
|
"modularFlags",
|
||||||
|
@@ -20,51 +20,6 @@ import unittest
|
|||||||
from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import
|
from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import
|
||||||
|
|
||||||
|
|
||||||
class TestSignatureToElements(unittest.TestCase):
|
|
||||||
|
|
||||||
def signatureToElements(self, signature):
|
|
||||||
return InteriorNode().signatureToElements(signature)
|
|
||||||
|
|
||||||
def test_signatureToElements_1(self):
|
|
||||||
expected = [
|
|
||||||
'package:java',
|
|
||||||
'package:lang',
|
|
||||||
'class:ProcessBuilder',
|
|
||||||
'class:Redirect',
|
|
||||||
'class:1',
|
|
||||||
'member:<init>()V',
|
|
||||||
]
|
|
||||||
self.assertEqual(
|
|
||||||
expected,
|
|
||||||
self.signatureToElements(
|
|
||||||
'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V'))
|
|
||||||
|
|
||||||
def test_signatureToElements_2(self):
|
|
||||||
expected = [
|
|
||||||
'package:java',
|
|
||||||
'package:lang',
|
|
||||||
'class:Object',
|
|
||||||
'member:hashCode()I',
|
|
||||||
]
|
|
||||||
self.assertEqual(
|
|
||||||
expected,
|
|
||||||
self.signatureToElements('Ljava/lang/Object;->hashCode()I'))
|
|
||||||
|
|
||||||
def test_signatureToElements_3(self):
|
|
||||||
expected = [
|
|
||||||
'package:java',
|
|
||||||
'package:lang',
|
|
||||||
'class:CharSequence',
|
|
||||||
'class:',
|
|
||||||
'class:ExternalSyntheticLambda0',
|
|
||||||
'member:<init>(Ljava/lang/CharSequence;)V',
|
|
||||||
]
|
|
||||||
self.assertEqual(
|
|
||||||
expected,
|
|
||||||
self.signatureToElements(
|
|
||||||
'Ljava/lang/CharSequence$$ExternalSyntheticLambda0;'
|
|
||||||
'-><init>(Ljava/lang/CharSequence;)V'))
|
|
||||||
|
|
||||||
#pylint: disable=line-too-long
|
#pylint: disable=line-too-long
|
||||||
class TestDetectOverlaps(unittest.TestCase):
|
class TestDetectOverlaps(unittest.TestCase):
|
||||||
|
|
||||||
@@ -239,18 +194,6 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
|
|||||||
}
|
}
|
||||||
self.assertEqual(expected, subset)
|
self.assertEqual(expected, subset)
|
||||||
|
|
||||||
def test_extract_subset_invalid_pattern_wildcard_and_member(self):
|
|
||||||
monolithic = self.read_flag_trie_from_string(
|
|
||||||
TestDetectOverlaps.extractInput)
|
|
||||||
|
|
||||||
patterns = 'Ljava/lang/*;->hashCode()I'
|
|
||||||
|
|
||||||
with self.assertRaises(Exception) as context:
|
|
||||||
self.extract_subset_from_monolithic_flags_as_dict_from_string(
|
|
||||||
monolithic, patterns)
|
|
||||||
self.assertTrue('contains wildcard * and member signature hashCode()I'
|
|
||||||
in str(context.exception))
|
|
||||||
|
|
||||||
def test_read_trie_duplicate(self):
|
def test_read_trie_duplicate(self):
|
||||||
with self.assertRaises(Exception) as context:
|
with self.assertRaises(Exception) as context:
|
||||||
self.read_flag_trie_from_string("""
|
self.read_flag_trie_from_string("""
|
||||||
@@ -369,6 +312,8 @@ Ljava/lang/Object;->hashCode()I,blocked
|
|||||||
mismatches = compare_signature_flags(monolithic, modular)
|
mismatches = compare_signature_flags(monolithic, modular)
|
||||||
expected = []
|
expected = []
|
||||||
self.assertEqual(expected, mismatches)
|
self.assertEqual(expected, mismatches)
|
||||||
|
|
||||||
|
|
||||||
#pylint: enable=line-too-long
|
#pylint: enable=line-too-long
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
Reference in New Issue
Block a user