280bae6d20
Previously, the sdk snapshot would include all the entries from the
stub-flags.csv and all-flags.csv modular files generated by a single
bootclasspath_fragment. That included implementation details, i.e.
class members that are not part of a stable API or the hidden API which
meant that the sdk snapshots were implementation dependent.
This change removes the implementation details from the modular flag
files, i.e. those entries that are only marked as "blocked". When
comparing the files against the corresponding subset of the monolithic
files it assumes that any entries missing from the modular flag files
are blocked.
Bug: 194063708
Test: atest --host verify_overlaps_test signature_patterns_test
m out/soong/hiddenapi/hiddenapi-flags.csv
- manually change files to cause difference in flags to check
that it detects the differences.
Change-Id: I6b67b2253cf029d6830b58a06ebb0c8fcaa0dd71
380 lines
15 KiB
Python
Executable File
380 lines
15 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Copyright (C) 2018 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""
|
|
Verify that one set of hidden API flags is a subset of another.
|
|
"""
|
|
|
|
import argparse
|
|
import csv
|
|
import sys
|
|
from itertools import chain
|
|
|
|
class InteriorNode:
|
|
"""
|
|
An interior node in a trie.
|
|
|
|
Each interior node has a dict that maps from an element of a signature to
|
|
either another interior node or a leaf. Each interior node represents either
|
|
a package, class or nested class. Class members are represented by a Leaf.
|
|
|
|
Associating the set of flags [public-api] with the signature
|
|
"Ljava/lang/Object;->String()Ljava/lang/String;" will cause the following
|
|
nodes to be created:
|
|
Node()
|
|
^- package:java -> Node()
|
|
^- package:lang -> Node()
|
|
^- class:Object -> Node()
|
|
^- member:String()Ljava/lang/String; -> Leaf([public-api])
|
|
|
|
Associating the set of flags [blocked,core-platform-api] with the signature
|
|
"Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;"
|
|
will cause the following nodes to be created:
|
|
Node()
|
|
^- package:java -> Node()
|
|
^- package:lang -> Node()
|
|
^- class:Character -> Node()
|
|
^- class:UnicodeScript -> Node()
|
|
^- member:of(I)Ljava/lang/Character$UnicodeScript;
|
|
-> Leaf([blocked,core-platform-api])
|
|
|
|
Attributes:
|
|
nodes: a dict from an element of the signature to the Node/Leaf
|
|
containing the next element/value.
|
|
"""
|
|
def __init__(self):
|
|
self.nodes = {}
|
|
|
|
def signatureToElements(self, signature):
|
|
"""
|
|
Split a signature or a prefix into a number of elements:
|
|
1. The packages (excluding the leading L preceding the first package).
|
|
2. The class names, from outermost to innermost.
|
|
3. The member signature.
|
|
|
|
e.g. Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;
|
|
will be broken down into these elements:
|
|
1. package:java
|
|
2. package:lang
|
|
3. class:Character
|
|
4. class:UnicodeScript
|
|
5. member:of(I)Ljava/lang/Character$UnicodeScript;
|
|
"""
|
|
# Remove the leading L.
|
|
# - java/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;
|
|
text = signature.removeprefix("L")
|
|
# Split the signature between qualified class name and the class member
|
|
# signature.
|
|
# 0 - java/lang/Character$UnicodeScript
|
|
# 1 - of(I)Ljava/lang/Character$UnicodeScript;
|
|
parts = text.split(";->")
|
|
member = parts[1:]
|
|
# Split the qualified class name into packages, and class name.
|
|
# 0 - java
|
|
# 1 - lang
|
|
# 2 - Character$UnicodeScript
|
|
elements = parts[0].split("/")
|
|
packages = elements[0:-1]
|
|
className = elements[-1]
|
|
if className == "*" or className == "**":
|
|
# Cannot specify a wildcard and target a specific member
|
|
if len(member) != 0:
|
|
raise Exception("Invalid signature %s: contains wildcard %s and member signature %s"
|
|
% (signature, className, member[0]))
|
|
wildcard = [className]
|
|
# Assemble the parts into a single list, adding prefixes to identify
|
|
# the different parts.
|
|
# 0 - package:java
|
|
# 1 - package:lang
|
|
# 2 - *
|
|
return list(chain(map(lambda x : "package:" + x, packages),
|
|
wildcard))
|
|
else:
|
|
# Split the class name into outer / inner classes
|
|
# 0 - Character
|
|
# 1 - UnicodeScript
|
|
classes = className.split("$")
|
|
# Assemble the parts into a single list, adding prefixes to identify
|
|
# the different parts.
|
|
# 0 - package:java
|
|
# 1 - package:lang
|
|
# 2 - class:Character
|
|
# 3 - class:UnicodeScript
|
|
# 4 - member:of(I)Ljava/lang/Character$UnicodeScript;
|
|
return list(chain(map(lambda x : "package:" + x, packages),
|
|
map(lambda x : "class:" + x, classes),
|
|
map(lambda x : "member:" + x, member)))
|
|
|
|
def add(self, signature, value):
|
|
"""
|
|
Associate the value with the specific signature.
|
|
:param signature: the member signature
|
|
:param value: the value to associated with the signature
|
|
:return: n/a
|
|
"""
|
|
# Split the signature into elements.
|
|
elements = self.signatureToElements(signature)
|
|
# Find the Node associated with the deepest class.
|
|
node = self
|
|
for element in elements[:-1]:
|
|
if element in node.nodes:
|
|
node = node.nodes[element]
|
|
else:
|
|
next = InteriorNode()
|
|
node.nodes[element] = next
|
|
node = next
|
|
# Add a Leaf containing the value and associate it with the member
|
|
# signature within the class.
|
|
lastElement = elements[-1]
|
|
if not lastElement.startswith("member:"):
|
|
raise Exception("Invalid signature: %s, does not identify a specific member" % signature)
|
|
if lastElement in node.nodes:
|
|
raise Exception("Duplicate signature: %s" % signature)
|
|
node.nodes[lastElement] = Leaf(value)
|
|
|
|
def getMatchingRows(self, pattern):
|
|
"""
|
|
Get the values (plural) associated with the pattern.
|
|
|
|
e.g. If the pattern is a full signature then this will return a list
|
|
containing the value associated with that signature.
|
|
|
|
If the pattern is a class then this will return a list containing the
|
|
values associated with all members of that class.
|
|
|
|
If the pattern is a package then this will return a list containing the
|
|
values associated with all the members of all the classes in that
|
|
package and sub-packages.
|
|
|
|
If the pattern ends with "*" then the preceding part is treated as a
|
|
package and this will return a list containing the values associated
|
|
with all the members of all the classes in that package.
|
|
|
|
If the pattern ends with "**" then the preceding part is treated
|
|
as a package and this will return a list containing the values
|
|
associated with all the members of all the classes in that package and
|
|
all sub-packages.
|
|
|
|
:param pattern: the pattern which could be a complete signature or a
|
|
class, or package wildcard.
|
|
:return: an iterable containing all the values associated with the
|
|
pattern.
|
|
"""
|
|
elements = self.signatureToElements(pattern)
|
|
node = self
|
|
# Include all values from this node and all its children.
|
|
selector = lambda x : True
|
|
lastElement = elements[-1]
|
|
if lastElement == "*" or lastElement == "**":
|
|
elements = elements[:-1]
|
|
if lastElement == "*":
|
|
# Do not include values from sub-packages.
|
|
selector = lambda x : not x.startswith("package:")
|
|
for element in elements:
|
|
if element in node.nodes:
|
|
node = node.nodes[element]
|
|
else:
|
|
return []
|
|
return chain.from_iterable(node.values(selector))
|
|
|
|
def values(self, selector):
|
|
"""
|
|
:param selector: a function that can be applied to a key in the nodes
|
|
attribute to determine whether to return its values.
|
|
:return: A list of iterables of all the values associated with this
|
|
node and its children.
|
|
"""
|
|
values = []
|
|
self.appendValues(values, selector)
|
|
return values
|
|
|
|
def appendValues(self, values, selector):
|
|
"""
|
|
Append the values associated with this node and its children to the
|
|
list.
|
|
|
|
For each item (key, child) in nodes the child node's values are returned
|
|
if and only if the selector returns True when called on its key. A child
|
|
node's values are all the values associated with it and all its
|
|
descendant nodes.
|
|
|
|
:param selector: a function that can be applied to a key in the nodes
|
|
attribute to determine whether to return its values.
|
|
:param values: a list of a iterables of values.
|
|
"""
|
|
for key, node in self.nodes.items():
|
|
if selector(key):
|
|
node.appendValues(values, lambda x : True)
|
|
|
|
class Leaf:
|
|
"""
|
|
A leaf of the trie
|
|
|
|
Attributes:
|
|
value: the value associated with this leaf.
|
|
"""
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
def values(self, selector):
|
|
"""
|
|
:return: A list of a list of the value associated with this node.
|
|
"""
|
|
return [[self.value]]
|
|
|
|
def appendValues(self, values, selector):
|
|
"""
|
|
Appends a list of the value associated with this node to the list.
|
|
:param values: a list of a iterables of values.
|
|
"""
|
|
values.append([self.value])
|
|
|
|
def dict_reader(input):
|
|
return csv.DictReader(input, delimiter=',', quotechar='|', fieldnames=['signature'])
|
|
|
|
def read_flag_trie_from_file(file):
|
|
with open(file, 'r') as stream:
|
|
return read_flag_trie_from_stream(stream)
|
|
|
|
def read_flag_trie_from_stream(stream):
|
|
trie = InteriorNode()
|
|
reader = dict_reader(stream)
|
|
for row in reader:
|
|
signature = row['signature']
|
|
trie.add(signature, row)
|
|
return trie
|
|
|
|
def extract_subset_from_monolithic_flags_as_dict_from_file(monolithicTrie, patternsFile):
|
|
"""
|
|
Extract a subset of flags from the dict containing all the monolithic flags.
|
|
|
|
:param monolithicFlagsDict: the dict containing all the monolithic flags.
|
|
:param patternsFile: a file containing a list of signature patterns that
|
|
define the subset.
|
|
:return: the dict from signature to row.
|
|
"""
|
|
with open(patternsFile, 'r') as stream:
|
|
return extract_subset_from_monolithic_flags_as_dict_from_stream(monolithicTrie, stream)
|
|
|
|
def extract_subset_from_monolithic_flags_as_dict_from_stream(monolithicTrie, stream):
|
|
"""
|
|
Extract a subset of flags from the trie containing all the monolithic flags.
|
|
|
|
:param monolithicTrie: the trie containing all the monolithic flags.
|
|
:param stream: a stream containing a list of signature patterns that define
|
|
the subset.
|
|
:return: the dict from signature to row.
|
|
"""
|
|
dict = {}
|
|
for pattern in stream:
|
|
pattern = pattern.rstrip()
|
|
rows = monolithicTrie.getMatchingRows(pattern)
|
|
for row in rows:
|
|
signature = row['signature']
|
|
dict[signature] = row
|
|
return dict
|
|
|
|
def read_signature_csv_from_stream_as_dict(stream):
|
|
"""
|
|
Read the csv contents from the stream into a dict. The first column is assumed to be the
|
|
signature and used as the key. The whole row is stored as the value.
|
|
|
|
:param stream: the csv contents to read
|
|
:return: the dict from signature to row.
|
|
"""
|
|
dict = {}
|
|
reader = dict_reader(stream)
|
|
for row in reader:
|
|
signature = row['signature']
|
|
dict[signature] = row
|
|
return dict
|
|
|
|
def read_signature_csv_from_file_as_dict(csvFile):
|
|
"""
|
|
Read the csvFile into a dict. The first column is assumed to be the
|
|
signature and used as the key. The whole row is stored as the value.
|
|
|
|
:param csvFile: the csv file to read
|
|
:return: the dict from signature to row.
|
|
"""
|
|
with open(csvFile, 'r') as f:
|
|
return read_signature_csv_from_stream_as_dict(f)
|
|
|
|
def compare_signature_flags(monolithicFlagsDict, modularFlagsDict):
|
|
"""
|
|
Compare the signature flags between the two dicts.
|
|
|
|
:param monolithicFlagsDict: the dict containing the subset of the monolithic
|
|
flags that should be equal to the modular flags.
|
|
:param modularFlagsDict:the dict containing the flags produced by a single
|
|
bootclasspath_fragment module.
|
|
:return: list of mismatches., each mismatch is a tuple where the first item
|
|
is the signature, and the second and third items are lists of the flags from
|
|
modular dict, and monolithic dict respectively.
|
|
"""
|
|
mismatchingSignatures = []
|
|
# Create a sorted set of all the signatures from both the monolithic and
|
|
# modular dicts.
|
|
allSignatures = sorted(set(chain(monolithicFlagsDict.keys(), modularFlagsDict.keys())))
|
|
for signature in allSignatures:
|
|
monolithicRow = monolithicFlagsDict.get(signature, {})
|
|
monolithicFlags = monolithicRow.get(None, [])
|
|
if signature in modularFlagsDict:
|
|
modularRow = modularFlagsDict.get(signature, {})
|
|
modularFlags = modularRow.get(None, [])
|
|
else:
|
|
modularFlags = ["blocked"]
|
|
if monolithicFlags != modularFlags:
|
|
mismatchingSignatures.append((signature, modularFlags, monolithicFlags))
|
|
return mismatchingSignatures
|
|
|
|
def main(argv):
|
|
args_parser = argparse.ArgumentParser(description='Verify that sets of hidden API flags are each a subset of the monolithic flag file.')
|
|
args_parser.add_argument('monolithicFlags', help='The monolithic flag file')
|
|
args_parser.add_argument('modularFlags', nargs=argparse.REMAINDER, help='Flags produced by individual bootclasspath_fragment modules')
|
|
args = args_parser.parse_args(argv[1:])
|
|
|
|
# Read in all the flags into the trie
|
|
monolithicFlagsPath = args.monolithicFlags
|
|
monolithicTrie = read_flag_trie_from_file(monolithicFlagsPath)
|
|
|
|
# For each subset specified on the command line, create dicts for the flags
|
|
# provided by the subset and the corresponding flags from the complete set
|
|
# of flags and compare them.
|
|
failed = False
|
|
for modularPair in args.modularFlags:
|
|
parts = modularPair.split(":")
|
|
modularFlagsPath = parts[0]
|
|
modularPatternsPath = parts[1]
|
|
modularFlagsDict = read_signature_csv_from_file_as_dict(modularFlagsPath)
|
|
monolithicFlagsSubsetDict = extract_subset_from_monolithic_flags_as_dict_from_file(monolithicTrie, modularPatternsPath)
|
|
mismatchingSignatures = compare_signature_flags(monolithicFlagsSubsetDict, modularFlagsDict)
|
|
if mismatchingSignatures:
|
|
failed = True
|
|
print("ERROR: Hidden API flags are inconsistent:")
|
|
print("< " + modularFlagsPath)
|
|
print("> " + monolithicFlagsPath)
|
|
for mismatch in mismatchingSignatures:
|
|
signature = mismatch[0]
|
|
print()
|
|
print("< " + ",".join([signature]+ mismatch[1]))
|
|
print("> " + ",".join([signature]+ mismatch[2]))
|
|
|
|
if failed:
|
|
sys.exit(1)
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv)
|