Separate creation of signature patterns from overlap checking

Previously, the signatures used to select the subset of the monolithic
flags were simply the signatures read from the modular flags file. This
change moves the creation of the signature list into a separate script
that outputs the signatures to a file and then passes the path through
Soong from the bootclasspath_fragment modules that create it to the
platform_bootclasspath module that uses it to compare the modular
flags against the monolithic flags.

Currently, the signatures are the full signatures but follow up changes
will replace them with patterns (hence the name) that avoids having to
include implementation details in the hidden API flags that are output
as part of a bootclasspath_fragment's snapshot.

This change moves the stub flags related code next to the all flags
related code as they are treated in a similar way.

Bug: 194063708
Test: atest --host verify_overlaps_test signature_patterns_test
      m out/soong/hiddenapi/hiddenapi-flags.csv
      - manually change files to cause difference in flags to check
        that it detects the differences.
Change-Id: I2855bf6d05c91b8a09591664185750361c7e644f

scripts/hiddenapi/Android.bp

@@ -104,3 +104,39 @@ python_test_host {
         unit_test: true,
     },
 }
+
+python_binary_host {
+    name: "signature_patterns",
+    main: "signature_patterns.py",
+    srcs: ["signature_patterns.py"],
+    version: {
+        py2: {
+            enabled: false,
+        },
+        py3: {
+            enabled: true,
+            embedded_launcher: true,
+        },
+    },
+}
+
+python_test_host {
+    name: "signature_patterns_test",
+    main: "signature_patterns_test.py",
+    srcs: [
+        "signature_patterns.py",
+        "signature_patterns_test.py",
+    ],
+    version: {
+        py2: {
+            enabled: false,
+        },
+        py3: {
+            enabled: true,
+            embedded_launcher: true,
+        },
+    },
+    test_options: {
+        unit_test: true,
+    },
+}

scripts/hiddenapi/signature_patterns.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Generate a set of signature patterns from the modular flags generated by a
+bootclasspath_fragment that can be used to select a subset of monolithic flags
+against which the modular flags can be compared.
+"""
+
+import argparse
+import csv
+
+def dict_reader(input):
+    return csv.DictReader(input, delimiter=',', quotechar='|', fieldnames=['signature'])
+
+def produce_patterns_from_file(file):
+    with open(file, 'r') as f:
+        return produce_patterns_from_stream(f)
+
+def produce_patterns_from_stream(stream):
+    patterns = []
+    allFlagsReader = dict_reader(stream)
+    for row in allFlagsReader:
+        signature = row['signature']
+        patterns.append(signature)
+    return patterns
+
+def main(args):
+    args_parser = argparse.ArgumentParser(description='Generate a set of signature patterns that select a subset of monolithic hidden API files.')
+    args_parser.add_argument('--flags', help='The stub flags file which contains an entry for every dex member')
+    args_parser.add_argument('--output', help='Generated signature prefixes')
+    args = args_parser.parse_args(args)
+
+    # Read in all the patterns into a list.
+    patterns = produce_patterns_from_file(args.flags)
+
+    # Write out all the patterns.
+    with open(args.output, 'w') as outputFile:
+        for pattern in patterns:
+            outputFile.write(pattern)
+            outputFile.write("\n")
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

scripts/hiddenapi/signature_patterns_test.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for signature_patterns.py."""
+import io
+import unittest
+
+from signature_patterns import *
+
+class TestGeneratedPatterns(unittest.TestCase):
+
+    def produce_patterns_from_string(self, csv):
+        with io.StringIO(csv) as f:
+            return produce_patterns_from_stream(f)
+
+    def test_generate(self):
+        patterns = self.produce_patterns_from_string('''
+Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
+Ljava/lang/Object;->toString()Ljava/lang/String;,blocked
+''')
+        expected = [
+            "Ljava/lang/Object;->hashCode()I",
+            "Ljava/lang/Object;->toString()Ljava/lang/String;",
+        ]
+        self.assertEqual(expected, patterns)
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)

scripts/hiddenapi/verify_overlaps.py

@@ -24,16 +24,30 @@ from itertools import chain
 def dict_reader(input):
     return csv.DictReader(input, delimiter=',', quotechar='|', fieldnames=['signature'])
 
-def extract_subset_from_monolithic_flags_as_dict(monolithicFlagsDict, signatures):
+def extract_subset_from_monolithic_flags_as_dict_from_file(monolithicFlagsDict, patternsFile):
     """
     Extract a subset of flags from the dict containing all the monolithic flags.
 
     :param monolithicFlagsDict: the dict containing all the monolithic flags.
-    :param signatures: a list of signature that define the subset.
+    :param patternsFile: a file containing a list of signature patterns that
+    define the subset.
+    :return: the dict from signature to row.
+    """
+    with open(patternsFile, 'r') as stream:
+        return extract_subset_from_monolithic_flags_as_dict_from_stream(monolithicFlagsDict, stream)
+
+def extract_subset_from_monolithic_flags_as_dict_from_stream(monolithicFlagsDict, stream):
+    """
+    Extract a subset of flags from the dict containing all the monolithic flags.
+
+    :param monolithicFlagsDict: the dict containing all the monolithic flags.
+    :param stream: a stream containing a list of signature patterns that define
+    the subset.
     :return: the dict from signature to row.
     """
     dict = {}
-    for signature in signatures:
+    for signature in stream:
+        signature = signature.rstrip()
         dict[signature] = monolithicFlagsDict.get(signature, {})
     return dict
 
@@ -102,9 +116,12 @@ def main(argv):
     # provided by the subset and the corresponding flags from the complete set of
     # flags and compare them.
     failed = False
-    for modularFlagsPath in args.modularFlags:
+    for modularPair in args.modularFlags:
+        parts = modularPair.split(":")
+        modularFlagsPath = parts[0]
+        modularPatternsPath = parts[1]
         modularFlagsDict = read_signature_csv_from_file_as_dict(modularFlagsPath)
-        monolithicFlagsSubsetDict = extract_subset_from_monolithic_flags_as_dict(monolithicFlagsDict, modularFlagsDict.keys())
+        monolithicFlagsSubsetDict = extract_subset_from_monolithic_flags_as_dict_from_file(monolithicFlagsDict, modularPatternsPath)
         mismatchingSignatures = compare_signature_flags(monolithicFlagsSubsetDict, modularFlagsDict)
         if mismatchingSignatures:
             failed = True

scripts/hiddenapi/verify_overlaps_test.py

@@ -26,6 +26,10 @@ class TestDetectOverlaps(unittest.TestCase):
         with io.StringIO(csv) as f:
             return read_signature_csv_from_stream_as_dict(f)
 
+    def extract_subset_from_monolithic_flags_as_dict_from_string(self, monolithic, patterns):
+        with io.StringIO(patterns) as f:
+            return extract_subset_from_monolithic_flags_as_dict_from_stream(monolithic, f)
+
     extractInput = '''
 Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
 Ljava/lang/Object;->toString()Ljava/lang/String;,blocked
@@ -36,7 +40,10 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,blocked
         modular = self.read_signature_csv_from_string_as_dict('''
 Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
 ''')
-        subset = extract_subset_from_monolithic_flags_as_dict(monolithic, modular.keys())
+
+        patterns = 'Ljava/lang/Object;->hashCode()I'
+
+        subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(monolithic, patterns)
         expected = {
             'Ljava/lang/Object;->hashCode()I': {
                 None: ['public-api', 'system-api', 'test-api'],