Add analyze_bcpf.py script to help manage bootclasspath_fragments am: 4dcf65951b am: a87d148823

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/2020741

Change-Id: Ifdbd09d2ccae7cc0beb57b12e037a7edc409da34
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

scripts/hiddenapi/Android.bp

@@ -18,6 +18,48 @@ package {
     default_applicable_licenses: ["Android-Apache-2.0"],
 }
 
+python_binary_host {
+    name: "analyze_bcpf",
+    main: "analyze_bcpf.py",
+    srcs: ["analyze_bcpf.py"],
+    libs: [
+        "signature_trie",
+    ],
+    version: {
+        py2: {
+            enabled: false,
+        },
+        py3: {
+            enabled: true,
+            embedded_launcher: true,
+        },
+    },
+}
+
+python_test_host {
+    name: "analyze_bcpf_test",
+    main: "analyze_bcpf_test.py",
+    srcs: [
+        "analyze_bcpf.py",
+        "analyze_bcpf_test.py",
+    ],
+    libs: [
+        "signature_trie",
+    ],
+    version: {
+        py2: {
+            enabled: false,
+        },
+        py3: {
+            enabled: true,
+            embedded_launcher: true,
+        },
+    },
+    test_options: {
+        unit_test: true,
+    },
+}
+
 python_binary_host {
     name: "merge_csv",
     main: "merge_csv.py",

scripts/hiddenapi/analyze_bcpf.py

@@ -0,0 +1,794 @@
+#!/usr/bin/env -S python -u
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Analyze bootclasspath_fragment usage."""
+import argparse
+import dataclasses
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import textwrap
+import typing
+import sys
+
+_STUB_FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-stub-flags.txt"
+
+_FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-flags.csv"
+
+_INCONSISTENT_FLAGS = "ERROR: Hidden API flags are inconsistent:"
+
+
+class BuildOperation:
+
+    def __init__(self, popen):
+        self.popen = popen
+        self.returncode = None
+
+    def lines(self):
+        """Return an iterator over the lines output by the build operation.
+
+        The lines have had any trailing white space, including the newline
+        stripped.
+        """
+        return newline_stripping_iter(self.popen.stdout.readline)
+
+    def wait(self, *args, **kwargs):
+        self.popen.wait(*args, **kwargs)
+        self.returncode = self.popen.returncode
+
+
+@dataclasses.dataclass()
+class FlagDiffs:
+    """Encapsulates differences in flags reported by the build"""
+
+    # Map from member signature to the (module flags, monolithic flags)
+    diffs: typing.Dict[str, typing.Tuple[str, str]]
+
+
+@dataclasses.dataclass()
+class ModuleInfo:
+    """Provides access to the generated module-info.json file.
+
+    This is used to find the location of the file within which specific modules
+    are defined.
+    """
+
+    modules: typing.Dict[str, typing.Dict[str, typing.Any]]
+
+    @staticmethod
+    def load(filename):
+        with open(filename, "r", encoding="utf8") as f:
+            j = json.load(f)
+            return ModuleInfo(j)
+
+    def _module(self, module_name):
+        """Find module by name in module-info.json file"""
+        if module_name in self.modules:
+            return self.modules[module_name]
+
+        raise Exception(f"Module {module_name} could not be found")
+
+    def module_path(self, module_name):
+        module = self._module(module_name)
+        # The "path" is actually a list of paths, one for each class of module
+        # but as the modules are all created from bp files if a module does
+        # create multiple classes of make modules they should all have the same
+        # path.
+        paths = module["path"]
+        unique_paths = set(paths)
+        if len(unique_paths) != 1:
+            raise Exception(f"Expected module '{module_name}' to have a "
+                            f"single unique path but found {unique_paths}")
+        return paths[0]
+
+
+@dataclasses.dataclass
+class FileChange:
+    path: str
+
+    description: str
+
+    def __lt__(self, other):
+        return self.path < other.path
+
+
+@dataclasses.dataclass
+class HiddenApiPropertyChange:
+
+    property_name: str
+
+    values: typing.List[str]
+
+    property_comment: str = ""
+
+    def snippet(self, indent):
+        snippet = "\n"
+        snippet += format_comment_as_text(self.property_comment, indent)
+        snippet += f"{indent}{self.property_name}: ["
+        if self.values:
+            snippet += "\n"
+            for value in self.values:
+                snippet += f'{indent}    "{value}",\n'
+            snippet += f"{indent}"
+        snippet += "],\n"
+        return snippet
+
+
+@dataclasses.dataclass()
+class Result:
+    """Encapsulates the result of the analysis."""
+
+    # The diffs in the flags.
+    diffs: typing.Optional[FlagDiffs] = None
+
+    # The bootclasspath_fragment hidden API properties changes.
+    property_changes: typing.List[HiddenApiPropertyChange] = dataclasses.field(
+        default_factory=list)
+
+    # The list of file changes.
+    file_changes: typing.List[FileChange] = dataclasses.field(
+        default_factory=list)
+
+
+@dataclasses.dataclass()
+class BcpfAnalyzer:
+    # Directory pointed to by ANDROID_BUILD_OUT
+    top_dir: str
+
+    # Directory pointed to by OUT_DIR of {top_dir}/out if that is not set.
+    out_dir: str
+
+    # Directory pointed to by ANDROID_PRODUCT_OUT.
+    product_out_dir: str
+
+    # The name of the bootclasspath_fragment module.
+    bcpf: str
+
+    # The name of the apex module containing {bcpf}, only used for
+    # informational purposes.
+    apex: str
+
+    # The name of the sdk module containing {bcpf}, only used for
+    # informational purposes.
+    sdk: str
+
+    # All the signatures, loaded from all-flags.csv, initialized by
+    # load_all_flags().
+    _signatures: typing.Set[str] = dataclasses.field(default_factory=set)
+
+    # All the classes, loaded from all-flags.csv, initialized by
+    # load_all_flags().
+    _classes: typing.Set[str] = dataclasses.field(default_factory=set)
+
+    # Information loaded from module-info.json, initialized by
+    # load_module_info().
+    module_info: ModuleInfo = None
+
+    @staticmethod
+    def reformat_report_test(text):
+        return re.sub(r"(.)\n([^\s])", r"\1 \2", text)
+
+    def report(self, text, **kwargs):
+        # Concatenate lines that are not separated by a blank line together to
+        # eliminate formatting applied to the supplied text to adhere to python
+        # line length limitations.
+        text = self.reformat_report_test(text)
+        logging.info("%s", text, **kwargs)
+
+    def run_command(self, cmd, *args, **kwargs):
+        cmd_line = " ".join(cmd)
+        logging.debug("Running %s", cmd_line)
+        subprocess.run(
+            cmd,
+            *args,
+            check=True,
+            cwd=self.top_dir,
+            stderr=subprocess.STDOUT,
+            stdout=log_stream_for_subprocess(),
+            text=True,
+            **kwargs)
+
+    @property
+    def signatures(self):
+        if not self._signatures:
+            raise Exception("signatures has not been initialized")
+        return self._signatures
+
+    @property
+    def classes(self):
+        if not self._classes:
+            raise Exception("classes has not been initialized")
+        return self._classes
+
+    def load_all_flags(self):
+        all_flags = self.find_bootclasspath_fragment_output_file(
+            "all-flags.csv")
+
+        # Extract the set of signatures and a separate set of classes produced
+        # by the bootclasspath_fragment.
+        with open(all_flags, "r", encoding="utf8") as f:
+            for line in newline_stripping_iter(f.readline):
+                signature = self.line_to_signature(line)
+                self._signatures.add(signature)
+                class_name = self.signature_to_class(signature)
+                self._classes.add(class_name)
+
+    def load_module_info(self):
+        module_info_file = os.path.join(self.product_out_dir,
+                                        "module-info.json")
+        self.report(f"""
+Making sure that {module_info_file} is up to date.
+""")
+        output = self.build_file_read_output(module_info_file)
+        lines = output.lines()
+        for line in lines:
+            logging.debug("%s", line)
+        output.wait(timeout=10)
+        if output.returncode:
+            raise Exception(f"Error building {module_info_file}")
+        abs_module_info_file = os.path.join(self.top_dir, module_info_file)
+        self.module_info = ModuleInfo.load(abs_module_info_file)
+
+    @staticmethod
+    def line_to_signature(line):
+        return line.split(",")[0]
+
+    @staticmethod
+    def signature_to_class(signature):
+        return signature.split(";->")[0]
+
+    @staticmethod
+    def to_parent_package(pkg_or_class):
+        return pkg_or_class.rsplit("/", 1)[0]
+
+    def module_path(self, module_name):
+        return self.module_info.module_path(module_name)
+
+    def module_out_dir(self, module_name):
+        module_path = self.module_path(module_name)
+        return os.path.join(self.out_dir, "soong/.intermediates", module_path,
+                            module_name)
+
+    def find_bootclasspath_fragment_output_file(self, basename):
+        # Find the output file of the bootclasspath_fragment with the specified
+        # base name.
+        found_file = ""
+        bcpf_out_dir = self.module_out_dir(self.bcpf)
+        for (dirpath, _, filenames) in os.walk(bcpf_out_dir):
+            for f in filenames:
+                if f == basename:
+                    found_file = os.path.join(dirpath, f)
+                    break
+        if not found_file:
+            raise Exception(f"Could not find {basename} in {bcpf_out_dir}")
+        return found_file
+
+    def analyze(self):
+        """Analyze a bootclasspath_fragment module.
+
+        Provides help in resolving any existing issues and provides
+        optimizations that can be applied.
+        """
+        self.report(f"Analyzing bootclasspath_fragment module {self.bcpf}")
+        self.report(f"""
+Run this tool to help initialize a bootclasspath_fragment module. Before you
+start make sure that:
+
+1. The current checkout is up to date.
+
+2. The environment has been initialized using lunch, e.g.
+   lunch aosp_arm64-userdebug
+
+3. You have added a bootclasspath_fragment module to the appropriate Android.bp
+file. Something like this:
+
+   bootclasspath_fragment {{
+     name: "{self.bcpf}",
+     contents: [
+       "...",
+     ],
+
+     // The bootclasspath_fragments that provide APIs on which this depends.
+     fragments: [
+       {{
+         apex: "com.android.art",
+         module: "art-bootclasspath-fragment",
+       }},
+     ],
+   }}
+
+4. You have added it to the platform_bootclasspath module in
+frameworks/base/boot/Android.bp. Something like this:
+
+   platform_bootclasspath {{
+     name: "platform-bootclasspath",
+     fragments: [
+       ...
+       {{
+         apex: "{self.apex}",
+         module: "{self.bcpf}",
+       }},
+     ],
+   }}
+
+5. You have added an sdk module. Something like this:
+
+   sdk {{
+     name: "{self.sdk}",
+     bootclasspath_fragments: ["{self.bcpf}"],
+   }}
+""")
+
+        # Make sure that the module-info.json file is up to date.
+        self.load_module_info()
+
+        self.report("""
+Cleaning potentially stale files.
+""")
+        # Remove the out/soong/hiddenapi files.
+        shutil.rmtree(f"{self.out_dir}/soong/hiddenapi", ignore_errors=True)
+
+        # Remove any bootclasspath_fragment output files.
+        shutil.rmtree(self.module_out_dir(self.bcpf), ignore_errors=True)
+
+        self.build_monolithic_stubs_flags()
+
+        result = Result()
+
+        self.build_monolithic_flags(result)
+
+        # If there were any changes that need to be made to the Android.bp
+        # file then report them.
+        if result.property_changes:
+            bcpf_dir = self.module_info.module_path(self.bcpf)
+            bcpf_bp_file = os.path.join(self.top_dir, bcpf_dir, "Android.bp")
+            hiddenapi_snippet = ""
+            for property_change in result.property_changes:
+                hiddenapi_snippet += property_change.snippet("        ")
+
+            # Remove leading and trailing blank lines.
+            hiddenapi_snippet = hiddenapi_snippet.strip("\n")
+
+            result.file_changes.append(
+                self.new_file_change(
+                    bcpf_bp_file, f"""
+Add the following snippet into the {self.bcpf} bootclasspath_fragment module
+in the {bcpf_dir}/Android.bp file. If the hidden_api block already exists then
+merge these properties into it.
+
+    hidden_api: {{
+{hiddenapi_snippet}
+    }},
+"""))
+
+        if result.file_changes:
+            self.report("""
+The following modifications need to be made:""")
+            result.file_changes.sort()
+            for file_change in result.file_changes:
+                self.report(f"""
+    {file_change.path}
+        {file_change.description}
+""".lstrip("\n"))
+
+    def new_file_change(self, file, description):
+        return FileChange(
+            path=os.path.relpath(file, self.top_dir), description=description)
+
+    def check_inconsistent_flag_lines(self, significant, module_line,
+                                      monolithic_line, separator_line):
+        if not (module_line.startswith("< ") and
+                monolithic_line.startswith("> ") and not separator_line):
+            # Something went wrong.
+            self.report(f"""Invalid build output detected:
+  module_line: "{module_line}"
+  monolithic_line: "{monolithic_line}"
+  separator_line: "{separator_line}"
+""")
+            sys.exit(1)
+
+        if significant:
+            logging.debug("%s", module_line)
+            logging.debug("%s", monolithic_line)
+            logging.debug("%s", separator_line)
+
+    def scan_inconsistent_flags_report(self, lines):
+        """Scans a hidden API flags report
+
+        The hidden API inconsistent flags report which looks something like
+        this.
+
+        < out/soong/.intermediates/.../filtered-stub-flags.csv
+        > out/soong/hiddenapi/hiddenapi-stub-flags.txt
+
+        < Landroid/compat/Compatibility;->clearOverrides()V
+        > Landroid/compat/Compatibility;->clearOverrides()V,core-platform-api
+
+        """
+
+        # The basic format of an entry in the inconsistent flags report is:
+        #   <module specific flag>
+        #   <monolithic flag>
+        #   <separator>
+        #
+        # Wrap the lines iterator in an iterator which returns a tuple
+        # consisting of the three separate lines.
+        triples = zip(lines, lines, lines)
+
+        module_line, monolithic_line, separator_line = next(triples)
+        significant = False
+        bcpf_dir = self.module_info.module_path(self.bcpf)
+        if os.path.join(bcpf_dir, self.bcpf) in module_line:
+            # These errors are related to the bcpf being analyzed so
+            # keep them.
+            significant = True
+        else:
+            self.report(f"Filtering out errors related to {module_line}")
+
+        self.check_inconsistent_flag_lines(significant, module_line,
+                                           monolithic_line, separator_line)
+
+        diffs = {}
+        for module_line, monolithic_line, separator_line in triples:
+            self.check_inconsistent_flag_lines(significant, module_line,
+                                               monolithic_line, "")
+
+            module_parts = module_line.removeprefix("< ").split(",")
+            module_signature = module_parts[0]
+            module_flags = module_parts[1:]
+
+            monolithic_parts = monolithic_line.removeprefix("> ").split(",")
+            monolithic_signature = monolithic_parts[0]
+            monolithic_flags = monolithic_parts[1:]
+
+            if module_signature != monolithic_signature:
+                # Something went wrong.
+                self.report(f"""Inconsistent signatures detected:
+  module_signature: "{module_signature}"
+  monolithic_signature: "{monolithic_signature}"
+""")
+                sys.exit(1)
+
+            diffs[module_signature] = (module_flags, monolithic_flags)
+
+            if separator_line:
+                # If the separator line is not blank then it is the end of the
+                # current report, and possibly the start of another.
+                return separator_line, diffs
+
+        return "", diffs
+
+    def build_file_read_output(self, filename):
+        # Make sure the filename is relative to top if possible as the build
+        # may be using relative paths as the target.
+        rel_filename = filename.removeprefix(self.top_dir)
+        cmd = ["build/soong/soong_ui.bash", "--make-mode", rel_filename]
+        cmd_line = " ".join(cmd)
+        logging.debug("%s", cmd_line)
+        # pylint: disable=consider-using-with
+        output = subprocess.Popen(
+            cmd,
+            cwd=self.top_dir,
+            stderr=subprocess.STDOUT,
+            stdout=subprocess.PIPE,
+            text=True,
+        )
+        return BuildOperation(popen=output)
+
+    def build_hiddenapi_flags(self, filename):
+        output = self.build_file_read_output(filename)
+
+        lines = output.lines()
+        diffs = None
+        for line in lines:
+            logging.debug("%s", line)
+            while line == _INCONSISTENT_FLAGS:
+                line, diffs = self.scan_inconsistent_flags_report(lines)
+
+        output.wait(timeout=10)
+        if output.returncode != 0:
+            logging.debug("Command failed with %s", output.returncode)
+        else:
+            logging.debug("Command succeeded")
+
+        return diffs
+
+    def build_monolithic_stubs_flags(self):
+        self.report(f"""
+Attempting to build {_STUB_FLAGS_FILE} to verify that the
+bootclasspath_fragment has the correct API stubs available...
+""")
+
+        # Build the hiddenapi-stubs-flags.txt file.
+        diffs = self.build_hiddenapi_flags(_STUB_FLAGS_FILE)
+        if diffs:
+            self.report(f"""
+There is a discrepancy between the stub API derived flags created by the
+bootclasspath_fragment and the platform_bootclasspath. See preceding error
+messages to see which flags are inconsistent. The inconsistencies can occur for
+a couple of reasons:
+
+If you are building against prebuilts of the Android SDK, e.g. by using
+TARGET_BUILD_APPS then the prebuilt versions of the APIs this
+bootclasspath_fragment depends upon are out of date and need updating. See
+go/update-prebuilts for help.
+
+Otherwise, this is happening because there are some stub APIs that are either
+provided by or used by the contents of the bootclasspath_fragment but which are
+not available to it. There are 4 ways to handle this:
+
+1. A java_sdk_library in the contents property will automatically make its stub
+   APIs available to the bootclasspath_fragment so nothing needs to be done.
+
+2. If the API provided by the bootclasspath_fragment is created by an api_only
+   java_sdk_library (or a java_library that compiles files generated by a
+   separate droidstubs module then it cannot be added to the contents and
+   instead must be added to the api.stubs property, e.g.
+
+   bootclasspath_fragment {{
+     name: "{self.bcpf}",
+     ...
+     api: {{
+       stubs: ["$MODULE-api-only"],"
+     }},
+   }}
+
+3. If the contents use APIs provided by another bootclasspath_fragment then
+   it needs to be added to the fragments property, e.g.
+
+   bootclasspath_fragment {{
+     name: "{self.bcpf}",
+     ...
+     // The bootclasspath_fragments that provide APIs on which this depends.
+     fragments: [
+       ...
+       {{
+         apex: "com.android.other",
+         module: "com.android.other-bootclasspath-fragment",
+       }},
+     ],
+   }}
+
+4. If the contents use APIs from a module that is not part of another
+   bootclasspath_fragment then it must be added to the additional_stubs
+   property, e.g.
+
+   bootclasspath_fragment {{
+     name: "{self.bcpf}",
+     ...
+     additional_stubs: ["android-non-updatable"],
+   }}
+
+   Like the api.stubs property these are typically java_sdk_library modules but
+   can be java_library too.
+
+   Note: The "android-non-updatable" is treated as if it was a java_sdk_library
+   which it is not at the moment but will be in future.
+""")
+
+        return diffs
+
+    def build_monolithic_flags(self, result):
+        self.report(f"""
+Attempting to build {_FLAGS_FILE} to verify that the
+bootclasspath_fragment has the correct hidden API flags...
+""")
+
+        # Build the hiddenapi-flags.csv file and extract any differences in
+        # the flags between this bootclasspath_fragment and the monolithic
+        # files.
+        result.diffs = self.build_hiddenapi_flags(_FLAGS_FILE)
+
+        # Load information from the bootclasspath_fragment's all-flags.csv file.
+        self.load_all_flags()
+
+        if result.diffs:
+            self.report(f"""
+There is a discrepancy between the hidden API flags created by the
+bootclasspath_fragment and the platform_bootclasspath. See preceding error
+messages to see which flags are inconsistent. The inconsistencies can occur for
+a couple of reasons:
+
+If you are building against prebuilts of this bootclasspath_fragment then the
+prebuilt version of the sdk snapshot (specifically the hidden API flag files)
+are inconsistent with the prebuilt version of the apex {self.apex}. Please
+ensure that they are both updated from the same build.
+
+1. There are custom hidden API flags specified in the one of the files in
+   frameworks/base/boot/hiddenapi which apply to the bootclasspath_fragment but
+   which are not supplied to the bootclasspath_fragment module.
+
+2. The bootclasspath_fragment specifies invalid "package_prefixes" or
+   "split_packages" properties that match packages and classes that it does not
+   provide.
+
+""")
+
+            # Check to see if there are any hiddenapi related properties that
+            # need to be added to the
+            self.report("""
+Checking custom hidden API flags....
+""")
+            self.check_frameworks_base_boot_hidden_api_files(result)
+
+    def report_hidden_api_flag_file_changes(self, result, property_name,
+                                            flags_file, rel_bcpf_flags_file,
+                                            bcpf_flags_file):
+        matched_signatures = set()
+        # Open the flags file to read the flags from.
+        with open(flags_file, "r", encoding="utf8") as f:
+            for signature in newline_stripping_iter(f.readline):
+                if signature in self.signatures:
+                    # The signature is provided by the bootclasspath_fragment so
+                    # it will need to be moved to the bootclasspath_fragment
+                    # specific file.
+                    matched_signatures.add(signature)
+
+        # If the bootclasspath_fragment specific flags file is not empty
+        # then it contains flags. That could either be new flags just moved
+        # from frameworks/base or previous contents of the file. In either
+        # case the file must not be removed.
+        if matched_signatures:
+            insert = textwrap.indent("\n".join(matched_signatures),
+                                     "            ")
+            result.file_changes.append(
+                self.new_file_change(
+                    flags_file, f"""Remove the following entries:
+{insert}
+"""))
+
+            result.file_changes.append(
+                self.new_file_change(
+                    bcpf_flags_file, f"""Add the following entries:
+{insert}
+"""))
+
+            result.property_changes.append(
+                HiddenApiPropertyChange(
+                    property_name=property_name,
+                    values=[rel_bcpf_flags_file],
+                ))
+
+    def check_frameworks_base_boot_hidden_api_files(self, result):
+        hiddenapi_dir = os.path.join(self.top_dir,
+                                     "frameworks/base/boot/hiddenapi")
+        for basename in sorted(os.listdir(hiddenapi_dir)):
+            if not (basename.startswith("hiddenapi-") and
+                    basename.endswith(".txt")):
+                continue
+
+            flags_file = os.path.join(hiddenapi_dir, basename)
+
+            logging.debug("Checking %s for flags related to %s", flags_file,
+                          self.bcpf)
+
+            # Map the file name in frameworks/base/boot/hiddenapi into a
+            # slightly more meaningful name for use by the
+            # bootclasspath_fragment.
+            if basename == "hiddenapi-max-target-o.txt":
+                basename = "hiddenapi-max-target-o-low-priority.txt"
+            elif basename == "hiddenapi-max-target-r-loprio.txt":
+                basename = "hiddenapi-max-target-r-low-priority.txt"
+
+            property_name = basename.removeprefix("hiddenapi-")
+            property_name = property_name.removesuffix(".txt")
+            property_name = property_name.replace("-", "_")
+
+            rel_bcpf_flags_file = f"hiddenapi/{basename}"
+            bcpf_dir = self.module_info.module_path(self.bcpf)
+            bcpf_flags_file = os.path.join(self.top_dir, bcpf_dir,
+                                           rel_bcpf_flags_file)
+
+            self.report_hidden_api_flag_file_changes(result, property_name,
+                                                     flags_file,
+                                                     rel_bcpf_flags_file,
+                                                     bcpf_flags_file)
+
+
+def newline_stripping_iter(iterator):
+    """Return an iterator over the iterator that strips trailing white space."""
+    lines = iter(iterator, "")
+    lines = (line.rstrip() for line in lines)
+    return lines
+
+
+def format_comment_as_text(text, indent):
+    return "".join(
+        [f"{line}\n" for line in format_comment_as_lines(text, indent)])
+
+
+def format_comment_as_lines(text, indent):
+    lines = textwrap.wrap(text.strip("\n"), width=77 - len(indent))
+    lines = [f"{indent}// {line}" for line in lines]
+    return lines
+
+
+def log_stream_for_subprocess():
+    stream = subprocess.DEVNULL
+    for handler in logging.root.handlers:
+        if handler.level == logging.DEBUG:
+            if isinstance(handler, logging.StreamHandler):
+                stream = handler.stream
+    return stream
+
+
+def main(argv):
+    args_parser = argparse.ArgumentParser(
+        description="Analyze a bootclasspath_fragment module.")
+    args_parser.add_argument(
+        "--bcpf",
+        help="The bootclasspath_fragment module to analyze",
+        required=True,
+    )
+    args_parser.add_argument(
+        "--apex",
+        help="The apex module to which the bootclasspath_fragment belongs. It "
+        "is not strictly necessary at the moment but providing it will "
+        "allow this script to give more useful messages and it may be"
+        "required in future.",
+        default="SPECIFY-APEX-OPTION")
+    args_parser.add_argument(
+        "--sdk",
+        help="The sdk module to which the bootclasspath_fragment belongs. It "
+        "is not strictly necessary at the moment but providing it will "
+        "allow this script to give more useful messages and it may be"
+        "required in future.",
+        default="SPECIFY-SDK-OPTION")
+    args = args_parser.parse_args(argv[1:])
+    top_dir = os.environ["ANDROID_BUILD_TOP"] + "/"
+    out_dir = os.environ.get("OUT_DIR", os.path.join(top_dir, "out"))
+    product_out_dir = os.environ.get("ANDROID_PRODUCT_OUT", top_dir)
+    # Make product_out_dir relative to the top so it can be used as part of a
+    # build target.
+    product_out_dir = product_out_dir.removeprefix(top_dir)
+    log_fd, abs_log_file = tempfile.mkstemp(
+        suffix="_analyze_bcpf.log", text=True)
+
+    with os.fdopen(log_fd, "w") as log_file:
+        # Set up debug logging to the log file.
+        logging.basicConfig(
+            level=logging.DEBUG,
+            format="%(levelname)-8s %(message)s",
+            stream=log_file)
+
+        # define a Handler which writes INFO messages or higher to the
+        # sys.stdout with just the message.
+        console = logging.StreamHandler()
+        console.setLevel(logging.INFO)
+        console.setFormatter(logging.Formatter("%(message)s"))
+        # add the handler to the root logger
+        logging.getLogger("").addHandler(console)
+
+        print(f"Writing log to {abs_log_file}")
+        try:
+            analyzer = BcpfAnalyzer(
+                top_dir=top_dir,
+                out_dir=out_dir,
+                product_out_dir=product_out_dir,
+                bcpf=args.bcpf,
+                apex=args.apex,
+                sdk=args.sdk,
+            )
+            analyzer.analyze()
+        finally:
+            print(f"Log written to {abs_log_file}")
+
+
+if __name__ == "__main__":
+    main(sys.argv)

scripts/hiddenapi/analyze_bcpf_test.py

@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for analyzing bootclasspath_fragment modules."""
+import os.path
+import shutil
+import tempfile
+import unittest
+import unittest.mock
+
+import analyze_bcpf as ab
+
+_FRAMEWORK_HIDDENAPI = "frameworks/base/boot/hiddenapi"
+_MAX_TARGET_O = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-o.txt"
+_MAX_TARGET_P = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-p.txt"
+_MAX_TARGET_Q = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-q.txt"
+_MAX_TARGET_R = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-r-loprio.txt"
+
+_MULTI_LINE_COMMENT = """
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut arcu justo,
+bibendum eu malesuada vel, fringilla in odio. Etiam gravida ultricies sem
+tincidunt luctus.""".replace("\n", " ").strip()
+
+
+class FakeBuildOperation(ab.BuildOperation):
+
+    def __init__(self, lines, return_code):
+        ab.BuildOperation.__init__(self, None)
+        self._lines = lines
+        self.returncode = return_code
+
+    def lines(self):
+        return iter(self._lines)
+
+    def wait(self, *args, **kwargs):
+        return
+
+
+class TestAnalyzeBcpf(unittest.TestCase):
+
+    def setUp(self):
+        # Create a temporary directory
+        self.test_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        # Remove the directory after the test
+        shutil.rmtree(self.test_dir)
+
+    @staticmethod
+    def write_abs_file(abs_path, contents):
+        os.makedirs(os.path.dirname(abs_path), exist_ok=True)
+        with open(abs_path, "w", encoding="utf8") as f:
+            print(contents.removeprefix("\n"), file=f, end="")
+
+    def populate_fs(self, fs):
+        for path, contents in fs.items():
+            abs_path = os.path.join(self.test_dir, path)
+            self.write_abs_file(abs_path, contents)
+
+    def create_analyzer_for_test(self,
+                                 fs=None,
+                                 bcpf="bcpf",
+                                 apex="apex",
+                                 sdk="sdk"):
+        if fs:
+            self.populate_fs(fs)
+
+        top_dir = self.test_dir
+        out_dir = os.path.join(self.test_dir, "out")
+        product_out_dir = "out/product"
+
+        bcpf_dir = f"{bcpf}-dir"
+        modules = {bcpf: {"path": [bcpf_dir]}}
+        module_info = ab.ModuleInfo(modules)
+
+        analyzer = ab.BcpfAnalyzer(
+            top_dir=top_dir,
+            out_dir=out_dir,
+            product_out_dir=product_out_dir,
+            bcpf=bcpf,
+            apex=apex,
+            sdk=sdk,
+            module_info=module_info,
+        )
+        analyzer.load_all_flags()
+        return analyzer
+
+    def test_reformat_report_text(self):
+        lines = """
+99. An item in a numbered list
+that traverses multiple lines.
+
+   An indented example
+   that should not be reformatted.
+"""
+        reformatted = ab.BcpfAnalyzer.reformat_report_test(lines)
+        self.assertEqual(
+            """
+99. An item in a numbered list that traverses multiple lines.
+
+   An indented example
+   that should not be reformatted.
+""", reformatted)
+
+    def test_build_flags(self):
+        lines = """
+ERROR: Hidden API flags are inconsistent:
+< out/soong/.intermediates/bcpf-dir/bcpf-dir/filtered-flags.csv
+> out/soong/hiddenapi/hiddenapi-flags.csv
+
+< Lacme/test/Class;-><init>()V,blocked
+> Lacme/test/Class;-><init>()V,max-target-o
+
+< Lacme/test/Other;->getThing()Z,blocked
+> Lacme/test/Other;->getThing()Z,max-target-p
+
+< Lacme/test/Widget;-><init()V,blocked
+> Lacme/test/Widget;-><init()V,max-target-q
+
+< Lacme/test/Gadget;->NAME:Ljava/lang/String;,blocked
+> Lacme/test/Gadget;->NAME:Ljava/lang/String;,lo-prio,max-target-r
+16:37:32 ninja failed with: exit status 1
+""".strip().splitlines()
+        operation = FakeBuildOperation(lines=lines, return_code=1)
+
+        fs = {
+            _MAX_TARGET_O:
+                """
+Lacme/items/Magnet;->size:I
+Lacme/test/Class;-><init>()V
+""",
+            _MAX_TARGET_P:
+                """
+Lacme/items/Rocket;->size:I
+Lacme/test/Other;->getThing()Z
+""",
+            _MAX_TARGET_Q:
+                """
+Lacme/items/Rock;->size:I
+Lacme/test/Widget;-><init()V
+""",
+            _MAX_TARGET_R:
+                """
+Lacme/items/Lever;->size:I
+Lacme/test/Gadget;->NAME:Ljava/lang/String;
+""",
+            "bcpf-dir/hiddenapi/hiddenapi-max-target-p.txt":
+                """
+Lacme/old/Class;->getWidget()Lacme/test/Widget;
+""",
+            "out/soong/.intermediates/bcpf-dir/bcpf/all-flags.csv":
+                """
+Lacme/test/Gadget;->NAME:Ljava/lang/String;,blocked
+Lacme/test/Widget;-><init()V,blocked
+Lacme/test/Class;-><init>()V,blocked
+Lacme/test/Other;->getThing()Z,blocked
+""",
+        }
+
+        analyzer = self.create_analyzer_for_test(fs)
+
+        # Override the build_file_read_output() method to just return a fake
+        # build operation.
+        analyzer.build_file_read_output = unittest.mock.Mock(
+            return_value=operation)
+
+        # Override the run_command() method to do nothing.
+        analyzer.run_command = unittest.mock.Mock()
+
+        result = ab.Result()
+
+        analyzer.build_monolithic_flags(result)
+        expected_diffs = {
+            "Lacme/test/Gadget;->NAME:Ljava/lang/String;":
+                (["blocked"], ["lo-prio", "max-target-r"]),
+            "Lacme/test/Widget;-><init()V": (["blocked"], ["max-target-q"]),
+            "Lacme/test/Class;-><init>()V": (["blocked"], ["max-target-o"]),
+            "Lacme/test/Other;->getThing()Z": (["blocked"], ["max-target-p"])
+        }
+        self.assertEqual(expected_diffs, result.diffs, msg="flag differences")
+
+        expected_property_changes = [
+            ab.HiddenApiPropertyChange(
+                property_name="max_target_o_low_priority",
+                values=["hiddenapi/hiddenapi-max-target-o-low-priority.txt"],
+                property_comment=""),
+            ab.HiddenApiPropertyChange(
+                property_name="max_target_p",
+                values=["hiddenapi/hiddenapi-max-target-p.txt"],
+                property_comment=""),
+            ab.HiddenApiPropertyChange(
+                property_name="max_target_q",
+                values=["hiddenapi/hiddenapi-max-target-q.txt"],
+                property_comment=""),
+            ab.HiddenApiPropertyChange(
+                property_name="max_target_r_low_priority",
+                values=["hiddenapi/hiddenapi-max-target-r-low-priority.txt"],
+                property_comment=""),
+        ]
+        self.assertEqual(
+            expected_property_changes,
+            result.property_changes,
+            msg="property changes")
+
+        expected_file_changes = [
+            ab.FileChange(
+                path="bcpf-dir/hiddenapi/"
+                "hiddenapi-max-target-o-low-priority.txt",
+                description="""Add the following entries:
+            Lacme/test/Class;-><init>()V
+""",
+            ),
+            ab.FileChange(
+                path="bcpf-dir/hiddenapi/hiddenapi-max-target-p.txt",
+                description="""Add the following entries:
+            Lacme/test/Other;->getThing()Z
+""",
+            ),
+            ab.FileChange(
+                path="bcpf-dir/hiddenapi/hiddenapi-max-target-q.txt",
+                description="""Add the following entries:
+            Lacme/test/Widget;-><init()V
+"""),
+            ab.FileChange(
+                path="bcpf-dir/hiddenapi/"
+                "hiddenapi-max-target-r-low-priority.txt",
+                description="""Add the following entries:
+            Lacme/test/Gadget;->NAME:Ljava/lang/String;
+"""),
+            ab.FileChange(
+                path="frameworks/base/boot/hiddenapi/"
+                "hiddenapi-max-target-o.txt",
+                description="""Remove the following entries:
+            Lacme/test/Class;-><init>()V
+"""),
+            ab.FileChange(
+                path="frameworks/base/boot/hiddenapi/"
+                "hiddenapi-max-target-p.txt",
+                description="""Remove the following entries:
+            Lacme/test/Other;->getThing()Z
+"""),
+            ab.FileChange(
+                path="frameworks/base/boot/hiddenapi/"
+                "hiddenapi-max-target-q.txt",
+                description="""Remove the following entries:
+            Lacme/test/Widget;-><init()V
+"""),
+            ab.FileChange(
+                path="frameworks/base/boot/hiddenapi/"
+                "hiddenapi-max-target-r-loprio.txt",
+                description="""Remove the following entries:
+            Lacme/test/Gadget;->NAME:Ljava/lang/String;
+""")
+        ]
+        result.file_changes.sort()
+        self.assertEqual(
+            expected_file_changes, result.file_changes, msg="file_changes")
+
+
+class TestHiddenApiPropertyChange(unittest.TestCase):
+
+    def setUp(self):
+        # Create a temporary directory
+        self.test_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        # Remove the directory after the test
+        shutil.rmtree(self.test_dir)
+
+    def check_change_snippet(self, change, expected):
+        snippet = change.snippet("        ")
+        self.assertEqual(expected, snippet)
+
+    def test_change_property_with_value_no_comment(self):
+        change = ab.HiddenApiPropertyChange(
+            property_name="split_packages",
+            values=["android.provider"],
+        )
+
+        self.check_change_snippet(
+            change, """
+        split_packages: [
+            "android.provider",
+        ],
+""")
+
+    def test_change_property_with_value_and_comment(self):
+        change = ab.HiddenApiPropertyChange(
+            property_name="split_packages",
+            values=["android.provider"],
+            property_comment=_MULTI_LINE_COMMENT,
+        )
+
+        self.check_change_snippet(
+            change, """
+        // Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut arcu
+        // justo, bibendum eu malesuada vel, fringilla in odio. Etiam gravida
+        // ultricies sem tincidunt luctus.
+        split_packages: [
+            "android.provider",
+        ],
+""")
+
+    def test_change_without_value_or_empty_comment(self):
+        change = ab.HiddenApiPropertyChange(
+            property_name="split_packages",
+            values=[],
+            property_comment="Single line comment.",
+        )
+
+        self.check_change_snippet(
+            change, """
+        // Single line comment.
+        split_packages: [],
+""")
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=3)