Add analyze_bcpf.py script to help manage bootclasspath_fragments am: 4dcf65951b
Original change: https://android-review.googlesource.com/c/platform/build/soong/+/2020741 Change-Id: I92c0a9d5b9d63abf04af465840ff6f0d13ef03e5 Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Paul Duffin
Automerger Merge Worker
@@ -18,6 +18,48 @@ package {
|
||||
default_applicable_licenses: ["Android-Apache-2.0"],
|
||||
}
|
||||
|
||||
python_binary_host {
|
||||
name: "analyze_bcpf",
|
||||
main: "analyze_bcpf.py",
|
||||
srcs: ["analyze_bcpf.py"],
|
||||
libs: [
|
||||
"signature_trie",
|
||||
],
|
||||
version: {
|
||||
py2: {
|
||||
enabled: false,
|
||||
},
|
||||
py3: {
|
||||
enabled: true,
|
||||
embedded_launcher: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
python_test_host {
|
||||
name: "analyze_bcpf_test",
|
||||
main: "analyze_bcpf_test.py",
|
||||
srcs: [
|
||||
"analyze_bcpf.py",
|
||||
"analyze_bcpf_test.py",
|
||||
],
|
||||
libs: [
|
||||
"signature_trie",
|
||||
],
|
||||
version: {
|
||||
py2: {
|
||||
enabled: false,
|
||||
},
|
||||
py3: {
|
||||
enabled: true,
|
||||
embedded_launcher: true,
|
||||
},
|
||||
},
|
||||
test_options: {
|
||||
unit_test: true,
|
||||
},
|
||||
}
|
||||
|
||||
python_binary_host {
|
||||
name: "merge_csv",
|
||||
main: "merge_csv.py",
|
||||
|
||||
794
scripts/hiddenapi/analyze_bcpf.py
Normal file
794
scripts/hiddenapi/analyze_bcpf.py
Normal file
@@ -0,0 +1,794 @@
|
||||
#!/usr/bin/env -S python -u
|
||||
#
|
||||
# Copyright (C) 2022 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Analyze bootclasspath_fragment usage."""
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import textwrap
|
||||
import typing
|
||||
import sys
|
||||
|
||||
_STUB_FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-stub-flags.txt"
|
||||
|
||||
_FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-flags.csv"
|
||||
|
||||
_INCONSISTENT_FLAGS = "ERROR: Hidden API flags are inconsistent:"
|
||||
|
||||
|
||||
class BuildOperation:
|
||||
|
||||
def __init__(self, popen):
|
||||
self.popen = popen
|
||||
self.returncode = None
|
||||
|
||||
def lines(self):
|
||||
"""Return an iterator over the lines output by the build operation.
|
||||
|
||||
The lines have had any trailing white space, including the newline
|
||||
stripped.
|
||||
"""
|
||||
return newline_stripping_iter(self.popen.stdout.readline)
|
||||
|
||||
def wait(self, *args, **kwargs):
|
||||
self.popen.wait(*args, **kwargs)
|
||||
self.returncode = self.popen.returncode
|
||||
|
||||
|
||||
@dataclasses.dataclass()
|
||||
class FlagDiffs:
|
||||
"""Encapsulates differences in flags reported by the build"""
|
||||
|
||||
# Map from member signature to the (module flags, monolithic flags)
|
||||
diffs: typing.Dict[str, typing.Tuple[str, str]]
|
||||
|
||||
|
||||
@dataclasses.dataclass()
|
||||
class ModuleInfo:
|
||||
"""Provides access to the generated module-info.json file.
|
||||
|
||||
This is used to find the location of the file within which specific modules
|
||||
are defined.
|
||||
"""
|
||||
|
||||
modules: typing.Dict[str, typing.Dict[str, typing.Any]]
|
||||
|
||||
@staticmethod
|
||||
def load(filename):
|
||||
with open(filename, "r", encoding="utf8") as f:
|
||||
j = json.load(f)
|
||||
return ModuleInfo(j)
|
||||
|
||||
def _module(self, module_name):
|
||||
"""Find module by name in module-info.json file"""
|
||||
if module_name in self.modules:
|
||||
return self.modules[module_name]
|
||||
|
||||
raise Exception(f"Module {module_name} could not be found")
|
||||
|
||||
def module_path(self, module_name):
|
||||
module = self._module(module_name)
|
||||
# The "path" is actually a list of paths, one for each class of module
|
||||
# but as the modules are all created from bp files if a module does
|
||||
# create multiple classes of make modules they should all have the same
|
||||
# path.
|
||||
paths = module["path"]
|
||||
unique_paths = set(paths)
|
||||
if len(unique_paths) != 1:
|
||||
raise Exception(f"Expected module '{module_name}' to have a "
|
||||
f"single unique path but found {unique_paths}")
|
||||
return paths[0]
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class FileChange:
|
||||
path: str
|
||||
|
||||
description: str
|
||||
|
||||
def __lt__(self, other):
|
||||
return self.path < other.path
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class HiddenApiPropertyChange:
|
||||
|
||||
property_name: str
|
||||
|
||||
values: typing.List[str]
|
||||
|
||||
property_comment: str = ""
|
||||
|
||||
def snippet(self, indent):
|
||||
snippet = "\n"
|
||||
snippet += format_comment_as_text(self.property_comment, indent)
|
||||
snippet += f"{indent}{self.property_name}: ["
|
||||
if self.values:
|
||||
snippet += "\n"
|
||||
for value in self.values:
|
||||
snippet += f'{indent} "{value}",\n'
|
||||
snippet += f"{indent}"
|
||||
snippet += "],\n"
|
||||
return snippet
|
||||
|
||||
|
||||
@dataclasses.dataclass()
|
||||
class Result:
|
||||
"""Encapsulates the result of the analysis."""
|
||||
|
||||
# The diffs in the flags.
|
||||
diffs: typing.Optional[FlagDiffs] = None
|
||||
|
||||
# The bootclasspath_fragment hidden API properties changes.
|
||||
property_changes: typing.List[HiddenApiPropertyChange] = dataclasses.field(
|
||||
default_factory=list)
|
||||
|
||||
# The list of file changes.
|
||||
file_changes: typing.List[FileChange] = dataclasses.field(
|
||||
default_factory=list)
|
||||
|
||||
|
||||
@dataclasses.dataclass()
|
||||
class BcpfAnalyzer:
|
||||
# Directory pointed to by ANDROID_BUILD_OUT
|
||||
top_dir: str
|
||||
|
||||
# Directory pointed to by OUT_DIR of {top_dir}/out if that is not set.
|
||||
out_dir: str
|
||||
|
||||
# Directory pointed to by ANDROID_PRODUCT_OUT.
|
||||
product_out_dir: str
|
||||
|
||||
# The name of the bootclasspath_fragment module.
|
||||
bcpf: str
|
||||
|
||||
# The name of the apex module containing {bcpf}, only used for
|
||||
# informational purposes.
|
||||
apex: str
|
||||
|
||||
# The name of the sdk module containing {bcpf}, only used for
|
||||
# informational purposes.
|
||||
sdk: str
|
||||
|
||||
# All the signatures, loaded from all-flags.csv, initialized by
|
||||
# load_all_flags().
|
||||
_signatures: typing.Set[str] = dataclasses.field(default_factory=set)
|
||||
|
||||
# All the classes, loaded from all-flags.csv, initialized by
|
||||
# load_all_flags().
|
||||
_classes: typing.Set[str] = dataclasses.field(default_factory=set)
|
||||
|
||||
# Information loaded from module-info.json, initialized by
|
||||
# load_module_info().
|
||||
module_info: ModuleInfo = None
|
||||
|
||||
@staticmethod
|
||||
def reformat_report_test(text):
|
||||
return re.sub(r"(.)\n([^\s])", r"\1 \2", text)
|
||||
|
||||
def report(self, text, **kwargs):
|
||||
# Concatenate lines that are not separated by a blank line together to
|
||||
# eliminate formatting applied to the supplied text to adhere to python
|
||||
# line length limitations.
|
||||
text = self.reformat_report_test(text)
|
||||
logging.info("%s", text, **kwargs)
|
||||
|
||||
def run_command(self, cmd, *args, **kwargs):
|
||||
cmd_line = " ".join(cmd)
|
||||
logging.debug("Running %s", cmd_line)
|
||||
subprocess.run(
|
||||
cmd,
|
||||
*args,
|
||||
check=True,
|
||||
cwd=self.top_dir,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdout=log_stream_for_subprocess(),
|
||||
text=True,
|
||||
**kwargs)
|
||||
|
||||
@property
|
||||
def signatures(self):
|
||||
if not self._signatures:
|
||||
raise Exception("signatures has not been initialized")
|
||||
return self._signatures
|
||||
|
||||
@property
|
||||
def classes(self):
|
||||
if not self._classes:
|
||||
raise Exception("classes has not been initialized")
|
||||
return self._classes
|
||||
|
||||
def load_all_flags(self):
|
||||
all_flags = self.find_bootclasspath_fragment_output_file(
|
||||
"all-flags.csv")
|
||||
|
||||
# Extract the set of signatures and a separate set of classes produced
|
||||
# by the bootclasspath_fragment.
|
||||
with open(all_flags, "r", encoding="utf8") as f:
|
||||
for line in newline_stripping_iter(f.readline):
|
||||
signature = self.line_to_signature(line)
|
||||
self._signatures.add(signature)
|
||||
class_name = self.signature_to_class(signature)
|
||||
self._classes.add(class_name)
|
||||
|
||||
def load_module_info(self):
|
||||
module_info_file = os.path.join(self.product_out_dir,
|
||||
"module-info.json")
|
||||
self.report(f"""
|
||||
Making sure that {module_info_file} is up to date.
|
||||
""")
|
||||
output = self.build_file_read_output(module_info_file)
|
||||
lines = output.lines()
|
||||
for line in lines:
|
||||
logging.debug("%s", line)
|
||||
output.wait(timeout=10)
|
||||
if output.returncode:
|
||||
raise Exception(f"Error building {module_info_file}")
|
||||
abs_module_info_file = os.path.join(self.top_dir, module_info_file)
|
||||
self.module_info = ModuleInfo.load(abs_module_info_file)
|
||||
|
||||
@staticmethod
|
||||
def line_to_signature(line):
|
||||
return line.split(",")[0]
|
||||
|
||||
@staticmethod
|
||||
def signature_to_class(signature):
|
||||
return signature.split(";->")[0]
|
||||
|
||||
@staticmethod
|
||||
def to_parent_package(pkg_or_class):
|
||||
return pkg_or_class.rsplit("/", 1)[0]
|
||||
|
||||
def module_path(self, module_name):
|
||||
return self.module_info.module_path(module_name)
|
||||
|
||||
def module_out_dir(self, module_name):
|
||||
module_path = self.module_path(module_name)
|
||||
return os.path.join(self.out_dir, "soong/.intermediates", module_path,
|
||||
module_name)
|
||||
|
||||
def find_bootclasspath_fragment_output_file(self, basename):
|
||||
# Find the output file of the bootclasspath_fragment with the specified
|
||||
# base name.
|
||||
found_file = ""
|
||||
bcpf_out_dir = self.module_out_dir(self.bcpf)
|
||||
for (dirpath, _, filenames) in os.walk(bcpf_out_dir):
|
||||
for f in filenames:
|
||||
if f == basename:
|
||||
found_file = os.path.join(dirpath, f)
|
||||
break
|
||||
if not found_file:
|
||||
raise Exception(f"Could not find {basename} in {bcpf_out_dir}")
|
||||
return found_file
|
||||
|
||||
def analyze(self):
|
||||
"""Analyze a bootclasspath_fragment module.
|
||||
|
||||
Provides help in resolving any existing issues and provides
|
||||
optimizations that can be applied.
|
||||
"""
|
||||
self.report(f"Analyzing bootclasspath_fragment module {self.bcpf}")
|
||||
self.report(f"""
|
||||
Run this tool to help initialize a bootclasspath_fragment module. Before you
|
||||
start make sure that:
|
||||
|
||||
1. The current checkout is up to date.
|
||||
|
||||
2. The environment has been initialized using lunch, e.g.
|
||||
lunch aosp_arm64-userdebug
|
||||
|
||||
3. You have added a bootclasspath_fragment module to the appropriate Android.bp
|
||||
file. Something like this:
|
||||
|
||||
bootclasspath_fragment {{
|
||||
name: "{self.bcpf}",
|
||||
contents: [
|
||||
"...",
|
||||
],
|
||||
|
||||
// The bootclasspath_fragments that provide APIs on which this depends.
|
||||
fragments: [
|
||||
{{
|
||||
apex: "com.android.art",
|
||||
module: "art-bootclasspath-fragment",
|
||||
}},
|
||||
],
|
||||
}}
|
||||
|
||||
4. You have added it to the platform_bootclasspath module in
|
||||
frameworks/base/boot/Android.bp. Something like this:
|
||||
|
||||
platform_bootclasspath {{
|
||||
name: "platform-bootclasspath",
|
||||
fragments: [
|
||||
...
|
||||
{{
|
||||
apex: "{self.apex}",
|
||||
module: "{self.bcpf}",
|
||||
}},
|
||||
],
|
||||
}}
|
||||
|
||||
5. You have added an sdk module. Something like this:
|
||||
|
||||
sdk {{
|
||||
name: "{self.sdk}",
|
||||
bootclasspath_fragments: ["{self.bcpf}"],
|
||||
}}
|
||||
""")
|
||||
|
||||
# Make sure that the module-info.json file is up to date.
|
||||
self.load_module_info()
|
||||
|
||||
self.report("""
|
||||
Cleaning potentially stale files.
|
||||
""")
|
||||
# Remove the out/soong/hiddenapi files.
|
||||
shutil.rmtree(f"{self.out_dir}/soong/hiddenapi", ignore_errors=True)
|
||||
|
||||
# Remove any bootclasspath_fragment output files.
|
||||
shutil.rmtree(self.module_out_dir(self.bcpf), ignore_errors=True)
|
||||
|
||||
self.build_monolithic_stubs_flags()
|
||||
|
||||
result = Result()
|
||||
|
||||
self.build_monolithic_flags(result)
|
||||
|
||||
# If there were any changes that need to be made to the Android.bp
|
||||
# file then report them.
|
||||
if result.property_changes:
|
||||
bcpf_dir = self.module_info.module_path(self.bcpf)
|
||||
bcpf_bp_file = os.path.join(self.top_dir, bcpf_dir, "Android.bp")
|
||||
hiddenapi_snippet = ""
|
||||
for property_change in result.property_changes:
|
||||
hiddenapi_snippet += property_change.snippet(" ")
|
||||
|
||||
# Remove leading and trailing blank lines.
|
||||
hiddenapi_snippet = hiddenapi_snippet.strip("\n")
|
||||
|
||||
result.file_changes.append(
|
||||
self.new_file_change(
|
||||
bcpf_bp_file, f"""
|
||||
Add the following snippet into the {self.bcpf} bootclasspath_fragment module
|
||||
in the {bcpf_dir}/Android.bp file. If the hidden_api block already exists then
|
||||
merge these properties into it.
|
||||
|
||||
hidden_api: {{
|
||||
{hiddenapi_snippet}
|
||||
}},
|
||||
"""))
|
||||
|
||||
if result.file_changes:
|
||||
self.report("""
|
||||
The following modifications need to be made:""")
|
||||
result.file_changes.sort()
|
||||
for file_change in result.file_changes:
|
||||
self.report(f"""
|
||||
{file_change.path}
|
||||
{file_change.description}
|
||||
""".lstrip("\n"))
|
||||
|
||||
def new_file_change(self, file, description):
|
||||
return FileChange(
|
||||
path=os.path.relpath(file, self.top_dir), description=description)
|
||||
|
||||
def check_inconsistent_flag_lines(self, significant, module_line,
|
||||
monolithic_line, separator_line):
|
||||
if not (module_line.startswith("< ") and
|
||||
monolithic_line.startswith("> ") and not separator_line):
|
||||
# Something went wrong.
|
||||
self.report(f"""Invalid build output detected:
|
||||
module_line: "{module_line}"
|
||||
monolithic_line: "{monolithic_line}"
|
||||
separator_line: "{separator_line}"
|
||||
""")
|
||||
sys.exit(1)
|
||||
|
||||
if significant:
|
||||
logging.debug("%s", module_line)
|
||||
logging.debug("%s", monolithic_line)
|
||||
logging.debug("%s", separator_line)
|
||||
|
||||
def scan_inconsistent_flags_report(self, lines):
|
||||
"""Scans a hidden API flags report
|
||||
|
||||
The hidden API inconsistent flags report which looks something like
|
||||
this.
|
||||
|
||||
< out/soong/.intermediates/.../filtered-stub-flags.csv
|
||||
> out/soong/hiddenapi/hiddenapi-stub-flags.txt
|
||||
|
||||
< Landroid/compat/Compatibility;->clearOverrides()V
|
||||
> Landroid/compat/Compatibility;->clearOverrides()V,core-platform-api
|
||||
|
||||
"""
|
||||
|
||||
# The basic format of an entry in the inconsistent flags report is:
|
||||
# <module specific flag>
|
||||
# <monolithic flag>
|
||||
# <separator>
|
||||
#
|
||||
# Wrap the lines iterator in an iterator which returns a tuple
|
||||
# consisting of the three separate lines.
|
||||
triples = zip(lines, lines, lines)
|
||||
|
||||
module_line, monolithic_line, separator_line = next(triples)
|
||||
significant = False
|
||||
bcpf_dir = self.module_info.module_path(self.bcpf)
|
||||
if os.path.join(bcpf_dir, self.bcpf) in module_line:
|
||||
# These errors are related to the bcpf being analyzed so
|
||||
# keep them.
|
||||
significant = True
|
||||
else:
|
||||
self.report(f"Filtering out errors related to {module_line}")
|
||||
|
||||
self.check_inconsistent_flag_lines(significant, module_line,
|
||||
monolithic_line, separator_line)
|
||||
|
||||
diffs = {}
|
||||
for module_line, monolithic_line, separator_line in triples:
|
||||
self.check_inconsistent_flag_lines(significant, module_line,
|
||||
monolithic_line, "")
|
||||
|
||||
module_parts = module_line.removeprefix("< ").split(",")
|
||||
module_signature = module_parts[0]
|
||||
module_flags = module_parts[1:]
|
||||
|
||||
monolithic_parts = monolithic_line.removeprefix("> ").split(",")
|
||||
monolithic_signature = monolithic_parts[0]
|
||||
monolithic_flags = monolithic_parts[1:]
|
||||
|
||||
if module_signature != monolithic_signature:
|
||||
# Something went wrong.
|
||||
self.report(f"""Inconsistent signatures detected:
|
||||
module_signature: "{module_signature}"
|
||||
monolithic_signature: "{monolithic_signature}"
|
||||
""")
|
||||
sys.exit(1)
|
||||
|
||||
diffs[module_signature] = (module_flags, monolithic_flags)
|
||||
|
||||
if separator_line:
|
||||
# If the separator line is not blank then it is the end of the
|
||||
# current report, and possibly the start of another.
|
||||
return separator_line, diffs
|
||||
|
||||
return "", diffs
|
||||
|
||||
def build_file_read_output(self, filename):
|
||||
# Make sure the filename is relative to top if possible as the build
|
||||
# may be using relative paths as the target.
|
||||
rel_filename = filename.removeprefix(self.top_dir)
|
||||
cmd = ["build/soong/soong_ui.bash", "--make-mode", rel_filename]
|
||||
cmd_line = " ".join(cmd)
|
||||
logging.debug("%s", cmd_line)
|
||||
# pylint: disable=consider-using-with
|
||||
output = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=self.top_dir,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
return BuildOperation(popen=output)
|
||||
|
||||
def build_hiddenapi_flags(self, filename):
|
||||
output = self.build_file_read_output(filename)
|
||||
|
||||
lines = output.lines()
|
||||
diffs = None
|
||||
for line in lines:
|
||||
logging.debug("%s", line)
|
||||
while line == _INCONSISTENT_FLAGS:
|
||||
line, diffs = self.scan_inconsistent_flags_report(lines)
|
||||
|
||||
output.wait(timeout=10)
|
||||
if output.returncode != 0:
|
||||
logging.debug("Command failed with %s", output.returncode)
|
||||
else:
|
||||
logging.debug("Command succeeded")
|
||||
|
||||
return diffs
|
||||
|
||||
def build_monolithic_stubs_flags(self):
|
||||
self.report(f"""
|
||||
Attempting to build {_STUB_FLAGS_FILE} to verify that the
|
||||
bootclasspath_fragment has the correct API stubs available...
|
||||
""")
|
||||
|
||||
# Build the hiddenapi-stubs-flags.txt file.
|
||||
diffs = self.build_hiddenapi_flags(_STUB_FLAGS_FILE)
|
||||
if diffs:
|
||||
self.report(f"""
|
||||
There is a discrepancy between the stub API derived flags created by the
|
||||
bootclasspath_fragment and the platform_bootclasspath. See preceding error
|
||||
messages to see which flags are inconsistent. The inconsistencies can occur for
|
||||
a couple of reasons:
|
||||
|
||||
If you are building against prebuilts of the Android SDK, e.g. by using
|
||||
TARGET_BUILD_APPS then the prebuilt versions of the APIs this
|
||||
bootclasspath_fragment depends upon are out of date and need updating. See
|
||||
go/update-prebuilts for help.
|
||||
|
||||
Otherwise, this is happening because there are some stub APIs that are either
|
||||
provided by or used by the contents of the bootclasspath_fragment but which are
|
||||
not available to it. There are 4 ways to handle this:
|
||||
|
||||
1. A java_sdk_library in the contents property will automatically make its stub
|
||||
APIs available to the bootclasspath_fragment so nothing needs to be done.
|
||||
|
||||
2. If the API provided by the bootclasspath_fragment is created by an api_only
|
||||
java_sdk_library (or a java_library that compiles files generated by a
|
||||
separate droidstubs module then it cannot be added to the contents and
|
||||
instead must be added to the api.stubs property, e.g.
|
||||
|
||||
bootclasspath_fragment {{
|
||||
name: "{self.bcpf}",
|
||||
...
|
||||
api: {{
|
||||
stubs: ["$MODULE-api-only"],"
|
||||
}},
|
||||
}}
|
||||
|
||||
3. If the contents use APIs provided by another bootclasspath_fragment then
|
||||
it needs to be added to the fragments property, e.g.
|
||||
|
||||
bootclasspath_fragment {{
|
||||
name: "{self.bcpf}",
|
||||
...
|
||||
// The bootclasspath_fragments that provide APIs on which this depends.
|
||||
fragments: [
|
||||
...
|
||||
{{
|
||||
apex: "com.android.other",
|
||||
module: "com.android.other-bootclasspath-fragment",
|
||||
}},
|
||||
],
|
||||
}}
|
||||
|
||||
4. If the contents use APIs from a module that is not part of another
|
||||
bootclasspath_fragment then it must be added to the additional_stubs
|
||||
property, e.g.
|
||||
|
||||
bootclasspath_fragment {{
|
||||
name: "{self.bcpf}",
|
||||
...
|
||||
additional_stubs: ["android-non-updatable"],
|
||||
}}
|
||||
|
||||
Like the api.stubs property these are typically java_sdk_library modules but
|
||||
can be java_library too.
|
||||
|
||||
Note: The "android-non-updatable" is treated as if it was a java_sdk_library
|
||||
which it is not at the moment but will be in future.
|
||||
""")
|
||||
|
||||
return diffs
|
||||
|
||||
def build_monolithic_flags(self, result):
|
||||
self.report(f"""
|
||||
Attempting to build {_FLAGS_FILE} to verify that the
|
||||
bootclasspath_fragment has the correct hidden API flags...
|
||||
""")
|
||||
|
||||
# Build the hiddenapi-flags.csv file and extract any differences in
|
||||
# the flags between this bootclasspath_fragment and the monolithic
|
||||
# files.
|
||||
result.diffs = self.build_hiddenapi_flags(_FLAGS_FILE)
|
||||
|
||||
# Load information from the bootclasspath_fragment's all-flags.csv file.
|
||||
self.load_all_flags()
|
||||
|
||||
if result.diffs:
|
||||
self.report(f"""
|
||||
There is a discrepancy between the hidden API flags created by the
|
||||
bootclasspath_fragment and the platform_bootclasspath. See preceding error
|
||||
messages to see which flags are inconsistent. The inconsistencies can occur for
|
||||
a couple of reasons:
|
||||
|
||||
If you are building against prebuilts of this bootclasspath_fragment then the
|
||||
prebuilt version of the sdk snapshot (specifically the hidden API flag files)
|
||||
are inconsistent with the prebuilt version of the apex {self.apex}. Please
|
||||
ensure that they are both updated from the same build.
|
||||
|
||||
1. There are custom hidden API flags specified in the one of the files in
|
||||
frameworks/base/boot/hiddenapi which apply to the bootclasspath_fragment but
|
||||
which are not supplied to the bootclasspath_fragment module.
|
||||
|
||||
2. The bootclasspath_fragment specifies invalid "package_prefixes" or
|
||||
"split_packages" properties that match packages and classes that it does not
|
||||
provide.
|
||||
|
||||
""")
|
||||
|
||||
# Check to see if there are any hiddenapi related properties that
|
||||
# need to be added to the
|
||||
self.report("""
|
||||
Checking custom hidden API flags....
|
||||
""")
|
||||
self.check_frameworks_base_boot_hidden_api_files(result)
|
||||
|
||||
def report_hidden_api_flag_file_changes(self, result, property_name,
|
||||
flags_file, rel_bcpf_flags_file,
|
||||
bcpf_flags_file):
|
||||
matched_signatures = set()
|
||||
# Open the flags file to read the flags from.
|
||||
with open(flags_file, "r", encoding="utf8") as f:
|
||||
for signature in newline_stripping_iter(f.readline):
|
||||
if signature in self.signatures:
|
||||
# The signature is provided by the bootclasspath_fragment so
|
||||
# it will need to be moved to the bootclasspath_fragment
|
||||
# specific file.
|
||||
matched_signatures.add(signature)
|
||||
|
||||
# If the bootclasspath_fragment specific flags file is not empty
|
||||
# then it contains flags. That could either be new flags just moved
|
||||
# from frameworks/base or previous contents of the file. In either
|
||||
# case the file must not be removed.
|
||||
if matched_signatures:
|
||||
insert = textwrap.indent("\n".join(matched_signatures),
|
||||
" ")
|
||||
result.file_changes.append(
|
||||
self.new_file_change(
|
||||
flags_file, f"""Remove the following entries:
|
||||
{insert}
|
||||
"""))
|
||||
|
||||
result.file_changes.append(
|
||||
self.new_file_change(
|
||||
bcpf_flags_file, f"""Add the following entries:
|
||||
{insert}
|
||||
"""))
|
||||
|
||||
result.property_changes.append(
|
||||
HiddenApiPropertyChange(
|
||||
property_name=property_name,
|
||||
values=[rel_bcpf_flags_file],
|
||||
))
|
||||
|
||||
def check_frameworks_base_boot_hidden_api_files(self, result):
|
||||
hiddenapi_dir = os.path.join(self.top_dir,
|
||||
"frameworks/base/boot/hiddenapi")
|
||||
for basename in sorted(os.listdir(hiddenapi_dir)):
|
||||
if not (basename.startswith("hiddenapi-") and
|
||||
basename.endswith(".txt")):
|
||||
continue
|
||||
|
||||
flags_file = os.path.join(hiddenapi_dir, basename)
|
||||
|
||||
logging.debug("Checking %s for flags related to %s", flags_file,
|
||||
self.bcpf)
|
||||
|
||||
# Map the file name in frameworks/base/boot/hiddenapi into a
|
||||
# slightly more meaningful name for use by the
|
||||
# bootclasspath_fragment.
|
||||
if basename == "hiddenapi-max-target-o.txt":
|
||||
basename = "hiddenapi-max-target-o-low-priority.txt"
|
||||
elif basename == "hiddenapi-max-target-r-loprio.txt":
|
||||
basename = "hiddenapi-max-target-r-low-priority.txt"
|
||||
|
||||
property_name = basename.removeprefix("hiddenapi-")
|
||||
property_name = property_name.removesuffix(".txt")
|
||||
property_name = property_name.replace("-", "_")
|
||||
|
||||
rel_bcpf_flags_file = f"hiddenapi/{basename}"
|
||||
bcpf_dir = self.module_info.module_path(self.bcpf)
|
||||
bcpf_flags_file = os.path.join(self.top_dir, bcpf_dir,
|
||||
rel_bcpf_flags_file)
|
||||
|
||||
self.report_hidden_api_flag_file_changes(result, property_name,
|
||||
flags_file,
|
||||
rel_bcpf_flags_file,
|
||||
bcpf_flags_file)
|
||||
|
||||
|
||||
def newline_stripping_iter(iterator):
|
||||
"""Return an iterator over the iterator that strips trailing white space."""
|
||||
lines = iter(iterator, "")
|
||||
lines = (line.rstrip() for line in lines)
|
||||
return lines
|
||||
|
||||
|
||||
def format_comment_as_text(text, indent):
|
||||
return "".join(
|
||||
[f"{line}\n" for line in format_comment_as_lines(text, indent)])
|
||||
|
||||
|
||||
def format_comment_as_lines(text, indent):
|
||||
lines = textwrap.wrap(text.strip("\n"), width=77 - len(indent))
|
||||
lines = [f"{indent}// {line}" for line in lines]
|
||||
return lines
|
||||
|
||||
|
||||
def log_stream_for_subprocess():
|
||||
stream = subprocess.DEVNULL
|
||||
for handler in logging.root.handlers:
|
||||
if handler.level == logging.DEBUG:
|
||||
if isinstance(handler, logging.StreamHandler):
|
||||
stream = handler.stream
|
||||
return stream
|
||||
|
||||
|
||||
def main(argv):
|
||||
args_parser = argparse.ArgumentParser(
|
||||
description="Analyze a bootclasspath_fragment module.")
|
||||
args_parser.add_argument(
|
||||
"--bcpf",
|
||||
help="The bootclasspath_fragment module to analyze",
|
||||
required=True,
|
||||
)
|
||||
args_parser.add_argument(
|
||||
"--apex",
|
||||
help="The apex module to which the bootclasspath_fragment belongs. It "
|
||||
"is not strictly necessary at the moment but providing it will "
|
||||
"allow this script to give more useful messages and it may be"
|
||||
"required in future.",
|
||||
default="SPECIFY-APEX-OPTION")
|
||||
args_parser.add_argument(
|
||||
"--sdk",
|
||||
help="The sdk module to which the bootclasspath_fragment belongs. It "
|
||||
"is not strictly necessary at the moment but providing it will "
|
||||
"allow this script to give more useful messages and it may be"
|
||||
"required in future.",
|
||||
default="SPECIFY-SDK-OPTION")
|
||||
args = args_parser.parse_args(argv[1:])
|
||||
top_dir = os.environ["ANDROID_BUILD_TOP"] + "/"
|
||||
out_dir = os.environ.get("OUT_DIR", os.path.join(top_dir, "out"))
|
||||
product_out_dir = os.environ.get("ANDROID_PRODUCT_OUT", top_dir)
|
||||
# Make product_out_dir relative to the top so it can be used as part of a
|
||||
# build target.
|
||||
product_out_dir = product_out_dir.removeprefix(top_dir)
|
||||
log_fd, abs_log_file = tempfile.mkstemp(
|
||||
suffix="_analyze_bcpf.log", text=True)
|
||||
|
||||
with os.fdopen(log_fd, "w") as log_file:
|
||||
# Set up debug logging to the log file.
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(levelname)-8s %(message)s",
|
||||
stream=log_file)
|
||||
|
||||
# define a Handler which writes INFO messages or higher to the
|
||||
# sys.stdout with just the message.
|
||||
console = logging.StreamHandler()
|
||||
console.setLevel(logging.INFO)
|
||||
console.setFormatter(logging.Formatter("%(message)s"))
|
||||
# add the handler to the root logger
|
||||
logging.getLogger("").addHandler(console)
|
||||
|
||||
print(f"Writing log to {abs_log_file}")
|
||||
try:
|
||||
analyzer = BcpfAnalyzer(
|
||||
top_dir=top_dir,
|
||||
out_dir=out_dir,
|
||||
product_out_dir=product_out_dir,
|
||||
bcpf=args.bcpf,
|
||||
apex=args.apex,
|
||||
sdk=args.sdk,
|
||||
)
|
||||
analyzer.analyze()
|
||||
finally:
|
||||
print(f"Log written to {abs_log_file}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv)
|
||||
331
scripts/hiddenapi/analyze_bcpf_test.py
Normal file
331
scripts/hiddenapi/analyze_bcpf_test.py
Normal file
@@ -0,0 +1,331 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (C) 2022 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the 'License');
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an 'AS IS' BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Unit tests for analyzing bootclasspath_fragment modules."""
|
||||
import os.path
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
import analyze_bcpf as ab
|
||||
|
||||
_FRAMEWORK_HIDDENAPI = "frameworks/base/boot/hiddenapi"
|
||||
_MAX_TARGET_O = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-o.txt"
|
||||
_MAX_TARGET_P = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-p.txt"
|
||||
_MAX_TARGET_Q = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-q.txt"
|
||||
_MAX_TARGET_R = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-r-loprio.txt"
|
||||
|
||||
_MULTI_LINE_COMMENT = """
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut arcu justo,
|
||||
bibendum eu malesuada vel, fringilla in odio. Etiam gravida ultricies sem
|
||||
tincidunt luctus.""".replace("\n", " ").strip()
|
||||
|
||||
|
||||
class FakeBuildOperation(ab.BuildOperation):
|
||||
|
||||
def __init__(self, lines, return_code):
|
||||
ab.BuildOperation.__init__(self, None)
|
||||
self._lines = lines
|
||||
self.returncode = return_code
|
||||
|
||||
def lines(self):
|
||||
return iter(self._lines)
|
||||
|
||||
def wait(self, *args, **kwargs):
|
||||
return
|
||||
|
||||
|
||||
class TestAnalyzeBcpf(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# Create a temporary directory
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
# Remove the directory after the test
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
@staticmethod
|
||||
def write_abs_file(abs_path, contents):
|
||||
os.makedirs(os.path.dirname(abs_path), exist_ok=True)
|
||||
with open(abs_path, "w", encoding="utf8") as f:
|
||||
print(contents.removeprefix("\n"), file=f, end="")
|
||||
|
||||
def populate_fs(self, fs):
|
||||
for path, contents in fs.items():
|
||||
abs_path = os.path.join(self.test_dir, path)
|
||||
self.write_abs_file(abs_path, contents)
|
||||
|
||||
def create_analyzer_for_test(self,
|
||||
fs=None,
|
||||
bcpf="bcpf",
|
||||
apex="apex",
|
||||
sdk="sdk"):
|
||||
if fs:
|
||||
self.populate_fs(fs)
|
||||
|
||||
top_dir = self.test_dir
|
||||
out_dir = os.path.join(self.test_dir, "out")
|
||||
product_out_dir = "out/product"
|
||||
|
||||
bcpf_dir = f"{bcpf}-dir"
|
||||
modules = {bcpf: {"path": [bcpf_dir]}}
|
||||
module_info = ab.ModuleInfo(modules)
|
||||
|
||||
analyzer = ab.BcpfAnalyzer(
|
||||
top_dir=top_dir,
|
||||
out_dir=out_dir,
|
||||
product_out_dir=product_out_dir,
|
||||
bcpf=bcpf,
|
||||
apex=apex,
|
||||
sdk=sdk,
|
||||
module_info=module_info,
|
||||
)
|
||||
analyzer.load_all_flags()
|
||||
return analyzer
|
||||
|
||||
def test_reformat_report_text(self):
|
||||
lines = """
|
||||
99. An item in a numbered list
|
||||
that traverses multiple lines.
|
||||
|
||||
An indented example
|
||||
that should not be reformatted.
|
||||
"""
|
||||
reformatted = ab.BcpfAnalyzer.reformat_report_test(lines)
|
||||
self.assertEqual(
|
||||
"""
|
||||
99. An item in a numbered list that traverses multiple lines.
|
||||
|
||||
An indented example
|
||||
that should not be reformatted.
|
||||
""", reformatted)
|
||||
|
||||
def test_build_flags(self):
|
||||
lines = """
|
||||
ERROR: Hidden API flags are inconsistent:
|
||||
< out/soong/.intermediates/bcpf-dir/bcpf-dir/filtered-flags.csv
|
||||
> out/soong/hiddenapi/hiddenapi-flags.csv
|
||||
|
||||
< Lacme/test/Class;-><init>()V,blocked
|
||||
> Lacme/test/Class;-><init>()V,max-target-o
|
||||
|
||||
< Lacme/test/Other;->getThing()Z,blocked
|
||||
> Lacme/test/Other;->getThing()Z,max-target-p
|
||||
|
||||
< Lacme/test/Widget;-><init()V,blocked
|
||||
> Lacme/test/Widget;-><init()V,max-target-q
|
||||
|
||||
< Lacme/test/Gadget;->NAME:Ljava/lang/String;,blocked
|
||||
> Lacme/test/Gadget;->NAME:Ljava/lang/String;,lo-prio,max-target-r
|
||||
16:37:32 ninja failed with: exit status 1
|
||||
""".strip().splitlines()
|
||||
operation = FakeBuildOperation(lines=lines, return_code=1)
|
||||
|
||||
fs = {
|
||||
_MAX_TARGET_O:
|
||||
"""
|
||||
Lacme/items/Magnet;->size:I
|
||||
Lacme/test/Class;-><init>()V
|
||||
""",
|
||||
_MAX_TARGET_P:
|
||||
"""
|
||||
Lacme/items/Rocket;->size:I
|
||||
Lacme/test/Other;->getThing()Z
|
||||
""",
|
||||
_MAX_TARGET_Q:
|
||||
"""
|
||||
Lacme/items/Rock;->size:I
|
||||
Lacme/test/Widget;-><init()V
|
||||
""",
|
||||
_MAX_TARGET_R:
|
||||
"""
|
||||
Lacme/items/Lever;->size:I
|
||||
Lacme/test/Gadget;->NAME:Ljava/lang/String;
|
||||
""",
|
||||
"bcpf-dir/hiddenapi/hiddenapi-max-target-p.txt":
|
||||
"""
|
||||
Lacme/old/Class;->getWidget()Lacme/test/Widget;
|
||||
""",
|
||||
"out/soong/.intermediates/bcpf-dir/bcpf/all-flags.csv":
|
||||
"""
|
||||
Lacme/test/Gadget;->NAME:Ljava/lang/String;,blocked
|
||||
Lacme/test/Widget;-><init()V,blocked
|
||||
Lacme/test/Class;-><init>()V,blocked
|
||||
Lacme/test/Other;->getThing()Z,blocked
|
||||
""",
|
||||
}
|
||||
|
||||
analyzer = self.create_analyzer_for_test(fs)
|
||||
|
||||
# Override the build_file_read_output() method to just return a fake
|
||||
# build operation.
|
||||
analyzer.build_file_read_output = unittest.mock.Mock(
|
||||
return_value=operation)
|
||||
|
||||
# Override the run_command() method to do nothing.
|
||||
analyzer.run_command = unittest.mock.Mock()
|
||||
|
||||
result = ab.Result()
|
||||
|
||||
analyzer.build_monolithic_flags(result)
|
||||
expected_diffs = {
|
||||
"Lacme/test/Gadget;->NAME:Ljava/lang/String;":
|
||||
(["blocked"], ["lo-prio", "max-target-r"]),
|
||||
"Lacme/test/Widget;-><init()V": (["blocked"], ["max-target-q"]),
|
||||
"Lacme/test/Class;-><init>()V": (["blocked"], ["max-target-o"]),
|
||||
"Lacme/test/Other;->getThing()Z": (["blocked"], ["max-target-p"])
|
||||
}
|
||||
self.assertEqual(expected_diffs, result.diffs, msg="flag differences")
|
||||
|
||||
expected_property_changes = [
|
||||
ab.HiddenApiPropertyChange(
|
||||
property_name="max_target_o_low_priority",
|
||||
values=["hiddenapi/hiddenapi-max-target-o-low-priority.txt"],
|
||||
property_comment=""),
|
||||
ab.HiddenApiPropertyChange(
|
||||
property_name="max_target_p",
|
||||
values=["hiddenapi/hiddenapi-max-target-p.txt"],
|
||||
property_comment=""),
|
||||
ab.HiddenApiPropertyChange(
|
||||
property_name="max_target_q",
|
||||
values=["hiddenapi/hiddenapi-max-target-q.txt"],
|
||||
property_comment=""),
|
||||
ab.HiddenApiPropertyChange(
|
||||
property_name="max_target_r_low_priority",
|
||||
values=["hiddenapi/hiddenapi-max-target-r-low-priority.txt"],
|
||||
property_comment=""),
|
||||
]
|
||||
self.assertEqual(
|
||||
expected_property_changes,
|
||||
result.property_changes,
|
||||
msg="property changes")
|
||||
|
||||
expected_file_changes = [
|
||||
ab.FileChange(
|
||||
path="bcpf-dir/hiddenapi/"
|
||||
"hiddenapi-max-target-o-low-priority.txt",
|
||||
description="""Add the following entries:
|
||||
Lacme/test/Class;-><init>()V
|
||||
""",
|
||||
),
|
||||
ab.FileChange(
|
||||
path="bcpf-dir/hiddenapi/hiddenapi-max-target-p.txt",
|
||||
description="""Add the following entries:
|
||||
Lacme/test/Other;->getThing()Z
|
||||
""",
|
||||
),
|
||||
ab.FileChange(
|
||||
path="bcpf-dir/hiddenapi/hiddenapi-max-target-q.txt",
|
||||
description="""Add the following entries:
|
||||
Lacme/test/Widget;-><init()V
|
||||
"""),
|
||||
ab.FileChange(
|
||||
path="bcpf-dir/hiddenapi/"
|
||||
"hiddenapi-max-target-r-low-priority.txt",
|
||||
description="""Add the following entries:
|
||||
Lacme/test/Gadget;->NAME:Ljava/lang/String;
|
||||
"""),
|
||||
ab.FileChange(
|
||||
path="frameworks/base/boot/hiddenapi/"
|
||||
"hiddenapi-max-target-o.txt",
|
||||
description="""Remove the following entries:
|
||||
Lacme/test/Class;-><init>()V
|
||||
"""),
|
||||
ab.FileChange(
|
||||
path="frameworks/base/boot/hiddenapi/"
|
||||
"hiddenapi-max-target-p.txt",
|
||||
description="""Remove the following entries:
|
||||
Lacme/test/Other;->getThing()Z
|
||||
"""),
|
||||
ab.FileChange(
|
||||
path="frameworks/base/boot/hiddenapi/"
|
||||
"hiddenapi-max-target-q.txt",
|
||||
description="""Remove the following entries:
|
||||
Lacme/test/Widget;-><init()V
|
||||
"""),
|
||||
ab.FileChange(
|
||||
path="frameworks/base/boot/hiddenapi/"
|
||||
"hiddenapi-max-target-r-loprio.txt",
|
||||
description="""Remove the following entries:
|
||||
Lacme/test/Gadget;->NAME:Ljava/lang/String;
|
||||
""")
|
||||
]
|
||||
result.file_changes.sort()
|
||||
self.assertEqual(
|
||||
expected_file_changes, result.file_changes, msg="file_changes")
|
||||
|
||||
|
||||
class TestHiddenApiPropertyChange(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# Create a temporary directory
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
# Remove the directory after the test
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
def check_change_snippet(self, change, expected):
|
||||
snippet = change.snippet(" ")
|
||||
self.assertEqual(expected, snippet)
|
||||
|
||||
def test_change_property_with_value_no_comment(self):
|
||||
change = ab.HiddenApiPropertyChange(
|
||||
property_name="split_packages",
|
||||
values=["android.provider"],
|
||||
)
|
||||
|
||||
self.check_change_snippet(
|
||||
change, """
|
||||
split_packages: [
|
||||
"android.provider",
|
||||
],
|
||||
""")
|
||||
|
||||
def test_change_property_with_value_and_comment(self):
|
||||
change = ab.HiddenApiPropertyChange(
|
||||
property_name="split_packages",
|
||||
values=["android.provider"],
|
||||
property_comment=_MULTI_LINE_COMMENT,
|
||||
)
|
||||
|
||||
self.check_change_snippet(
|
||||
change, """
|
||||
// Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut arcu
|
||||
// justo, bibendum eu malesuada vel, fringilla in odio. Etiam gravida
|
||||
// ultricies sem tincidunt luctus.
|
||||
split_packages: [
|
||||
"android.provider",
|
||||
],
|
||||
""")
|
||||
|
||||
def test_change_without_value_or_empty_comment(self):
|
||||
change = ab.HiddenApiPropertyChange(
|
||||
property_name="split_packages",
|
||||
values=[],
|
||||
property_comment="Single line comment.",
|
||||
)
|
||||
|
||||
self.check_change_snippet(
|
||||
change, """
|
||||
// Single line comment.
|
||||
split_packages: [],
|
||||
""")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=3)
|
||||
Reference in New Issue
Block a user