StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge changes I4a43604b,I7200f5f9,If3402a0a am: 55c090ebe0 am: ef5dc401ae am: e886de863b

Browse Source 
Original change: https://android-review.googlesource.com/c/platform/build/+/1413820

Change-Id: Ibc696d4ce4083df1a74939dcdef2cd6957fff355
This commit is contained in:
Joe Onorato
2020-09-11 07:17:50 +00:00
committed by Automerger Merge Worker
parent d5ad93ed52 e886de863b
commit f57c53d1e0
2 changed files with 709 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
core/Makefile
View File

@@ -495,9 +495,20 @@ $(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
--title="Remaining Android.mk files for $(TARGET_DEVICE)-$(TARGET_BUILD_VARIANT)" \
--codesearch=$(PRIVATE_CODE_SEARCH_BASE_URL) \
--out_dir="$(OUT_DIR)" \
--mode=html \
> $@
$(call dist-for-goals,droidcore,$(MK2BP_REMAINING_HTML))
MK2BP_REMAINING_CSV := $(PRODUCT_OUT)/mk2bp_remaining.csv
$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
@rm -f $@
$(hide) $(MK2BP_CATALOG_SCRIPT) \
--device=$(TARGET_DEVICE) \
--out_dir="$(OUT_DIR)" \
--mode=csv \
> $@
$(call dist-for-goals,droidcore,$(MK2BP_REMAINING_CSV))
# -----------------------------------------------------------------
# Modules use -Wno-error, or added default -Wall -Werror
WALL_WERROR := $(PRODUCT_OUT)/wall_werror.txt

337
tools/mk2bp_catalog.py
View File

@@ -168,22 +168,24 @@ def is_google(dirname):
return True
return False
def make_annotation_link(annotations, analysis, modules):
if analysis:
return "<a href='javascript:update_details(%d)'>%s</a>" % (
annotations.Add(analysis, modules),
len(analysis)
)
else:
return "";
def is_clean(makefile):
for analysis in makefile.analyses.values():
if analysis:
return False
return True
def clean_and_only_blocked_by_clean(soong, all_makefiles, makefile):
if not is_clean(makefile):
return False
modules = soong.reverse_makefiles[makefile.filename]
for module in modules:
for dep in soong.transitive_deps(module):
for filename in soong.makefiles.get(dep, []):
m = all_makefiles.get(filename)
if m and not is_clean(m):
return False
return True
class Annotations(object):
def __init__(self):
self.entries = []
@@ -205,6 +207,7 @@ class SoongData(object):
self.makefiles = dict()
self.reverse_makefiles = dict()
self.installed = dict()
self.reverse_installed = dict()
self.modules = set()
for (module, module_type, problem, dependencies, makefiles, installed) in reader:
@@ -222,6 +225,29 @@ class SoongData(object):
self.reverse_makefiles.setdefault(f, []).append(module)
for f in installed.strip().split(' '):
self.installed[f] = module
self.reverse_installed.setdefault(module, []).append(f)
def transitive_deps(self, module):
results = set()
def traverse(module):
for dep in self.deps.get(module, []):
if not dep in results:
results.add(dep)
traverse(module)
traverse(module)
return results
def contains_unblocked_modules(self, filename):
for m in self.reverse_makefiles[filename]:
if len(self.deps[m]) == 0:
return True
return False
def contains_blocked_modules(self, filename):
for m in self.reverse_makefiles[filename]:
if len(self.deps[m]) > 0:
return True
return False
def count_deps(depsdb, module, seen):
"""Based on the depsdb, count the number of transitive dependencies.
@@ -237,18 +263,6 @@ def count_deps(depsdb, module, seen):
count += 1 + count_deps(depsdb, dep, seen)
return count
def contains_unblocked_modules(soong, modules):
for m in modules:
if len(soong.deps[m]) == 0:
return True
return False
def contains_blocked_modules(soong, modules):
for m in modules:
if len(soong.deps[m]) > 0:
return True
return False
OTHER_PARTITON = "_other"
HOST_PARTITON = "_host"
@@ -273,6 +287,27 @@ def format_module_link(module):
def format_module_list(modules):
return "".join(["<div>%s</div>" % format_module_link(m) for m in modules])
def print_analysis_header(link, title):
print("""
<a name="%(link)s"></a>
<h2>%(title)s</h2>
<table>
<tr>
<th class="RowTitle">Directory</th>
<th class="Count">Total</th>
<th class="Count Clean">Easy</th>
<th class="Count Clean">Unblocked Clean</th>
<th class="Count Unblocked">Unblocked</th>
<th class="Count Blocked">Blocked</th>
<th class="Count Clean">Clean</th>
""" % {
"link": link,
"title": title
})
for analyzer in ANALYZERS:
print("""<th class="Count Warning">%s</th>""" % analyzer.title)
print(" </tr>")
def main():
parser = argparse.ArgumentParser(description="Info about remaining Android.mk files.")
parser.add_argument("--device", type=str, required=True,
@@ -287,6 +322,9 @@ def main():
help="Equivalent of $OUT_DIR, which will also be checked if"
+ " --out_dir is unset. If neither is set, default is"
+ " 'out'.")
parser.add_argument("--mode", type=str,
default="html",
help="output format: csv or html")
args = parser.parse_args()
@@ -297,14 +335,11 @@ def main():
args.out_dir = args.out_dir[:-1]
TARGET_DEVICE = args.device
HOST_OUT_ROOT = args.out_dir + "host"
global HOST_OUT_ROOT
HOST_OUT_ROOT = args.out_dir + "/host"
global PRODUCT_OUT
PRODUCT_OUT = args.out_dir + "/target/product/%s" % TARGET_DEVICE
if args.title:
page_title = args.title
else:
page_title = "Remaining Android.mk files"
# Read target information
# TODO: Pull from configurable location. This is also slightly different because it's
# only a single build, where as the tree scanning we do below is all Android.mk files.
@@ -312,10 +347,35 @@ def main():
% PRODUCT_OUT, "r", errors="ignore") as csvfile:
soong = SoongData(csv.reader(csvfile))
# Read the makefiles
all_makefiles = dict()
for filename, modules in soong.reverse_makefiles.items():
if filename.startswith(args.out_dir + "/"):
continue
all_makefiles[filename] = Makefile(filename)
if args.mode == "html":
HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()
elif args.mode == "csv":
CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()
class HtmlProcessor(object):
def __init__(self, args, soong, all_makefiles):
self.args = args
self.soong = soong
self.all_makefiles = all_makefiles
self.annotations = Annotations()
def execute(self):
if self.args.title:
page_title = self.args.title
else:
page_title = "Remaining Android.mk files"
# Which modules are installed where
modules_by_partition = dict()
partitions = set()
for installed, module in soong.installed.items():
for installed, module in self.soong.installed.items():
partition = get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, installed)
modules_by_partition.setdefault(partition, []).append(module)
partitions.add(partition)
@@ -343,7 +403,7 @@ def main():
overflow: hidden;
}
#tables {
padding: 0 20px 0 20px;
padding: 0 20px 40px 20px;
overflow: scroll;
flex: 2 2 600px;
}
@@ -359,7 +419,7 @@ def main():
h2 {
margin: 12px 0 4px 0;
}
.DirName {
.RowTitle {
text-align: left;
width: 200px;
min-width: 200px;
@@ -395,6 +455,10 @@ def main():
padding: 2px 4px;
border-right: 2px solid white;
}
tr.TotalRow td {
background-color: white;
border-right-color: white;
}
tr.AospDir td {
background-color: #e6f4ea;
border-right-color: #e6f4ea;
@@ -501,6 +565,7 @@ def main():
print("""
<span class='NavSpacer'></span><span class='NavSpacer'> </span>
<a href='#summary'>Overall Summary</a>
</div>
<div id="container">
<div id="tables">
@@ -529,6 +594,16 @@ def main():
<th>Total</th>
<td>The total number of makefiles in this each directory.</td>
</tr>
<tr>
<th class="Clean">Easy</th>
<td>The number of makefiles that have no warnings themselves, and also
none of their dependencies have warnings either.</td>
</tr>
<tr>
<th class="Clean">Unblocked Clean</th>
<td>The number of makefiles that are both Unblocked and Clean.</td>
</tr>
<tr>
<th class="Unblocked">Unblocked</th>
<td>Makefiles containing one or more modules that don't have any
@@ -597,21 +672,22 @@ def main():
</div>
""")
annotations = Annotations()
overall_summary = Summary()
# For each partition
makefiles_for_partitions = dict()
for partition in sorted(partitions):
modules = modules_by_partition[partition]
makefiles = set(itertools.chain.from_iterable(
[soong.makefiles[module] for module in modules]))
[self.soong.makefiles[module] for module in modules]))
# Read makefiles
summary = Summary()
for filename in makefiles:
if not filename.startswith(args.out_dir + "/"):
summary.Add(Makefile(filename))
makefile = self.all_makefiles.get(filename)
if makefile:
summary.Add(makefile)
overall_summary.Add(makefile)
# Categorize directories by who is responsible
aosp_dirs = []
@@ -625,66 +701,24 @@ def main():
else:
partner_dirs.append(dirname)
print("""
<a name="partition_%(partition)s"></a>
<h2>%(partition)s</h2>
<table>
<tr>
<th class="DirName">Directory</th>
<th class="Count">Total</th>
<th class="Count Unblocked">Unblocked</th>
<th class="Count Blocked">Blocked</th>
<th class="Count Clean">Clean</th>
""" % {
"partition": partition
})
print_analysis_header("partition_" + partition, partition)
for analyzer in ANALYZERS:
print("""<th class="Count Warning">%s</th>""" % analyzer.title)
print(" </tr>")
for dirgroup, rowclass in [(aosp_dirs, "AospDir"),
(google_dirs, "GoogleDir"),
(partner_dirs, "PartnerDir"),]:
for dirname in dirgroup:
makefiles = summary.directories[dirname]
self.print_analysis_row(summary, modules,
dirname, rowclass, summary.directories[dirname])
all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles]
clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
if is_clean(makefile)]
unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
if contains_unblocked_modules(soong,
soong.reverse_makefiles[makefile.filename])]
blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
if contains_blocked_modules(soong,
soong.reverse_makefiles[makefile.filename])]
print("""
<tr class="%(rowclass)s">
<td class="DirName">%(dirname)s</td>
<td class="Count">%(makefiles)s</td>
<td class="Count">%(unblocked)s</td>
<td class="Count">%(blocked)s</td>
<td class="Count">%(clean)s</td>
""" % {
"rowclass": rowclass,
"dirname": dirname,
"makefiles": make_annotation_link(annotations, all_makefiles, modules),
"unblocked": make_annotation_link(annotations, unblocked_makefiles, modules),
"blocked": make_annotation_link(annotations, blocked_makefiles, modules),
"clean": make_annotation_link(annotations, clean_makefiles, modules),
})
for analyzer in ANALYZERS:
analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)]
print("""<td class="Count">%s</td>"""
% make_annotation_link(annotations, analyses, modules))
print(" </tr>")
self.print_analysis_row(summary, modules,
"Total", "TotalRow",
set(itertools.chain.from_iterable(summary.directories.values())))
print("""
</table>
""")
module_details = [(count_deps(soong.deps, m, []), -count_deps(soong.reverse_deps, m, []), m)
module_details = [(count_deps(self.soong.deps, m, []),
-count_deps(self.soong.reverse_deps, m, []), m)
for m in modules]
module_details.sort()
module_details = [m[2] for m in module_details]
@@ -699,7 +733,7 @@ def main():
altRow = True
for module in module_details:
analyses = set()
for filename in soong.makefiles[module]:
for filename in self.soong.makefiles[module]:
makefile = summary.makefiles.get(filename)
if makefile:
for analyzer, analysis in makefile.analyses.items():
@@ -711,13 +745,23 @@ def main():
print(" <td><a name='module_%s'></a>%s</td>" % (module, module))
print(" <td class='AnalysisCol'>%s</td>" % " ".join(["<span class='Analysis'>%s</span>" % title
for title in analyses]))
print(" <td>%s</td>" % count_deps(soong.deps, module, []))
print(" <td>%s</td>" % format_module_list(soong.deps.get(module, [])))
print(" <td>%s</td>" % count_deps(soong.reverse_deps, module, []))
print(" <td>%s</td>" % format_module_list(soong.reverse_deps.get(module, [])))
print(" <td>%s</td>" % count_deps(self.soong.deps, module, []))
print(" <td>%s</td>" % format_module_list(self.soong.deps.get(module, [])))
print(" <td>%s</td>" % count_deps(self.soong.reverse_deps, module, []))
print(" <td>%s</td>" % format_module_list(self.soong.reverse_deps.get(module, [])))
print("</tr>")
print("""</table>""")
print_analysis_header("summary", "Overall Summary")
modules = [module for installed, module in self.soong.installed.items()]
self.print_analysis_row(overall_summary, modules,
"All Makefiles", "TotalRow",
set(itertools.chain.from_iterable(overall_summary.directories.values())))
print("""
</table>
""")
print("""
<script type="text/javascript">
function close_details() {
@@ -843,16 +887,16 @@ def main():
var ANALYSIS = [
""" % {
"codesearch": args.codesearch,
"codesearch": self.args.codesearch,
})
for entry, mods in annotations.entries:
for entry, mods in self.annotations.entries:
print(" [")
for analysis in entry:
print(" new Analysis('%(filename)s', %(modules)s, [%(line_matches)s])," % {
"filename": analysis.filename,
#"modules": json.dumps([m for m in mods if m in filename in soong.makefiles[m]]),
#"modules": json.dumps([m for m in mods if m in filename in self.soong.makefiles[m]]),
"modules": json.dumps(
[m for m in soong.reverse_makefiles[analysis.filename] if m in mods]),
[m for m in self.soong.reverse_makefiles[analysis.filename] if m in mods]),
"line_matches": ", ".join([
"new LineMatch(%d, %s)" % (lineno, json.dumps(text))
for lineno, text in analysis.line_matches]),
@@ -862,10 +906,10 @@ def main():
];
var MODULE_DATA = {
""")
for module in soong.modules:
for module in self.soong.modules:
print(" '%(name)s': new Module(%(deps)s)," % {
"name": module,
"deps": json.dumps(soong.deps[module]),
"deps": json.dumps(self.soong.deps[module]),
})
print("""
};
@@ -887,6 +931,105 @@ def main():
</html>
""")
def traverse_ready_makefiles(self, summary, makefiles):
return [Analysis(makefile.filename, []) for makefile in makefiles
if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile)]
def print_analysis_row(self, summary, modules, rowtitle, rowclass, makefiles):
all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles]
clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
if is_clean(makefile)]
easy_makefiles = self.traverse_ready_makefiles(summary, makefiles)
unblocked_clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
if (self.soong.contains_unblocked_modules(makefile.filename)
and is_clean(makefile))]
unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
if self.soong.contains_unblocked_modules(makefile.filename)]
blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
if self.soong.contains_blocked_modules(makefile.filename)]
print("""
<tr class="%(rowclass)s">
<td class="RowTitle">%(rowtitle)s</td>
<td class="Count">%(makefiles)s</td>
<td class="Count">%(easy)s</td>
<td class="Count">%(unblocked_clean)s</td>
<td class="Count">%(unblocked)s</td>
<td class="Count">%(blocked)s</td>
<td class="Count">%(clean)s</td>
""" % {
"rowclass": rowclass,
"rowtitle": rowtitle,
"makefiles": self.make_annotation_link(all_makefiles, modules),
"unblocked": self.make_annotation_link(unblocked_makefiles, modules),
"blocked": self.make_annotation_link(blocked_makefiles, modules),
"clean": self.make_annotation_link(clean_makefiles, modules),
"unblocked_clean": self.make_annotation_link(unblocked_clean_makefiles, modules),
"easy": self.make_annotation_link(easy_makefiles, modules),
})
for analyzer in ANALYZERS:
analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)]
print("""<td class="Count">%s</td>"""
% self.make_annotation_link(analyses, modules))
print(" </tr>")
def make_annotation_link(self, analysis, modules):
if analysis:
return "<a href='javascript:update_details(%d)'>%s</a>" % (
self.annotations.Add(analysis, modules),
len(analysis)
)
else:
return "";
class CsvProcessor(object):
def __init__(self, args, soong, all_makefiles):
self.args = args
self.soong = soong
self.all_makefiles = all_makefiles
def execute(self):
csvout = csv.writer(sys.stdout)
# Title row
row = ["Filename", "Module", "Partitions", "Easy", "Unblocked Clean", "Unblocked",
"Blocked", "Clean"]
for analyzer in ANALYZERS:
row.append(analyzer.title)
csvout.writerow(row)
# Makefile & module data
for filename in sorted(self.all_makefiles.keys()):
makefile = self.all_makefiles[filename]
for module in self.soong.reverse_makefiles[filename]:
row = [filename, module]
# Partitions
row.append(";".join(sorted(set([get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT,
installed)
for installed
in self.soong.reverse_installed.get(module, [])]))))
# Easy
row.append(1
if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile)
else "")
# Unblocked Clean
row.append(1
if (self.soong.contains_unblocked_modules(makefile.filename) and is_clean(makefile))
else "")
# Unblocked
row.append(1 if self.soong.contains_unblocked_modules(makefile.filename) else "")
# Blocked
row.append(1 if self.soong.contains_blocked_modules(makefile.filename) else "")
# Clean
row.append(1 if is_clean(makefile) else "")
# Analysis
for analyzer in ANALYZERS:
row.append(1 if makefile.analyses.get(analyzer) else "")
# Write results
csvout.writerow(row)
if __name__ == "__main__":
main()