From 934bd8dcfcc9de8de015dd49c317456b16279a0f Mon Sep 17 00:00:00 2001 From: Joe Onorato Date: Thu, 16 Jul 2020 18:10:48 -0700 Subject: [PATCH 1/3] Add per-partition summaries and "easy" transitions to mk2bp_catalog "Easy" means makefiles that are both clean themselves and only depend on clean targets themselves too. Test: m out/target/product/$(get_build_var TARGET_DEVICE)/mk2bp_remaining.html Change-Id: If3402a0ada5d9db5fb04c617d197ab28d695b03c --- tools/mk2bp_catalog.py | 185 +++++++++++++++++++++++++++++------------ 1 file changed, 133 insertions(+), 52 deletions(-) diff --git a/tools/mk2bp_catalog.py b/tools/mk2bp_catalog.py index 83abd62512..e85ae18faa 100755 --- a/tools/mk2bp_catalog.py +++ b/tools/mk2bp_catalog.py @@ -119,6 +119,15 @@ class Summary(object): self.makefiles[makefile.filename] = makefile self.directories.setdefault(directory_group(makefile.filename), []).append(makefile) + def IsClean(self, filename): + """Also returns true if filename isn't present, with the assumption that it's already + converted. + """ + makefile = self.makefiles.get(filename) + if not makefile: + return True + return is_clean(makefile) + class Makefile(object): def __init__(self, filename): self.filename = filename @@ -223,6 +232,16 @@ class SoongData(object): for f in installed.strip().split(' '): self.installed[f] = module + def transitive_deps(self, module): + results = set() + def traverse(module): + for dep in self.deps.get(module, []): + if not dep in results: + results.add(dep) + traverse(module) + traverse(module) + return results + def count_deps(depsdb, module, seen): """Based on the depsdb, count the number of transitive dependencies. @@ -273,6 +292,84 @@ def format_module_link(module): def format_module_list(modules): return "".join(["
%s
" % format_module_link(m) for m in modules]) +def traverse_ready_makefiles(soong, summary, makefiles): + def clean_and_only_blocked_by_clean(makefile): + if not is_clean(makefile): + return False + modules = soong.reverse_makefiles[makefile.filename] + for module in modules: + for dep in soong.transitive_deps(module): + for m in soong.makefiles.get(dep, []): + if not summary.IsClean(m): + return False + return True + + return [Analysis(makefile.filename, []) for makefile in makefiles + if clean_and_only_blocked_by_clean(makefile)] + +def print_analysis_header(link, title): + print(""" + +

%(title)s

+ + + + + + + + + + """ % { + "link": link, + "title": title + }) + for analyzer in ANALYZERS: + print("""""" % analyzer.title) + print(" ") + + +def print_analysis_row(soong, summary, annotations, modules, rowtitle, rowclass, makefiles): + all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles] + clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if is_clean(makefile)] + easy_makefiles = traverse_ready_makefiles(soong, summary, makefiles) + unblocked_clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if (contains_unblocked_modules(soong, soong.reverse_makefiles[makefile.filename]) + and is_clean(makefile))] + unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if contains_unblocked_modules(soong, + soong.reverse_makefiles[makefile.filename])] + blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if contains_blocked_modules(soong, soong.reverse_makefiles[makefile.filename])] + + print(""" + + + + + + + + + """ % { + "rowclass": rowclass, + "rowtitle": rowtitle, + "makefiles": make_annotation_link(annotations, all_makefiles, modules), + "unblocked": make_annotation_link(annotations, unblocked_makefiles, modules), + "blocked": make_annotation_link(annotations, blocked_makefiles, modules), + "clean": make_annotation_link(annotations, clean_makefiles, modules), + "unblocked_clean": make_annotation_link(annotations, unblocked_clean_makefiles, modules), + "easy": make_annotation_link(annotations, easy_makefiles, modules), + }) + + for analyzer in ANALYZERS: + analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)] + print("""""" + % make_annotation_link(annotations, analyses, modules)) + + print(" ") + def main(): parser = argparse.ArgumentParser(description="Info about remaining Android.mk files.") parser.add_argument("--device", type=str, required=True, @@ -297,7 +394,7 @@ def main(): args.out_dir = args.out_dir[:-1] TARGET_DEVICE = args.device - HOST_OUT_ROOT = args.out_dir + "host" + HOST_OUT_ROOT = args.out_dir + "/host" PRODUCT_OUT = args.out_dir + "/target/product/%s" % TARGET_DEVICE if args.title: @@ -343,7 +440,7 @@ def main(): overflow: hidden; } #tables { - padding: 0 20px 0 20px; + padding: 0 20px 40px 20px; overflow: scroll; flex: 2 2 600px; } @@ -359,7 +456,7 @@ def main(): h2 { margin: 12px 0 4px 0; } - .DirName { + .RowTitle { text-align: left; width: 200px; min-width: 200px; @@ -395,6 +492,10 @@ def main(): padding: 2px 4px; border-right: 2px solid white; } + tr.TotalRow td { + background-color: white; + border-right-color: white; + } tr.AospDir td { background-color: #e6f4ea; border-right-color: #e6f4ea; @@ -501,6 +602,7 @@ def main(): print(""" + Overall Summary
@@ -529,6 +631,16 @@ def main():
+ + + + + + + + + ") print("""
DirectoryTotalEasyUnblocked CleanUnblockedBlockedClean%s
%(rowtitle)s%(makefiles)s%(easy)s%(unblocked_clean)s%(unblocked)s%(blocked)s%(clean)s%s
Total The total number of makefiles in this each directory.
EasyThe number of makefiles that have no warnings themselves, and also + none of their dependencies have warnings either.
Unblocked CleanThe number of makefiles that are both Unblocked and Clean.
Unblocked Makefiles containing one or more modules that don't have any @@ -598,6 +710,7 @@ def main(): """) annotations = Annotations() + overall_summary = Summary() # For each partition makefiles_for_partitions = dict() @@ -612,6 +725,7 @@ def main(): for filename in makefiles: if not filename.startswith(args.out_dir + "/"): summary.Add(Makefile(filename)) + overall_summary.Add(Makefile(filename)) # Categorize directories by who is responsible aosp_dirs = [] @@ -625,61 +739,18 @@ def main(): else: partner_dirs.append(dirname) - print(""" - -

%(partition)s

- - - - - - - - """ % { - "partition": partition - }) + print_analysis_header("partition_" + partition, partition) - for analyzer in ANALYZERS: - print("""""" % analyzer.title) - - print(" ") for dirgroup, rowclass in [(aosp_dirs, "AospDir"), (google_dirs, "GoogleDir"), (partner_dirs, "PartnerDir"),]: for dirname in dirgroup: - makefiles = summary.directories[dirname] + print_analysis_row(soong, summary, annotations, modules, + dirname, rowclass, summary.directories[dirname]) - all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles] - clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles - if is_clean(makefile)] - unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles - if contains_unblocked_modules(soong, - soong.reverse_makefiles[makefile.filename])] - blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles - if contains_blocked_modules(soong, - soong.reverse_makefiles[makefile.filename])] - - print(""" - - - - - - - """ % { - "rowclass": rowclass, - "dirname": dirname, - "makefiles": make_annotation_link(annotations, all_makefiles, modules), - "unblocked": make_annotation_link(annotations, unblocked_makefiles, modules), - "blocked": make_annotation_link(annotations, blocked_makefiles, modules), - "clean": make_annotation_link(annotations, clean_makefiles, modules), - }) - for analyzer in ANALYZERS: - analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)] - print("""""" - % make_annotation_link(annotations, analyses, modules)) - - print(" ") + print_analysis_row(soong, summary, annotations, modules, + "Total", "TotalRow", + set(itertools.chain.from_iterable(summary.directories.values()))) print("""
DirectoryTotalUnblockedBlockedClean%s
%(dirname)s%(makefiles)s%(unblocked)s%(blocked)s%(clean)s%s
""") @@ -718,6 +789,16 @@ def main(): print("
""") + print_analysis_header("summary", "Overall Summary") + + modules = [module for installed, module in soong.installed.items()] + print_analysis_row(soong, overall_summary, annotations, modules, + "All Makefiles", "TotalRow", + set(itertools.chain.from_iterable(overall_summary.directories.values()))) + print(""" + + """) + print(""" + for entry, mods in self.annotations.entries: + print(" [") + for analysis in entry: + print(" new Analysis('%(filename)s', %(modules)s, [%(line_matches)s])," % { + "filename": analysis.filename, + #"modules": json.dumps([m for m in mods if m in filename in self.soong.makefiles[m]]), + "modules": json.dumps( + [m for m in self.soong.reverse_makefiles[analysis.filename] if m in mods]), + "line_matches": ", ".join([ + "new LineMatch(%d, %s)" % (lineno, json.dumps(text)) + for lineno, text in analysis.line_matches]), + }) + print(" ],") + print(""" + ]; + var MODULE_DATA = { + """) + for module in self.soong.modules: + print(" '%(name)s': new Module(%(deps)s)," % { + "name": module, + "deps": json.dumps(self.soong.deps[module]), + }) + print(""" + }; + - """) + """) - print(""" - -
-
- - - + print(""" +
+
+
+ + + +
+
-
-
- - - """) + + + """) + + def traverse_ready_makefiles(self, summary, makefiles): + def clean_and_only_blocked_by_clean(makefile): + if not is_clean(makefile): + return False + modules = self.soong.reverse_makefiles[makefile.filename] + for module in modules: + for dep in self.soong.transitive_deps(module): + for m in self.soong.makefiles.get(dep, []): + if not summary.IsClean(m): + return False + return True + + return [Analysis(makefile.filename, []) for makefile in makefiles + if clean_and_only_blocked_by_clean(makefile)] + + def print_analysis_row(self, summary, modules, rowtitle, rowclass, makefiles): + all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles] + clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if is_clean(makefile)] + easy_makefiles = self.traverse_ready_makefiles(summary, makefiles) + unblocked_clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if (self.soong.contains_unblocked_modules(makefile.filename) + and is_clean(makefile))] + unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if self.soong.contains_unblocked_modules(makefile.filename)] + blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if self.soong.contains_blocked_modules(makefile.filename)] + + print(""" + + %(rowtitle)s + %(makefiles)s + %(easy)s + %(unblocked_clean)s + %(unblocked)s + %(blocked)s + %(clean)s + """ % { + "rowclass": rowclass, + "rowtitle": rowtitle, + "makefiles": self.make_annotation_link(all_makefiles, modules), + "unblocked": self.make_annotation_link(unblocked_makefiles, modules), + "blocked": self.make_annotation_link(blocked_makefiles, modules), + "clean": self.make_annotation_link(clean_makefiles, modules), + "unblocked_clean": self.make_annotation_link(unblocked_clean_makefiles, modules), + "easy": self.make_annotation_link(easy_makefiles, modules), + }) + + for analyzer in ANALYZERS: + analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)] + print("""%s""" + % self.make_annotation_link(analyses, modules)) + + print(" ") + + def make_annotation_link(self, analysis, modules): + if analysis: + return "%s" % ( + self.annotations.Add(analysis, modules), + len(analysis) + ) + else: + return ""; if __name__ == "__main__": main() From 3198607cfd960dbcd82d199f217547a658835db9 Mon Sep 17 00:00:00 2001 From: Joe Onorato Date: Mon, 10 Aug 2020 09:58:49 -0700 Subject: [PATCH 3/3] Add CSV output of remaining makefiles for bp converstion Test: m out/target/product/$(get_build_var TARGET_DEVICE)/mk2bp_remaining.html out/target/product/$(get_build_var TARGET_DEVICE)/mk2bp_remaining.csv Change-Id: I4a43604b2d536387c787b02a4627a0ad9e2a6ff9 --- core/Makefile | 11 ++++++ tools/mk2bp_catalog.py | 90 +++++++++++++++++++++++++++++++----------- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/core/Makefile b/core/Makefile index 4e03c92090..3030832cd2 100644 --- a/core/Makefile +++ b/core/Makefile @@ -495,9 +495,20 @@ $(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) --title="Remaining Android.mk files for $(TARGET_DEVICE)-$(TARGET_BUILD_VARIANT)" \ --codesearch=$(PRIVATE_CODE_SEARCH_BASE_URL) \ --out_dir="$(OUT_DIR)" \ + --mode=html \ > $@ $(call dist-for-goals,droidcore,$(MK2BP_REMAINING_HTML)) +MK2BP_REMAINING_CSV := $(PRODUCT_OUT)/mk2bp_remaining.csv +$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) + @rm -f $@ + $(hide) $(MK2BP_CATALOG_SCRIPT) \ + --device=$(TARGET_DEVICE) \ + --out_dir="$(OUT_DIR)" \ + --mode=csv \ + > $@ +$(call dist-for-goals,droidcore,$(MK2BP_REMAINING_CSV)) + # ----------------------------------------------------------------- # Modules use -Wno-error, or added default -Wall -Werror WALL_WERROR := $(PRODUCT_OUT)/wall_werror.txt diff --git a/tools/mk2bp_catalog.py b/tools/mk2bp_catalog.py index e41f3eada0..c2afb9b948 100755 --- a/tools/mk2bp_catalog.py +++ b/tools/mk2bp_catalog.py @@ -119,15 +119,6 @@ class Summary(object): self.makefiles[makefile.filename] = makefile self.directories.setdefault(directory_group(makefile.filename), []).append(makefile) - def IsClean(self, filename): - """Also returns true if filename isn't present, with the assumption that it's already - converted. - """ - makefile = self.makefiles.get(filename) - if not makefile: - return True - return is_clean(makefile) - class Makefile(object): def __init__(self, filename): self.filename = filename @@ -183,6 +174,18 @@ def is_clean(makefile): return False return True +def clean_and_only_blocked_by_clean(soong, all_makefiles, makefile): + if not is_clean(makefile): + return False + modules = soong.reverse_makefiles[makefile.filename] + for module in modules: + for dep in soong.transitive_deps(module): + for filename in soong.makefiles.get(dep, []): + m = all_makefiles.get(filename) + if m and not is_clean(m): + return False + return True + class Annotations(object): def __init__(self): self.entries = [] @@ -204,6 +207,7 @@ class SoongData(object): self.makefiles = dict() self.reverse_makefiles = dict() self.installed = dict() + self.reverse_installed = dict() self.modules = set() for (module, module_type, problem, dependencies, makefiles, installed) in reader: @@ -221,6 +225,7 @@ class SoongData(object): self.reverse_makefiles.setdefault(f, []).append(module) for f in installed.strip().split(' '): self.installed[f] = module + self.reverse_installed.setdefault(module, []).append(f) def transitive_deps(self, module): results = set() @@ -317,6 +322,9 @@ def main(): help="Equivalent of $OUT_DIR, which will also be checked if" + " --out_dir is unset. If neither is set, default is" + " 'out'.") + parser.add_argument("--mode", type=str, + default="html", + help="output format: csv or html") args = parser.parse_args() @@ -346,7 +354,10 @@ def main(): continue all_makefiles[filename] = Makefile(filename) - HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute() + if args.mode == "html": + HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute() + elif args.mode == "csv": + CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute() class HtmlProcessor(object): def __init__(self, args, soong, all_makefiles): @@ -921,19 +932,8 @@ class HtmlProcessor(object): """) def traverse_ready_makefiles(self, summary, makefiles): - def clean_and_only_blocked_by_clean(makefile): - if not is_clean(makefile): - return False - modules = self.soong.reverse_makefiles[makefile.filename] - for module in modules: - for dep in self.soong.transitive_deps(module): - for m in self.soong.makefiles.get(dep, []): - if not summary.IsClean(m): - return False - return True - return [Analysis(makefile.filename, []) for makefile in makefiles - if clean_and_only_blocked_by_clean(makefile)] + if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile)] def print_analysis_row(self, summary, modules, rowtitle, rowclass, makefiles): all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles] @@ -984,6 +984,52 @@ class HtmlProcessor(object): else: return ""; +class CsvProcessor(object): + def __init__(self, args, soong, all_makefiles): + self.args = args + self.soong = soong + self.all_makefiles = all_makefiles + + def execute(self): + csvout = csv.writer(sys.stdout) + + # Title row + row = ["Filename", "Module", "Partitions", "Easy", "Unblocked Clean", "Unblocked", + "Blocked", "Clean"] + for analyzer in ANALYZERS: + row.append(analyzer.title) + csvout.writerow(row) + + # Makefile & module data + for filename in sorted(self.all_makefiles.keys()): + makefile = self.all_makefiles[filename] + for module in self.soong.reverse_makefiles[filename]: + row = [filename, module] + # Partitions + row.append(";".join(sorted(set([get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, + installed) + for installed + in self.soong.reverse_installed.get(module, [])])))) + # Easy + row.append(1 + if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile) + else "") + # Unblocked Clean + row.append(1 + if (self.soong.contains_unblocked_modules(makefile.filename) and is_clean(makefile)) + else "") + # Unblocked + row.append(1 if self.soong.contains_unblocked_modules(makefile.filename) else "") + # Blocked + row.append(1 if self.soong.contains_blocked_modules(makefile.filename) else "") + # Clean + row.append(1 if is_clean(makefile) else "") + # Analysis + for analyzer in ANALYZERS: + row.append(1 if makefile.analyses.get(analyzer) else "") + # Write results + csvout.writerow(row) + if __name__ == "__main__": main()