diff --git a/core/Makefile b/core/Makefile index 669745a894..65f51fc42e 100644 --- a/core/Makefile +++ b/core/Makefile @@ -495,9 +495,20 @@ $(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) --title="Remaining Android.mk files for $(TARGET_DEVICE)-$(TARGET_BUILD_VARIANT)" \ --codesearch=$(PRIVATE_CODE_SEARCH_BASE_URL) \ --out_dir="$(OUT_DIR)" \ + --mode=html \ > $@ $(call dist-for-goals,droidcore,$(MK2BP_REMAINING_HTML)) +MK2BP_REMAINING_CSV := $(PRODUCT_OUT)/mk2bp_remaining.csv +$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) + @rm -f $@ + $(hide) $(MK2BP_CATALOG_SCRIPT) \ + --device=$(TARGET_DEVICE) \ + --out_dir="$(OUT_DIR)" \ + --mode=csv \ + > $@ +$(call dist-for-goals,droidcore,$(MK2BP_REMAINING_CSV)) + # ----------------------------------------------------------------- # Modules use -Wno-error, or added default -Wall -Werror WALL_WERROR := $(PRODUCT_OUT)/wall_werror.txt diff --git a/tools/mk2bp_catalog.py b/tools/mk2bp_catalog.py index 83abd62512..c2afb9b948 100755 --- a/tools/mk2bp_catalog.py +++ b/tools/mk2bp_catalog.py @@ -168,22 +168,24 @@ def is_google(dirname): return True return False -def make_annotation_link(annotations, analysis, modules): - if analysis: - return "%s" % ( - annotations.Add(analysis, modules), - len(analysis) - ) - else: - return ""; - - def is_clean(makefile): for analysis in makefile.analyses.values(): if analysis: return False return True +def clean_and_only_blocked_by_clean(soong, all_makefiles, makefile): + if not is_clean(makefile): + return False + modules = soong.reverse_makefiles[makefile.filename] + for module in modules: + for dep in soong.transitive_deps(module): + for filename in soong.makefiles.get(dep, []): + m = all_makefiles.get(filename) + if m and not is_clean(m): + return False + return True + class Annotations(object): def __init__(self): self.entries = [] @@ -205,6 +207,7 @@ class SoongData(object): self.makefiles = dict() self.reverse_makefiles = dict() self.installed = dict() + self.reverse_installed = dict() self.modules = set() for (module, module_type, problem, dependencies, makefiles, installed) in reader: @@ -222,6 +225,29 @@ class SoongData(object): self.reverse_makefiles.setdefault(f, []).append(module) for f in installed.strip().split(' '): self.installed[f] = module + self.reverse_installed.setdefault(module, []).append(f) + + def transitive_deps(self, module): + results = set() + def traverse(module): + for dep in self.deps.get(module, []): + if not dep in results: + results.add(dep) + traverse(module) + traverse(module) + return results + + def contains_unblocked_modules(self, filename): + for m in self.reverse_makefiles[filename]: + if len(self.deps[m]) == 0: + return True + return False + + def contains_blocked_modules(self, filename): + for m in self.reverse_makefiles[filename]: + if len(self.deps[m]) > 0: + return True + return False def count_deps(depsdb, module, seen): """Based on the depsdb, count the number of transitive dependencies. @@ -237,18 +263,6 @@ def count_deps(depsdb, module, seen): count += 1 + count_deps(depsdb, dep, seen) return count -def contains_unblocked_modules(soong, modules): - for m in modules: - if len(soong.deps[m]) == 0: - return True - return False - -def contains_blocked_modules(soong, modules): - for m in modules: - if len(soong.deps[m]) > 0: - return True - return False - OTHER_PARTITON = "_other" HOST_PARTITON = "_host" @@ -273,6 +287,27 @@ def format_module_link(module): def format_module_list(modules): return "".join(["
%s
" % format_module_link(m) for m in modules]) +def print_analysis_header(link, title): + print(""" + +

%(title)s

+ + + + + + + + + + """ % { + "link": link, + "title": title + }) + for analyzer in ANALYZERS: + print("""""" % analyzer.title) + print(" ") + def main(): parser = argparse.ArgumentParser(description="Info about remaining Android.mk files.") parser.add_argument("--device", type=str, required=True, @@ -287,6 +322,9 @@ def main(): help="Equivalent of $OUT_DIR, which will also be checked if" + " --out_dir is unset. If neither is set, default is" + " 'out'.") + parser.add_argument("--mode", type=str, + default="html", + help="output format: csv or html") args = parser.parse_args() @@ -297,14 +335,11 @@ def main(): args.out_dir = args.out_dir[:-1] TARGET_DEVICE = args.device - HOST_OUT_ROOT = args.out_dir + "host" + global HOST_OUT_ROOT + HOST_OUT_ROOT = args.out_dir + "/host" + global PRODUCT_OUT PRODUCT_OUT = args.out_dir + "/target/product/%s" % TARGET_DEVICE - if args.title: - page_title = args.title - else: - page_title = "Remaining Android.mk files" - # Read target information # TODO: Pull from configurable location. This is also slightly different because it's # only a single build, where as the tree scanning we do below is all Android.mk files. @@ -312,580 +347,688 @@ def main(): % PRODUCT_OUT, "r", errors="ignore") as csvfile: soong = SoongData(csv.reader(csvfile)) - # Which modules are installed where - modules_by_partition = dict() - partitions = set() - for installed, module in soong.installed.items(): - partition = get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, installed) - modules_by_partition.setdefault(partition, []).append(module) - partitions.add(partition) + # Read the makefiles + all_makefiles = dict() + for filename, modules in soong.reverse_makefiles.items(): + if filename.startswith(args.out_dir + "/"): + continue + all_makefiles[filename] = Makefile(filename) - print(""" - - - %(page_title)s - - - -

%(page_title)s

-
- Help - - Partitions: - """ % { - "page_title": page_title, - }) - for partition in sorted(partitions): - print("%s" % (partition, partition)) + if args.mode == "html": + HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute() + elif args.mode == "csv": + CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute() - print(""" - -
-
-
- -
-

- This page analyzes the remaining Android.mk files in the Android Source tree. -

- The modules are first broken down by which of the device filesystem partitions - they are installed to. This also includes host tools and testcases which don't - actually reside in their own partition but convenitely group together. -

- The makefiles for each partition are further are grouped into a set of directories - aritrarily picked to break down the problem size by owners. -

    -
  • AOSP directories are colored green.
    • -
  • Google directories are colored blue.
    • -
  • Other partner directories are colored red.
    • -
- Each of the makefiles are scanned for issues that are likely to come up during - conversion to soong. Clicking the number in each cell shows additional information, - including the line that triggered the warning. -

-

DirectoryTotalEasyUnblocked CleanUnblockedBlockedClean%s
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
TotalThe total number of makefiles in this each directory.
UnblockedMakefiles containing one or more modules that don't have any - additional dependencies pending before conversion.
BlockedMakefiles containiong one or more modules which do have - additional prerequesite depenedencies that are not yet converted.
CleanThe number of makefiles that have none of the following warnings.
ifeq / ifneqMakefiles that use ifeq or ifneq. i.e. - conditionals.
Wacky IncludesMakefiles that include files other than the standard build-system - defined template and macros.
Calls base_rulesMakefiles that include base_rules.mk directly.
Calls defineMakefiles that define their own macros. Some of these are easy to convert - to soong defaults, but others are complex.
Has ../Makefiles containing the string "../" outside of a comment. These likely - access files outside their directories.
dist-for-goalsMakefiles that call dist-for-goals directly.
.PHONYMakefiles that declare .PHONY targets.
renderscriptMakefiles defining targets that depend on .rscript source files.
vts srcMakefiles defining targets that depend on .vts source files.
COPY_HEADERSMakefiles using LOCAL_COPY_HEADERS.
-

- Following the list of directories is a list of the modules that are installed on - each partition. Potential issues from their makefiles are listed, as well as the - total number of dependencies (both blocking that module and blocked by that module) - and the list of direct dependencies. Note: The number is the number of all transitive - dependencies and the list of modules is only the direct dependencies. - - """) +class HtmlProcessor(object): + def __init__(self, args, soong, all_makefiles): + self.args = args + self.soong = soong + self.all_makefiles = all_makefiles + self.annotations = Annotations() - annotations = Annotations() + def execute(self): + if self.args.title: + page_title = self.args.title + else: + page_title = "Remaining Android.mk files" - # For each partition - makefiles_for_partitions = dict() - for partition in sorted(partitions): - modules = modules_by_partition[partition] - - makefiles = set(itertools.chain.from_iterable( - [soong.makefiles[module] for module in modules])) - - # Read makefiles - summary = Summary() - for filename in makefiles: - if not filename.startswith(args.out_dir + "/"): - summary.Add(Makefile(filename)) - - # Categorize directories by who is responsible - aosp_dirs = [] - google_dirs = [] - partner_dirs = [] - for dirname in sorted(summary.directories.keys()): - if is_aosp(dirname): - aosp_dirs.append(dirname) - elif is_google(dirname): - google_dirs.append(dirname) - else: - partner_dirs.append(dirname) + # Which modules are installed where + modules_by_partition = dict() + partitions = set() + for installed, module in self.soong.installed.items(): + partition = get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, installed) + modules_by_partition.setdefault(partition, []).append(module) + partitions.add(partition) print(""" - -

%(partition)s

- - - - - - - + + + %(page_title)s + + + +

%(page_title)s

+
+ Help + + Partitions: """ % { - "partition": partition + "page_title": page_title, }) + for partition in sorted(partitions): + print("%s" % (partition, partition)) - for analyzer in ANALYZERS: - print("""
""" % analyzer.title) - - print(" ") - for dirgroup, rowclass in [(aosp_dirs, "AospDir"), - (google_dirs, "GoogleDir"), - (partner_dirs, "PartnerDir"),]: - for dirname in dirgroup: - makefiles = summary.directories[dirname] - - all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles] - clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles - if is_clean(makefile)] - unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles - if contains_unblocked_modules(soong, - soong.reverse_makefiles[makefile.filename])] - blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles - if contains_blocked_modules(soong, - soong.reverse_makefiles[makefile.filename])] - - print(""" - - - - - - - """ % { - "rowclass": rowclass, - "dirname": dirname, - "makefiles": make_annotation_link(annotations, all_makefiles, modules), - "unblocked": make_annotation_link(annotations, unblocked_makefiles, modules), - "blocked": make_annotation_link(annotations, blocked_makefiles, modules), - "clean": make_annotation_link(annotations, clean_makefiles, modules), - }) - for analyzer in ANALYZERS: - analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)] - print("""""" - % make_annotation_link(annotations, analyses, modules)) - - print(" ") print(""" -
DirectoryTotalUnblockedBlockedClean%s
%(dirname)s%(makefiles)s%(unblocked)s%(blocked)s%(clean)s%s
+ + Overall Summary + +
+
+ +
+

+ This page analyzes the remaining Android.mk files in the Android Source tree. +

+ The modules are first broken down by which of the device filesystem partitions + they are installed to. This also includes host tools and testcases which don't + actually reside in their own partition but convenitely group together. +

+ The makefiles for each partition are further are grouped into a set of directories + aritrarily picked to break down the problem size by owners. +

    +
  • AOSP directories are colored green.
    • +
  • Google directories are colored blue.
    • +
  • Other partner directories are colored red.
    • +
+ Each of the makefiles are scanned for issues that are likely to come up during + conversion to soong. Clicking the number in each cell shows additional information, + including the line that triggered the warning. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TotalThe total number of makefiles in this each directory.
EasyThe number of makefiles that have no warnings themselves, and also + none of their dependencies have warnings either.
Unblocked CleanThe number of makefiles that are both Unblocked and Clean.
UnblockedMakefiles containing one or more modules that don't have any + additional dependencies pending before conversion.
BlockedMakefiles containiong one or more modules which do have + additional prerequesite depenedencies that are not yet converted.
CleanThe number of makefiles that have none of the following warnings.
ifeq / ifneqMakefiles that use ifeq or ifneq. i.e. + conditionals.
Wacky IncludesMakefiles that include files other than the standard build-system + defined template and macros.
Calls base_rulesMakefiles that include base_rules.mk directly.
Calls defineMakefiles that define their own macros. Some of these are easy to convert + to soong defaults, but others are complex.
Has ../Makefiles containing the string "../" outside of a comment. These likely + access files outside their directories.
dist-for-goalsMakefiles that call dist-for-goals directly.
.PHONYMakefiles that declare .PHONY targets.
renderscriptMakefiles defining targets that depend on .rscript source files.
vts srcMakefiles defining targets that depend on .vts source files.
COPY_HEADERSMakefiles using LOCAL_COPY_HEADERS.
+

+ Following the list of directories is a list of the modules that are installed on + each partition. Potential issues from their makefiles are listed, as well as the + total number of dependencies (both blocking that module and blocked by that module) + and the list of direct dependencies. Note: The number is the number of all transitive + dependencies and the list of modules is only the direct dependencies. +

""") - module_details = [(count_deps(soong.deps, m, []), -count_deps(soong.reverse_deps, m, []), m) - for m in modules] - module_details.sort() - module_details = [m[2] for m in module_details] - print(""" - """) - print("") - print(" ") - print(" ") - print(" ") - print(" ") - print("") - altRow = True - for module in module_details: - analyses = set() - for filename in soong.makefiles[module]: - makefile = summary.makefiles.get(filename) + overall_summary = Summary() + + # For each partition + for partition in sorted(partitions): + modules = modules_by_partition[partition] + + makefiles = set(itertools.chain.from_iterable( + [self.soong.makefiles[module] for module in modules])) + + # Read makefiles + summary = Summary() + for filename in makefiles: + makefile = self.all_makefiles.get(filename) if makefile: - for analyzer, analysis in makefile.analyses.items(): - if analysis: - analyses.add(analyzer.title) + summary.Add(makefile) + overall_summary.Add(makefile) - altRow = not altRow - print("" % ("Alt" if altRow else "",)) - print(" " % (module, module)) - print(" " % " ".join(["%s" % title - for title in analyses])) - print(" " % count_deps(soong.deps, module, [])) - print(" " % format_module_list(soong.deps.get(module, []))) - print(" " % count_deps(soong.reverse_deps, module, [])) - print(" " % format_module_list(soong.reverse_deps.get(module, []))) + # Categorize directories by who is responsible + aosp_dirs = [] + google_dirs = [] + partner_dirs = [] + for dirname in sorted(summary.directories.keys()): + if is_aosp(dirname): + aosp_dirs.append(dirname) + elif is_google(dirname): + google_dirs.append(dirname) + else: + partner_dirs.append(dirname) + + print_analysis_header("partition_" + partition, partition) + + for dirgroup, rowclass in [(aosp_dirs, "AospDir"), + (google_dirs, "GoogleDir"), + (partner_dirs, "PartnerDir"),]: + for dirname in dirgroup: + self.print_analysis_row(summary, modules, + dirname, rowclass, summary.directories[dirname]) + + self.print_analysis_row(summary, modules, + "Total", "TotalRow", + set(itertools.chain.from_iterable(summary.directories.values()))) + print(""" +
Module NameIssuesBlocked ByBlocking
%s%s%s%s%s%s
+ """) + + module_details = [(count_deps(self.soong.deps, m, []), + -count_deps(self.soong.reverse_deps, m, []), m) + for m in modules] + module_details.sort() + module_details = [m[2] for m in module_details] + print(""" + """) + print("") + print(" ") + print(" ") + print(" ") + print(" ") print("") - print("""
Module NameIssuesBlocked ByBlocking
""") + altRow = True + for module in module_details: + analyses = set() + for filename in self.soong.makefiles[module]: + makefile = summary.makefiles.get(filename) + if makefile: + for analyzer, analysis in makefile.analyses.items(): + if analysis: + analyses.add(analyzer.title) - print(""" - + for entry, mods in self.annotations.entries: + print(" [") + for analysis in entry: + print(" new Analysis('%(filename)s', %(modules)s, [%(line_matches)s])," % { + "filename": analysis.filename, + #"modules": json.dumps([m for m in mods if m in filename in self.soong.makefiles[m]]), + "modules": json.dumps( + [m for m in self.soong.reverse_makefiles[analysis.filename] if m in mods]), + "line_matches": ", ".join([ + "new LineMatch(%d, %s)" % (lineno, json.dumps(text)) + for lineno, text in analysis.line_matches]), + }) + print(" ],") + print(""" + ]; + var MODULE_DATA = { + """) + for module in self.soong.modules: + print(" '%(name)s': new Module(%(deps)s)," % { + "name": module, + "deps": json.dumps(self.soong.deps[module]), + }) + print(""" + }; + - """) + """) - print(""" -
-
-
- - - + print(""" +
+
+
+ + + +
+
-
-
- - - """) + + + """) + + def traverse_ready_makefiles(self, summary, makefiles): + return [Analysis(makefile.filename, []) for makefile in makefiles + if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile)] + + def print_analysis_row(self, summary, modules, rowtitle, rowclass, makefiles): + all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles] + clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if is_clean(makefile)] + easy_makefiles = self.traverse_ready_makefiles(summary, makefiles) + unblocked_clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if (self.soong.contains_unblocked_modules(makefile.filename) + and is_clean(makefile))] + unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if self.soong.contains_unblocked_modules(makefile.filename)] + blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if self.soong.contains_blocked_modules(makefile.filename)] + + print(""" + + %(rowtitle)s + %(makefiles)s + %(easy)s + %(unblocked_clean)s + %(unblocked)s + %(blocked)s + %(clean)s + """ % { + "rowclass": rowclass, + "rowtitle": rowtitle, + "makefiles": self.make_annotation_link(all_makefiles, modules), + "unblocked": self.make_annotation_link(unblocked_makefiles, modules), + "blocked": self.make_annotation_link(blocked_makefiles, modules), + "clean": self.make_annotation_link(clean_makefiles, modules), + "unblocked_clean": self.make_annotation_link(unblocked_clean_makefiles, modules), + "easy": self.make_annotation_link(easy_makefiles, modules), + }) + + for analyzer in ANALYZERS: + analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)] + print("""%s""" + % self.make_annotation_link(analyses, modules)) + + print(" ") + + def make_annotation_link(self, analysis, modules): + if analysis: + return "%s" % ( + self.annotations.Add(analysis, modules), + len(analysis) + ) + else: + return ""; + +class CsvProcessor(object): + def __init__(self, args, soong, all_makefiles): + self.args = args + self.soong = soong + self.all_makefiles = all_makefiles + + def execute(self): + csvout = csv.writer(sys.stdout) + + # Title row + row = ["Filename", "Module", "Partitions", "Easy", "Unblocked Clean", "Unblocked", + "Blocked", "Clean"] + for analyzer in ANALYZERS: + row.append(analyzer.title) + csvout.writerow(row) + + # Makefile & module data + for filename in sorted(self.all_makefiles.keys()): + makefile = self.all_makefiles[filename] + for module in self.soong.reverse_makefiles[filename]: + row = [filename, module] + # Partitions + row.append(";".join(sorted(set([get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, + installed) + for installed + in self.soong.reverse_installed.get(module, [])])))) + # Easy + row.append(1 + if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile) + else "") + # Unblocked Clean + row.append(1 + if (self.soong.contains_unblocked_modules(makefile.filename) and is_clean(makefile)) + else "") + # Unblocked + row.append(1 if self.soong.contains_unblocked_modules(makefile.filename) else "") + # Blocked + row.append(1 if self.soong.contains_blocked_modules(makefile.filename) else "") + # Clean + row.append(1 if is_clean(makefile) else "") + # Analysis + for analyzer in ANALYZERS: + row.append(1 if makefile.analyses.get(analyzer) else "") + # Write results + csvout.writerow(row) if __name__ == "__main__": main()