Merge changes I4a43604b,I7200f5f9,If3402a0a am: 55c090ebe0 am: ef5dc401ae am: e886de863b

Original change: https://android-review.googlesource.com/c/platform/build/+/1413820

Change-Id: Ibc696d4ce4083df1a74939dcdef2cd6957fff355

core/Makefile

@@ -495,9 +495,20 @@ $(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
 		--title="Remaining Android.mk files for $(TARGET_DEVICE)-$(TARGET_BUILD_VARIANT)" \
 		--codesearch=$(PRIVATE_CODE_SEARCH_BASE_URL) \
 		--out_dir="$(OUT_DIR)" \
+		--mode=html \
 		> $@
 $(call dist-for-goals,droidcore,$(MK2BP_REMAINING_HTML))
 
+MK2BP_REMAINING_CSV := $(PRODUCT_OUT)/mk2bp_remaining.csv
+$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
+	@rm -f $@
+	$(hide) $(MK2BP_CATALOG_SCRIPT) \
+		--device=$(TARGET_DEVICE) \
+		--out_dir="$(OUT_DIR)" \
+		--mode=csv \
+		> $@
+$(call dist-for-goals,droidcore,$(MK2BP_REMAINING_CSV))
+
 # -----------------------------------------------------------------
 # Modules use -Wno-error, or added default -Wall -Werror
 WALL_WERROR := $(PRODUCT_OUT)/wall_werror.txt

tools/mk2bp_catalog.py

@@ -168,22 +168,24 @@ def is_google(dirname):
       return True
   return False
 
-def make_annotation_link(annotations, analysis, modules):
-  if analysis:
-    return "<a href='javascript:update_details(%d)'>%s</a>" % (
-      annotations.Add(analysis, modules),
-      len(analysis)
-    )
-  else:
-    return "";
-
-
 def is_clean(makefile):
   for analysis in makefile.analyses.values():
     if analysis:
       return False
   return True
 
+def clean_and_only_blocked_by_clean(soong, all_makefiles, makefile):
+  if not is_clean(makefile):
+    return False
+  modules = soong.reverse_makefiles[makefile.filename]
+  for module in modules:
+    for dep in soong.transitive_deps(module):
+      for filename in soong.makefiles.get(dep, []):
+        m = all_makefiles.get(filename)
+        if m and not is_clean(m):
+          return False
+  return True
+
 class Annotations(object):
   def __init__(self):
     self.entries = []
@@ -205,6 +207,7 @@ class SoongData(object):
     self.makefiles = dict()
     self.reverse_makefiles = dict()
     self.installed = dict()
+    self.reverse_installed = dict()
     self.modules = set()
 
     for (module, module_type, problem, dependencies, makefiles, installed) in reader:
@@ -222,6 +225,29 @@ class SoongData(object):
         self.reverse_makefiles.setdefault(f, []).append(module)
       for f in installed.strip().split(' '):
         self.installed[f] = module
+        self.reverse_installed.setdefault(module, []).append(f)
+
+  def transitive_deps(self, module):
+    results = set()
+    def traverse(module):
+      for dep in self.deps.get(module, []):
+        if not dep in results:
+          results.add(dep)
+          traverse(module)
+    traverse(module)
+    return results
+
+  def contains_unblocked_modules(self, filename):
+    for m in self.reverse_makefiles[filename]:
+      if len(self.deps[m]) == 0:
+        return True
+    return False
+
+  def contains_blocked_modules(self, filename):
+    for m in self.reverse_makefiles[filename]:
+      if len(self.deps[m]) > 0:
+        return True
+    return False
 
 def count_deps(depsdb, module, seen):
   """Based on the depsdb, count the number of transitive dependencies.
@@ -237,18 +263,6 @@ def count_deps(depsdb, module, seen):
       count += 1 + count_deps(depsdb, dep, seen)
   return count
 
-def contains_unblocked_modules(soong, modules):
-  for m in modules:
-    if len(soong.deps[m]) == 0:
-      return True
-  return False
-
-def contains_blocked_modules(soong, modules):
-  for m in modules:
-    if len(soong.deps[m]) > 0:
-      return True
-  return False
-
 OTHER_PARTITON = "_other"
 HOST_PARTITON = "_host"
 
@@ -273,6 +287,27 @@ def format_module_link(module):
 def format_module_list(modules):
   return "".join(["<div>%s</div>" % format_module_link(m) for m in modules])
 
+def print_analysis_header(link, title):
+  print("""
+    <a name="%(link)s"></a>
+    <h2>%(title)s</h2>
+    <table>
+      <tr>
+        <th class="RowTitle">Directory</th>
+        <th class="Count">Total</th>
+        <th class="Count Clean">Easy</th>
+        <th class="Count Clean">Unblocked Clean</th>
+        <th class="Count Unblocked">Unblocked</th>
+        <th class="Count Blocked">Blocked</th>
+        <th class="Count Clean">Clean</th>
+  """ % {
+    "link": link,
+    "title": title
+  })
+  for analyzer in ANALYZERS:
+    print("""<th class="Count Warning">%s</th>""" % analyzer.title)
+  print("      </tr>")
+
 def main():
   parser = argparse.ArgumentParser(description="Info about remaining Android.mk files.")
   parser.add_argument("--device", type=str, required=True,
@@ -287,6 +322,9 @@ def main():
                       help="Equivalent of $OUT_DIR, which will also be checked if"
                         + " --out_dir is unset. If neither is set, default is"
                         + " 'out'.")
+  parser.add_argument("--mode", type=str,
+                      default="html",
+                      help="output format: csv or html")
 
   args = parser.parse_args()
 
@@ -297,14 +335,11 @@ def main():
     args.out_dir = args.out_dir[:-1]
 
   TARGET_DEVICE = args.device
-  HOST_OUT_ROOT = args.out_dir + "host"
+  global HOST_OUT_ROOT
+  HOST_OUT_ROOT = args.out_dir + "/host"
+  global PRODUCT_OUT
   PRODUCT_OUT = args.out_dir + "/target/product/%s" % TARGET_DEVICE
 
-  if args.title:
-    page_title = args.title
-  else:
-    page_title = "Remaining Android.mk files"
-
   # Read target information
   # TODO: Pull from configurable location. This is also slightly different because it's
   # only a single build, where as the tree scanning we do below is all Android.mk files.
@@ -312,10 +347,35 @@ def main():
       % PRODUCT_OUT, "r", errors="ignore") as csvfile:
     soong = SoongData(csv.reader(csvfile))
 
+  # Read the makefiles
+  all_makefiles = dict()
+  for filename, modules in soong.reverse_makefiles.items():
+    if filename.startswith(args.out_dir + "/"):
+      continue
+    all_makefiles[filename] = Makefile(filename)
+
+  if args.mode == "html":
+    HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()
+  elif args.mode == "csv":
+    CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()
+
+class HtmlProcessor(object):
+  def __init__(self, args, soong, all_makefiles):
+    self.args = args
+    self.soong = soong
+    self.all_makefiles = all_makefiles
+    self.annotations = Annotations()
+
+  def execute(self):
+    if self.args.title:
+      page_title = self.args.title
+    else:
+      page_title = "Remaining Android.mk files"
+
     # Which modules are installed where
     modules_by_partition = dict()
     partitions = set()
-  for installed, module in soong.installed.items():
+    for installed, module in self.soong.installed.items():
       partition = get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, installed)
       modules_by_partition.setdefault(partition, []).append(module)
       partitions.add(partition)
@@ -343,7 +403,7 @@ def main():
             overflow: hidden;
           }
           #tables {
-          padding: 0 20px 0 20px;
+            padding: 0 20px 40px 20px;
             overflow: scroll;
             flex: 2 2 600px;
           }
@@ -359,7 +419,7 @@ def main():
           h2 {
             margin: 12px 0 4px 0;
           }
-        .DirName {
+          .RowTitle {
             text-align: left;
             width: 200px;
             min-width: 200px;
@@ -395,6 +455,10 @@ def main():
             padding: 2px 4px;
             border-right: 2px solid white;
           }
+          tr.TotalRow td {
+            background-color: white;
+            border-right-color: white;
+          }
           tr.AospDir td {
             background-color: #e6f4ea;
             border-right-color: #e6f4ea;
@@ -501,6 +565,7 @@ def main():
 
     print("""
           <span class='NavSpacer'></span><span class='NavSpacer'> </span>
+          <a href='#summary'>Overall Summary</a>
         </div>
         <div id="container">
           <div id="tables">
@@ -529,6 +594,16 @@ def main():
                 <th>Total</th>
                 <td>The total number of makefiles in this each directory.</td>
               </tr>
+              <tr>
+                <th class="Clean">Easy</th>
+                <td>The number of makefiles that have no warnings themselves, and also
+                    none of their dependencies have warnings either.</td>
+              </tr>
+              <tr>
+                <th class="Clean">Unblocked Clean</th>
+                <td>The number of makefiles that are both Unblocked and Clean.</td>
+              </tr>
+
               <tr>
                 <th class="Unblocked">Unblocked</th>
                 <td>Makefiles containing one or more modules that don't have any
@@ -597,21 +672,22 @@ def main():
           </div>
     """)
 
-  annotations = Annotations()
+    overall_summary = Summary()
 
     # For each partition
-  makefiles_for_partitions = dict()
     for partition in sorted(partitions):
       modules = modules_by_partition[partition]
 
       makefiles = set(itertools.chain.from_iterable(
-        [soong.makefiles[module] for module in modules]))
+          [self.soong.makefiles[module] for module in modules]))
 
       # Read makefiles
       summary = Summary()
       for filename in makefiles:
-      if not filename.startswith(args.out_dir + "/"):
+        makefile = self.all_makefiles.get(filename)
-        summary.Add(Makefile(filename))
+        if makefile:
+          summary.Add(makefile)
+          overall_summary.Add(makefile)
 
       # Categorize directories by who is responsible
       aosp_dirs = []
@@ -625,66 +701,24 @@ def main():
         else:
           partner_dirs.append(dirname)
 
-    print("""
+      print_analysis_header("partition_" + partition, partition)
-      <a name="partition_%(partition)s"></a>
-      <h2>%(partition)s</h2>
-      <table>
-        <tr>
-          <th class="DirName">Directory</th>
-          <th class="Count">Total</th>
-          <th class="Count Unblocked">Unblocked</th>
-          <th class="Count Blocked">Blocked</th>
-          <th class="Count Clean">Clean</th>
-    """ % {
-      "partition": partition
-    })
 
-    for analyzer in ANALYZERS:
-      print("""<th class="Count Warning">%s</th>""" % analyzer.title)
-
-    print("      </tr>")
       for dirgroup, rowclass in [(aosp_dirs, "AospDir"),
                                  (google_dirs, "GoogleDir"),
                                  (partner_dirs, "PartnerDir"),]:
         for dirname in dirgroup:
-        makefiles = summary.directories[dirname]
+          self.print_analysis_row(summary, modules,
+                               dirname, rowclass, summary.directories[dirname])
 
-        all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles]
+      self.print_analysis_row(summary, modules,
-        clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
+                           "Total", "TotalRow",
-            if is_clean(makefile)]
+                           set(itertools.chain.from_iterable(summary.directories.values())))
-        unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
-            if contains_unblocked_modules(soong,
-              soong.reverse_makefiles[makefile.filename])]
-        blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
-            if contains_blocked_modules(soong,
-              soong.reverse_makefiles[makefile.filename])]
-
-        print("""
-          <tr class="%(rowclass)s">
-            <td class="DirName">%(dirname)s</td>
-            <td class="Count">%(makefiles)s</td>
-            <td class="Count">%(unblocked)s</td>
-            <td class="Count">%(blocked)s</td>
-            <td class="Count">%(clean)s</td>
-        """ % {
-          "rowclass": rowclass,
-          "dirname": dirname,
-          "makefiles": make_annotation_link(annotations, all_makefiles, modules),
-          "unblocked": make_annotation_link(annotations, unblocked_makefiles, modules),
-          "blocked": make_annotation_link(annotations, blocked_makefiles, modules),
-          "clean": make_annotation_link(annotations, clean_makefiles, modules),
-        })
-        for analyzer in ANALYZERS:
-          analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)]
-          print("""<td class="Count">%s</td>"""
-              % make_annotation_link(annotations, analyses, modules))
-
-        print("      </tr>")
       print("""
         </table>
       """)
 
-    module_details = [(count_deps(soong.deps, m, []), -count_deps(soong.reverse_deps, m, []), m)
+      module_details = [(count_deps(self.soong.deps, m, []),
+                         -count_deps(self.soong.reverse_deps, m, []), m)
                  for m in modules]
       module_details.sort()
       module_details = [m[2] for m in module_details]
@@ -699,7 +733,7 @@ def main():
       altRow = True
       for module in module_details:
         analyses = set()
-      for filename in soong.makefiles[module]:
+        for filename in self.soong.makefiles[module]:
           makefile = summary.makefiles.get(filename)
           if makefile:
             for analyzer, analysis in makefile.analyses.items():
@@ -711,13 +745,23 @@ def main():
         print("  <td><a name='module_%s'></a>%s</td>" % (module, module))
         print("  <td class='AnalysisCol'>%s</td>" % " ".join(["<span class='Analysis'>%s</span>" % title
             for title in analyses]))
-      print("  <td>%s</td>" % count_deps(soong.deps, module, []))
+        print("  <td>%s</td>" % count_deps(self.soong.deps, module, []))
-      print("  <td>%s</td>" % format_module_list(soong.deps.get(module, [])))
+        print("  <td>%s</td>" % format_module_list(self.soong.deps.get(module, [])))
-      print("  <td>%s</td>" % count_deps(soong.reverse_deps, module, []))
+        print("  <td>%s</td>" % count_deps(self.soong.reverse_deps, module, []))
-      print("  <td>%s</td>" % format_module_list(soong.reverse_deps.get(module, [])))
+        print("  <td>%s</td>" % format_module_list(self.soong.reverse_deps.get(module, [])))
         print("</tr>")
       print("""</table>""")
 
+    print_analysis_header("summary", "Overall Summary")
+
+    modules = [module for installed, module in self.soong.installed.items()]
+    self.print_analysis_row(overall_summary, modules,
+                         "All Makefiles", "TotalRow",
+                         set(itertools.chain.from_iterable(overall_summary.directories.values())))
+    print("""
+        </table>
+    """)
+
     print("""
       <script type="text/javascript">
       function close_details() {
@@ -843,16 +887,16 @@ def main():
 
       var ANALYSIS = [
       """ % {
-        "codesearch": args.codesearch,
+          "codesearch": self.args.codesearch,
       })
-  for entry, mods in annotations.entries:
+    for entry, mods in self.annotations.entries:
       print("  [")
       for analysis in entry:
         print("    new Analysis('%(filename)s', %(modules)s, [%(line_matches)s])," % {
           "filename": analysis.filename,
-        #"modules": json.dumps([m for m in mods if m in filename in soong.makefiles[m]]),
+          #"modules": json.dumps([m for m in mods if m in filename in self.soong.makefiles[m]]),
           "modules": json.dumps(
-            [m for m in soong.reverse_makefiles[analysis.filename] if m in mods]),
+              [m for m in self.soong.reverse_makefiles[analysis.filename] if m in mods]),
           "line_matches": ", ".join([
               "new LineMatch(%d, %s)" % (lineno, json.dumps(text))
               for lineno, text in analysis.line_matches]),
@@ -862,10 +906,10 @@ def main():
       ];
       var MODULE_DATA = {
     """)
-  for module in soong.modules:
+    for module in self.soong.modules:
       print("      '%(name)s': new Module(%(deps)s)," % {
         "name": module,
-      "deps": json.dumps(soong.deps[module]),
+        "deps": json.dumps(self.soong.deps[module]),
       })
     print("""
       };
@@ -887,6 +931,105 @@ def main():
     </html>
     """)
 
+  def traverse_ready_makefiles(self, summary, makefiles):
+    return [Analysis(makefile.filename, []) for makefile in makefiles
+        if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile)]
+
+  def print_analysis_row(self, summary, modules, rowtitle, rowclass, makefiles):
+    all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles]
+    clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
+        if is_clean(makefile)]
+    easy_makefiles = self.traverse_ready_makefiles(summary, makefiles)
+    unblocked_clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
+        if (self.soong.contains_unblocked_modules(makefile.filename)
+            and is_clean(makefile))]
+    unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
+        if self.soong.contains_unblocked_modules(makefile.filename)]
+    blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
+        if self.soong.contains_blocked_modules(makefile.filename)]
+
+    print("""
+      <tr class="%(rowclass)s">
+        <td class="RowTitle">%(rowtitle)s</td>
+        <td class="Count">%(makefiles)s</td>
+        <td class="Count">%(easy)s</td>
+        <td class="Count">%(unblocked_clean)s</td>
+        <td class="Count">%(unblocked)s</td>
+        <td class="Count">%(blocked)s</td>
+        <td class="Count">%(clean)s</td>
+    """ % {
+      "rowclass": rowclass,
+      "rowtitle": rowtitle,
+      "makefiles": self.make_annotation_link(all_makefiles, modules),
+      "unblocked": self.make_annotation_link(unblocked_makefiles, modules),
+      "blocked": self.make_annotation_link(blocked_makefiles, modules),
+      "clean": self.make_annotation_link(clean_makefiles, modules),
+      "unblocked_clean": self.make_annotation_link(unblocked_clean_makefiles, modules),
+      "easy": self.make_annotation_link(easy_makefiles, modules),
+    })
+
+    for analyzer in ANALYZERS:
+      analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)]
+      print("""<td class="Count">%s</td>"""
+          % self.make_annotation_link(analyses, modules))
+
+    print("      </tr>")
+
+  def make_annotation_link(self, analysis, modules):
+    if analysis:
+      return "<a href='javascript:update_details(%d)'>%s</a>" % (
+        self.annotations.Add(analysis, modules),
+        len(analysis)
+      )
+    else:
+      return "";
+
+class CsvProcessor(object):
+  def __init__(self, args, soong, all_makefiles):
+    self.args = args
+    self.soong = soong
+    self.all_makefiles = all_makefiles
+
+  def execute(self):
+    csvout = csv.writer(sys.stdout)
+
+    # Title row
+    row = ["Filename", "Module", "Partitions", "Easy", "Unblocked Clean", "Unblocked",
+           "Blocked", "Clean"]
+    for analyzer in ANALYZERS:
+      row.append(analyzer.title)
+    csvout.writerow(row)
+
+    # Makefile & module data
+    for filename in sorted(self.all_makefiles.keys()):
+      makefile = self.all_makefiles[filename]
+      for module in self.soong.reverse_makefiles[filename]:
+        row = [filename, module]
+        # Partitions
+        row.append(";".join(sorted(set([get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT,
+                                         installed)
+                                        for installed
+                                        in self.soong.reverse_installed.get(module, [])]))))
+        # Easy
+        row.append(1
+            if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile)
+            else "")
+        # Unblocked Clean
+        row.append(1
+            if (self.soong.contains_unblocked_modules(makefile.filename) and is_clean(makefile))
+            else "")
+        # Unblocked
+        row.append(1 if self.soong.contains_unblocked_modules(makefile.filename) else "")
+        # Blocked
+        row.append(1 if self.soong.contains_blocked_modules(makefile.filename) else "")
+        # Clean
+        row.append(1 if is_clean(makefile) else "")
+        # Analysis
+        for analyzer in ANALYZERS:
+          row.append(1 if makefile.analyses.get(analyzer) else "")
+        # Write results
+        csvout.writerow(row)
+
 if __name__ == "__main__":
   main()