Generate ninja files for api assembly and run ninja.

Change-Id: I003536e4ed4481acbdc041a6450106f8459d5978

orchestrator/core/api_assembly.py

@@ -14,11 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import json
 import os
+import sys
 
-def assemble_apis(inner_trees):
+import api_assembly_cc
+import ninja_tools
 
+
+ContributionData = collections.namedtuple("ContributionData", ("inner_tree", "json_data"))
+
+def assemble_apis(context, inner_trees):
     # Find all of the contributions from the inner tree
     contribution_files_dict = inner_trees.for_each_tree(api_contribution_files_for_inner_tree)
 
@@ -27,25 +34,33 @@ def assemble_apis(inner_trees):
     contributions = []
     for tree_key, filenames in contribution_files_dict.items():
         for filename in filenames:
-            contribution_data = load_contribution_file(filename)
+            json_data = load_contribution_file(filename)
-            if not contribution_data:
+            if not json_data:
                 continue
             # TODO: Validate the configs, especially that the domains match what we asked for
             # from the lunch config.
-            contributions.append(contribution_data)
+            contributions.append(ContributionData(inner_trees.get(tree_key), json_data))
 
     # Group contributions by language and API surface
     stub_libraries = collate_contributions(contributions)
 
-    # Iterate through all of the stub libraries and generate rules to assemble them
+    # Initialize the ninja file writer
-    # and Android.bp/BUILD files to make those available to inner trees.
+    with open(context.out.api_ninja_file(), "w") as ninja_file:
-    # TODO: Parallelize? Skip unnecessary work?
+        ninja = ninja_tools.Ninja(context, ninja_file)
-    ninja_file = NinjaFile() # TODO: parameters?
-    build_file = BuildFile() # TODO: parameters?
-    for stub_library in stub_libraries:
-        STUB_LANGUAGE_HANDLERS[stub_library.language](ninja_file, build_file, stub_library)
 
-    # TODO: Handle host_executables separately or as a StubLibrary language?
+        # Initialize the build file writer
+        build_file = BuildFile() # TODO: parameters?
+
+        # Iterate through all of the stub libraries and generate rules to assemble them
+        # and Android.bp/BUILD files to make those available to inner trees.
+        # TODO: Parallelize? Skip unnecessary work?
+        for stub_library in stub_libraries:
+            STUB_LANGUAGE_HANDLERS[stub_library.language](context, ninja, build_file, stub_library)
+
+        # TODO: Handle host_executables separately or as a StubLibrary language?
+
+        # Finish writing the ninja file
+        ninja.write()
 
 
 def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):
@@ -72,7 +87,8 @@ def load_contribution_file(filename):
 
 
 class StubLibraryContribution(object):
-    def __init__(self, api_domain, library_contribution):
+    def __init__(self, inner_tree, api_domain, library_contribution):
+        self.inner_tree = inner_tree
         self.api_domain = api_domain
         self.library_contribution = library_contribution
 
@@ -96,54 +112,42 @@ def collate_contributions(contributions):
     grouped = {}
     for contribution in contributions:
         for language in STUB_LANGUAGE_HANDLERS.keys():
-            for library in contribution.get(language, []):
+            for library in contribution.json_data.get(language, []):
-                key = (language, contribution["name"], contribution["version"], library["name"])
+                key = (language, contribution.json_data["name"],
+                        contribution.json_data["version"], library["name"])
                 stub_library = grouped.get(key)
                 if not stub_library:
-                    stub_library = StubLibrary(language, contribution["name"],
+                    stub_library = StubLibrary(language, contribution.json_data["name"],
-                            contribution["version"], library["name"])
+                            contribution.json_data["version"], library["name"])
                     grouped[key] = stub_library
-                stub_library.add_contribution(StubLibraryContribution(
+                stub_library.add_contribution(StubLibraryContribution(contribution.inner_tree,
-                        contribution["api_domain"], library))
+                        contribution.json_data["api_domain"], library))
     return list(grouped.values())
 
 
-def assemble_cc_api_library(ninja_file, build_file, stub_library):
+def assemble_java_api_library(context, ninja, build_file, stub_library):
-    print("assembling cc_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
+    print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface,
-            stub_library.name))
+            stub_library.api_surface_version, stub_library.name))
     for contrib in stub_library.contributions:
         print("  %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
     # TODO: Implement me
 
 
-def assemble_java_api_library(ninja_file, build_file, stub_library):
+def assemble_resource_api_library(context, ninja, build_file, stub_library):
-    print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
+    print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface,
-            stub_library.name))
+            stub_library.api_surface_version, stub_library.name))
-    for contrib in stub_library.contributions:
-        print("  %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
-    # TODO: Implement me
-
-
-def assemble_resource_api_library(ninja_file, build_file, stub_library):
-    print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
-            stub_library.name))
     for contrib in stub_library.contributions:
         print("  %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
     # TODO: Implement me
 
 
 STUB_LANGUAGE_HANDLERS = {
-    "cc_libraries": assemble_cc_api_library,
+    "cc_libraries": api_assembly_cc.assemble_cc_api_library,
     "java_libraries": assemble_java_api_library,
     "resource_libraries": assemble_resource_api_library,
 }
 
 
-class NinjaFile(object):
-    "Generator for build actions and dependencies."
-    pass
-
-
 class BuildFile(object):
     "Abstract generator for Android.bp files and BUILD files."
     pass

orchestrator/core/api_assembly_cc.py

@@ -0,0 +1,55 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+def assemble_cc_api_library(context, ninja, build_file, stub_library):
+    print("\nassembling cc_api_library %s-%s %s from:" % (stub_library.api_surface,
+        stub_library.api_surface_version, stub_library.name))
+    for contrib in stub_library.contributions:
+        print("  %s %s" % (contrib.api_domain, contrib.library_contribution))
+
+    staging_dir = context.out.api_library_dir(stub_library.api_surface,
+            stub_library.api_surface_version, stub_library.name)
+    work_dir = context.out.api_library_work_dir(stub_library.api_surface,
+            stub_library.api_surface_version, stub_library.name)
+    print("staging_dir=%s" % (staging_dir))
+    print("work_dir=%s" % (work_dir))
+
+    # Generate rules to copy headers
+    includes = []
+    include_dir = os.path.join(staging_dir, "include")
+    for contrib in stub_library.contributions:
+        for headers in contrib.library_contribution["headers"]:
+            root = headers["root"]
+            for file in headers["files"]:
+                # TODO: Deal with collisions of the same name from multiple contributions
+                include = os.path.join(include_dir, file)
+                ninja.add_copy_file(include, os.path.join(contrib.inner_tree.root, root, file))
+                includes.append(include)
+
+    # Generate rule to run ndkstubgen
+
+
+    # Generate rule to compile stubs to library
+
+    # Generate phony rule to build the library
+    # TODO: This name probably conflictgs with something
+    ninja.add_phony("-".join((stub_library.api_surface, str(stub_library.api_surface_version),
+            stub_library.name)), includes)
+
+    # Generate build files
+

orchestrator/core/final_packaging.py

@@ -0,0 +1,29 @@
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ninja_tools
+import ninja_syntax # Has to be after ninja_tools because of the path hack
+
+def final_packaging(context):
+    """Pull together all of the previously defined rules into the final build stems."""
+
+    with open(context.out.outer_ninja_file(), "w") as ninja_file:
+        ninja = ninja_tools.Ninja(context, ninja_file)
+
+        # Add the api surfaces file
+        ninja.add_subninja(ninja_syntax.Subninja(context.out.api_ninja_file(), chDir=None))
+
+        # Finish writing the ninja file
+        ninja.write()

orchestrator/core/inner_tree.py

@@ -56,13 +56,13 @@ class InnerTreeKey(object):
 
 
 class InnerTree(object):
-    def __init__(self, root, product):
+    def __init__(self, context, root, product):
         """Initialize with the inner tree root (relative to the workspace root)"""
         self.root = root
         self.product = product
         self.domains = {}
         # TODO: Base directory on OUT_DIR
-        self.out = OutDirLayout(os.path.join("out", "trees", root))
+        self.out = OutDirLayout(context.out.inner_tree_dir(root))
 
     def __str__(self):
         return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
@@ -134,7 +134,14 @@ class InnerTrees(object):
         return result
 
 
+    def get(self, tree_key):
+        """Get an inner tree for tree_key"""
+        return self.trees.get(tree_key)
+
 class OutDirLayout(object):
+    """Encapsulates the logic about the layout of the inner tree out directories.
+    See also context.OutDir for outer tree out dir contents."""
+
     def __init__(self, root):
         "Initialize with the root of the OUT_DIR for the inner tree."
         self._root = root

orchestrator/core/ninja_runner.py

@@ -0,0 +1,36 @@
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+
+def run_ninja(context, targets):
+    """Run ninja.
+    """
+
+    # Construct the command
+    cmd = [
+            context.tools.ninja(),
+            "-f",
+            context.out.outer_ninja_file(),
+        ] + targets
+
+    # Run the command
+    process = subprocess.run(cmd, shell=False)
+
+    # TODO: Probably want better handling of inner tree failures
+    if process.returncode:
+        sys.stderr.write("Build error in outer tree.\nstopping multitree build.\n")
+        sys.exit(1)
+

orchestrator/core/ninja_tools.py

@@ -0,0 +1,46 @@
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+# Workaround for python include path
+_ninja_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "ninja"))
+if _ninja_dir not in sys.path:
+    sys.path.append(_ninja_dir)
+import ninja_writer
+from ninja_syntax import Variable, BuildAction, Rule, Pool, Subninja, Line
+
+
+class Ninja(ninja_writer.Writer):
+    """Some higher level constructs on top of raw ninja writing.
+    TODO: Not sure where these should be."""
+    def __init__(self, context, file):
+        super(Ninja, self).__init__(file)
+        self._context = context
+        self._did_copy_file = False
+
+    def add_copy_file(self, copy_to, copy_from):
+        if not self._did_copy_file:
+            self._did_copy_file = True
+            rule = Rule("copy_file")
+            rule.add_variable("command", "mkdir -p ${out_dir} && " + self._context.tools.acp()
+                    + " -f ${in} ${out}")
+            self.add_rule(rule)
+        build_action = BuildAction(copy_to, "copy_file", inputs=[copy_from,],
+                implicits=[self._context.tools.acp()])
+        build_action.add_variable("out_dir", os.path.dirname(copy_to))
+        self.add_build_action(build_action)
+
+

orchestrator/core/orchestrator.py

@@ -22,9 +22,12 @@ sys.dont_write_bytecode = True
 import api_assembly
 import api_domain
 import api_export
+import final_packaging
 import inner_tree
 import interrogate
 import lunch
+import ninja_runner
+import utils
 
 EXIT_STATUS_OK = 0
 EXIT_STATUS_ERROR = 1
@@ -33,14 +36,14 @@ API_DOMAIN_SYSTEM = "system"
 API_DOMAIN_VENDOR = "vendor"
 API_DOMAIN_MODULE = "module"
 
-def process_config(lunch_config):
+def process_config(context, lunch_config):
     """Returns a InnerTrees object based on the configuration requested in the lunch config."""
     def add(domain_name, tree_root, product):
         tree_key = inner_tree.InnerTreeKey(tree_root, product)
         if tree_key in trees:
             tree = trees[tree_key]
         else:
-            tree = inner_tree.InnerTree(tree_root, product)
+            tree = inner_tree.InnerTree(context, tree_root, product)
             trees[tree_key] = tree
         domain = api_domain.ApiDomain(domain_name, tree, product)
         domains[domain_name] = domain
@@ -68,6 +71,9 @@ def build():
     # Load lunch combo
     #
 
+    # Choose the out directory, set up error handling, etc.
+    context = utils.Context(utils.choose_out_dir(), utils.Errors(sys.stderr))
+
     # Read the config file
     try:
         config_file, config, variant = lunch.load_current_config()
@@ -77,7 +83,7 @@ def build():
     sys.stdout.write(lunch.make_config_header(config_file, config, variant))
 
     # Construct the trees and domains dicts
-    inner_trees = process_config(config)
+    inner_trees = process_config(context, config)
 
     #
     # 1. Interrogate the trees
@@ -93,7 +99,7 @@ def build():
     #
     # 2b. API Surface Assembly
     #
-    api_assembly.assemble_apis(inner_trees)
+    api_assembly.assemble_apis(context, inner_trees)
 
     #
     # 3a. API Domain Analysis
@@ -102,11 +108,17 @@ def build():
     #
     # 3b. Final Packaging Rules
     #
+    final_packaging.final_packaging(context)
 
     #
     # 4. Build Execution
     #
-
+    # TODO: Decide what we want the UX for selecting targets to be across
+    # branches... since there are very likely to be conflicting soong short
+    # names.
+    print("Running ninja...")
+    targets = ["public_api-1-libhwui", "public_api-1-libc"]
+    ninja_runner.run_ninja(context, targets)
 
     #
     # Success!

orchestrator/core/utils.py

@@ -0,0 +1,120 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import platform
+
+class Context(object):
+    """Mockable container for global state."""
+    def __init__(self, out_root, errors):
+        self.out = OutDir(out_root)
+        self.errors = errors
+        self.tools = HostTools()
+
+class TestContext(Context):
+    "Context for testing. The real Context is manually constructed in orchestrator.py."
+
+    def __init__(self, test_work_dir, test_name):
+        super(MockContext, self).__init__(os.path.join(test_work_dir, test_name),
+                Errors(None))
+
+
+class OutDir(object):
+    """Encapsulates the logic about the out directory at the outer-tree level.
+    See also inner_tree.OutDirLayout for inner tree out dir contents."""
+
+    def __init__(self, root):
+        "Initialize with the root of the OUT_DIR for the outer tree."
+        self._root = root
+        self._intermediates = "intermediates"
+
+    def root(self):
+        return self._root
+
+    def inner_tree_dir(self, tree_root):
+        """Root directory for inner tree inside the out dir."""
+        return os.path.join(self._root, "trees", tree_root)
+
+    def api_ninja_file(self):
+        """The ninja file that assembles API surfaces."""
+        return os.path.join(self._root, "api_surfaces.ninja")
+
+    def api_library_dir(self, surface, version, library):
+        """Directory for all the contents of a library inside an API surface, including
+        the build files.  Any intermediates should go in api_library_work_dir."""
+        return os.path.join(self._root, "api_surfaces", surface, str(version), library)
+
+    def api_library_work_dir(self, surface, version, library):
+        """Intermediates / scratch directory for library inside an API surface."""
+        return os.path.join(self._root, self._intermediates, "api_surfaces", surface, str(version),
+                library)
+
+    def outer_ninja_file(self):
+        return os.path.join(self._root, "multitree.ninja")
+
+
+class Errors(object):
+    """Class for reporting and tracking errors."""
+    def __init__(self, stream):
+        """Initialize Error reporter with a file-like object."""
+        self._stream = stream
+        self._all = []
+
+    def error(self, message):
+        """Record the error message."""
+        s = str(s)
+        if s[-1] != "\n":
+            s += "\n"
+        self._all.append(s)
+        if self._stream:
+            self._stream.write(s)
+
+    def had_error(self):
+        """Return if there were any errors reported."""
+        return len(self._all)
+
+    def get_errors(self):
+        """Get all errors that were reported."""
+        return self._all
+
+
+class HostTools(object):
+    def __init__(self):
+        if platform.system() == "Linux":
+            self._arch = "linux-x86"
+        else:
+            raise Exception("Orchestrator running on an unknown system: %s" % platform.system())
+
+        # Some of these are called a lot, so pre-compute the strings to save memory
+        self._prebuilts = os.path.join("build", "prebuilts", "build-tools", self._arch, "bin")
+        self._acp = os.path.join(self._prebuilts, "acp")
+        self._ninja = os.path.join(self._prebuilts, "ninja")
+
+    def acp(self):
+        return self._acp
+
+    def ninja(self):
+        return self._ninja
+
+
+def choose_out_dir():
+    """Get the root of the out dir, either from the environment or by picking
+    a default."""
+    result = os.environ.get("OUT_DIR")
+    if result:
+        return result
+    else:
+        return "out"

orchestrator/ninja/ninja_syntax.py

@@ -159,7 +159,7 @@ class Subninja(Node):
     self.chDir = chDir
 
   # TODO(spandandas): Update the syntax when aosp/2064612 lands
-  def stream() -> Iterator[str]:
+  def stream(self) -> Iterator[str]:
     yield f"subninja {self.subninja}"
 
 class Line(Node):

orchestrator/ninja/ninja_writer.py

@@ -49,6 +49,10 @@ class Writer:
   def add_subninja(self, subninja: Subninja):
     self.nodes.append(subninja)
 
+  def add_phony(self, name, deps):
+    build_action = BuildAction(name, "phony", inputs=deps)
+    self.add_build_action(build_action)
+
   def write(self):
     for node in self.nodes:
       for line in node.stream():