Merge "orchestrator: inner-tree path can be a list." am: ef1d2f06c6

Original change: https://android-review.googlesource.com/c/platform/build/+/2138856

Change-Id: I75c88dcbbac1caf69727a08c400679909aa7c968
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

envsetup.sh

@@ -455,10 +455,19 @@ function multitree_lunch()
 {
     local code
     local results
+    # Lunch must be run in the topdir, but this way we get a clear error
+    # message, instead of FileNotFound.
+    local T=$(multitree_gettop)
+    if [ -n "$T" ]; then
+      "$T/build/build/make/orchestrator/core/orchestrator.py" "$@"
+    else
+      _multitree_lunch_error
+      return 1
+    fi
     if $(echo "$1" | grep -q '^-') ; then
         # Calls starting with a -- argument are passed directly and the function
         # returns with the lunch.py exit code.
-        build/build/make/orchestrator/core/lunch.py "$@"
+        "${T}/build/build/make/orchestrator/core/lunch.py" "$@"
         code=$?
         if [[ $code -eq 2 ]] ; then
           echo 1>&2
@@ -469,7 +478,7 @@ function multitree_lunch()
         fi
     else
         # All other calls go through the --lunch variant of lunch.py
-        results=($(build/build/make/orchestrator/core/lunch.py --lunch "$@"))
+        results=($(${T}/build/build/make/orchestrator/core/lunch.py --lunch "$@"))
         code=$?
         if [[ $code -eq 2 ]] ; then
           echo 1>&2
@@ -1817,7 +1826,8 @@ function _wrap_build()
 function _trigger_build()
 (
     local -r bc="$1"; shift
-    if T="$(gettop)"; then
+    local T=$(gettop)
+    if [ -n "$T" ]; then
       _wrap_build "$T/build/soong/soong_ui.bash" --build-mode --${bc} --dir="$(pwd)" "$@"
     else
       >&2 echo "Couldn't locate the top of the tree. Try setting TOP."
@@ -1877,8 +1887,9 @@ function _multitree_lunch_error()
 
 function multitree_build()
 {
-    if T="$(multitree_gettop)"; then
+    local T=$(multitree_gettop)
-      "$T/build/build/orchestrator/core/orchestrator.py" "$@"
+    if [ -n "$T" ]; then
+      "$T/build/build/make/orchestrator/core/orchestrator.py" "$@"
     else
       _multitree_lunch_error
       return 1

orchestrator/core/inner_tree.py

@@ -14,11 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import subprocess
 import sys
 import textwrap
 
+
 class InnerTreeKey(object):
     """Trees are identified uniquely by their root and the TARGET_PRODUCT they will use to build.
     If a single tree uses two different prdoucts, then we won't make assumptions about
@@ -26,21 +28,33 @@ class InnerTreeKey(object):
     TODO: This is true for soong. It's more likely that bazel could do analysis for two
     products at the same time in a single tree, so there's an optimization there to do
     eventually."""
+
     def __init__(self, root, product):
+        if isinstance(root, list):
+            self.melds = root[1:]
+            root = root[0]
+        else:
+            self.melds = []
         self.root = root
         self.product = product
 
     def __str__(self):
-        return "TreeKey(root=%s product=%s)" % (enquote(self.root), enquote(self.product))
+        return (f"TreeKey(root={enquote(self.root)} "
+                f"product={enquote(self.product)}")
 
     def __hash__(self):
         return hash((self.root, self.product))
 
     def _cmp(self, other):
+        assert isinstance(other, InnerTreeKey)
         if self.root < other.root:
             return -1
         if self.root > other.root:
             return 1
+        if self.melds < other.melds:
+            return -1
+        if self.melds > other.melds:
+            return 1
         if self.product == other.product:
             return 0
         if self.product is None:
@@ -71,13 +85,16 @@ class InnerTreeKey(object):
 
 
 class InnerTree(object):
-    def __init__(self, context, root, product):
+    def __init__(self, context, paths, product):
         """Initialize with the inner tree root (relative to the workspace root)"""
-        self.root = root
+        if not isinstance(paths, list):
+            paths = [paths]
+        self.root = paths[0]
+        self.meld_dirs = paths[1:]
         self.product = product
         self.domains = {}
         # TODO: Base directory on OUT_DIR
-        out_root = context.out.inner_tree_dir(root)
+        out_root = context.out.inner_tree_dir(self.root)
         if product:
             out_root += "_" + product
         else:
@@ -85,9 +102,10 @@ class InnerTree(object):
         self.out = OutDirLayout(out_root)
 
     def __str__(self):
-        return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
+        return (f"InnerTree(root={enquote(self.root)} "
-                enquote(self.product),
+                f"product={enquote(self.product)} "
-                " ".join([enquote(d) for d in sorted(self.domains.keys())]))
+                f"domains={enquote(list(self.domains.keys()))} "
+                f"meld={enquote(self.meld_dirs)})")
 
     def invoke(self, args):
         """Call the inner tree command for this inner tree. Exits on failure."""
@@ -97,8 +115,9 @@ class InnerTree(object):
         # so we can print a good error message
         inner_build_tool = os.path.join(self.root, ".inner_build")
         if not os.access(inner_build_tool, os.X_OK):
-            sys.stderr.write(("Unable to execute %s. Is there an inner tree or lunch combo"
+            sys.stderr.write(
-                    + " misconfiguration?\n") % inner_build_tool)
+                f"Unable to execute {inner_build_tool}. Is there an inner tree "
+                "or lunch combo misconfiguration?\n")
             sys.exit(1)
 
         # TODO: This is where we should set up the shared trees
@@ -115,8 +134,9 @@ class InnerTree(object):
 
         # TODO: Probably want better handling of inner tree failures
         if process.returncode:
-            sys.stderr.write("Build error in inner tree: %s\nstopping multitree build.\n"
+            sys.stderr.write(
-                    % self.root)
+                f"Build error in inner tree: {self.root}\nstopping "
+                "multitree build.\n")
             sys.exit(1)
 
 
@@ -127,19 +147,19 @@ class InnerTrees(object):
 
     def __str__(self):
         "Return a debugging dump of this object"
-        return textwrap.dedent("""\
+
-        InnerTrees {
+        def _vals(values):
+            return ("\n" + " " * 16).join(sorted([str(t) for t in values]))
+
+        return textwrap.dedent(f"""\
+        InnerTrees {{
             trees: [
-                %(trees)s
+                {_vals(self.trees.values())}
             ]
             domains: [
-                %(domains)s
+                {_vals(self.domains.values())}
             ]
-        }""" % {
+        }}""")
-            "trees": "\n        ".join(sorted([str(t) for t in self.trees.values()])),
-            "domains": "\n        ".join(sorted([str(d) for d in self.domains.values()])),
-        })
-
 
     def for_each_tree(self, func, cookie=None):
         """Call func for each of the inner trees once for each product that will be built in it.
@@ -153,7 +173,6 @@ class InnerTrees(object):
             result[key] = func(key, self.trees[key], cookie)
         return result
 
-
     def get(self, tree_key):
         """Get an inner tree for tree_key"""
         return self.trees.get(tree_key)
@@ -188,6 +207,4 @@ class OutDirLayout(object):
 
 
 def enquote(s):
-    return "None" if s is None else "\"%s\"" % s
+    return json.dumps(s)
-
-

orchestrator/core/lunch.py

@@ -240,9 +240,12 @@ def make_config_header(config_file, config, variant):
     trees = [("Component", "Path", "Product"),
              ("---------", "----", "-------")]
     entry = config.get("system", None)
+
     def add_config_tuple(trees, entry, name):
         if entry:
-            trees.append((name, entry.get("tree"), entry.get("product", "")))
+            trees.append(
+                (name, entry.get("inner-tree"), entry.get("product", "")))
+
     add_config_tuple(trees, config.get("system"), "system")
     add_config_tuple(trees, config.get("vendor"), "vendor")
     for k, v in config.get("modules", {}).items():

orchestrator/core/orchestrator.py

@@ -55,14 +55,16 @@ def process_config(context, lunch_config):
 
     system_entry = lunch_config.get("system")
     if system_entry:
-        add(API_DOMAIN_SYSTEM, system_entry["tree"], system_entry["product"])
+        add(API_DOMAIN_SYSTEM, system_entry["inner-tree"],
+            system_entry["product"])
 
     vendor_entry = lunch_config.get("vendor")
     if vendor_entry:
-        add(API_DOMAIN_VENDOR, vendor_entry["tree"], vendor_entry["product"])
+        add(API_DOMAIN_VENDOR, vendor_entry["inner-tree"],
+            vendor_entry["product"])
 
     for module_name, module_entry in lunch_config.get("modules", []).items():
-        add(module_name, module_entry["tree"], None)
+        add(module_name, module_entry["inner-tree"], None)
 
     return inner_tree.InnerTrees(trees, domains)
 

orchestrator/core/utils.py

@@ -28,7 +28,7 @@ class TestContext(Context):
     "Context for testing. The real Context is manually constructed in orchestrator.py."
 
     def __init__(self, test_work_dir, test_name):
-        super(MockContext, self).__init__(os.path.join(test_work_dir, test_name),
+        super(TestContext, self).__init__(os.path.join(test_work_dir, test_name),
                 Errors(None))
 
 

orchestrator/multitree_combos/aosp_cf_arm64_phone.mcombo

@@ -1,16 +1,16 @@
 {
     "lunchable": true,
     "system": {
-        "tree": "master",
+        "inner-tree": "aosp-master-with-phones",
         "product": "aosp_cf_arm64_phone"
     },
     "vendor": {
-        "tree": "master",
+        "inner-tree": "aosp-master-with-phones",
         "product": "aosp_cf_arm64_phone"
     },
     "modules": {
         "com.android.bionic": {
-            "tree": "sc-mainline-prod"
+            "inner-tree": "aosp-master-with-phones"
         }
     }
 }

orchestrator/multitree_combos/test.mcombo

@@ -1,16 +1,16 @@
 {
     "lunchable": true,
     "system": {
-        "tree": "inner_tree_system",
+        "inner-tree": "inner_tree_system",
         "product": "system_lunch_product"
     },
     "vendor": {
-        "tree": "inner_tree_vendor",
+        "inner-tree": "inner_tree_vendor",
         "product": "vendor_lunch_product"
     },
     "modules": {
         "com.android.something": {
-            "tree": "inner_tree_module"
+            "inner-tree": ["inner_tree_module", "sc-common"]
         }
     }
 }