StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Orchestrator can build end to end.

Browse Source 
This reduces the scope of the demo to just building and installing
a single .so, but it makes the demo actually build that single .so.

Next up, writing some unit tests and fleshing out functionality.

Test: see the README
Change-Id: I560904b786fbf69d3a83dbb08d496dba5a3192ca
This commit is contained in:
Joe Onorato
2022-05-13 12:10:23 -07:00
parent 4117e78575
commit c35895676c
16 changed files with 277 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
orchestrator/core/api_assembly.py
View File

@@ -34,7 +34,7 @@ def assemble_apis(context, inner_trees):
contributions = []
for tree_key, filenames in contribution_files_dict.items():
for filename in filenames:
json_data = load_contribution_file(filename)
json_data = load_contribution_file(context, filename)
if not json_data:
continue
# TODO: Validate the configs, especially that the domains match what we asked for
@@ -76,13 +76,14 @@ def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):
return result
def load_contribution_file(filename):
def load_contribution_file(context, filename):
"Load and return the API contribution at filename. On error report error and return None."
with open(filename) as f:
try:
return json.load(f)
except json.decoder.JSONDecodeError as ex:
# TODO: Error reporting
context.errors.error(ex.msg, filename, ex.lineno, ex.colno)
raise ex

7
orchestrator/core/api_assembly_cc.py
View File

@@ -17,17 +17,10 @@
import os
def assemble_cc_api_library(context, ninja, build_file, stub_library):
print("\nassembling cc_api_library %s-%s %s from:" % (stub_library.api_surface,
stub_library.api_surface_version, stub_library.name))
for contrib in stub_library.contributions:
print(" %s %s" % (contrib.api_domain, contrib.library_contribution))
staging_dir = context.out.api_library_dir(stub_library.api_surface,
stub_library.api_surface_version, stub_library.name)
work_dir = context.out.api_library_work_dir(stub_library.api_surface,
stub_library.api_surface_version, stub_library.name)
print("staging_dir=%s" % (staging_dir))
print("work_dir=%s" % (work_dir))
# Generate rules to copy headers
includes = []

90
orchestrator/core/final_packaging.py
View File

@@ -13,10 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import sys
import ninja_tools
import ninja_syntax # Has to be after ninja_tools because of the path hack
def final_packaging(context):
def final_packaging(context, inner_trees):
"""Pull together all of the previously defined rules into the final build stems."""
with open(context.out.outer_ninja_file(), "w") as ninja_file:
@@ -25,5 +29,89 @@ def final_packaging(context):
# Add the api surfaces file
ninja.add_subninja(ninja_syntax.Subninja(context.out.api_ninja_file(), chDir=None))
# For each inner tree
for tree in inner_trees.keys():
# TODO: Verify that inner_tree.ninja was generated
# Read and verify file
build_targets = read_build_targets_json(context, tree)
if not build_targets:
continue
# Generate the ninja and build files for this inner tree
generate_cross_domain_build_rules(context, ninja, tree, build_targets)
# Finish writing the ninja file
ninja.write()
def read_build_targets_json(context, tree):
"""Read and validate the build_targets.json file for the given tree."""
try:
f = open(tree.out.build_targets_file())
except FileNotFoundError:
# It's allowed not to have any artifacts (e.g. if a tree is a light tree with only APIs)
return None
data = None
with f:
try:
data = json.load(f)
except json.decoder.JSONDecodeError as ex:
sys.stderr.write("Error parsing file: %s\n" % tree.out.build_targets_file())
# TODO: Error reporting
raise ex
# TODO: Better error handling
# TODO: Validate json schema
return data
def generate_cross_domain_build_rules(context, ninja, tree, build_targets):
"Generate the ninja and build files for the inner tree."
# Include the inner tree's inner_tree.ninja
ninja.add_subninja(ninja_syntax.Subninja(tree.out.main_ninja_file(), chDir=tree.root))
# Generate module rules and files
for module in build_targets.get("modules", []):
generate_shared_module(context, ninja, tree, module)
# Generate staging rules
staging_dir = context.out.staging_dir()
for staged in build_targets.get("staging", []):
# TODO: Enforce that dest isn't in disallowed subdir of out or absolute
dest = staged["dest"]
dest = os.path.join(staging_dir, dest)
if "src" in staged and "obj" in staged:
context.errors.error("Can't have both \"src\" and \"obj\" tags in \"staging\" entry."
) # TODO: Filename and line if possible
if "src" in staged:
ninja.add_copy_file(dest, os.path.join(tree.root, staged["src"]))
elif "obj" in staged:
ninja.add_copy_file(dest, os.path.join(tree.out.root(), staged["obj"]))
ninja.add_global_phony("staging", [dest])
# Generate dist rules
dist_dir = context.out.dist_dir()
for disted in build_targets.get("dist", []):
# TODO: Enforce that dest absolute
dest = disted["dest"]
dest = os.path.join(dist_dir, dest)
ninja.add_copy_file(dest, os.path.join(tree.root, disted["src"]))
ninja.add_global_phony("dist", [dest])
def generate_shared_module(context, ninja, tree, module):
"""Generate ninja rules for the given build_targets.json defined module."""
module_name = module["name"]
module_type = module["type"]
share_dir = context.out.module_share_dir(module_type, module_name)
src_file = os.path.join(tree.root, module["file"])
if module_type == "apex":
ninja.add_copy_file(os.path.join(share_dir, module_name + ".apex"), src_file)
# TODO: Generate build file
else:
# TODO: Better error handling
raise Exception("Invalid module type: %s" % module)

45
orchestrator/core/inner_tree.py
View File

@@ -36,23 +36,38 @@ class InnerTreeKey(object):
def __hash__(self):
return hash((self.root, self.product))
def _cmp(self, other):
if self.root < other.root:
return -1
if self.root > other.root:
return 1
if self.product == other.product:
return 0
if self.product is None:
return -1
if other.product is None:
return 1
if self.product < other.product:
return -1
return 1
def __eq__(self, other):
return (self.root == other.root and self.product == other.product)
return self._cmp(other) == 0
def __ne__(self, other):
return not self.__eq__(other)
return self._cmp(other) != 0
def __lt__(self, other):
return (self.root, self.product) < (other.root, other.product)
return self._cmp(other) < 0
def __le__(self, other):
return (self.root, self.product) <= (other.root, other.product)
return self._cmp(other) <= 0
def __gt__(self, other):
return (self.root, self.product) > (other.root, other.product)
return self._cmp(other) > 0
def __ge__(self, other):
return (self.root, self.product) >= (other.root, other.product)
return self._cmp(other) >= 0
class InnerTree(object):
@@ -62,7 +77,12 @@ class InnerTree(object):
self.product = product
self.domains = {}
# TODO: Base directory on OUT_DIR
self.out = OutDirLayout(context.out.inner_tree_dir(root))
out_root = context.out.inner_tree_dir(root)
if product:
out_root += "_" + product
else:
out_root += "_unbundled"
self.out = OutDirLayout(out_root)
def __str__(self):
return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
@@ -138,6 +158,11 @@ class InnerTrees(object):
"""Get an inner tree for tree_key"""
return self.trees.get(tree_key)
def keys(self):
"Get the keys for the inner trees in name order."
return [self.trees[k] for k in sorted(self.trees.keys())]
class OutDirLayout(object):
"""Encapsulates the logic about the layout of the inner tree out directories.
See also context.OutDir for outer tree out dir contents."""
@@ -155,6 +180,12 @@ class OutDirLayout(object):
def api_contributions_dir(self):
return os.path.join(self._root, "api_contributions")
def build_targets_file(self):
return os.path.join(self._root, "build_targets.json")
def main_ninja_file(self):
return os.path.join(self._root, "inner_tree.ninja")
def enquote(s):
return "None" if s is None else "\"%s\"" % s

1
orchestrator/core/ninja_runner.py
View File

@@ -14,6 +14,7 @@
# limitations under the License.
import subprocess
import sys
def run_ninja(context, targets):
"""Run ninja.

13
orchestrator/core/ninja_tools.py
View File

@@ -30,6 +30,7 @@ class Ninja(ninja_writer.Writer):
super(Ninja, self).__init__(file)
self._context = context
self._did_copy_file = False
self._phonies = {}
def add_copy_file(self, copy_to, copy_from):
if not self._did_copy_file:
@@ -43,4 +44,16 @@ class Ninja(ninja_writer.Writer):
build_action.add_variable("out_dir", os.path.dirname(copy_to))
self.add_build_action(build_action)
def add_global_phony(self, name, deps):
"""Add a phony target where there are multiple places that will want to add to
the same phony. If you can, to save memory, use add_phony instead of this function."""
if type(deps) not in (list, tuple):
raise Exception("Assertion failed: bad type of deps: %s" % type(deps))
self._phonies.setdefault(name, []).extend(deps)
def write(self):
for phony, deps in self._phonies.items():
self.add_phony(phony, deps)
super(Ninja, self).write()

28
orchestrator/core/orchestrator.py
View File

@@ -24,6 +24,7 @@ import api_domain
import api_export
import final_packaging
import inner_tree
import tree_analysis
import interrogate
import lunch
import ninja_runner
@@ -67,14 +68,10 @@ def process_config(context, lunch_config):
def build():
#
# Load lunch combo
#
# Choose the out directory, set up error handling, etc.
context = utils.Context(utils.choose_out_dir(), utils.Errors(sys.stderr))
# Read the config file
# Read the lunch config file
try:
config_file, config, variant = lunch.load_current_config()
except lunch.ConfigException as ex:
@@ -85,44 +82,31 @@ def build():
# Construct the trees and domains dicts
inner_trees = process_config(context, config)
#
# 1. Interrogate the trees
#
inner_trees.for_each_tree(interrogate.interrogate_tree)
# TODO: Detect bazel-only mode
#
# 2a. API Export
#
inner_trees.for_each_tree(api_export.export_apis_from_tree)
#
# 2b. API Surface Assembly
#
api_assembly.assemble_apis(context, inner_trees)
#
# 3a. API Domain Analysis
#
# 3a. Inner tree analysis
tree_analysis.analyze_trees(context, inner_trees)
#
# 3b. Final Packaging Rules
#
final_packaging.final_packaging(context)
final_packaging.final_packaging(context, inner_trees)
#
# 4. Build Execution
#
# TODO: Decide what we want the UX for selecting targets to be across
# branches... since there are very likely to be conflicting soong short
# names.
print("Running ninja...")
targets = ["public_api-1-libhwui", "public_api-1-libc"]
targets = ["staging", "system"]
ninja_runner.run_ninja(context, targets)
#
# Success!
#
return EXIT_STATUS_OK
def main(argv):

24
orchestrator/core/tree_analysis.py Normal file
View File

@@ -0,0 +1,24 @@
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
def analyze_trees(context, inner_trees):
inner_trees.for_each_tree(run_analysis)
def run_analysis(tree_key, inner_tree, cookie):
inner_tree.invoke(["analyze"])

41
orchestrator/core/utils.py
View File

@@ -38,33 +38,42 @@ class OutDir(object):
def __init__(self, root):
"Initialize with the root of the OUT_DIR for the outer tree."
self._root = root
self._out_root = root
self._intermediates = "intermediates"
def root(self):
return self._root
return self._out_root
def inner_tree_dir(self, tree_root):
"""Root directory for inner tree inside the out dir."""
return os.path.join(self._root, "trees", tree_root)
return os.path.join(self._out_root, "trees", tree_root)
def api_ninja_file(self):
"""The ninja file that assembles API surfaces."""
return os.path.join(self._root, "api_surfaces.ninja")
return os.path.join(self._out_root, "api_surfaces.ninja")
def api_library_dir(self, surface, version, library):
"""Directory for all the contents of a library inside an API surface, including
the build files. Any intermediates should go in api_library_work_dir."""
return os.path.join(self._root, "api_surfaces", surface, str(version), library)
return os.path.join(self._out_root, "api_surfaces", surface, str(version), library)
def api_library_work_dir(self, surface, version, library):
"""Intermediates / scratch directory for library inside an API surface."""
return os.path.join(self._root, self._intermediates, "api_surfaces", surface, str(version),
library)
return os.path.join(self._out_root, self._intermediates, "api_surfaces", surface,
str(version), library)
def outer_ninja_file(self):
return os.path.join(self._root, "multitree.ninja")
return os.path.join(self._out_root, "multitree.ninja")
def module_share_dir(self, module_type, module_name):
return os.path.join(self._out_root, "shared", module_type, module_name)
def staging_dir(self):
return os.path.join(self._out_root, "staging")
def dist_dir(self):
"The DIST_DIR provided or out/dist" # TODO: Look at DIST_DIR
return os.path.join(self._out_root, "dist")
class Errors(object):
"""Class for reporting and tracking errors."""
@@ -73,9 +82,21 @@ class Errors(object):
self._stream = stream
self._all = []
def error(self, message):
def error(self, message, file=None, line=None, col=None):
"""Record the error message."""
s = str(s)
s = ""
if file:
s += str(file)
s += ":"
if line:
s += str(line)
s += ":"
if col:
s += str(col)
s += ":"
if s:
s += " "
s += str(message)
if s[-1] != "\n":
s += "\n"
self._all.append(s)