diff --git a/core/build_id.rbc b/core/build_id.rbc new file mode 100644 index 0000000000..4f33833575 --- /dev/null +++ b/core/build_id.rbc @@ -0,0 +1,21 @@ + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file has been manually converted from build_id.mk +def init(g): + + # BUILD_ID is usually used to specify the branch name (like "MAIN") or a branch name and a release candidate + # (like "CRB01"). It must be a single word, and is capitalized by convention. + g["BUILD_ID"]="AOSP.MASTER" \ No newline at end of file diff --git a/core/envsetup.rbc b/core/envsetup.rbc new file mode 100644 index 0000000000..2924ee1525 --- /dev/null +++ b/core/envsetup.rbc @@ -0,0 +1,199 @@ + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":build_id.rbc|init", _build_id_init="init") + +def _all_versions(): + """Returns all known versions.""" + versions = ["OPR1", "OPD1", "OPD2","OPM1", "OPM2", "PPR1", "PPD1", "PPD2", "PPM1", "PPM2", "QPR1" ] + for v in ("Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"): + for e in ("P1A", "P1B", "P2A", "P2B", "D1A", "D1B", "D2A", "D2B", "Q1A", "Q1B", "Q2A", "Q2B", "Q3A", "Q3B"): + versions.append(v+e) + return versions + +def _allowed_versions(all_versions, min_version, max_version, default_version): + """Checks that version range and default versions is valid, returns all versions in range.""" + for v in (min_version, max_version, default_version): + if v not in all_versions: + fail("% is invalid" % v) + + min_i = all_versions.index(min_version) + max_i = all_versions.index(max_version) + def_i = all_versions.index(default_version) + if min_i > max_i: + fail("%s should come before %s in the version list" % (min_version, max_version)) + if def_i < min_i or def_i > max_i: + fail("%s should come between % and %s" % (default_version, min_version, max_version)) + return all_versions[min_i:max_i+1] + +# This function is a manual conversion of the version_defaults.mk +def _versions_default(g, all_versions): + """Handle various build version information. + + Guarantees that the following are defined: + PLATFORM_VERSION + PLATFORM_SDK_VERSION + PLATFORM_VERSION_CODENAME + DEFAULT_APP_TARGET_SDK + BUILD_ID + BUILD_NUMBER + PLATFORM_SECURITY_PATCH + PLATFORM_VNDK_VERSION + PLATFORM_SYSTEMSDK_VERSIONS + """ + + # If build_id.rbc exists, it may override some of the defaults. + # Note that build.prop target also wants INTERNAL_BUILD_ID_MAKEFILE to be set if the file exists. + if _build_id_init != None: + _build_id_init(g) + g["INTERNAL_BUILD_ID_MAKEFILE"] = "build/make/core/build_id" + + allowed_versions = _allowed_versions(all_versions, v_min, v_max, v_default) + g.setdefault("TARGET_PLATFORM_VERSION", v_default) + if g["TARGET_PLATFORM_VERSION"] not in allowed_versions: + fail("% is not valid, must be one of %s" % (g["TARGET_PLATFORM_VERSION"], allowed_versions)) + + g["DEFAULT_PLATFORM_VERSION"] = v_default + g["PLATFORM_VERSION_LAST_STABLE"] = 11 + g.setdefault("PLATFORM_VERSION_CODENAME", g["TARGET_PLATFORM_VERSION"]) + # TODO(asmundak): set PLATFORM_VERSION_ALL_CODENAMES + + g.setdefault("PLATFORM_VERSION", + g["PLATFORM_VERSION_LAST_STABLE"] if g["PLATFORM_VERSION_CODENAME"] == "REL" else g["PLATFORM_VERSION_CODENAME"]) + g.setdefault("PLATFORM_SDK_VERSION", 30) + if g["PLATFORM_VERSION_CODENAME"] == "REL": + g["PLATFORM_PREVIEW_SDK_VERSION"] = 0 + else: + g.setdefault("PLATFORM_PREVIEW_SDK_VERSION", 1) + + g.setdefault("DEFAULT_APP_TARGET_SDK", + g["PLATFORM_SDK_VERSION"] if g["PLATFORM_VERSION_CODENAME"] == "REL" else g["PLATFORM_VERSION_CODENAME"]) + g.setdefault("PLATFORM_VNDK_VERSION", + g["PLATFORM_SDK_VERSION"] if g["PLATFORM_VERSION_CODENAME"] == "REL" else g["PLATFORM_VERSION_CODENAME"]) + g.setdefault("PLATFORM_SYSTEMSDK_MIN_VERSION", 28) + versions = [str(i) for i in range(g["PLATFORM_SYSTEMSDK_MIN_VERSION"], g["PLATFORM_SDK_VERSION"] + 1)] + versions.append(g["PLATFORM_VERSION_CODENAME"]) + g["PLATFORM_SYSTEMSDK_VERSIONS"] = sorted(versions) + + # Used to indicate the security patch that has been applied to the device. + # It must signify that the build includes all security patches issued up through the designated Android Public Security Bulletin. + # It must be of the form "YYYY-MM-DD" on production devices. + # It must match one of the Android Security Patch Level strings of the Public Security Bulletins. + # If there is no $PLATFORM_SECURITY_PATCH set, keep it empty. + g.setdefault("PLATFORM_SECURITY_PATCH", "2021-03-05") + dt = 'TZ="GMT" %s' % g["PLATFORM_SECURITY_PATCH"] + g.setdefault("PLATFORM_SECURITY_PATCH_TIMESTAMP", rblf_shell("date -d '%s' +%%s" % dt)) + + # Used to indicate the base os applied to the device. Can be an arbitrary string, but must be a single word. + # If there is no $PLATFORM_BASE_OS set, keep it empty. + g.setdefault("PLATFORM_BASE_OS", "") + + # Used to signify special builds. E.g., branches and/or releases, like "M5-RC7". Can be an arbitrary string, but + # must be a single word and a valid file name. If there is no BUILD_ID set, make it obvious. + g.setdefault("BUILD_ID", "UNKNOWN") + + # BUILD_NUMBER should be set to the source control value that represents the current state of the source code. + # E.g., a perforce changelist number or a git hash. Can be an arbitrary string (to allow for source control that + # uses something other than numbers), but must be a single word and a valid file name. + # + # If no BUILD_NUMBER is set, create a useful "I am an engineering build from this date/time" value. Make it start + # with a non-digit so that anyone trying to parse it as an integer will probably get "0". + g.setdefault("BUILD_NUMBER", "eng.%s.%s" % (g["USER"], "TIMESTAMP")) + + # Used to set minimum supported target sdk version. Apps targeting SDK version lower than the set value will result + # in a warning being shown when any activity from the app is started. + g.setdefault("PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION", 23) + +def init(g): + all_versions = _all_versions() + _versions_default(g, all_versions) + for v in all_versions: + g["IS_AT_LEAST" + v] = True + if v == g["TARGET_PLATFORM_VERSION"]: + break + + # --------------------------------------------------------------- + # If you update the build system such that the environment setup or buildspec.mk need to be updated, + # increment this number, and people who haven't re-run those will have to do so before they can build. + # Make sure to also update the corresponding value in buildspec.mk.default and envsetup.sh. + g["CORRECT_BUILD_ENV_SEQUENCE_NUMBER"] = 13 + + g.setdefault("TARGET_PRODUCT", "aosp_arm") + g.setdefault("TARGET_BUILD_VARIANT", "eng") + + g.setdefault("TARGET_BUILD_APPS", []) + g["TARGET_BUILD_UNBUNDLED"] = (g["TARGET_BUILD_APPS"] != []) or (getattr(g, "TARGET_BUILD_UNBUNDLED_IMAGE", "") != "") + + # --------------------------------------------------------------- + # Set up configuration for host machine. We don't do cross-compiles except for arm, so the HOST + # is whatever we are running on. + host = rblf_shell("uname -sm") + if host.find("Linux") >= 0: + g["HOST_OS"] = "linux" + elif host.find("Darwin") >= 0: + g["HOST_OS"] = "darwin" + else: + fail("Cannot run on %s OS" % host) + + # TODO(asmundak): set g.HOST_OS_EXTRA + + g["BUILD_OS"] = g["HOST_OS"] + + # TODO(asmundak): check cross-OS build + + if host.find("x86_64") >= 0: + g["HOST_ARCH"] = "x86_64" + g["HOST_2ND_ARCH"] = "x86" + g["HOST_IS_64_BIT"] = True + elif host.find("i686") >= 0 or host.find("x86") >= 0: + fail("Building on a 32-bit x86 host is not supported: %s" % host) + elif g["HOST_OS"] == "darwin": + g["HOST_2ND_ARCH"] = "" + + g["HOST_2ND_ARCH_VAR_PREFIX"] = "2ND_" + g["HOST_2ND_ARCH_MODULE_SUFFIX"] = "_32" + g["HOST_CROSS_2ND_ARCH_VAR_PREFIX"] = "2ND_" + g["HOST_CROSS_2ND_ARCH_MODULE_SUFFIX"] = "_64" + g["TARGET_2ND_ARCH_VAR_PREFIX"] = "2ND_" + + # TODO(asmundak): combo-related stuff + + # on windows, the tools have .exe at the end, and we depend on the + # host config stuff being done first + g["BUILD_ARCH"] = g["HOST_ARCH"] + g["BUILD_2ND_ARCH"] = g["HOST_2ND_ARCH"] + + # the host build defaults to release, and it must be release or debug + g.setdefault("HOST_BUILD_TYPE", "release") + if g["HOST_BUILD_TYPE"] != "release" and g["HOST_BUILD_TYPE"] != "debug": + fail("HOST_BUILD_TYPE must be either release or debug, not '%s'" % g["HOST_BUILD_TYPE"]) + + # TODO(asmundak): a lot more, but not needed for the product configuration + + g["ART_APEX_JARS"] = [ + "com.android.art:core-oj", + "com.android.art:core-libart", + "com.android.art:okhttp", + "com.android.art:bouncycastle", + "com.android.art:apache-xml" + ] + + if g.get("TARGET_BUILD_TYPE", "") != "debug": + g["TARGET_BUILD_TYPE"] = "release" + + +v_default = "SP1A" +v_min = "SP1A" +v_max = "SP1A" diff --git a/core/product_config.rbc b/core/product_config.rbc new file mode 100644 index 0000000000..6e968222d7 --- /dev/null +++ b/core/product_config.rbc @@ -0,0 +1,483 @@ + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//build/make/core:envsetup.rbc", _envsetup_init="init") +"""Runtime functions.""" + +def _global_init(): + """Returns dict created from the runtime environment.""" + globals = dict() + + # Environment variables + for k in dir(rblf_env): + globals[k] = getattr(rblf_env, k) + + # Variables set as var=value command line arguments + for k in dir(rblf_cli): + globals[k] = getattr(rblf_cli, k) + + globals.setdefault("PRODUCT_SOONG_NAMESPACES", []) + _envsetup_init(globals) + + # Variables that should be defined. + mandatory_vars = [ + "PLATFORM_VERSION_CODENAME", "PLATFORM_VERSION", + "PRODUCT_SOONG_NAMESPACES", + # TODO(asmundak): do we need TARGET_ARCH? AOSP does not reference it + "TARGET_BUILD_TYPE", "TARGET_BUILD_VARIANT", "TARGET_PRODUCT", + ] + for bv in mandatory_vars: + if not bv in globals: + fail(bv, " is not defined") + + return globals + +_globals_base = _global_init() + +def __print_attr(attr, value): + if not value: + return + if type(value) == "list": + if _options.rearrange: + value = __printvars_rearrange_list(value) + if _options.format == "pretty": + print(attr, "=", repr(value)) + elif _options.format == "make": + print(attr, ":=", " ".join(value)) + else: + if _options.format == "pretty": + print(attr, "=", repr(value)) + elif _options.format == "make": + print(attr, ":=", value) + else: + fail("bad output format", _options.format) + + +def _printvars(globals, cfg): + """Prints known configuration variables.""" + for attr, val in sorted(cfg.items()): + __print_attr(attr, val) + if _options.print_globals: + for attr, val in sorted(globals.items()): + print() + if attr not in _globals_base: + __print_attr(attr, val) + + +def __printvars_rearrange_list(l): + """Rearrange value list: return only distinct elements, maybe sorted.""" + seen = { item: 0 for item in l} + return sorted(seen.keys()) if _options.rearrange == "sort" else seen.keys() + +def _product_configuration(top_pcm_name, top_pcm): + """Creates configuration.""" + + # Product configuration is created by traversing product's inheritance + # tree. It is traversed twice. + # First, beginning with top-level module we execute a module and find + # its ancestors, repeating this recursively. At the end of this phase + # we get the full inheritance tree. + # Second, we traverse the tree in the postfix order (i.e., visiting a + # node after its ancestors) to calculate the product configuration. + # + # PCM means "Product Configuration Module", i.e., a Starlark file + # whose body consists of a single init function. + + globals = dict(**_globals_base) + + config_postfix = [] # Configs in postfix order + # Each PCM is represented by a quadruple of function, config, children names + # and readyness (that is, the configurations from inherited PCMs have been + # substituted). + configs = { top_pcm_name: (top_pcm, None, [], False)} # All known PCMs + + stash = [] # Configs to push once their descendants are done + + # Stack maintaining PCMs to be processed. An item in the stack + # is a pair of PCMs name and its height in the product inheritance tree. + pcm_stack = [] + pcm_stack.append((top_pcm_name, 0)) + pcm_count = 0 + # Run it until pcm_stack is exhausted, but no more than N times + for n in range(1000): + if not pcm_stack: + break + (name, height) = pcm_stack.pop() + pcm, cfg, c, _ = configs[name] + # each PCM is executed once + if cfg != None: + continue + # Push ancestors until we reach this node's height + config_postfix.extend([stash.pop() for i in range(len(stash) - height)]) + + # Run this one, obtaining its configuration and child PCMs. + if _options.trace_modules: + print("%s:" % n[0]) + + # The handle passed to the PCM consists of config and inheritance state.dict of inherited modules + # and a list containing the current default value of a list variable. + handle = __h_new() + pcm(globals, handle) + children = __h_inherited_modules(handle) + if _options.trace_modules: + print(" ", " ".join(children.keys())) + configs[name] = (pcm, __h_cfg(handle), children.keys(), False) + pcm_count = pcm_count+1 + + if len(children) == 0: + # Leaf PCM goes straight to the config_postfix + config_postfix.append(name) + continue + + # Stash this PCM, process children in the sorted order + stash.append(name) + for child_name in sorted(children, reverse=True): + if child_name not in configs: + configs[child_name] = (children[child_name], None, [], False) + pcm_stack.append((child_name, len(stash))) + + # Flush the stash + config_postfix.extend([stash.pop() for i in range(len(stash))]) + if len(config_postfix) != pcm_count: + fail("Ran %d modules but postfix tree has only %d entries" % (pcm_count, len(config_postfix))) + + + if _options.trace_modules: + print("\n---Postfix---") + for x in config_postfix: + print(" ", x) + + # Traverse the tree from the bottom, evaluating inherited values + for pcm_name in config_postfix: + pcm, cfg, children_names, ready = configs[pcm_name] + # Should run + if cfg == None: + fail("%s: has not been run" % pcm_name) + # Ready once + if ready: + continue + # Children should be ready + for child_name in children_names: + if not configs[child_name][3]: + fail("%s: child is not ready" % child_name) + + # if _options.trace_modules: + # print(">%s: %s" % (pcm_name, cfg)) + + _substitute_inherited(configs, pcm_name, cfg) + _percolate_inherited(configs, pcm_name, cfg, children_names) + configs[pcm_name] = pcm, cfg, children_names, True + # if _options.trace_modules: + # print("<%s: %s" % (pcm_name, cfg)) + + return globals, configs[top_pcm_name][1] + + +def _substitute_inherited(configs, pcm_name, cfg): + """Substitutes inherited values in all the configuration settings.""" + for attr, val in cfg.items(): + trace_it = attr in _options.trace_variables + if trace_it: + old_val = val + + # TODO(asmundak): should we handle single vars? + if type(val) != "list": + continue + + if trace_it: + new_val = _value_expand(configs, attr, val) + if new_val != old_val: + print("%s(i): %s=%s (was %s)" % (pcm_name, attr, new_val, old_val)) + cfg[attr] = new_val + continue + + cfg[attr] = _value_expand(configs, attr, val) + + + +def _value_expand(configs, attr, values_list): + """Expands references to inherited values in a given list.""" + result = [] + expanded={} + for item in values_list: + # Inherited values are 1-tuples + if type(item) != "tuple": + result.append(item) + continue + child_name = item[0] + if child_name in expanded: + continue + expanded[child_name] = True + child = configs[child_name] + if not child[3]: + fail("%s should be ready" % child_name) + __move_items(result, child[1], attr) + + return result + + +def _percolate_inherited(configs, cfg_name, cfg, children_names): + """Percolates the settings that are present only in children.""" + percolated_attrs = {} + for child_name in children_names: + child_cfg = configs[child_name][1] + for attr, value in child_cfg.items(): + if type(value) != "list": + if attr in percolated_attrs or not attr in cfg: + cfg[attr] = value + percolated_attrs[attr] = True + continue + if attr in percolated_attrs: + # We already are percolating this one, just add this list + __move_items(cfg[attr], child_cfg, attr) + elif not attr in cfg: + percolated_attrs[attr] = True + cfg[attr] = [] + __move_items(cfg[attr], child_cfg, attr) + + for attr in _options.trace_variables: + if attr in percolated_attrs: + print("%s: %s^=%s" % (cfg_name, attr, cfg[attr])) + + +def __move_items(to_list, from_cfg, attr): + l = from_cfg.get(attr, []) + if l: + to_list.extend(l) + from_cfg[attr] = [] + + +def _indirect(pcm_name): + """Returns configuration item for the inherited module.""" + return (pcm_name,) + + +def _addprefix(prefix, string_or_list): + """Adds prefix and returns a list. + + If string_or_list is a list, prepends prefix to each element. + Otherwise, string_or_list is considered to be a string which + is split into words and then prefix is prepended to each one. + + Args: + prefix + string_or_list + + """ + return [ prefix + x for x in __words(string_or_list)] + + +def _addsuffix(suffix, string_or_list): + """Adds suffix and returns a list. + + If string_or_list is a list, appends suffix to each element. + Otherwise, string_or_list is considered to be a string which + is split into words and then suffix is appended to each one. + + Args: + suffix + string_or_list + """ + return [ x + suffix for x in __words(string_or_list)] + + +def __words(string_or_list): + if type(string_or_list) == "list": + return string_or_list + return string_or_list.split() + + +def __h_new(): + """Constructs a handle which is passed to PCM.""" + return (dict(), dict(), list()) + +def __h_inherited_modules(handle): + return handle[1] + + +def __h_cfg(handle): + return handle[0] + + +def _setdefault(handle, attr): + """Sets given attribute's value if it has not been set.""" + cfg = handle[0] + if cfg.get(attr) == None: + cfg[attr] = list(handle[2]) + return cfg[attr] + +def _inherit(handle, pcm_name, pcm): + """Records inheritance.""" + cfg, inherited, default_lv = handle + inherited[pcm_name]=pcm + default_lv.append(_indirect(pcm_name)) + # Add inherited module reference to all configuration values + for attr, val in cfg.items(): + if type(val) == "list": + val.append(_indirect(pcm_name)) + + +def _copy_if_exists(path_pair): + """If from file exists, returns [from:to] pair.""" + l = path_pair.split(":", 2) + # Check that l[0] exists + return [":".join(l)] if rblf_file_exists(l[0]) else [] + +def _enforce_product_packages_exist(pkg_string_or_list): + """Makes including non-existent modules in PRODUCT_PACKAGES an error.""" + #TODO(asmundak) + pass + + +def _file_wildcard_exists(file_pattern): + """Return True if there are files matching given bash pattern.""" + return len(rblf_wildcard(file_pattern)) > 0 + + +def _find_and_copy(pattern, from_dir, to_dir): + """Return a copy list for the files matching the pattern.""" + return ["%s/%s:%s/%s" % (from_dir, f, to_dir, f) for f in rblf_wildcard(pattern, from_dir)] + + +def _filter_out(pattern, text): + """Return all the words from `text' that do not match any word in `pattern'. + + Args: + pattern: string or list of words. '%' stands for wildcard (in regex terms, '.*') + text: string or list of words + Return: + list of words + """ + rex = __mk2regex(__words(pattern)) + res = [] + for w in __words(text): + if not _regex_match(rex, w): + res.append(w) + return res + + +def _filter(pattern, text): + """Return all the words in `text` that match `pattern`. + + Args: + pattern: strings of words or a list. A word can contain '%', + which stands for any sequence of characters. + text: string or list of words. + """ + rex = __mk2regex(__words(pattern)) + res = [] + for w in __words(text): + if _regex_match(rex, w): + res.append(w) + return res + + +def __mk2regex(words): + """Returns regular expression equivalent to Make pattern.""" + + # TODO(asmundak): this will mishandle '\%' + return "^(" + "|".join([w.replace("%", ".*", 1) for w in words]) + ")" + + +def _regex_match(regex, w): + return rblf_regex(regex, w) + + +def _require_artifacts_in_path(paths, allowed_paths): + """TODO.""" + #print("require_artifacts_in_path(", __words(paths), ",", __words(allowed_paths), ")") + pass + + +def _require_artifacts_in_path_relaxed(paths, allowed_paths): + """TODO.""" + pass + + +def _expand_wildcard(pattern): + """Expands shell wildcard pattern.""" + return rblf_wildcard(pattern) + + +def _mkerror(file, message=""): + """Prints error and stops.""" + fail("%s: %s. Stop" % (file, message)) + + +def _mkwarning(file, message=""): + """Prints warning.""" + print("%s: warning: %s" % (file, message)) + + +def _mkinfo(file, message=""): + """Prints info.""" + print(message) + + +def __get_options(): + """Returns struct containing runtime global settings.""" + settings = dict( + format = "pretty", + print_globals = False, + rearrange = "", + trace_modules = False, + trace_variables = [], + ) + for x in getattr(rblf_cli, "RBC_OUT", "").split(","): + if x == "sort" or x == "unique": + if settings["rearrange"]: + fail("RBC_OUT: either sort or unique is allowed (and sort implies unique)") + settings["rearrange"] = x + elif x == "pretty" or x == "make": + settings["format"] = x + elif x == "global": + settings["print_globals"] = True + elif x != "": + fail("RBC_OUT: got %s, should be one of: [pretty|make] [sort|unique]" % x) + for x in getattr(rblf_cli, "RBC_DEBUG", "").split(","): + if x == "!trace": + settings["trace_modules"] = True + elif x != "": + settings["trace_variables"].append(x) + return struct(**settings) + +# Settings used during debugging. +_options = __get_options() +rblf = struct(addprefix=_addprefix, + addsuffix=_addsuffix, + copy_if_exists=_copy_if_exists, + cfg=__h_cfg, + enforce_product_packages_exist=_enforce_product_packages_exist, + expand_wildcard=_expand_wildcard, + file_exists=rblf_file_exists, + file_wildcard_exists=_file_wildcard_exists, + filter=_filter, + filter_out=_filter_out, + find_and_copy=_find_and_copy, + global_init=_global_init, + inherit=_inherit, + indirect=_indirect, + mkinfo=_mkinfo, + mkerror=_mkerror, + mkwarning=_mkwarning, + printvars=_printvars, + product_configuration=_product_configuration, + require_artifacts_in_path=_require_artifacts_in_path, + require_artifacts_in_path_relaxed=_require_artifacts_in_path_relaxed, + setdefault=_setdefault, + shell=rblf_shell, + warning=_mkwarning, + ) + diff --git a/tests/device.rbc b/tests/device.rbc new file mode 100644 index 0000000000..5d4e70c2b1 --- /dev/null +++ b/tests/device.rbc @@ -0,0 +1,42 @@ + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Top-level test configuration. +# Converted from the following makefile +### PRODUCT_PACKAGES += dev +### PRODUCT_HOST_PACKAGES += host +### $(call inherit-product, $(LOCAL_PATH)/part1.mk) +### PRODUCT_COPY_FILES += device_from:device_to +### include $(LOCAL_PATH)/include1.mk +### PRODUCT_PACKAGES += dev_after +### PRODUCT_COPY_FILES += $(call find-copy-subdir-files,audio_platform_info*.xml,device/google/redfin/audio,$(TARGET_COPY_OUT_VENDOR)/etc) xyz + +load("//build/make/core:product_config.rbc", "rblf") +load(":part1.rbc", _part1_init = "init") +load(":include1.rbc", _include1_init = "init") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.setdefault(handle, "PRODUCT_PACKAGES") + cfg["PRODUCT_PACKAGES"] += ["dev"] + rblf.setdefault(handle, "PRODUCT_HOST_PACKAGES") + cfg["PRODUCT_HOST_PACKAGES"] += ["host"] + rblf.inherit(handle, "test/part1", _part1_init) + rblf.setdefault(handle, "PRODUCT_COPY_FILES") + cfg["PRODUCT_COPY_FILES"] += ["device_from:device_to"] + _include1_init(g, handle) + cfg["PRODUCT_PACKAGES"] += ["dev_after"] + cfg["PRODUCT_COPY_FILES"] += (rblf.find_and_copy("audio_platform_info*.xml", "device/google/redfin/audio", "||VENDOR-PATH-PH||/etc") + + ["xyz"]) diff --git a/tests/include1.rbc b/tests/include1.rbc new file mode 100644 index 0000000000..c0c9b3ba9f --- /dev/null +++ b/tests/include1.rbc @@ -0,0 +1,25 @@ + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Included file (not inherited) +# Converted from makefile +### PRODUCT_PACKAGES += inc + +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.setdefault(handle, "PRODUCT_PACKAGES") + cfg["PRODUCT_PACKAGES"] += ["inc"] diff --git a/tests/part1.rbc b/tests/part1.rbc new file mode 100644 index 0000000000..3e50751026 --- /dev/null +++ b/tests/part1.rbc @@ -0,0 +1,28 @@ + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Part configuration +# Converted from +### PRODUCT_COPY_FILES += part_from:part_to +### PRODUCT_PRODUCT_PROPERTIES += part_properties + +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.setdefault(handle, "PRODUCT_COPY_FILES") + cfg["PRODUCT_COPY_FILES"] += ["part_from:part_to"] + rblf.setdefault(handle, "PRODUCT_PRODUCT_PROPERTIES") + cfg["PRODUCT_PRODUCT_PROPERTIES"] += ["part_properties"] diff --git a/tests/run.rbc b/tests/run.rbc new file mode 100644 index 0000000000..b13f835bc1 --- /dev/null +++ b/tests/run.rbc @@ -0,0 +1,50 @@ + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Run test configuration and verify its result. +# The main configuration file is device.rbc. +# It inherits part1.rbc and also includes include1.rbc +# TODO(asmundak): more tests are needed to verify that: +# * multi-level inheritance works as expected +# * all runtime functions (wildcard, regex, etc.) work + +load("//build/make/core:product_config.rbc", "rblf") +load(":device.rbc", "init") + +def assert_eq(expected, actual): + if expected != actual: + fail("Expected %s, got %s" % (expected, actual)) + + +globals, config = rblf.product_configuration("test/device", init) +assert_eq( + { + "PRODUCT_COPY_FILES": [ + "part_from:part_to", + "device_from:device_to", + "device/google/redfin/audio/audio_platform_info_noextcodec_snd.xml:||VENDOR-PATH-PH||/etc/audio_platform_info_noextcodec_snd.xml", + "xyz" + ], + "PRODUCT_HOST_PACKAGES": ["host"], + "PRODUCT_PACKAGES": [ + "dev", + "inc", + "dev_after" + ], + "PRODUCT_PRODUCT_PROPERTIES": ["part_properties"] + }, + { k:v for k, v in sorted(config.items()) } +) diff --git a/tools/rbcrun/Android.bp b/tools/rbcrun/Android.bp new file mode 100644 index 0000000000..b5f71552a4 --- /dev/null +++ b/tools/rbcrun/Android.bp @@ -0,0 +1,36 @@ +// +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +blueprint_go_binary { + name: "rbcrun", + srcs: ["cmd/rbcrun.go"], + deps: ["rbcrun-module"], +} + +bootstrap_go_package { + name: "rbcrun-module", + srcs: [ + "host.go", + ], + testSrcs: [ + "host_test.go", + ], + pkgPath: "rbcrun", + deps: [ + "go-starlark-starlark", + "go-starlark-starlarkstruct", + "go-starlark-starlarktest", + ], +} diff --git a/tools/rbcrun/README.md b/tools/rbcrun/README.md new file mode 100644 index 0000000000..fb58c897aa --- /dev/null +++ b/tools/rbcrun/README.md @@ -0,0 +1,84 @@ +# Roboleaf configuration files interpreter + +Reads and executes Roboleaf product configuration files. + +## Usage + +`rbcrun` *options* *VAR=value*... [ *file* ] + +A Roboleaf configuration file is a Starlark script. Usually it is read from *file*. The option `-c` allows to provide a +script directly on the command line. The option `-f` is there to allow the name of a file script to contain (`=`). +(i.e., `my=file.rbc` sets `my` to `file.rbc`, `-f my=file.rbc` runs the script from `my=file.rbc`). + +### Options + +`-d` *dir*\ +Root directory for load("//path",...) + +`-c` *text*\ +Read script from *text* + +`--perf` *file*\ +Gather performance statistics and save it to *file*. Use \ +` go tool prof -top`*file*\ +to show top CPU users + +`-f` *file*\ +File to run. + +## Extensions + +The runner allows Starlark scripts to use the following features that Bazel's Starlark interpreter does not support: + +### Load statement URI + +Starlark does not define the format of the load statement's first argument. +The Roboleaf configuration interpreter supports the format that Bazel uses +(`":file"` or `"//path:file"`). In addition, it allows the URI to end with +`"|symbol"` which defines a single variable `symbol` with `None` value if a +module does not exist. Thus, + +``` +load(":mymodule.rbc|init", mymodule_init="init") +``` + +will load the module `mymodule.rbc` and export a symbol `init` in it as +`mymodule_init` if `mymodule.rbc` exists. If `mymodule.rbc` is missing, +`mymodule_init` will be set to `None` + +### Predefined Symbols + +#### rblf_env + +A `struct` containing environment variables. E.g., `rblf_env.USER` is the username when running on Unix. + +#### rblf_cli + +A `struct` containing the variable set by the interpreter's command line. That is, running + +``` +rbcrun FOO=bar myfile.rbc +``` + +will have the value of `rblf_cli.FOO` be `"bar"` + +### Predefined Functions + +#### rblf_file_exists(*file*) + +Returns `True` if *file* exists + +#### rblf_wildcard(*glob*, *top* = None) + +Expands *glob*. If *top* is supplied, expands "*top*/*glob*", then removes +"*top*/" prefix from the matching file names. + +#### rblf_regex(*pattern*, *text*) + +Returns *True* if *text* matches *pattern*. + +#### rblf_shell(*command*) + +Runs `sh -c "`*command*`"`, reads its output, converts all newlines into spaces, chops trailing newline returns this +string. This is equivalent to Make's +`shell` builtin function. *This function will be eventually removed*. diff --git a/tools/rbcrun/cmd/rbcrun.go b/tools/rbcrun/cmd/rbcrun.go new file mode 100644 index 0000000000..784856259b --- /dev/null +++ b/tools/rbcrun/cmd/rbcrun.go @@ -0,0 +1,98 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "go.starlark.net/starlark" + "os" + "rbcrun" + "strings" +) + +var ( + execprog = flag.String("c", "", "execute program `prog`") + rootdir = flag.String("d", ".", "the value of // for load paths") + file = flag.String("f", "", "file to execute") + perfFile = flag.String("perf", "", "save performance data") +) + +func main() { + flag.Parse() + filename := *file + var src interface{} + var env []string + + rc := 0 + for _, arg := range flag.Args() { + if strings.Contains(arg, "=") { + env = append(env, arg) + } else if filename == "" { + filename = arg + } else { + quit("only one file can be executed\n") + } + } + if *execprog != "" { + if filename != "" { + quit("either -c or file name should be present\n") + } + filename = "" + src = *execprog + } + if filename == "" { + if len(env) > 0 { + fmt.Fprintln(os.Stderr, + "no file to run -- if your file's name contains '=', use -f to specify it") + } + flag.Usage() + os.Exit(1) + } + if stat, err := os.Stat(*rootdir); os.IsNotExist(err) || !stat.IsDir() { + quit("%s is not a directory\n", *rootdir) + } + if *perfFile != "" { + pprof, err := os.Create(*perfFile) + if err != nil { + quit("%s: err", *perfFile) + } + defer pprof.Close() + if err := starlark.StartProfile(pprof); err != nil { + quit("%s\n", err) + } + } + rbcrun.LoadPathRoot = *rootdir + err := rbcrun.Run(filename, src, env) + if *perfFile != "" { + if err2 := starlark.StopProfile(); err2 != nil { + fmt.Fprintln(os.Stderr, err2) + rc = 1 + } + } + if err != nil { + if evalErr, ok := err.(*starlark.EvalError); ok { + quit("%s\n", evalErr.Backtrace()) + } else { + quit("%s\n", err) + } + } + os.Exit(rc) +} + +func quit(format string, s ...interface{}) { + fmt.Fprintln(os.Stderr, format, s) + os.Exit(2) +} diff --git a/tools/rbcrun/go.mod b/tools/rbcrun/go.mod new file mode 100644 index 0000000000..a029eb4ec2 --- /dev/null +++ b/tools/rbcrun/go.mod @@ -0,0 +1,10 @@ +module rbcrun + +require ( + github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect + go.starlark.net v0.0.0-20201006213952-227f4aabceb5 +) + +replace go.starlark.net => ../../../../external/starlark-go + +go 1.15 diff --git a/tools/rbcrun/go.sum b/tools/rbcrun/go.sum new file mode 100644 index 0000000000..db4d51e9dd --- /dev/null +++ b/tools/rbcrun/go.sum @@ -0,0 +1,75 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/tools/rbcrun/host.go b/tools/rbcrun/host.go new file mode 100644 index 0000000000..f1697f1f3b --- /dev/null +++ b/tools/rbcrun/host.go @@ -0,0 +1,263 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rbcrun + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +const callerDirKey = "callerDir" + +var LoadPathRoot = "." +var shellPath string + +type modentry struct { + globals starlark.StringDict + err error +} + +var moduleCache = make(map[string]*modentry) + +var builtins starlark.StringDict + +func moduleName2AbsPath(moduleName string, callerDir string) (string, error) { + path := moduleName + if ix := strings.LastIndex(path, ":"); ix >= 0 { + path = path[0:ix] + string(os.PathSeparator) + path[ix+1:] + } + if strings.HasPrefix(path, "//") { + return filepath.Abs(filepath.Join(LoadPathRoot, path[2:])) + } else if strings.HasPrefix(moduleName, ":") { + return filepath.Abs(filepath.Join(callerDir, path[1:])) + } else { + return filepath.Abs(path) + } +} + +// loader implements load statement. The format of the loaded module URI is +// [//path]:base[|symbol] +// The file path is $ROOT/path/base if path is present, /base otherwise. +// The presence of `|symbol` indicates that the loader should return a single 'symbol' +// bound to None if file is missing. +func loader(thread *starlark.Thread, module string) (starlark.StringDict, error) { + pipePos := strings.LastIndex(module, "|") + mustLoad := pipePos < 0 + var defaultSymbol string + if !mustLoad { + defaultSymbol = module[pipePos+1:] + module = module[:pipePos] + } + modulePath, err := moduleName2AbsPath(module, thread.Local(callerDirKey).(string)) + if err != nil { + return nil, err + } + e, ok := moduleCache[modulePath] + if e == nil { + if ok { + return nil, fmt.Errorf("cycle in load graph") + } + + // Add a placeholder to indicate "load in progress". + moduleCache[modulePath] = nil + + // Decide if we should load. + if !mustLoad { + if _, err := os.Stat(modulePath); err == nil { + mustLoad = true + } + } + + // Load or return default + if mustLoad { + childThread := &starlark.Thread{Name: "exec " + module, Load: thread.Load} + // Cheating for the sake of testing: + // propagate starlarktest's Reporter key, otherwise testing + // the load function may cause panic in starlarktest code. + const testReporterKey = "Reporter" + if v := thread.Local(testReporterKey); v != nil { + childThread.SetLocal(testReporterKey, v) + } + + childThread.SetLocal(callerDirKey, filepath.Dir(modulePath)) + globals, err := starlark.ExecFile(childThread, modulePath, nil, builtins) + e = &modentry{globals, err} + } else { + e = &modentry{starlark.StringDict{defaultSymbol: starlark.None}, nil} + } + + // Update the cache. + moduleCache[modulePath] = e + } + return e.globals, e.err +} + +// fileExists returns True if file with given name exists. +func fileExists(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, + kwargs []starlark.Tuple) (starlark.Value, error) { + var path string + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &path); err != nil { + return starlark.None, err + } + if stat, err := os.Stat(path); err != nil || stat.IsDir() { + return starlark.False, nil + } + return starlark.True, nil +} + +// regexMatch(pattern, s) returns True if s matches pattern (a regex) +func regexMatch(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, + kwargs []starlark.Tuple) (starlark.Value, error) { + var pattern, s string + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 2, &pattern, &s); err != nil { + return starlark.None, err + } + match, err := regexp.MatchString(pattern, s) + if err != nil { + return starlark.None, err + } + if match { + return starlark.True, nil + } + return starlark.False, nil +} + +// wildcard(pattern, top=None) expands shell's glob pattern. If 'top' is present, +// the 'top/pattern' is globbed and then 'top/' prefix is removed. +func wildcard(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, + kwargs []starlark.Tuple) (starlark.Value, error) { + var pattern string + var top string + + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &pattern, &top); err != nil { + return starlark.None, err + } + + var files []string + var err error + if top == "" { + if files, err = filepath.Glob(pattern); err != nil { + return starlark.None, err + } + } else { + prefix := top + string(filepath.Separator) + if files, err = filepath.Glob(prefix + pattern); err != nil { + return starlark.None, err + } + for i := range files { + files[i] = strings.TrimPrefix(files[i], prefix) + } + } + return makeStringList(files), nil +} + +// shell(command) runs OS shell with given command and returns back +// its output the same way as Make's $(shell ) function. The end-of-lines +// ("\n" or "\r\n") are replaced with " " in the result, and the trailing +// end-of-line is removed. +func shell(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, + kwargs []starlark.Tuple) (starlark.Value, error) { + var command string + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &command); err != nil { + return starlark.None, err + } + if shellPath == "" { + return starlark.None, + fmt.Errorf("cannot run shell, SHELL environment variable is not set (running on Windows?)") + } + cmd := exec.Command(shellPath, "-c", command) + // We ignore command's status + bytes, _ := cmd.Output() + output := string(bytes) + if strings.HasSuffix(output, "\n") { + output = strings.TrimSuffix(output, "\n") + } else { + output = strings.TrimSuffix(output, "\r\n") + } + + return starlark.String( + strings.ReplaceAll( + strings.ReplaceAll(output, "\r\n", " "), + "\n", " ")), nil +} + +func makeStringList(items []string) *starlark.List { + elems := make([]starlark.Value, len(items)) + for i, item := range items { + elems[i] = starlark.String(item) + } + return starlark.NewList(elems) +} + +// propsetFromEnv constructs a propset from the array of KEY=value strings +func structFromEnv(env []string) *starlarkstruct.Struct { + sd := make(map[string]starlark.Value, len(env)) + for _, x := range env { + kv := strings.SplitN(x, "=", 2) + sd[kv[0]] = starlark.String(kv[1]) + } + return starlarkstruct.FromStringDict(starlarkstruct.Default, sd) +} + +func setup(env []string) { + // Create the symbols that aid makefile conversion. See README.md + builtins = starlark.StringDict{ + "struct": starlark.NewBuiltin("struct", starlarkstruct.Make), + "rblf_cli": structFromEnv(env), + "rblf_env": structFromEnv(os.Environ()), + // To convert makefile's $(wildcard foo) + "rblf_file_exists": starlark.NewBuiltin("rblf_file_exists", fileExists), + // To convert makefile's $(filter ...)/$(filter-out) + "rblf_regex": starlark.NewBuiltin("rblf_regex", regexMatch), + // To convert makefile's $(shell cmd) + "rblf_shell": starlark.NewBuiltin("rblf_shell", shell), + // To convert makefile's $(wildcard foo*) + "rblf_wildcard": starlark.NewBuiltin("rblf_wildcard", wildcard), + } + + // NOTE(asmundak): OS-specific. + shellPath, _ = os.LookupEnv("SHELL") +} + +// Parses, resolves, and executes a Starlark file. +// filename and src parameters are as for starlark.ExecFile: +// * filename is the name of the file to execute, +// and the name that appears in error messages; +// * src is an optional source of bytes to use instead of filename +// (it can be a string, or a byte array, or an io.Reader instance) +// * commandVars is an array of "VAR=value" items. They are accessible from +// the starlark script as members of the `rblf_cli` propset. +func Run(filename string, src interface{}, commandVars []string) error { + setup(commandVars) + + mainThread := &starlark.Thread{ + Name: "main", + Print: func(_ *starlark.Thread, msg string) { fmt.Println(msg) }, + Load: loader, + } + absPath, err := filepath.Abs(filename) + if err == nil { + mainThread.SetLocal(callerDirKey, filepath.Dir(absPath)) + _, err = starlark.ExecFile(mainThread, absPath, src, builtins) + } + return err +} diff --git a/tools/rbcrun/host_test.go b/tools/rbcrun/host_test.go new file mode 100644 index 0000000000..3be5ee67aa --- /dev/null +++ b/tools/rbcrun/host_test.go @@ -0,0 +1,159 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rbcrun + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + "go.starlark.net/resolve" + "go.starlark.net/starlark" + "go.starlark.net/starlarktest" +) + +// In order to use "assert.star" from go/starlark.net/starlarktest in the tests, +// provide: +// * load function that handles "assert.star" +// * starlarktest.DataFile function that finds its location + +func init() { + starlarktestSetup() +} + +func starlarktestSetup() { + resolve.AllowLambda = true + starlarktest.DataFile = func(pkgdir, filename string) string { + // The caller expects this function to return the path to the + // data file. The implementation assumes that the source file + // containing the caller and the data file are in the same + // directory. It's ugly. Not sure what's the better way. + // TODO(asmundak): handle Bazel case + _, starlarktestSrcFile, _, _ := runtime.Caller(1) + if filepath.Base(starlarktestSrcFile) != "starlarktest.go" { + panic(fmt.Errorf("this function should be called from starlarktest.go, got %s", + starlarktestSrcFile)) + } + return filepath.Join(filepath.Dir(starlarktestSrcFile), filename) + } +} + +// Common setup for the tests: create thread, change to the test directory +func testSetup(t *testing.T, env []string) *starlark.Thread { + setup(env) + thread := &starlark.Thread{ + Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) { + if module == "assert.star" { + return starlarktest.LoadAssertModule() + } + return nil, fmt.Errorf("load not implemented") + }} + starlarktest.SetReporter(thread, t) + if err := os.Chdir(dataDir()); err != nil { + t.Fatal(err) + } + return thread +} + +func dataDir() string { + _, thisSrcFile, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(thisSrcFile), "testdata") + +} + +func exerciseStarlarkTestFile(t *testing.T, starFile string) { + // In order to use "assert.star" from go/starlark.net/starlarktest in the tests, provide: + // * load function that handles "assert.star" + // * starlarktest.DataFile function that finds its location + setup(nil) + thread := &starlark.Thread{ + Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) { + if module == "assert.star" { + return starlarktest.LoadAssertModule() + } + return nil, fmt.Errorf("load not implemented") + }} + starlarktest.SetReporter(thread, t) + _, thisSrcFile, _, _ := runtime.Caller(0) + filename := filepath.Join(filepath.Dir(thisSrcFile), starFile) + if _, err := starlark.ExecFile(thread, filename, nil, builtins); err != nil { + if err, ok := err.(*starlark.EvalError); ok { + t.Fatal(err.Backtrace()) + } + t.Fatal(err) + } +} + +func TestCliAndEnv(t *testing.T) { + // TODO(asmundak): convert this to use exerciseStarlarkTestFile + if err := os.Setenv("TEST_ENVIRONMENT_FOO", "test_environment_foo"); err != nil { + t.Fatal(err) + } + thread := testSetup(t, []string{"CLI_FOO=foo"}) + if _, err := starlark.ExecFile(thread, "cli_and_env.star", nil, builtins); err != nil { + if err, ok := err.(*starlark.EvalError); ok { + t.Fatal(err.Backtrace()) + } + t.Fatal(err) + } +} + +func TestFileOps(t *testing.T) { + // TODO(asmundak): convert this to use exerciseStarlarkTestFile + if err := os.Setenv("TEST_DATA_DIR", dataDir()); err != nil { + t.Fatal(err) + } + thread := testSetup(t, nil) + if _, err := starlark.ExecFile(thread, "file_ops.star", nil, builtins); err != nil { + if err, ok := err.(*starlark.EvalError); ok { + t.Fatal(err.Backtrace()) + } + t.Fatal(err) + } +} + +func TestLoad(t *testing.T) { + // TODO(asmundak): convert this to use exerciseStarlarkTestFile + thread := testSetup(t, nil) + thread.Load = func(thread *starlark.Thread, module string) (starlark.StringDict, error) { + if module == "assert.star" { + return starlarktest.LoadAssertModule() + } else { + return loader(thread, module) + } + } + dir := dataDir() + thread.SetLocal(callerDirKey, dir) + LoadPathRoot = filepath.Dir(dir) + if _, err := starlark.ExecFile(thread, "load.star", nil, builtins); err != nil { + if err, ok := err.(*starlark.EvalError); ok { + t.Fatal(err.Backtrace()) + } + t.Fatal(err) + } +} + +func TestRegex(t *testing.T) { + exerciseStarlarkTestFile(t, "testdata/regex.star") +} + +func TestShell(t *testing.T) { + if err := os.Setenv("TEST_DATA_DIR", dataDir()); err != nil { + t.Fatal(err) + } + exerciseStarlarkTestFile(t, "testdata/shell.star") +} diff --git a/tools/rbcrun/testdata/cli_and_env.star b/tools/rbcrun/testdata/cli_and_env.star new file mode 100644 index 0000000000..d6f464aa79 --- /dev/null +++ b/tools/rbcrun/testdata/cli_and_env.star @@ -0,0 +1,11 @@ +# Tests rblf_env access +load("assert.star", "assert") + + +def test(): + assert.eq(rblf_env.TEST_ENVIRONMENT_FOO, "test_environment_foo") + assert.fails(lambda: rblf_env.FOO_BAR_BAZ, ".*struct has no .FOO_BAR_BAZ attribute$") + assert.eq(rblf_cli.CLI_FOO, "foo") + + +test() diff --git a/tools/rbcrun/testdata/file_ops.star b/tools/rbcrun/testdata/file_ops.star new file mode 100644 index 0000000000..e1f1ac27ea --- /dev/null +++ b/tools/rbcrun/testdata/file_ops.star @@ -0,0 +1,18 @@ +# Tests file ops builtins +load("assert.star", "assert") + + +def test(): + myname = "file_ops.star" + assert.true(rblf_file_exists(myname), "the file %s does exist" % myname) + assert.true(not rblf_file_exists("no_such_file"), "the file no_such_file does not exist") + files = rblf_wildcard("*.star") + assert.true(myname in files, "expected %s in %s" % (myname, files)) + # RBCDATADIR is set by the caller to the path where this file resides + files = rblf_wildcard("*.star", rblf_env.TEST_DATA_DIR) + assert.true(myname in files, "expected %s in %s" % (myname, files)) + files = rblf_wildcard("*.xxx") + assert.true(len(files) == 0, "expansion should be empty but contains %s" % files) + + +test() diff --git a/tools/rbcrun/testdata/load.star b/tools/rbcrun/testdata/load.star new file mode 100644 index 0000000000..b14f2bbef1 --- /dev/null +++ b/tools/rbcrun/testdata/load.star @@ -0,0 +1,14 @@ +# Test load, simple and conditional +load("assert.star", "assert") +load(":module1.star", test1="test") +load("//testdata:module2.star", test2="test") +load(":module3|test", test3="test") + + +def test(): + assert.eq(test1, "module1") + assert.eq(test2, "module2") + assert.eq(test3, None) + + +test() diff --git a/tools/rbcrun/testdata/module1.star b/tools/rbcrun/testdata/module1.star new file mode 100644 index 0000000000..913fb7d7cb --- /dev/null +++ b/tools/rbcrun/testdata/module1.star @@ -0,0 +1,7 @@ +# Module loaded my load.star +load("assert.star", "assert") + +# Make sure that builtins are defined for the loaded module, too +assert.true(rblf_file_exists("module1.star")) +assert.true(not rblf_file_exists("no_such file")) +test = "module1" diff --git a/tools/rbcrun/testdata/module2.star b/tools/rbcrun/testdata/module2.star new file mode 100644 index 0000000000..f6818a2915 --- /dev/null +++ b/tools/rbcrun/testdata/module2.star @@ -0,0 +1,2 @@ +# Module loaded my load.star +test = "module2" diff --git a/tools/rbcrun/testdata/regex.star b/tools/rbcrun/testdata/regex.star new file mode 100644 index 0000000000..04e1d424e9 --- /dev/null +++ b/tools/rbcrun/testdata/regex.star @@ -0,0 +1,13 @@ +# Tests rblf_regex +load("assert.star", "assert") + + +def test(): + pattern = "^(foo.*bar|abc.*d|1.*)$" + for w in ("foobar", "fooxbar", "abcxd", "123"): + assert.true(rblf_regex(pattern, w), "%s should match %s" % (w, pattern)) + for w in ("afoobar", "abcde"): + assert.true(not rblf_regex(pattern, w), "%s should not match %s" % (w, pattern)) + + +test() diff --git a/tools/rbcrun/testdata/shell.star b/tools/rbcrun/testdata/shell.star new file mode 100644 index 0000000000..ad106974c4 --- /dev/null +++ b/tools/rbcrun/testdata/shell.star @@ -0,0 +1,5 @@ +# Tests "queue" data type +load("assert.star", "assert") + +assert.eq("load.star shell.star", rblf_shell("cd %s && ls -1 shell.star load.star 2>&1" % rblf_env.TEST_DATA_DIR)) +assert.eq("shell.star", rblf_shell("cd %s && echo shell.sta*" % rblf_env.TEST_DATA_DIR))