From f1f49bb9101448bd6b508625b1eae0ae30e127fa Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 1 Dec 2021 13:49:12 -0800 Subject: [PATCH] Pass input variables to product config Passing variables via a makefile instead of rblf_cli / rblf_env allows us to give them correct types while converting the makefile to starlark, as opposed to the variables always being strings when given via rblf_cli / rblf_env. This also allows us to remove some hand-converted starlark code. Bug: 201700692 Test: ./out/soong/rbcrun ./build/make/tests/run.rbc Change-Id: I58c4f20b29171c14e5ae759beb26a849426f6961 --- core/board_config.mk | 13 +- core/envsetup.mk | 10 + core/envsetup.rbc | 224 ------------------ core/product_config.mk | 12 +- core/product_config.rbc | 38 ++- .../build_id.rbc => tests/input_variables.rbc | 19 +- tests/run.rbc | 8 +- 7 files changed, 55 insertions(+), 269 deletions(-) delete mode 100644 core/envsetup.rbc rename core/build_id.rbc => tests/input_variables.rbc (51%) diff --git a/core/board_config.mk b/core/board_config.mk index 110da95e59..821607e950 100644 --- a/core/board_config.mk +++ b/core/board_config.mk @@ -221,16 +221,6 @@ else .KATI_READONLY := TARGET_DEVICE_DIR endif -# Dumps all variables that match [A-Z][A-Z0-9_]* to the file at $(1) -# It is used to print only the variables that are likely to be relevant to the -# board configuration. -define dump-public-variables -$(file >$(OUT_DIR)/dump-public-variables-temp.txt,$(subst $(space),$(newline),$(.VARIABLES)))\ -$(file >$(1),\ -$(foreach v, $(shell grep -he "^[A-Z][A-Z0-9_]*$$" $(OUT_DIR)/dump-public-variables-temp.txt | grep -vhE "^(SOONG_.*|LOCAL_PATH|TOPDIR|PRODUCT_COPY_OUT_.*)$$"),\ -$(v) := $(strip $($(v)))$(newline))) -endef - # TODO(colefaust) change this if to RBC_PRODUCT_CONFIG when # the board configuration is known to work on everything # the product config works on. @@ -238,8 +228,7 @@ ifndef RBC_BOARD_CONFIG include $(board_config_mk) else $(shell mkdir -p $(OUT_DIR)/rbc) - - $(call dump-public-variables, $(OUT_DIR)/rbc/make_vars_pre_board_config.mk) + $(call dump-variables-rbc, $(OUT_DIR)/rbc/make_vars_pre_board_config.mk) $(shell $(OUT_DIR)/soong/mk2rbc \ --mode=write -r --outdir $(OUT_DIR)/rbc \ diff --git a/core/envsetup.mk b/core/envsetup.mk index 3e40a0268e..21eac7f3b8 100644 --- a/core/envsetup.mk +++ b/core/envsetup.mk @@ -310,6 +310,16 @@ ifeq (true,$(EMMA_INSTRUMENT_FRAMEWORK)) endif ################################################################# +# Dumps all variables that match [A-Z][A-Z0-9_]* (with a few exceptions) +# to the file at $(1). It is used to print only the variables that are +# likely to be relevant to the product or board configuration. +define dump-variables-rbc +$(file >$(OUT_DIR)/dump-variables-rbc-temp.txt,$(subst $(space),$(newline),$(.VARIABLES)))\ +$(file >$(1),\ +$(foreach v, $(shell grep -he "^[A-Z][A-Z0-9_]*$$" $(OUT_DIR)/dump-variables-rbc-temp.txt | grep -vhE "^(SOONG_.*|LOCAL_PATH|TOPDIR|PRODUCT_COPY_OUT_.*)$$"),\ +$(v) := $(strip $($(v)))$(newline))) +endef + # Read the product specs so we can get TARGET_DEVICE and other # variables that we need in order to locate the output files. include $(BUILD_SYSTEM)/product_config.mk diff --git a/core/envsetup.rbc b/core/envsetup.rbc deleted file mode 100644 index 4cc98c80f0..0000000000 --- a/core/envsetup.rbc +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load(":build_id.rbc|init", _build_id_init = "init") - -def _all_versions(): - """Returns all known versions.""" - versions = ["OPR1", "OPD1", "OPD2", "OPM1", "OPM2", "PPR1", "PPD1", "PPD2", "PPM1", "PPM2", "QPR1"] - for v in ("Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"): - for e in ("P1A", "P1B", "P2A", "P2B", "D1A", "D1B", "D2A", "D2B", "Q1A", "Q1B", "Q2A", "Q2B", "Q3A", "Q3B"): - versions.append(v + e) - return versions - -def _allowed_versions(all_versions, min_version, max_version, default_version): - """Checks that version range and default versions is valid, returns all versions in range.""" - for v in (min_version, max_version, default_version): - if v not in all_versions: - fail("% is invalid" % v) - - min_i = all_versions.index(min_version) - max_i = all_versions.index(max_version) - def_i = all_versions.index(default_version) - if min_i > max_i: - fail("%s should come before %s in the version list" % (min_version, max_version)) - if def_i < min_i or def_i > max_i: - fail("%s should come between % and %s" % (default_version, min_version, max_version)) - return all_versions[min_i:max_i + 1] - -# This function is a manual conversion of the version_defaults.mk -def _versions_default(g, all_versions, v): - """Handle various build version information. - - Guarantees that the following are defined: - PLATFORM_VERSION - PLATFORM_SDK_VERSION - PLATFORM_VERSION_CODENAME - DEFAULT_APP_TARGET_SDK - BUILD_ID - BUILD_NUMBER - PLATFORM_SECURITY_PATCH - PLATFORM_VNDK_VERSION - PLATFORM_SYSTEMSDK_VERSIONS - """ - - # If build_id.rbc exists, it may override some of the defaults. - # Note that build.prop target also wants INTERNAL_BUILD_ID_MAKEFILE to be set if the file exists. - if _build_id_init != None: - _build_id_init(g) - g["INTERNAL_BUILD_ID_MAKEFILE"] = "build/make/core/build_id" - - allowed_versions = _allowed_versions(all_versions, v.min_platform_version, v.max_platform_version, v.default_platform_version) - g.setdefault("TARGET_PLATFORM_VERSION", v.default_platform_version) - if g["TARGET_PLATFORM_VERSION"] not in allowed_versions: - fail("% is not valid, must be one of %s" % (g["TARGET_PLATFORM_VERSION"], allowed_versions)) - - g["DEFAULT_PLATFORM_VERSION"] = v.default_platform_version - g["PLATFORM_VERSION_LAST_STABLE"] = v.platform_version_last_stable - target_platform_version = g["TARGET_PLATFORM_VERSION"] - if v.codenames[target_platform_version]: - g.setdefault("PLATFORM_VERSION_CODENAME", v.codenames[target_platform_version]) - else: - g.setdefault("PLATFORM_VERSION_CODENAME", target_platform_version) - # TODO(asmundak): set PLATFORM_VERSION_ALL_CODENAMES - - g.setdefault("PLATFORM_SDK_VERSION", v.platform_sdk_version) - version_codename = g["PLATFORM_VERSION_CODENAME"] - if version_codename == "REL": - g.setdefault("PLATFORM_VERSION", g["PLATFORM_VERSION_LAST_STABLE"]) - g["PLATFORM_PREVIEW_SDK_VERSION"] = 0 - g.setdefault("DEFAULT_APP_TARGET_SDK", g["PLATFORM_SDK_VERSION"]) - g.setdefault("PLATFORM_VNDK_VERSION", g["PLATFORM_SDK_VERSION"]) - else: - g.setdefault("PLATFORM_VERSION", version_codename) - g.setdefault("PLATFORM_PREVIEW_SDK_VERSION", 1) - g.setdefault("DEFAULT_APP_TARGET_SDK", version_codename) - g.setdefault("PLATFORM_VNDK_VERSION", version_codename) - - g.setdefault("PLATFORM_SYSTEMSDK_MIN_VERSION", 28) - versions = [str(i) for i in range(g["PLATFORM_SYSTEMSDK_MIN_VERSION"], g["PLATFORM_SDK_VERSION"] + 1)] - versions.append(version_codename) - g["PLATFORM_SYSTEMSDK_VERSIONS"] = sorted(versions) - - # Used to indicate the security patch that has been applied to the device. - # It must signify that the build includes all security patches issued up through the designated Android Public Security Bulletin. - # It must be of the form "YYYY-MM-DD" on production devices. - # It must match one of the Android Security Patch Level strings of the Public Security Bulletins. - # If there is no $PLATFORM_SECURITY_PATCH set, keep it empty. - - g.setdefault("PLATFORM_SECURITY_PATCH", v.platform_security_patch) - dt = 'TZ="GMT" %s' % g["PLATFORM_SECURITY_PATCH"] - g.setdefault("PLATFORM_SECURITY_PATCH_TIMESTAMP", rblf_shell("date -d '%s' +%%s" % dt)) - - # Used to indicate the base os applied to the device. Can be an arbitrary string, but must be a single word. - # If there is no $PLATFORM_BASE_OS set, keep it empty. - g.setdefault("PLATFORM_BASE_OS", "") - - # Used to signify special builds. E.g., branches and/or releases, like "M5-RC7". Can be an arbitrary string, but - # must be a single word and a valid file name. If there is no BUILD_ID set, make it obvious. - g.setdefault("BUILD_ID", "UNKNOWN") - - # BUILD_NUMBER should be set to the source control value that represents the current state of the source code. - # E.g., a perforce changelist number or a git hash. Can be an arbitrary string (to allow for source control that - # uses something other than numbers), but must be a single word and a valid file name. - # - # If no BUILD_NUMBER is set, create a useful "I am an engineering build from this date/time" value. Make it start - # with a non-digit so that anyone trying to parse it as an integer will probably get "0". - g.setdefault("BUILD_NUMBER", "eng.%s.%s" % (g["USER"], "TIMESTAMP")) - - # Used to set minimum supported target sdk version. Apps targeting SDK version lower than the set value will result - # in a warning being shown when any activity from the app is started. - g.setdefault("PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION", 23) - - # This is the sdk extension version of this tree. - g["PLATFORM_SDK_EXTENSION_VERSION"] = v.platform_sdk_extension_version - # This is the sdk extension version that PLATFORM_SDK_VERSION ships with. - g["PLATFORM_BASE_SDK_EXTENSION_VERSION"] = v.platform_base_sdk_extension_version - - -def init(g, v): - """Initializes globals. - - The code is the Starlark counterpart of the contents of the - envsetup.mk file. - Args: - g: globals dictionary - v: version info struct - """ - all_versions = _all_versions() - _versions_default(g, all_versions, v) - for v in all_versions: - g["IS_AT_LEAST" + v] = True - if v == g["TARGET_PLATFORM_VERSION"]: - break - - # --------------------------------------------------------------- - # If you update the build system such that the environment setup or buildspec.mk need to be updated, - # increment this number, and people who haven't re-run those will have to do so before they can build. - # Make sure to also update the corresponding value in buildspec.mk.default and envsetup.sh. - g["CORRECT_BUILD_ENV_SEQUENCE_NUMBER"] = 13 - - g.setdefault("TARGET_PRODUCT", "aosp_arm") - g.setdefault("TARGET_BUILD_VARIANT", "eng") - - g.setdefault("TARGET_BUILD_APPS", []) - g["TARGET_BUILD_UNBUNDLED"] = (g["TARGET_BUILD_APPS"] != []) or (getattr(g, "TARGET_BUILD_UNBUNDLED_IMAGE", "") != "") - - # --------------------------------------------------------------- - # Set up configuration for host machine. We don't do cross-compiles except for arm, so the HOST - # is whatever we are running on. - host = rblf_shell("uname -sm") - if host.find("Linux") >= 0: - g["HOST_OS"] = "linux" - elif host.find("Darwin") >= 0: - g["HOST_OS"] = "darwin" - else: - fail("Cannot run on %s OS" % host) - - # TODO(asmundak): set g.HOST_OS_EXTRA - - g["BUILD_OS"] = g["HOST_OS"] - - # TODO(asmundak): check cross-OS build - - if host.find("x86_64") >= 0: - g["HOST_ARCH"] = "x86_64" - g["HOST_2ND_ARCH"] = "x86" - g["HOST_IS_64_BIT"] = True - elif host.find("i686") >= 0 or host.find("x86") >= 0: - fail("Building on a 32-bit x86 host is not supported: %s" % host) - elif g["HOST_OS"] == "darwin": - g["HOST_2ND_ARCH"] = "" - - g["HOST_2ND_ARCH_VAR_PREFIX"] = "2ND_" - g["HOST_2ND_ARCH_MODULE_SUFFIX"] = "_32" - g["HOST_CROSS_2ND_ARCH_VAR_PREFIX"] = "2ND_" - g["HOST_CROSS_2ND_ARCH_MODULE_SUFFIX"] = "_64" - g["TARGET_2ND_ARCH_VAR_PREFIX"] = "2ND_" - - # TODO(asmundak): envsetup.mk lines 216-226: - # convert combo-related stuff from combo/select.mk - - # on windows, the tools have .exe at the end, and we depend on the - # host config stuff being done first - g["BUILD_ARCH"] = g["HOST_ARCH"] - g["BUILD_2ND_ARCH"] = g["HOST_2ND_ARCH"] - - # the host build defaults to release, and it must be release or debug - g.setdefault("HOST_BUILD_TYPE", "release") - if g["HOST_BUILD_TYPE"] not in ["release", "debug"]: - fail("HOST_BUILD_TYPE must be either release or debug, not '%s'" % g["HOST_BUILD_TYPE"]) - - g.update([ - ("TARGET_COPY_OUT_VENDOR", "||VENDOR-PATH-PH||"), - ("TARGET_COPY_OUT_PRODUCT", "||PRODUCT-PATH-PH||"), - ("TARGET_COPY_OUT_PRODUCT_SERVICES", "||PRODUCT-PATH-PH||"), - ("TARGET_COPY_OUT_SYSTEM_EXT", "||SYSTEM_EXT-PATH-PH||"), - ("TARGET_COPY_OUT_ODM", "||ODM-PATH-PH||"), - ("TARGET_COPY_OUT_VENDOR_DLKM", "||VENDOR_DLKM-PATH-PH||"), - ("TARGET_COPY_OUT_ODM_DLKM", "||ODM_DLKM-PATH-PH||"), - ]) - - # TODO(asmundak): there is more stuff in envsetup.mk lines 249-292, but - # it does not seem to affect product configuration. Revisit this. - g["ART_APEX_JARS"] = [ - "com.android.art:core-oj", - "com.android.art:core-libart", - "com.android.art:okhttp", - "com.android.art:bouncycastle", - "com.android.art:apache-xml", - ] - - if g.get("TARGET_BUILD_TYPE", "") != "debug": - g["TARGET_BUILD_TYPE"] = "release" diff --git a/core/product_config.mk b/core/product_config.mk index 2b444349b8..0e969fe047 100644 --- a/core/product_config.mk +++ b/core/product_config.mk @@ -206,12 +206,18 @@ endif ifndef RBC_PRODUCT_CONFIG $(call import-products, $(current_product_makefile)) else - $(shell build/soong/scripts/update_out $(OUT_DIR)/rbctemp.mk \ - build/soong/scripts/rbc-run $(current_product_makefile)) + $(shell mkdir -p $(OUT_DIR)/rbc) + $(call dump-variables-rbc, $(OUT_DIR)/rbc/make_vars_pre_product_config.mk) + + $(shell build/soong/scripts/update_out \ + $(OUT_DIR)/rbc/rbc_product_config_results.mk \ + build/soong/scripts/rbc-run \ + $(current_product_makefile) \ + $(OUT_DIR)/rbc/make_vars_pre_product_config.mk) ifneq ($(.SHELLSTATUS),0) $(error product configuration converter failed: $(.SHELLSTATUS)) endif - include $(OUT_DIR)/rbctemp.mk + include $(OUT_DIR)/rbc/rbc_product_config_results.mk PRODUCTS += $(current_product_makefile) endif endif # Import all or just the current product makefile diff --git a/core/product_config.rbc b/core/product_config.rbc index c53ddbf4ed..45eca9f567 100644 --- a/core/product_config.rbc +++ b/core/product_config.rbc @@ -12,26 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("//build/make/core:envsetup.rbc", _envsetup_init = "init") - """Runtime functions.""" _soong_config_namespaces_key = "$SOONG_CONFIG_NAMESPACES" _dist_for_goals_key = "$dist_for_goals" -def _init_globals(version_info): - """Returns dict created from the runtime environment.""" - globals = dict() +def _init_globals(input_variables_init): + """Initializes dictionaries of global variables. - # Environment variables - for k in dir(rblf_env): - globals[k] = getattr(rblf_env, k) + This function runs the given input_variables_init function, + passing it a globals dictionary and a handle as if it + were a regular product. It then returns 2 copies of + the globals dictionary, so that one can be kept around + to diff changes made to the other later. + """ + globals_base = {"PRODUCT_SOONG_NAMESPACES": []} + input_variables_init(globals_base, __h_new()) - # Variables set as var=value command line arguments - for k in dir(rblf_cli): - globals[k] = getattr(rblf_cli, k) - - globals.setdefault("PRODUCT_SOONG_NAMESPACES", []) - _envsetup_init(globals, version_info) + # Rerun input_variables_init to produce a copy + # of globals_base, because starlark doesn't support + # deep copying objects. + globals = {"PRODUCT_SOONG_NAMESPACES": []} + input_variables_init(globals, __h_new()) # Variables that should be defined. mandatory_vars = [ @@ -39,15 +40,14 @@ def _init_globals(version_info): "PLATFORM_VERSION", "PRODUCT_SOONG_NAMESPACES", # TODO(asmundak): do we need TARGET_ARCH? AOSP does not reference it - "TARGET_BUILD_TYPE", "TARGET_BUILD_VARIANT", "TARGET_PRODUCT", ] for bv in mandatory_vars: if not bv in globals: fail(bv, " is not defined") - return globals + return (globals, globals_base) def __print_attr(attr, value): # Allow using empty strings to clear variables, but not None values @@ -111,7 +111,7 @@ def __printvars_rearrange_list(value_list): seen = {item: 0 for item in value_list} return sorted(seen.keys()) if _options.rearrange == "sort" else seen.keys() -def _product_configuration(top_pcm_name, top_pcm, version_info): +def _product_configuration(top_pcm_name, top_pcm, input_variables_init): """Creates configuration.""" # Product configuration is created by traversing product's inheritance @@ -125,8 +125,7 @@ def _product_configuration(top_pcm_name, top_pcm, version_info): # PCM means "Product Configuration Module", i.e., a Starlark file # whose body consists of a single init function. - globals_base = _init_globals(version_info) - globals = dict(**globals_base) + globals, globals_base = _init_globals(input_variables_init) config_postfix = [] # Configs in postfix order @@ -725,7 +724,6 @@ rblf = struct( filter = _filter, filter_out = _filter_out, find_and_copy = _find_and_copy, - init_globals = _init_globals, inherit = _inherit, indirect = _indirect, mk2rbc_error = _mk2rbc_error, diff --git a/core/build_id.rbc b/tests/input_variables.rbc similarity index 51% rename from core/build_id.rbc rename to tests/input_variables.rbc index 4f33833575..0bb100f629 100644 --- a/core/build_id.rbc +++ b/tests/input_variables.rbc @@ -1,4 +1,3 @@ - # Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,9 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This file has been manually converted from build_id.mk -def init(g): +# This file was generated by running `m RBC_PRODUCT_CONFIG=1 nothing` +# and then copying it from out/rbc/out/rbc/make_vars_pre_product_config.rbc. +# It was manually trimmed down afterwards to just the variables we need. - # BUILD_ID is usually used to specify the branch name (like "MAIN") or a branch name and a release candidate - # (like "CRB01"). It must be a single word, and is capitalized by convention. - g["BUILD_ID"]="AOSP.MASTER" \ No newline at end of file +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + g["PLATFORM_VERSION_CODENAME"] = "Tiramisu" + g["PLATFORM_VERSION"] = "Tiramisu" + g["TARGET_BUILD_VARIANT"] = "userdebug" + g["TARGET_BUILD_TYPE"] = "release" + g["TARGET_PRODUCT"] = "aosp_arm64" + g["PLATFORM_SDK_VERSION"] = "31" diff --git a/tests/run.rbc b/tests/run.rbc index 2afee087fe..53eda160df 100644 --- a/tests/run.rbc +++ b/tests/run.rbc @@ -21,7 +21,7 @@ # * all runtime functions (wildcard, regex, etc.) work load("//build/make/core:product_config.rbc", "rblf") -load(":version_defaults.rbc", "version_defaults") +load(":input_variables.rbc", input_variables_init = "init") load(":product.rbc", "init") load(":board.rbc", board_init = "init") load(":board_input_vars.rbc", board_input_vars_init = "init") @@ -59,7 +59,7 @@ assert_eq("foo.c no_folder", rblf.notdir("src/foo.c no_folder")) assert_eq("", rblf.notdir("/")) assert_eq("", rblf.notdir("")) -(globals, config, globals_base) = rblf.product_configuration("test/device", init, version_defaults) +(globals, config, globals_base) = rblf.product_configuration("test/device", init, input_variables_init) assert_eq( { "PRODUCT_COPY_FILES": [ @@ -99,8 +99,8 @@ assert_eq( {k:v for k, v in sorted(ns.items()) } ) -assert_eq("S", globals["PLATFORM_VERSION"]) -assert_eq(30, globals["PLATFORM_SDK_VERSION"]) +assert_eq("Tiramisu", globals["PLATFORM_VERSION"]) +assert_eq("31", globals["PLATFORM_SDK_VERSION"]) assert_eq("xyz", rblf.soong_config_get(globals, "NS2", "v3")) assert_eq(None, rblf.soong_config_get(globals, "NS2", "nonexistant_var"))