diff --git a/core/Makefile b/core/Makefile index ab80516ffc..ccd4e17ef1 100644 --- a/core/Makefile +++ b/core/Makefile @@ -27,26 +27,28 @@ $(eval $(strip $(1)): PRIVATE_OUT := $(strip $(1))) $(eval $(strip $(1)): PRIVATE_FLAG_NAMES := $(strip $(2))) $(strip $(1)): mkdir -p $$(dir $$(PRIVATE_OUT)) - ( \ - echo '{' ; \ - echo 'flags: [' ; \ - $$(foreach flag, $$(PRIVATE_FLAG_NAMES), \ + echo '{' > $$(PRIVATE_OUT) + echo '"flags": [' >> $$(PRIVATE_OUT) + $$(foreach flag, $$(PRIVATE_FLAG_NAMES), \ + ( \ printf ' { "name": "%s", "value": "%s", ' \ '$$(flag)' \ '$$(_ALL_RELEASE_FLAGS.$$(flag).VALUE)' \ ; \ - printf '"set": "%s", "default": "%s", "declared": "%s", }' \ + printf '"set": "%s", "default": "%s", "declared": "%s" }' \ '$$(_ALL_RELEASE_FLAGS.$$(flag).SET_IN)' \ '$$(_ALL_RELEASE_FLAGS.$$(flag).DEFAULT)' \ '$$(_ALL_RELEASE_FLAGS.$$(flag).DECLARED_IN)' \ ; \ printf '$$(if $$(filter $$(lastword $$(PRIVATE_FLAG_NAMES)),$$(flag)),,$$(comma))\n' ; \ - ) \ - echo "]" ; \ - echo "}" \ - ) >> $$(PRIVATE_OUT) + ) >> $$(PRIVATE_OUT) \ + ) + echo "]" >> $$(PRIVATE_OUT) + echo "}" >> $$(PRIVATE_OUT) endef +_FLAG_PARTITIONS := product system system_ext vendor + $(foreach partition, $(_FLAG_PARTITIONS), \ $(eval BUILD_FLAG_SUMMARIES.$(partition) \ := $(TARGET_OUT_FLAGS)/$(partition)/etc/build_flags.json) \ diff --git a/core/release_config.bzl b/core/release_config.bzl index ecb9ec6022..134650851c 100644 --- a/core/release_config.bzl +++ b/core/release_config.bzl @@ -20,71 +20,101 @@ _flag_partitions = [ "vendor", ] -def _combine_dicts_no_duplicate_keys(dicts): - result = {} - for d in dicts: - for k, v in d.items(): - if k in result: - fail("Duplicate key: " + k) - result[k] = v - return result +ALL = ["all"] +PRODUCT = ["product"] +SYSTEM = ["system"] +SYSTEM_EXT = ["system_ext"] +VENDOR = ["vendor"] -def release_config(target_release, flag_definitions, config_maps, fail_if_no_release_config = True): - result = { - "_ALL_RELEASE_FLAGS": [flag.name for flag in flag_definitions], +_valid_types = ["NoneType", "bool", "list", "string", "int"] + +def flag(name, partitions, default): + "Declare a flag." + if not partitions: + fail("At least 1 partition is required") + if not name.startswith("RELEASE_"): + fail("Release flag names must start with RELEASE_") + if " " in name or "\t" in name or "\n" in name: + fail("Flag names must not contain whitespace: \"" + name + "\"") + for partition in partitions: + if partition == "all": + if len(partitions) > 1: + fail("\"all\" can't be combined with other partitions: " + str(partitions)) + elif partition not in _flag_partitions: + fail("Invalid partition: " + partition + ", allowed partitions: " + + str(_flag_partitions)) + if type(default) not in _valid_types: + fail("Invalid type of default for flag \"" + name + "\" (" + type(default) + ")") + return { + "name": name, + "partitions": partitions, + "default": default, } - all_flags = {} - for flag in flag_definitions: - if sorted(dir(flag)) != ["default", "name", "partitions"]: - fail("Flag structs must contain 3 fields: name, partitions, and default") - if not flag.partitions: - fail("At least 1 partition is required") - for partition in flag.partitions: - if partition == "all": - if len(flag.partitions) > 1: - fail("\"all\" can't be combined with other partitions: " + str(flag.partitions)) - elif partition not in _flag_partitions: - fail("Invalid partition: " + flag.partition + ", allowed partitions: " + str(_flag_partitions)) - if not flag.name.startswith("RELEASE_"): - fail("Release flag names must start with RELEASE_") - if " " in flag.name or "\t" in flag.name or "\n" in flag.name: - fail("Flag names must not contain whitespace.") - if flag.name in all_flags: - fail("Duplicate declaration of flag " + flag.name) - all_flags[flag.name] = True - default = flag.default - if type(default) == "bool": - default = "true" if default else "" +def value(name, value): + "Define the flag value for a particular configuration." + return { + "name": name, + "value": value, + } - result["_ALL_RELEASE_FLAGS." + flag.name + ".PARTITIONS"] = flag.partitions - result["_ALL_RELEASE_FLAGS." + flag.name + ".DEFAULT"] = default - result["_ALL_RELEASE_FLAGS." + flag.name + ".VALUE"] = default - - # If TARGET_RELEASE is set, fail if there is no matching release config - # If it isn't set, no release config files will be included and all flags - # will get their default values. - if target_release: - config_map = _combine_dicts_no_duplicate_keys(config_maps) - if target_release not in config_map: - fail("No release config found for TARGET_RELEASE: " + target_release + ". Available releases are: " + str(config_map.keys())) - release_config = config_map[target_release] - if sorted(dir(release_config)) != ["flags", "release_version"]: - fail("A release config must be a struct with a flags and release_version fields") - result["_RELEASE_VERSION"] = release_config.release_version - for flag in release_config.flags: - if sorted(dir(flag)) != ["name", "value"]: - fail("A flag must be a struct with name and value fields, got: " + str(sorted(dir(flag)))) - if flag.name not in all_flags: - fail("Undeclared build flag: " + flag.name) - value = flag.value - if type(value) == "bool": - value = "true" if value else "" - result["_ALL_RELEASE_FLAGS." + flag.name + ".VALUE"] = value - elif fail_if_no_release_config: - fail("FAIL_IF_NO_RELEASE_CONFIG was set and TARGET_RELEASE was not") +def _format_value(val): + "Format the starlark type correctly for make" + if type(val) == "NoneType": + return "" + elif type(val) == "bool": + return "true" if val else "" else: - # No TARGET_RELEASE means release version 0 - result["_RELEASE_VERSION"] = 0 + return val + +def release_config(all_flags, all_values): + "Return the make variables that should be set for this release config." + + # Validate flags + flag_names = [] + for flag in all_flags: + if flag["name"] in flag_names: + fail(flag["declared_in"] + ": Duplicate declaration of flag " + flag["name"]) + flag_names.append(flag["name"]) + + # Record which flags go on which partition + partitions = {} + for flag in all_flags: + for partition in flag["partitions"]: + if partition == "all": + for partition in _flag_partitions: + partitions.setdefault(partition, []).append(flag["name"]) + else: + partitions.setdefault(partition, []).append(flag["name"]) + + # Validate values + values = {} + for value in all_values: + if value["name"] not in flag_names: + fail(value["set_in"] + ": Value set for undeclared build flag: " + value["name"]) + values[value["name"]] = value + + # Collect values + result = { + "_ALL_RELEASE_FLAGS": sorted(flag_names), + } + for partition, names in partitions.items(): + result["_ALL_RELEASE_FLAGS.PARTITIONS." + partition] = names + for flag in all_flags: + if flag["name"] in values: + val = values[flag["name"]]["value"] + set_in = values[flag["name"]]["set_in"] + if type(val) not in _valid_types: + fail("Invalid type of value for flag \"" + flag["name"] + "\" (" + type(val) + ")") + else: + val = flag["default"] + set_in = flag["declared_in"] + val = _format_value(val) + result[flag["name"]] = val + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".PARTITIONS"] = flag["partitions"] + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".DEFAULT"] = _format_value(flag["default"]) + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".VALUE"] = val + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".DECLARED_IN"] = flag["declared_in"] + result["_ALL_RELEASE_FLAGS." + flag["name"] + ".SET_IN"] = set_in return result diff --git a/core/release_config.mk b/core/release_config.mk index ce57855aae..3cd8b4103b 100644 --- a/core/release_config.mk +++ b/core/release_config.mk @@ -12,6 +12,104 @@ # See the License for the specific language governing permissions and # limitations under the License. + +# ----------------------------------------------------------------- +# Choose the flag files +# ----------------------------------------------------------------- +# Do this first, because we're going to unset TARGET_RELEASE before +# including anyone, so they don't start making conditionals based on it. +# This logic is in make because starlark doesn't understand optional +# vendor files. + +# If this is a google source tree, restrict it to only the one file +# which has OWNERS control. If it isn't let others define their own. +# TODO: Remove wildcard for build/release one when all branch manifests +# have updated. +config_map_files := $(wildcard build/release/release_config_map.mk) \ + $(if $(wildcard vendor/google/release/release_config_map.mk), \ + vendor/google/release/release_config_map.mk, \ + $(sort \ + $(wildcard device/*/release/release_config_map.mk) \ + $(wildcard device/*/*/release/release_config_map.mk) \ + $(wildcard vendor/*/release/release_config_map.mk) \ + $(wildcard vendor/*/*/release/release_config_map.mk) \ + ) \ + ) + +# $1 config name +# $2 release config files +define declare-release-config + $(eval # No duplicates) + $(if $(filter $(_all_release_configs), $(strip $(1))), \ + $(error declare-release-config: config $(strip $(1)) declared in: $(_included) Previously declared here: $(_all_release_configs.$(strip $(1)).DECLARED_IN)) \ + ) + $(eval # Must have release config files) + $(if $(strip $(2)),, \ + $(error declare-release-config: config $(strip $(1)) must have release config files) \ + ) + $(eval _all_release_configs := $(sort $(_all_release_configs) $(strip $(1)))) + $(eval _all_release_configs.$(strip $(1)).DECLARED_IN := $(_included)) + $(eval _all_release_configs.$(strip $(1)).FILES := $(strip $(2))) +endef + +# Include the config map files +$(foreach f, $(config_map_files), \ + $(eval _included := $(f)) \ + $(eval include $(f)) \ +) + +# If TARGET_RELEASE is set, fail if there is no matching release config +# If it isn't set, no release config files will be included and all flags +# will get their default values. +ifneq ($(TARGET_RELEASE),) +ifeq ($(filter $(_all_release_configs), $(TARGET_RELEASE)),) + $(error No release config found for TARGET_RELEASE: $(TARGET_RELEASE). Available releases are: $(_all_release_configs)) +else + # Choose flag files + # Don't sort this, use it in the order they gave us. + flag_value_files := $(_all_release_configs.$(TARGET_RELEASE).FILES) +endif +else +# Useful for finding scripts etc that aren't passing or setting TARGET_RELEASE +ifneq ($(FAIL_IF_NO_RELEASE_CONFIG),) + $(error FAIL_IF_NO_RELEASE_CONFIG was set and TARGET_RELEASE was not) +endif +flag_value_files := +endif + +# Unset variables so they can't use them +define declare-release-config +$(error declare-release-config can only be called from inside release_config_map.mk files) +endef + +# TODO: Remove this check after enough people have sourced lunch that we don't +# need to worry about it trying to do get_build_vars TARGET_RELEASE. Maybe after ~9/2023 +ifneq ($(CALLED_FROM_SETUP),true) +define TARGET_RELEASE +$(error TARGET_RELEASE may not be accessed directly. Use individual flags.) +endef +else +TARGET_RELEASE:= +endif +.KATI_READONLY := TARGET_RELEASE + + +$(foreach config, $(_all_release_configs), \ + $(eval _all_release_configs.$(config).DECLARED_IN:= ) \ + $(eval _all_release_configs.$(config).FILES:= ) \ +) +_all_release_configs:= +config_map_files:= + + +# ----------------------------------------------------------------- +# Flag declarations and values +# ----------------------------------------------------------------- +# This part is in starlark. We generate a root starlark file that loads +# all of the flags declaration files that we found, and the flag_value_files +# that we chose from the config map above. Then we run that, and load the +# results of that into the make environment. + # If this is a google source tree, restrict it to only the one file # which has OWNERS control. If it isn't let others define their own. # TODO: Remove wildcard for build/release one when all branch manifests @@ -26,43 +124,25 @@ flag_declaration_files := $(wildcard build/release/build_flags.bzl) \ $(wildcard vendor/*/*/release/build_flags.bzl) \ ) \ ) -config_map_files := $(wildcard build/release/release_config_map.bzl) \ - $(if $(wildcard vendor/google/release/release_config_map.bzl), \ - vendor/google/release/release_config_map.bzl, \ - $(sort \ - $(wildcard device/*/release/release_config_map.bzl) \ - $(wildcard device/*/*/release/release_config_map.bzl) \ - $(wildcard vendor/*/release/release_config_map.bzl) \ - $(wildcard vendor/*/*/release/release_config_map.bzl) \ - ) \ - ) + # Because starlark can't find files with $(wildcard), write an entrypoint starlark script that # contains the result of the above wildcards for the starlark code to use. filename_to_starlark=$(subst /,_,$(subst .,_,$(1))) _c:=load("//build/make/core/release_config.bzl", "release_config") -_c+=$(foreach f,$(flag_declaration_files),$(newline)load("//$(f)", flags_$(call filename_to_starlark,$(f)) = "flags")) -_c+=$(foreach f,$(config_map_files),$(newline)load("//$(f)", config_maps_$(call filename_to_starlark,$(f)) = "config_maps")) -_c+=$(newline)all_flags = [] $(foreach f,$(flag_declaration_files),+ flags_$(call filename_to_starlark,$(f))) -_c+=$(newline)all_config_maps = [$(foreach f,$(config_map_files),config_maps_$(call filename_to_starlark,$(f))$(comma))] -_c+=$(newline)target_release = "$(TARGET_RELEASE)" -_c+=$(newline)fail_if_no_release_config = True if "$(FAIL_IF_NO_RELEASE_CONFIG)" else False -_c+=$(newline)variables_to_export_to_make = release_config(target_release, all_flags, all_config_maps, fail_if_no_release_config) +_c+=$(newline)def add(d, k, v): +_c+=$(newline)$(space)d = dict(d) +_c+=$(newline)$(space)d[k] = v +_c+=$(newline)$(space)return d +_c+=$(foreach f,$(flag_declaration_files),$(newline)load("$(f)", flags_$(call filename_to_starlark,$(f)) = "flags")) +_c+=$(newline)all_flags = [] $(foreach f,$(flag_declaration_files),+ [add(x, "declared_in", "$(f)") for x in flags_$(call filename_to_starlark,$(f))]) +_c+=$(foreach f,$(flag_value_files),$(newline)load("//$(f)", values_$(call filename_to_starlark,$(f)) = "values")) +_c+=$(newline)all_values = [] $(foreach f,$(flag_value_files),+ [add(x, "set_in", "$(f)") for x in values_$(call filename_to_starlark,$(f))]) +_c+=$(newline)variables_to_export_to_make = release_config(all_flags, all_values) $(file >$(OUT_DIR)/release_config_entrypoint.bzl,$(_c)) _c:= filename_to_starlark:= -# TODO: Remove this check after enough people have sourced lunch that we don't -# need to worry about it trying to do get_build_vars TARGET_RELEASE. Maybe after ~9/2023 -ifneq ($(CALLED_FROM_SETUP),true) -define TARGET_RELEASE -$(error TARGET_RELEASE may not be accessed directly. Use individual flags.) -endef -else -TARGET_RELEASE:= -endif -.KATI_READONLY := TARGET_RELEASE - # Exclude the entrypoint file as a dependency (by passing it as the 2nd argument) so that we don't # rerun kati every build. Kati will replay the $(file) command that generates it every build, # updating its timestamp. @@ -71,9 +151,3 @@ endif # outside of the source tree. $(call run-starlark,$(OUT_DIR)/release_config_entrypoint.bzl,$(OUT_DIR)/release_config_entrypoint.bzl,--allow_external_entrypoint) -# Set the flag values, and don't allow any one to modify them. -$(foreach flag, $(_ALL_RELEASE_FLAGS), \ - $(eval $(flag) := $(_ALL_RELEASE_FLAGS.$(flag).VALUE)) \ - $(eval .KATI_READONLY := $(flag)) \ -) -