From 7ddc50a066287b804a754951a597668d627478df Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Thu, 18 Oct 2018 16:11:40 -0700 Subject: [PATCH 1/2] Refactor common makefile fragments In preparation for setting up another Kati stage, move some common settings and tools into a common folder. This way it's a bit easier to see that they're safe to use, and that they shouldn't depend on anything outside of the common folder. Bug: 117463001 Test: build-aosp_arm.ninja is the same before and after Change-Id: Ief4b75a4dbe45b73ffd03bf32c60695c816d979d --- common/core.mk | 56 +++++++++++++++++++++++++++++++++++++ {core => common}/math.mk | 0 {core => common}/strings.mk | 0 core/config.mk | 36 ++++-------------------- core/definitions.mk | 18 +----------- core/main.mk | 29 +++++++++++++++++-- 6 files changed, 88 insertions(+), 51 deletions(-) create mode 100644 common/core.mk rename {core => common}/math.mk (100%) rename {core => common}/strings.mk (100%) diff --git a/common/core.mk b/common/core.mk new file mode 100644 index 0000000000..e5264b072d --- /dev/null +++ b/common/core.mk @@ -0,0 +1,56 @@ +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Only use ANDROID_BUILD_SHELL to wrap around bash. +# DO NOT use other shells such as zsh. +ifdef ANDROID_BUILD_SHELL +SHELL := $(ANDROID_BUILD_SHELL) +else +# Use bash, not whatever shell somebody has installed as /bin/sh +# This is repeated from main.mk, since envsetup.sh runs this file +# directly. +SHELL := /bin/bash +endif + +# Utility variables. +empty := +space := $(empty) $(empty) +comma := , +# Note that make will eat the newline just before endef. +define newline + + +endef +# The pound character "#" +define pound +# +endef +# Unfortunately you can't simply define backslash as \ or \\. +backslash := \a +backslash := $(patsubst %a,%,$(backslash)) + +# Prevent accidentally changing these variables +.KATI_READONLY := SHELL empty space comma newline pound backslash + +# Basic warning/error wrappers. These will be redefined to include the local +# module information when reading Android.mk files. +define pretty-warning +$(warning $(1)) +endef + +define pretty-error +$(error $(1)) +endef diff --git a/core/math.mk b/common/math.mk similarity index 100% rename from core/math.mk rename to common/math.mk diff --git a/core/strings.mk b/common/strings.mk similarity index 100% rename from core/strings.mk rename to common/strings.mk diff --git a/core/config.mk b/core/config.mk index aeb8aeeaee..5491234e68 100644 --- a/core/config.mk +++ b/core/config.mk @@ -17,36 +17,10 @@ $(warning ) $(error done) endif -# Only use ANDROID_BUILD_SHELL to wrap around bash. -# DO NOT use other shells such as zsh. -ifdef ANDROID_BUILD_SHELL -SHELL := $(ANDROID_BUILD_SHELL) -else -# Use bash, not whatever shell somebody has installed as /bin/sh -# This is repeated from main.mk, since envsetup.sh runs this file -# directly. -SHELL := /bin/bash -endif +BUILD_SYSTEM :=$= build/make/core +BUILD_SYSTEM_COMMON :=$= build/make/common -# Utility variables. -empty := -space := $(empty) $(empty) -comma := , -# Note that make will eat the newline just before endef. -define newline - - -endef -# The pound character "#" -define pound -# -endef -# Unfortunately you can't simply define backslash as \ or \\. -backslash := \a -backslash := $(patsubst %a,%,$(backslash)) - -# Prevent accidentally changing these variables -.KATI_READONLY := SHELL empty space comma newline pound backslash +include $(BUILD_SYSTEM_COMMON)/core.mk # Mark variables that should be coming as environment variables from soong_ui # as readonly @@ -138,9 +112,9 @@ endif # Set up efficient math functions which are used in make. # Here since this file is included by envsetup as well as during build. -include $(BUILD_SYSTEM)/math.mk +include $(BUILD_SYSTEM_COMMON)/math.mk -include $(BUILD_SYSTEM)/strings.mk +include $(BUILD_SYSTEM_COMMON)/strings.mk # Various mappings to avoid hard-coding paths all over the place include $(BUILD_SYSTEM)/pathmap.mk diff --git a/core/definitions.mk b/core/definitions.mk index 3538166d33..43a218942e 100644 --- a/core/definitions.mk +++ b/core/definitions.mk @@ -806,29 +806,13 @@ define echo-error echo -e "$(ESC_BOLD)$(1): $(ESC_ERROR)error:$(ESC_RESET)$(ESC_BOLD)" $(2) "$(ESC_RESET)" >&2 endef -# $(1): message to print -define pretty-warning -$(shell $(call echo-warning,$(LOCAL_MODULE_MAKEFILE),$(LOCAL_MODULE): $(1))) -endef - -# $(1): message to print -define pretty-error -$(shell $(call echo-error,$(LOCAL_MODULE_MAKEFILE),$(LOCAL_MODULE): $(1))) -$(error done) -endef - ########################################################### -## Output the command lines, or not +## Legacy showcommands compatibility ########################################################### -ifeq ($(strip $(SHOW_COMMANDS)),) define pretty @echo $1 endef -else -define pretty -endef -endif ########################################################### ## Commands for including the dependency files the compiler generates diff --git a/core/main.mk b/core/main.mk index ecb8c0da26..967f52bbfd 100644 --- a/core/main.mk +++ b/core/main.mk @@ -36,8 +36,6 @@ PWD := $(shell pwd) TOP := . TOPDIR := -BUILD_SYSTEM := $(TOPDIR)build/make/core - # This is the default target. It must be the first declared target. .PHONY: droid DEFAULT_GOAL := droid @@ -48,7 +46,7 @@ droid_targets: # Set up various standard variables based on configuration # and host information. -include $(BUILD_SYSTEM)/config.mk +include build/make/core/config.mk ifneq ($(filter $(dont_bother_goals), $(MAKECMDGOALS)),) dont_bother := true @@ -419,6 +417,19 @@ ifneq ($(PRODUCT_ENFORCE_RRO_TARGETS),) ENFORCE_RRO_SOURCES := endif +# Color-coded warnings including current module info +# $(1): message to print +define pretty-warning +$(shell $(call echo-warning,$(LOCAL_MODULE_MAKEFILE),$(LOCAL_MODULE): $(1))) +endef + +# Color-coded errors including current module info +# $(1): message to print +define pretty-error +$(shell $(call echo-error,$(LOCAL_MODULE_MAKEFILE),$(LOCAL_MODULE): $(1))) +$(error done) +endef + subdir_makefiles_inc := . FULL_BUILD := @@ -492,6 +503,18 @@ $(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] fi # All module makefiles have been included at this point. # ------------------------------------------------------------------- +# ------------------------------------------------------------------- +# Use basic warning/error messages now that LOCAL_MODULE_MAKEFILE +# and LOCAL_MODULE aren't useful anymore. +# ------------------------------------------------------------------- +define pretty-warning +$(warning $(1)) +endef + +define pretty-error +$(error $(1)) +endef + # ------------------------------------------------------------------- # Enforce to generate all RRO packages for modules having resource # overlays. From 78c40be5606d13520af1e4b9c2f8dc78a6691511 Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Wed, 17 Oct 2018 16:50:49 -0700 Subject: [PATCH 2/2] Create a new kati packaging step; move dist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of looking at `dist` and DIST_DIR directly in the Kati Build step, always write out information about every call to dist, then create the rules in another ckati run. So instead of having: dist: -> -> \______________↑ nodist: -----------> Always use another phony target in the Kati Build step: ---> \----> _dist_ Then in the packaging step (which is much faster), choose between dist and no dist: dist: _dist_ -> -> nodist: _dist_ Bug: 117463001 Test: m dist Change-Id: Ic96bb6356740300dd3113f6ed699e6a619360c40 --- Changes.md | 26 +++++++++++++++++ CleanSpec.mk | 3 ++ core/config.mk | 21 ++++++++++---- core/distdir.mk | 66 ++++++++++++++++++++++---------------------- core/main.mk | 2 ++ core/ninja_config.mk | 1 - packaging/distdir.mk | 45 ++++++++++++++++++++++++++++++ packaging/main.mk | 37 +++++++++++++++++++++++++ 8 files changed, 162 insertions(+), 39 deletions(-) create mode 100644 packaging/distdir.mk create mode 100644 packaging/main.mk diff --git a/Changes.md b/Changes.md index baa5e6e7ac..4aa7ea2abe 100644 --- a/Changes.md +++ b/Changes.md @@ -1,5 +1,31 @@ # Build System Changes for Android.mk Writers +## `DIST_DIR`, `dist_goal`, and `dist-for-goals` {#dist} + +`DIST_DIR` and `dist_goal` are no longer available when reading Android.mk +files (or other build tasks). Always use `dist-for-goals` instead, which takes +a PHONY goal, and a list of files to copy to `$DIST_DIR`. Whenever `dist` is +specified, and the goal would be built (either explicitly on the command line, +or as a dependency of something on the command line), that file will be copied +into `$DIST_DIR`. For example, + +``` make +$(call dist-for-goals,foo,bar/baz) +``` + +will copy `bar/baz` into `$DIST_DIR/baz` when `m foo dist` is run. + +#### Renames during copy + +Instead of specifying just a file, a destination name can be specified, +including subdirectories: + +``` make +$(call dist-for-goals,foo,bar/baz:logs/foo.log) +``` + +will copy `bar/baz` into `$DIST_DIR/logs/foo.log` when `m foo dist` is run. + ## `.PHONY` rule enforcement {#phony_targets} There are several new warnings/errors meant to ensure the proper use of diff --git a/CleanSpec.mk b/CleanSpec.mk index a9093d2243..39441e1082 100644 --- a/CleanSpec.mk +++ b/CleanSpec.mk @@ -512,6 +512,9 @@ $(call add-clean-step, rm -rf $(addsuffix /lib,\ # Remove strip.sh intermediates to save space $(call add-clean-step, find $(OUT_DIR) \( -name "*.so.debug" -o -name "*.so.dynsyms" -o -name "*.so.funcsyms" -o -name "*.so.keep_symbols" -o -name "*.so.mini_debuginfo.xz" \) -print0 | xargs -0 rm -f) +# Clean up old ninja files +$(call add-clean-step, rm -f $(OUT_DIR)/build-*-dist*.ninja) + # ************************************************ # NEWER CLEAN STEPS MUST BE AT THE END OF THE LIST # ************************************************ diff --git a/core/config.mk b/core/config.mk index 5491234e68..fae53bbd28 100644 --- a/core/config.mk +++ b/core/config.mk @@ -24,7 +24,13 @@ include $(BUILD_SYSTEM_COMMON)/core.mk # Mark variables that should be coming as environment variables from soong_ui # as readonly -.KATI_READONLY := OUT_DIR TMPDIR BUILD_DATETIME_FILE DIST_DIR +.KATI_READONLY := OUT_DIR TMPDIR BUILD_DATETIME_FILE +ifdef CALLED_FROM_SETUP + .KATI_READONLY := CALLED_FROM_SETUP DIST_DIR +endif +ifdef KATI_PACKAGE_MK_DIR + .KATI_READONLY := KATI_PACKAGE_MK_DIR +endif # Mark variables deprecated/obsolete CHANGES_URL := https://android.googlesource.com/platform/build/+/master/Changes.md @@ -87,16 +93,21 @@ $(KATI_obsolete_var \ # This is marked as obsolete in envsetup.mk after reading the BoardConfig.mk $(KATI_deprecate_export It is a global setting. See $(CHANGES_URL)#export_keyword) -CHANGES_URL := - # Used to force goals to build. Only use for conditionally defined goals. .PHONY: FORCE FORCE: ORIGINAL_MAKECMDGOALS := $(MAKECMDGOALS) -dist_goal := $(strip $(filter dist,$(MAKECMDGOALS))) -MAKECMDGOALS := $(strip $(filter-out dist,$(MAKECMDGOALS))) +ifdef CALLED_FROM_SETUP + dist_goal := $(strip $(filter dist,$(MAKECMDGOALS))) + MAKECMDGOALS := $(strip $(filter-out dist,$(MAKECMDGOALS))) + .KATI_READONLY := dist_goal +else + $(KATI_obsolete_var DIST_DIR dist_goal,Use dist-for-goals instead. See $(CHANGES_URL)#dist) +endif + +CHANGES_URL := UNAME := $(shell uname -sm) diff --git a/core/distdir.mk b/core/distdir.mk index c074186b8d..a2eabd2b3e 100644 --- a/core/distdir.mk +++ b/core/distdir.mk @@ -17,52 +17,52 @@ # When specifying "dist", the user has asked that we copy the important # files from this build into DIST_DIR. -ifdef dist_goal - -# $(1): source file -# $(2): destination file -# $(3): goals that should copy the file -# -define copy-one-dist-file -$(3): $(2) -$(2): $(1) - @echo "Dist: $$@" - $$(copy-file-to-new-target-with-cp) -endef - -# A global variable to remember all dist'ed src:dst pairs. -# So if a src:dst is already dist'ed by another goal, -# we should just establish the dependency and don't really call the -# copy-one-dist-file to avoid multiple rules for the same target. +# list of all goals that depend on any dist files +_all_dist_goals := +# pairs of goal:distfile +_all_dist_goal_output_pairs := +# pairs of srcfile:distfile _all_dist_src_dst_pairs := + # Other parts of the system should use this function to associate # certain files with certain goals. When those goals are built # and "dist" is specified, the marked files will be copied to DIST_DIR. # -# $(1): a list of goals (e.g. droid, sdk, pdk, ndk) +# $(1): a list of goals (e.g. droid, sdk, pdk, ndk). These must be PHONY # $(2): the dist files to add to those goals. If the file contains ':', # the text following the colon is the name that the file is copied # to under the dist directory. Subdirs are ok, and will be created # at copy time if necessary. define dist-for-goals +$(eval _all_dist_goals += $$(1)) \ $(foreach file,$(2), \ - $(eval fw := $(subst :,$(space),$(file))) \ - $(eval src := $(word 1,$(fw))) \ - $(eval dst := $(word 2,$(fw))) \ - $(eval dst := $(if $(dst),$(dst),$(notdir $(src)))) \ - $(if $(filter $(_all_dist_src_dst_pairs),$(src):$(dst)),\ - $(eval $(call add-dependency,$(1),$(DIST_DIR)/$(dst))),\ - $(eval $(call copy-one-dist-file,\ - $(src),$(DIST_DIR)/$(dst),$(1)))\ - $(eval _all_dist_src_dst_pairs += $(src):$(dst))\ - )\ -) + $(eval src := $(call word-colon,1,$(file))) \ + $(eval dst := $(call word-colon,2,$(file))) \ + $(if $(dst),,$(eval dst := $$(notdir $$(src)))) \ + $(eval _all_dist_src_dst_pairs += $$(src):$$(dst)) \ + $(foreach goal,$(1), \ + $(eval _all_dist_goal_output_pairs += $$(goal):$$(dst)))) endef -else # !dist_goal +#------------------------------------------------------------------ +# To be used at the end of the build to collect all the uses of +# dist-for-goals, and write them into a file for the packaging step to use. -# empty definition when not building dist -define dist-for-goals +# $(1): The file to write +define dist-write-file +$(strip \ + $(KATI_obsolete_var dist-for-goals,Cannot be used after dist-write-file) \ + $(foreach goal,$(sort $(_all_dist_goals)), \ + $(eval $$(goal): _dist_$$(goal))) \ + $(shell mkdir -p $(dir $(1))) \ + $(file >$(1).tmp, \ + DIST_GOAL_OUTPUT_PAIRS := $(sort $(_all_dist_goal_output_pairs)) \ + $(newline)DIST_SRC_DST_PAIRS := $(sort $(_all_dist_src_dst_pairs))) \ + $(shell if ! cmp -s $(1).tmp $(1); then \ + mv $(1).tmp $(1); \ + else \ + rm $(1).tmp; \ + fi)) endef -endif # !dist_goal +.KATI_READONLY := dist-for-goals dist-write-file diff --git a/core/main.mk b/core/main.mk index 967f52bbfd..3c4dd9ab7d 100644 --- a/core/main.mk +++ b/core/main.mk @@ -1490,6 +1490,8 @@ tidy_only: ndk: $(SOONG_OUT_DIR)/ndk.timestamp .PHONY: ndk +$(call dist-write-file,$(KATI_PACKAGE_MK_DIR)/dist.mk) + $(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] writing build rules ...) endif # KATI diff --git a/core/ninja_config.mk b/core/ninja_config.mk index ca2dceecec..2d44d8fc6c 100644 --- a/core/ninja_config.mk +++ b/core/ninja_config.mk @@ -28,7 +28,6 @@ PARSE_TIME_MAKE_GOALS := \ custom_images \ deps-license \ dicttool_aosp \ - dist \ dump-products \ eng \ fusion \ diff --git a/packaging/distdir.mk b/packaging/distdir.mk new file mode 100644 index 0000000000..f1ef336317 --- /dev/null +++ b/packaging/distdir.mk @@ -0,0 +1,45 @@ +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# From the Android.mk pass: +DIST_GOAL_OUTPUT_PAIRS := +DIST_SRC_DST_PAIRS := +include $(KATI_PACKAGE_MK_DIR)/dist.mk + +$(foreach pair,$(DIST_GOAL_OUTPUT_PAIRS), \ + $(eval goal := $(call word-colon,1,$(pair))) \ + $(eval output := $(call word-colon,2,$(pair))) \ + $(eval .PHONY: _dist_$$(goal)) \ + $(if $(call streq,$(DIST),true),\ + $(eval _dist_$$(goal): $$(DIST_DIR)/$$(output)))) + +define copy-one-dist-file +$(2): $(1) + @echo "Dist: $$@" + rm -f $$@ + cp $$< $$@ +endef + +ifeq ($(DIST),true) + $(foreach pair,$(DIST_SRC_DST_PAIRS), \ + $(eval src := $(call word-colon,1,$(pair))) \ + $(eval dst := $(DIST_DIR)/$(call word-colon,2,$(pair))) \ + $(eval $(call copy-one-dist-file,$(src),$(dst)))) +endif + +copy-one-dist-file := +DIST_GOAL_OUTPUT_PAIRS := +DIST_SRC_DST_PAIRS := diff --git a/packaging/main.mk b/packaging/main.mk new file mode 100644 index 0000000000..0b746a8f8e --- /dev/null +++ b/packaging/main.mk @@ -0,0 +1,37 @@ +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Create a default rule. This is unused currently, as the real default rule is +# still in the Kati build step. +.PHONY: _packaging_default_rule_ +_packaging_default_rule_: + +ifndef KATI +$(error Only Kati is supported.) +endif + +$(info [1/3] initializing packaging system ...) + +.KATI_READONLY := KATI_PACKAGE_MK_DIR + +include build/make/common/core.mk +include build/make/common/strings.mk + +$(info [2/3] including distdir.mk ...) + +include build/make/packaging/distdir.mk + +$(info [3/3] writing packaging rules ...)