Merge changes from topic "refactor_dist"

* changes:
  Create a new kati packaging step; move dist
  Refactor common makefile fragments

Changes.md

@@ -1,5 +1,31 @@
 # Build System Changes for Android.mk Writers
 
+## `DIST_DIR`, `dist_goal`, and `dist-for-goals`  {#dist}
+
+`DIST_DIR` and `dist_goal` are no longer available when reading Android.mk
+files (or other build tasks). Always use `dist-for-goals` instead, which takes
+a PHONY goal, and a list of files to copy to `$DIST_DIR`. Whenever `dist` is
+specified, and the goal would be built (either explicitly on the command line,
+or as a dependency of something on the command line), that file will be copied
+into `$DIST_DIR`. For example,
+
+``` make
+$(call dist-for-goals,foo,bar/baz)
+```
+
+will copy `bar/baz` into `$DIST_DIR/baz` when `m foo dist` is run.
+
+#### Renames during copy
+
+Instead of specifying just a file, a destination name can be specified,
+including subdirectories:
+
+``` make
+$(call dist-for-goals,foo,bar/baz:logs/foo.log)
+```
+
+will copy `bar/baz` into `$DIST_DIR/logs/foo.log` when `m foo dist` is run.
+
 ## `.PHONY` rule enforcement  {#phony_targets}
 
 There are several new warnings/errors meant to ensure the proper use of

CleanSpec.mk

@@ -512,6 +512,9 @@ $(call add-clean-step, rm -rf $(addsuffix /lib,\
 # Remove strip.sh intermediates to save space
 $(call add-clean-step, find $(OUT_DIR) \( -name "*.so.debug" -o -name "*.so.dynsyms" -o -name "*.so.funcsyms" -o -name "*.so.keep_symbols" -o -name "*.so.mini_debuginfo.xz" \) -print0 | xargs -0 rm -f)
 
+# Clean up old ninja files
+$(call add-clean-step, rm -f $(OUT_DIR)/build-*-dist*.ninja)
+
 # ************************************************
 # NEWER CLEAN STEPS MUST BE AT THE END OF THE LIST
 # ************************************************

common/core.mk

@@ -0,0 +1,56 @@
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Only use ANDROID_BUILD_SHELL to wrap around bash.
+# DO NOT use other shells such as zsh.
+ifdef ANDROID_BUILD_SHELL
+SHELL := $(ANDROID_BUILD_SHELL)
+else
+# Use bash, not whatever shell somebody has installed as /bin/sh
+# This is repeated from main.mk, since envsetup.sh runs this file
+# directly.
+SHELL := /bin/bash
+endif
+
+# Utility variables.
+empty :=
+space := $(empty) $(empty)
+comma := ,
+# Note that make will eat the newline just before endef.
+define newline
+
+
+endef
+# The pound character "#"
+define pound
+#
+endef
+# Unfortunately you can't simply define backslash as \ or \\.
+backslash := \a
+backslash := $(patsubst %a,%,$(backslash))
+
+# Prevent accidentally changing these variables
+.KATI_READONLY := SHELL empty space comma newline pound backslash
+
+# Basic warning/error wrappers. These will be redefined to include the local
+# module information when reading Android.mk files.
+define pretty-warning
+$(warning $(1))
+endef
+
+define pretty-error
+$(error $(1))
+endef

core/config.mk

@@ -17,40 +17,20 @@ $(warning )
 $(error done)
 endif
 
-# Only use ANDROID_BUILD_SHELL to wrap around bash.
+BUILD_SYSTEM :=$= build/make/core
-# DO NOT use other shells such as zsh.
+BUILD_SYSTEM_COMMON :=$= build/make/common
-ifdef ANDROID_BUILD_SHELL
-SHELL := $(ANDROID_BUILD_SHELL)
-else
-# Use bash, not whatever shell somebody has installed as /bin/sh
-# This is repeated from main.mk, since envsetup.sh runs this file
-# directly.
-SHELL := /bin/bash
-endif
 
-# Utility variables.
+include $(BUILD_SYSTEM_COMMON)/core.mk
-empty :=
-space := $(empty) $(empty)
-comma := ,
-# Note that make will eat the newline just before endef.
-define newline
-
-
-endef
-# The pound character "#"
-define pound
-#
-endef
-# Unfortunately you can't simply define backslash as \ or \\.
-backslash := \a
-backslash := $(patsubst %a,%,$(backslash))
-
-# Prevent accidentally changing these variables
-.KATI_READONLY := SHELL empty space comma newline pound backslash
 
 # Mark variables that should be coming as environment variables from soong_ui
 # as readonly
-.KATI_READONLY := OUT_DIR TMPDIR BUILD_DATETIME_FILE DIST_DIR
+.KATI_READONLY := OUT_DIR TMPDIR BUILD_DATETIME_FILE
+ifdef CALLED_FROM_SETUP
+  .KATI_READONLY := CALLED_FROM_SETUP DIST_DIR
+endif
+ifdef KATI_PACKAGE_MK_DIR
+  .KATI_READONLY := KATI_PACKAGE_MK_DIR
+endif
 
 # Mark variables deprecated/obsolete
 CHANGES_URL := https://android.googlesource.com/platform/build/+/master/Changes.md
@@ -113,16 +93,21 @@ $(KATI_obsolete_var \
 # This is marked as obsolete in envsetup.mk after reading the BoardConfig.mk
 $(KATI_deprecate_export It is a global setting. See $(CHANGES_URL)#export_keyword)
 
-CHANGES_URL :=
-
 # Used to force goals to build.  Only use for conditionally defined goals.
 .PHONY: FORCE
 FORCE:
 
 ORIGINAL_MAKECMDGOALS := $(MAKECMDGOALS)
 
-dist_goal := $(strip $(filter dist,$(MAKECMDGOALS)))
+ifdef CALLED_FROM_SETUP
-MAKECMDGOALS := $(strip $(filter-out dist,$(MAKECMDGOALS)))
+  dist_goal := $(strip $(filter dist,$(MAKECMDGOALS)))
+  MAKECMDGOALS := $(strip $(filter-out dist,$(MAKECMDGOALS)))
+  .KATI_READONLY := dist_goal
+else
+  $(KATI_obsolete_var DIST_DIR dist_goal,Use dist-for-goals instead. See $(CHANGES_URL)#dist)
+endif
+
+CHANGES_URL :=
 
 UNAME := $(shell uname -sm)
 
@@ -138,9 +123,9 @@ endif
 
 # Set up efficient math functions which are used in make.
 # Here since this file is included by envsetup as well as during build.
-include $(BUILD_SYSTEM)/math.mk
+include $(BUILD_SYSTEM_COMMON)/math.mk
 
-include $(BUILD_SYSTEM)/strings.mk
+include $(BUILD_SYSTEM_COMMON)/strings.mk
 
 # Various mappings to avoid hard-coding paths all over the place
 include $(BUILD_SYSTEM)/pathmap.mk

core/definitions.mk

@@ -806,29 +806,13 @@ define echo-error
 echo -e "$(ESC_BOLD)$(1): $(ESC_ERROR)error:$(ESC_RESET)$(ESC_BOLD)" $(2) "$(ESC_RESET)" >&2
 endef
 
-# $(1): message to print
-define pretty-warning
-$(shell $(call echo-warning,$(LOCAL_MODULE_MAKEFILE),$(LOCAL_MODULE): $(1)))
-endef
-
-# $(1): message to print
-define pretty-error
-$(shell $(call echo-error,$(LOCAL_MODULE_MAKEFILE),$(LOCAL_MODULE): $(1)))
-$(error done)
-endef
-
 ###########################################################
-## Output the command lines, or not
+## Legacy showcommands compatibility
 ###########################################################
 
-ifeq ($(strip $(SHOW_COMMANDS)),)
 define pretty
 @echo $1
 endef
-else
-define pretty
-endef
-endif
 
 ###########################################################
 ## Commands for including the dependency files the compiler generates

core/distdir.mk

@@ -17,52 +17,52 @@
 # When specifying "dist", the user has asked that we copy the important
 # files from this build into DIST_DIR.
 
-ifdef dist_goal
+# list of all goals that depend on any dist files
-
+_all_dist_goals :=
-# $(1): source file
+# pairs of goal:distfile
-# $(2): destination file
+_all_dist_goal_output_pairs :=
-# $(3): goals that should copy the file
+# pairs of srcfile:distfile
-#
-define copy-one-dist-file
-$(3): $(2)
-$(2): $(1)
-	@echo "Dist: $$@"
-	$$(copy-file-to-new-target-with-cp)
-endef
-
-# A global variable to remember all dist'ed src:dst pairs.
-# So if a src:dst is already dist'ed by another goal,
-# we should just establish the dependency and don't really call the
-# copy-one-dist-file to avoid multiple rules for the same target.
 _all_dist_src_dst_pairs :=
+
 # Other parts of the system should use this function to associate
 # certain files with certain goals.  When those goals are built
 # and "dist" is specified, the marked files will be copied to DIST_DIR.
 #
-# $(1): a list of goals  (e.g. droid, sdk, pdk, ndk)
+# $(1): a list of goals  (e.g. droid, sdk, pdk, ndk). These must be PHONY
 # $(2): the dist files to add to those goals.  If the file contains ':',
 #       the text following the colon is the name that the file is copied
 #       to under the dist directory.  Subdirs are ok, and will be created
 #       at copy time if necessary.
 define dist-for-goals
+$(eval _all_dist_goals += $$(1)) \
 $(foreach file,$(2), \
-  $(eval fw := $(subst :,$(space),$(file))) \
+  $(eval src := $(call word-colon,1,$(file))) \
-  $(eval src := $(word 1,$(fw))) \
+  $(eval dst := $(call word-colon,2,$(file))) \
-  $(eval dst := $(word 2,$(fw))) \
+  $(if $(dst),,$(eval dst := $$(notdir $$(src)))) \
-  $(eval dst := $(if $(dst),$(dst),$(notdir $(src)))) \
+  $(eval _all_dist_src_dst_pairs += $$(src):$$(dst)) \
-  $(if $(filter $(_all_dist_src_dst_pairs),$(src):$(dst)),\
+  $(foreach goal,$(1), \
-    $(eval $(call add-dependency,$(1),$(DIST_DIR)/$(dst))),\
+    $(eval _all_dist_goal_output_pairs += $$(goal):$$(dst))))
-    $(eval $(call copy-one-dist-file,\
-      $(src),$(DIST_DIR)/$(dst),$(1)))\
-      $(eval _all_dist_src_dst_pairs += $(src):$(dst))\
-  )\
-)
 endef
 
-else # !dist_goal
+#------------------------------------------------------------------
+# To be used at the end of the build to collect all the uses of
+# dist-for-goals, and write them into a file for the packaging step to use.
 
-# empty definition when not building dist
+# $(1): The file to write
-define dist-for-goals
+define dist-write-file
+$(strip \
+  $(KATI_obsolete_var dist-for-goals,Cannot be used after dist-write-file) \
+  $(foreach goal,$(sort $(_all_dist_goals)), \
+    $(eval $$(goal): _dist_$$(goal))) \
+  $(shell mkdir -p $(dir $(1))) \
+  $(file >$(1).tmp, \
+    DIST_GOAL_OUTPUT_PAIRS := $(sort $(_all_dist_goal_output_pairs)) \
+    $(newline)DIST_SRC_DST_PAIRS := $(sort $(_all_dist_src_dst_pairs))) \
+  $(shell if ! cmp -s $(1).tmp $(1); then \
+            mv $(1).tmp $(1); \
+          else \
+            rm $(1).tmp; \
+          fi))
 endef
 
-endif # !dist_goal
+.KATI_READONLY := dist-for-goals dist-write-file

core/main.mk

@@ -36,8 +36,6 @@ PWD := $(shell pwd)
 TOP := .
 TOPDIR :=
 
-BUILD_SYSTEM := $(TOPDIR)build/make/core
-
 # This is the default target.  It must be the first declared target.
 .PHONY: droid
 DEFAULT_GOAL := droid
@@ -48,7 +46,7 @@ droid_targets:
 
 # Set up various standard variables based on configuration
 # and host information.
-include $(BUILD_SYSTEM)/config.mk
+include build/make/core/config.mk
 
 ifneq ($(filter $(dont_bother_goals), $(MAKECMDGOALS)),)
 dont_bother := true
@@ -419,6 +417,19 @@ ifneq ($(PRODUCT_ENFORCE_RRO_TARGETS),)
 ENFORCE_RRO_SOURCES :=
 endif
 
+# Color-coded warnings including current module info
+# $(1): message to print
+define pretty-warning
+$(shell $(call echo-warning,$(LOCAL_MODULE_MAKEFILE),$(LOCAL_MODULE): $(1)))
+endef
+
+# Color-coded errors including current module info
+# $(1): message to print
+define pretty-error
+$(shell $(call echo-error,$(LOCAL_MODULE_MAKEFILE),$(LOCAL_MODULE): $(1)))
+$(error done)
+endef
+
 subdir_makefiles_inc := .
 FULL_BUILD :=
 
@@ -492,6 +503,18 @@ $(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] fi
 # All module makefiles have been included at this point.
 # -------------------------------------------------------------------
 
+# -------------------------------------------------------------------
+# Use basic warning/error messages now that LOCAL_MODULE_MAKEFILE
+# and LOCAL_MODULE aren't useful anymore.
+# -------------------------------------------------------------------
+define pretty-warning
+$(warning $(1))
+endef
+
+define pretty-error
+$(error $(1))
+endef
+
 # -------------------------------------------------------------------
 # Enforce to generate all RRO packages for modules having resource
 # overlays.
@@ -1467,6 +1490,8 @@ tidy_only:
 ndk: $(SOONG_OUT_DIR)/ndk.timestamp
 .PHONY: ndk
 
+$(call dist-write-file,$(KATI_PACKAGE_MK_DIR)/dist.mk)
+
 $(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] writing build rules ...)
 
 endif # KATI

core/ninja_config.mk

@@ -28,7 +28,6 @@ PARSE_TIME_MAKE_GOALS := \
 	custom_images \
 	deps-license \
 	dicttool_aosp \
-	dist \
 	dump-products \
 	eng \
 	fusion \

packaging/distdir.mk

@@ -0,0 +1,45 @@
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# From the Android.mk pass:
+DIST_GOAL_OUTPUT_PAIRS :=
+DIST_SRC_DST_PAIRS :=
+include $(KATI_PACKAGE_MK_DIR)/dist.mk
+
+$(foreach pair,$(DIST_GOAL_OUTPUT_PAIRS), \
+  $(eval goal := $(call word-colon,1,$(pair))) \
+  $(eval output := $(call word-colon,2,$(pair))) \
+  $(eval .PHONY: _dist_$$(goal)) \
+  $(if $(call streq,$(DIST),true),\
+    $(eval _dist_$$(goal): $$(DIST_DIR)/$$(output))))
+
+define copy-one-dist-file
+$(2): $(1)
+	@echo "Dist: $$@"
+	rm -f $$@
+	cp $$< $$@
+endef
+
+ifeq ($(DIST),true)
+  $(foreach pair,$(DIST_SRC_DST_PAIRS), \
+    $(eval src := $(call word-colon,1,$(pair))) \
+    $(eval dst := $(DIST_DIR)/$(call word-colon,2,$(pair))) \
+    $(eval $(call copy-one-dist-file,$(src),$(dst))))
+endif
+
+copy-one-dist-file :=
+DIST_GOAL_OUTPUT_PAIRS :=
+DIST_SRC_DST_PAIRS :=

packaging/main.mk

@@ -0,0 +1,37 @@
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Create a default rule. This is unused currently, as the real default rule is
+# still in the Kati build step.
+.PHONY: _packaging_default_rule_
+_packaging_default_rule_:
+
+ifndef KATI
+$(error Only Kati is supported.)
+endif
+
+$(info [1/3] initializing packaging system ...)
+
+.KATI_READONLY := KATI_PACKAGE_MK_DIR
+
+include build/make/common/core.mk
+include build/make/common/strings.mk
+
+$(info [2/3] including distdir.mk ...)
+
+include build/make/packaging/distdir.mk
+
+$(info [3/3] writing packaging rules ...)