From e88ef9299ad647f5a1da16213efd25e7adfc1c10 Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Fri, 5 Jun 2020 22:28:20 -0700 Subject: [PATCH] Rewrite test data logic to make it faster It turns out we have a few tests with thousands of test data files, which this logic does not handle very well. Just switching from `x := $(x) ...` to `x += ...` accomplishes the majority of these gains, but I rewrote the logic to be less function-call+eval based to hopefully make it easier to understand. For an aosp-master/aosp_coral-eng build, this brings the time spent in this logic from 7.6s to 0.3s (total kati runtime of 54s to 46s). Kati's peak memory use (maxrss) also shrunk from 6.1GB to 3.8GB. Bug: 158488548 Test: build-aosp_coral.ninja is the same before/after Change-Id: Ia2ec8b29b5c1dd12fa7ea10b4f80fe802c692725 Merged-In: Ia2ec8b29b5c1dd12fa7ea10b4f80fe802c692725 (cherry picked from commit c5ec8e95bc11083d3edd034379c624467c687c31) --- core/base_rules.mk | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/core/base_rules.mk b/core/base_rules.mk index 15d7a0ef24..999105b95a 100644 --- a/core/base_rules.mk +++ b/core/base_rules.mk @@ -574,17 +574,35 @@ ifneq ($(strip $(filter NATIVE_TESTS,$(LOCAL_MODULE_CLASS)) $(LOCAL_IS_FUZZ_TARG ifneq ($(strip $(LOCAL_TEST_DATA)),) ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE)) -my_test_data_pairs := $(strip $(foreach td,$(LOCAL_TEST_DATA), \ - $(eval _file := $(call word-colon,2,$(td))) \ - $(if $(_file), \ - $(eval _src_base := $(call word-colon,1,$(td))), \ - $(eval _src_base := $(LOCAL_PATH)) \ - $(eval _file := $(call word-colon,1,$(td)))) \ - $(if $(call streq,$(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)),, \ - $(if $(findstring ..,$(_file)),$(error $(LOCAL_MODULE_MAKEFILE): LOCAL_TEST_DATA may not include '..': $(_file))) \ - $(if $(filter /%,$(_src_base) $(_file)),$(error $(LOCAL_MODULE_MAKEFILE): LOCAL_TEST_DATA may not include absolute paths: $(_src_base) $(_file)))) \ - $(eval my_test_data_file_pairs := $(my_test_data_file_pairs) $(call append-path,$(_src_base),$(_file)):$(_file)) \ - $(call append-path,$(_src_base),$(_file)):$(call append-path,$(my_module_path),$(_file)))) +ifeq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)) + define copy_test_data_pairs + _src_base := $$(call word-colon,1,$$(td)) + _file := $$(call word-colon,2,$$(td)) + my_test_data_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(call append-path,$$(my_module_path),$$(_file)) + my_test_data_file_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(_file) + endef +else + define copy_test_data_pairs + _src_base := $$(call word-colon,1,$$(td)) + _file := $$(call word-colon,2,$$(td)) + ifndef _file + _file := $$(_src_base) + _src_base := $$(LOCAL_PATH) + endif + ifneq (,$$(findstring ..,$$(_file))) + $$(call pretty-error,LOCAL_TEST_DATA may not include '..': $$(_file)) + endif + ifneq (,$$(filter/%,$$(_src_base) $$(_file))) + $$(call pretty-error,LOCAL_TEST_DATA may not include absolute paths: $$(_src_base) $$(_file)) + endif + my_test_data_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(call append-path,$$(my_module_path),$$(_file)) + my_test_data_file_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(_file) + endef +endif + +$(foreach td,$(LOCAL_TEST_DATA),$(eval $(copy_test_data_pairs))) + +copy_test_data_pairs := my_installed_test_data := $(call copy-many-files,$(my_test_data_pairs)) $(LOCAL_INSTALLED_MODULE): $(my_installed_test_data)