StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite test data logic to make it faster

Browse Source 
It turns out we have a few tests with thousands of test data files,
which this logic does not handle very well. Just switching from `x :=
$(x) ...` to `x += ...` accomplishes the majority of these gains, but I
rewrote the logic to be less function-call+eval based to hopefully make
it easier to understand.

For an aosp-master/aosp_coral-eng build, this brings the time spent in
this logic from 7.6s to 0.3s (total kati runtime of 54s to 46s). Kati's
peak memory use (maxrss) also shrunk from 6.1GB to 3.8GB.

Bug: 158488548
Test: build-aosp_coral.ninja is the same before/after
Change-Id: Ia2ec8b29b5c1dd12fa7ea10b4f80fe802c692725
Merged-In: Ia2ec8b29b5c1dd12fa7ea10b4f80fe802c692725
(cherry picked from commit c5ec8e95bc)
This commit is contained in:
Dan Willemsen
2020-06-05 22:28:20 -07:00
parent fff4e1f5a4
commit e88ef9299a
1 changed files with 29 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
core/base_rules.mk
View File

@@ -574,17 +574,35 @@ ifneq ($(strip $(filter NATIVE_TESTS,$(LOCAL_MODULE_CLASS)) $(LOCAL_IS_FUZZ_TARG
ifneq ($(strip $(LOCAL_TEST_DATA)),) ifneq ($(strip $(LOCAL_TEST_DATA)),)
ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE)) ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE))
my_test_data_pairs := $(strip $(foreach td,$(LOCAL_TEST_DATA), \ ifeq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK))
$(eval _file := $(call word-colon,2,$(td))) \ define copy_test_data_pairs
$(if $(_file), \ _src_base := $$(call word-colon,1,$$(td))
$(eval _src_base := $(call word-colon,1,$(td))), \ _file := $$(call word-colon,2,$$(td))
$(eval _src_base := $(LOCAL_PATH)) \ my_test_data_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(call append-path,$$(my_module_path),$$(_file))
$(eval _file := $(call word-colon,1,$(td)))) \ my_test_data_file_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(_file)
$(if $(call streq,$(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)),, \ endef
$(if $(findstring ..,$(_file)),$(error $(LOCAL_MODULE_MAKEFILE): LOCAL_TEST_DATA may not include '..': $(_file))) \ else
$(if $(filter /%,$(_src_base) $(_file)),$(error $(LOCAL_MODULE_MAKEFILE): LOCAL_TEST_DATA may not include absolute paths: $(_src_base) $(_file)))) \ define copy_test_data_pairs
$(eval my_test_data_file_pairs := $(my_test_data_file_pairs) $(call append-path,$(_src_base),$(_file)):$(_file)) \ _src_base := $$(call word-colon,1,$$(td))
$(call append-path,$(_src_base),$(_file)):$(call append-path,$(my_module_path),$(_file)))) _file := $$(call word-colon,2,$$(td))
ifndef _file
_file := $$(_src_base)
_src_base := $$(LOCAL_PATH)
endif
ifneq (,$$(findstring ..,$$(_file)))
$$(call pretty-error,LOCAL_TEST_DATA may not include '..': $$(_file))
endif
ifneq (,$$(filter/%,$$(_src_base) $$(_file)))
$$(call pretty-error,LOCAL_TEST_DATA may not include absolute paths: $$(_src_base) $$(_file))
endif
my_test_data_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(call append-path,$$(my_module_path),$$(_file))
my_test_data_file_pairs += $$(call append-path,$$(_src_base),$$(_file)):$$(_file)
endef
endif
$(foreach td,$(LOCAL_TEST_DATA),$(eval $(copy_test_data_pairs)))
copy_test_data_pairs :=
my_installed_test_data := $(call copy-many-files,$(my_test_data_pairs)) my_installed_test_data := $(call copy-many-files,$(my_test_data_pairs))
$(LOCAL_INSTALLED_MODULE): $(my_installed_test_data) $(LOCAL_INSTALLED_MODULE): $(my_installed_test_data)