From a0e36d8cbaad6ead0102721fdedb3b4c50f0273c Mon Sep 17 00:00:00 2001 From: Jakub Czapiga Date: Tue, 6 Sep 2022 10:42:16 +0200 Subject: [PATCH] tests: Add support for tests build failures detection This patch introduces new target: junit.xml-unit-tests, which builds and runs unit-tests. It also creates build log containing build logs. This feature allows for one to see build failures in Jenkins dashboard. Signed-off-by: Jakub Czapiga Change-Id: I94184379dcc2ac10f1a47f4a9d205cacbeb640fe Reviewed-on: https://review.coreboot.org/c/coreboot/+/67372 Reviewed-by: Julius Werner Tested-by: build bot (Jenkins) --- payloads/libpayload/tests/Makefile.inc | 52 +++++++++++++++++++++++--- tests/Makefile.common | 2 + tests/Makefile.inc | 50 +++++++++++++++++++++++-- util/testing/Makefile.inc | 7 ++-- 4 files changed, 99 insertions(+), 12 deletions(-) diff --git a/payloads/libpayload/tests/Makefile.inc b/payloads/libpayload/tests/Makefile.inc index 2a067e0f9f..dede3404c5 100644 --- a/payloads/libpayload/tests/Makefile.inc +++ b/payloads/libpayload/tests/Makefile.inc @@ -202,19 +202,61 @@ $(TEST_KCONFIG_AUTOHEADER): $(TEST_DOTCONFIG) $(objk)/conf $(TEST_KCONFIG_AUTOCONFIG): $(TEST_KCONFIG_AUTOHEADER) true -.PHONY: $(alltests) $(addprefix clean-,$(alltests)) +.PHONY: $(alltests) $(addprefix clean-,$(alltests)) $(addprefix try-,$(alltests)) +.PHONY: $(addprefix build-,$(alltests)) $(addprefix run-,$(alltests)) .PHONY: unit-tests build-unit-tests run-unit-tests clean-unit-tests +.PHONY: junit.xml-unit-tests clean-junit.xml-unit-tests ifeq ($(JUNIT_OUTPUT),y) -$(alltests): export CMOCKA_MESSAGE_OUTPUT=xml -$(alltests): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml +$(addprefix run-,$(alltests)): export CMOCKA_MESSAGE_OUTPUT=xml +$(addprefix run-,$(alltests)): export CMOCKA_XML_FILE=$(testobj)/junit-libpayload-%g.xml endif -$(alltests): $$($$(@)-bin) +$(addprefix run-,$(alltests)): run-%: $$(%-bin) rm -f $(testobj)/junit-libpayload-$(subst /,_,$(patsubst $(testobj)/%/,%,$(dir $^)))\(*\).xml rm -f $(testobj)/$(subst /,_,$^).failed -$^ || echo failed > $(testobj)/$(subst /,_,$^).failed +$(addprefix build-,$(alltests)): build-%: $$(%-bin) + +$(alltests): run-$$(@) + +$(addprefix try-,$(alltests)): try-%: clean-% $(CMOCKA_LIB) $(TEST_KCONFIG_AUTOCONFIG) + mkdir -p $(testobj)/$* + echo "" >> $(testobj)/$*.tmp; \ + $(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "build-$*" >> $(testobj)/$*.tmp.2 2>&1 \ + && type="system-out" || type="failure"; \ + if [ $$type = "failure" ]; then \ + echo "" >> $(testobj)/$*.tmp; \ + else \ + echo "<$$type>" >> $(testobj)/$*.tmp; \ + fi; \ + echo '> $(testobj)/$*.tmp; \ + cat $(testobj)/$*.tmp.2 >> $(testobj)/$*.tmp; \ + echo "]]>" >> $(testobj)/$*.tmp; \ + rm -f $(testobj)/$*.tmp.2; \ + echo "" >> $(testobj)/$*.tmp; \ + if [ $$type != 'failure' ]; then \ + $(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "run-$*"; \ + fi + + +TESTS_BUILD_XML_FILE := $(testobj)/junit-libpayload-tests-build.xml + +$(TESTS_BUILD_XML_FILE): clean-junit.xml-unit-tests $(addprefix try-,$(alltests)) + mkdir -p $(dir $@) + echo '' > $@ + for tst in $(alltests); do \ + cat $(testobj)/$$tst.tmp >> $@; \ + done + echo "" >> $@ + +junit.xml-unit-tests: $(TESTS_BUILD_XML_FILE) + +clean-junit.xml-unit-tests: + rm -f $(TESTS_BUILD_XML_FILE) + + # Build a code coverage report by collecting all the gcov files into a single # report. If COV is not set, this might be a user error, and they're trying # to generate a coverage report without first having built and run the code @@ -254,7 +296,7 @@ run-unit-tests: $(alltests) exit 0; \ fi -$(addprefix clean-,$(alltests)): clean-% +$(addprefix clean-,$(alltests)): clean-%: rm -rf $(testobj)/$* clean-unit-tests: diff --git a/tests/Makefile.common b/tests/Makefile.common index 04bd6c94c3..595f870aca 100644 --- a/tests/Makefile.common +++ b/tests/Makefile.common @@ -170,3 +170,5 @@ $(TEST_KCONFIG_AUTOHEADER): $(TEST_DOTCONFIG) $(objutil)/kconfig/conf $(TEST_KCONFIG_AUTOCONFIG): $(TEST_KCONFIG_AUTOHEADER) true + +TEST_COMMON_DEPENDENCIES := $(CMOCKA_LIB) $(TEST_KCONFIG_AUTOCONFIG) diff --git a/tests/Makefile.inc b/tests/Makefile.inc index 10caf0a1e1..2c1a30bb8d 100644 --- a/tests/Makefile.inc +++ b/tests/Makefile.inc @@ -60,22 +60,64 @@ $(foreach test, $(alltests), \ DEPENDENCIES += $(addsuffix .d,$(basename $(all-test-objs))) -include $(DEPENDENCIES) -.PHONY: $(alltests) $(addprefix clean-,$(alltests)) +.PHONY: $(alltests) $(addprefix clean-,$(alltests)) $(addprefix try-,$(alltests)) +.PHONY: $(addprefix build-,$(alltests)) $(addprefix run-,$(alltests)) .PHONY: unit-tests build-unit-tests run-unit-tests clean-unit-tests +.PHONY: junit.xml-unit-tests clean-junit.xml-unit-tests # %g in CMOCKA_XML_FILE will be replaced with "__TEST_NAME__()" # by macro cb_run_group_tests(), which should be used for running tests. # __TEST_NAME__ contains test name including path e.g. tests_lib_rtc-test ifeq ($(JUNIT_OUTPUT),y) -$(alltests): export CMOCKA_MESSAGE_OUTPUT=xml -$(alltests): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml +$(addprefix run-,$(alltests)): export CMOCKA_MESSAGE_OUTPUT=xml +$(addprefix run-,$(alltests)): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml endif -$(alltests): $$($$(@)-bin) +$(addprefix run-,$(alltests)): run-%: $$(%-bin) rm -f $(testobj)/junit-$(subst /,_,$(patsubst $(testobj)/%/,%,$(dir $^)))\(*\).xml rm -f $(testobj)/$(subst /,_,$^).failed -$^ || echo failed > $(testobj)/$(subst /,_,$^).failed +$(addprefix build-,$(alltests)): build-%: $$(%-bin) + +$(alltests): run-$$(@) + +$(addprefix try-,$(alltests)): try-%: clean-% $(TEST_COMMON_DEPENDENCIES) + mkdir -p $(testobj)/$* + echo "" >> $(testobj)/$*.tmp; \ + $(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "build-$*" >> $(testobj)/$*.tmp.2 2>&1 \ + && type="system-out" || type="failure"; \ + if [ $$type = "failure" ]; then \ + echo "" >> $(testobj)/$*.tmp; \ + else \ + echo "<$$type>" >> $(testobj)/$*.tmp; \ + fi; \ + echo '> $(testobj)/$*.tmp; \ + cat $(testobj)/$*.tmp.2 >> $(testobj)/$*.tmp; \ + echo "]]>" >> $(testobj)/$*.tmp; \ + rm -f $(testobj)/$*.tmp.2; \ + echo "" >> $(testobj)/$*.tmp; \ + if [ $$type != 'failure' ]; then \ + $(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "run-$*"; \ + fi + + +TESTS_BUILD_XML_FILE := $(testobj)/junit-tests-build.xml + +$(TESTS_BUILD_XML_FILE): clean-junit.xml-unit-tests $(addprefix try-,$(alltests)) + mkdir -p $(dir $@) + echo '' > $@ + for tst in $(alltests); do \ + cat $(testobj)/$$tst.tmp >> $@; \ + done + echo "" >> $@ + +junit.xml-unit-tests: $(TESTS_BUILD_XML_FILE) + +clean-junit.xml-unit-tests: + rm -f $(TESTS_BUILD_XML_FILE) + + # Build a code coverage report by collecting all the gcov files into a single # report. If COV is not set, this might be a user error, and they're trying # to generate a coverage report without first having built and run the code diff --git a/util/testing/Makefile.inc b/util/testing/Makefile.inc index df6a25ef94..d9bc83fe48 100644 --- a/util/testing/Makefile.inc +++ b/util/testing/Makefile.inc @@ -104,9 +104,10 @@ endif $(MAKE) xcompile=$(COREBOOT_BUILD_DIR)/xcompile $(COREBOOT_BUILD_DIR)/xcompile $(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra124/lp0 BLD=tegra124_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml $(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra210/lp0 BLD=tegra120_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml - $(MAKE) unit-tests JUNIT_OUTPUT=y COV=1 - (cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) unit-tests coverage-report JUNIT_OUTPUT=y COV=1) - $(MAKE) coverage-report JUNIT_OUTPUT=y COV=1 + +$(MAKE) junit.xml-unit-tests COV=1 + +(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) junit.xml-unit-tests COV=1) + +(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) coverage-report COV=1) + +$(MAKE) coverage-report JUNIT_OUTPUT=y COV=1 find . -name 'tests.info' -exec cat {} + >$(COREBOOT_BUILD_DIR)/coverage.info test-basic: test-lint test-tools test-abuild test-payloads test-cleanup