tests: Add support for tests build failures detection
This patch introduces new target: junit.xml-unit-tests, which builds and runs unit-tests. It also creates build log containing build logs. This feature allows for one to see build failures in Jenkins dashboard. Signed-off-by: Jakub Czapiga <jacz@semihalf.com> Change-Id: I94184379dcc2ac10f1a47f4a9d205cacbeb640fe Reviewed-on: https://review.coreboot.org/c/coreboot/+/67372 Reviewed-by: Julius Werner <jwerner@chromium.org> Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
This commit is contained in:
Jakub Czapiga
Patrick Georgi
parent
b525ea726b
commit
a0e36d8cba
|
|
@ -202,19 +202,61 @@ $(TEST_KCONFIG_AUTOHEADER): $(TEST_DOTCONFIG) $(objk)/conf
|
|||
$(TEST_KCONFIG_AUTOCONFIG): $(TEST_KCONFIG_AUTOHEADER)
|
||||
true
|
||||
|
||||
.PHONY: $(alltests) $(addprefix clean-,$(alltests))
|
||||
.PHONY: $(alltests) $(addprefix clean-,$(alltests)) $(addprefix try-,$(alltests))
|
||||
.PHONY: $(addprefix build-,$(alltests)) $(addprefix run-,$(alltests))
|
||||
.PHONY: unit-tests build-unit-tests run-unit-tests clean-unit-tests
|
||||
.PHONY: junit.xml-unit-tests clean-junit.xml-unit-tests
|
||||
|
||||
ifeq ($(JUNIT_OUTPUT),y)
|
||||
$(alltests): export CMOCKA_MESSAGE_OUTPUT=xml
|
||||
$(alltests): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml
|
||||
$(addprefix run-,$(alltests)): export CMOCKA_MESSAGE_OUTPUT=xml
|
||||
$(addprefix run-,$(alltests)): export CMOCKA_XML_FILE=$(testobj)/junit-libpayload-%g.xml
|
||||
endif
|
||||
|
||||
$(alltests): $$($$(@)-bin)
|
||||
$(addprefix run-,$(alltests)): run-%: $$(%-bin)
|
||||
rm -f $(testobj)/junit-libpayload-$(subst /,_,$(patsubst $(testobj)/%/,%,$(dir $^)))\(*\).xml
|
||||
rm -f $(testobj)/$(subst /,_,$^).failed
|
||||
-$^ || echo failed > $(testobj)/$(subst /,_,$^).failed
|
||||
|
||||
$(addprefix build-,$(alltests)): build-%: $$(%-bin)
|
||||
|
||||
$(alltests): run-$$(@)
|
||||
|
||||
$(addprefix try-,$(alltests)): try-%: clean-% $(CMOCKA_LIB) $(TEST_KCONFIG_AUTOCONFIG)
|
||||
mkdir -p $(testobj)/$*
|
||||
echo "<testcase classname='libpayload_build_unit_test' name='$*'>" >> $(testobj)/$*.tmp; \
|
||||
$(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "build-$*" >> $(testobj)/$*.tmp.2 2>&1 \
|
||||
&& type="system-out" || type="failure"; \
|
||||
if [ $$type = "failure" ]; then \
|
||||
echo "<failure type='buildFailed'>" >> $(testobj)/$*.tmp; \
|
||||
else \
|
||||
echo "<$$type>" >> $(testobj)/$*.tmp; \
|
||||
fi; \
|
||||
echo '<![CDATA[' >> $(testobj)/$*.tmp; \
|
||||
cat $(testobj)/$*.tmp.2 >> $(testobj)/$*.tmp; \
|
||||
echo "]]></$$type>" >> $(testobj)/$*.tmp; \
|
||||
rm -f $(testobj)/$*.tmp.2; \
|
||||
echo "</testcase>" >> $(testobj)/$*.tmp; \
|
||||
if [ $$type != 'failure' ]; then \
|
||||
$(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "run-$*"; \
|
||||
fi
|
||||
|
||||
|
||||
TESTS_BUILD_XML_FILE := $(testobj)/junit-libpayload-tests-build.xml
|
||||
|
||||
$(TESTS_BUILD_XML_FILE): clean-junit.xml-unit-tests $(addprefix try-,$(alltests))
|
||||
mkdir -p $(dir $@)
|
||||
echo '<?xml version="1.0" encoding="utf-8"?><testsuite>' > $@
|
||||
for tst in $(alltests); do \
|
||||
cat $(testobj)/$$tst.tmp >> $@; \
|
||||
done
|
||||
echo "</testsuite>" >> $@
|
||||
|
||||
junit.xml-unit-tests: $(TESTS_BUILD_XML_FILE)
|
||||
|
||||
clean-junit.xml-unit-tests:
|
||||
rm -f $(TESTS_BUILD_XML_FILE)
|
||||
|
||||
|
||||
# Build a code coverage report by collecting all the gcov files into a single
|
||||
# report. If COV is not set, this might be a user error, and they're trying
|
||||
# to generate a coverage report without first having built and run the code
|
||||
|
|
@ -254,7 +296,7 @@ run-unit-tests: $(alltests)
|
|||
exit 0; \
|
||||
fi
|
||||
|
||||
$(addprefix clean-,$(alltests)): clean-%
|
||||
$(addprefix clean-,$(alltests)): clean-%:
|
||||
rm -rf $(testobj)/$*
|
||||
|
||||
clean-unit-tests:
|
||||
|
|
|
|||
|
|
@ -170,3 +170,5 @@ $(TEST_KCONFIG_AUTOHEADER): $(TEST_DOTCONFIG) $(objutil)/kconfig/conf
|
|||
|
||||
$(TEST_KCONFIG_AUTOCONFIG): $(TEST_KCONFIG_AUTOHEADER)
|
||||
true
|
||||
|
||||
TEST_COMMON_DEPENDENCIES := $(CMOCKA_LIB) $(TEST_KCONFIG_AUTOCONFIG)
|
||||
|
|
|
|||
|
|
@ -60,22 +60,64 @@ $(foreach test, $(alltests), \
|
|||
DEPENDENCIES += $(addsuffix .d,$(basename $(all-test-objs)))
|
||||
-include $(DEPENDENCIES)
|
||||
|
||||
.PHONY: $(alltests) $(addprefix clean-,$(alltests))
|
||||
.PHONY: $(alltests) $(addprefix clean-,$(alltests)) $(addprefix try-,$(alltests))
|
||||
.PHONY: $(addprefix build-,$(alltests)) $(addprefix run-,$(alltests))
|
||||
.PHONY: unit-tests build-unit-tests run-unit-tests clean-unit-tests
|
||||
.PHONY: junit.xml-unit-tests clean-junit.xml-unit-tests
|
||||
|
||||
# %g in CMOCKA_XML_FILE will be replaced with "__TEST_NAME__(<test-group-name>)"
|
||||
# by macro cb_run_group_tests(), which should be used for running tests.
|
||||
# __TEST_NAME__ contains test name including path e.g. tests_lib_rtc-test
|
||||
ifeq ($(JUNIT_OUTPUT),y)
|
||||
$(alltests): export CMOCKA_MESSAGE_OUTPUT=xml
|
||||
$(alltests): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml
|
||||
$(addprefix run-,$(alltests)): export CMOCKA_MESSAGE_OUTPUT=xml
|
||||
$(addprefix run-,$(alltests)): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml
|
||||
endif
|
||||
|
||||
$(alltests): $$($$(@)-bin)
|
||||
$(addprefix run-,$(alltests)): run-%: $$(%-bin)
|
||||
rm -f $(testobj)/junit-$(subst /,_,$(patsubst $(testobj)/%/,%,$(dir $^)))\(*\).xml
|
||||
rm -f $(testobj)/$(subst /,_,$^).failed
|
||||
-$^ || echo failed > $(testobj)/$(subst /,_,$^).failed
|
||||
|
||||
$(addprefix build-,$(alltests)): build-%: $$(%-bin)
|
||||
|
||||
$(alltests): run-$$(@)
|
||||
|
||||
$(addprefix try-,$(alltests)): try-%: clean-% $(TEST_COMMON_DEPENDENCIES)
|
||||
mkdir -p $(testobj)/$*
|
||||
echo "<testcase classname='coreboot_build_unit_test' name='$*'>" >> $(testobj)/$*.tmp; \
|
||||
$(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "build-$*" >> $(testobj)/$*.tmp.2 2>&1 \
|
||||
&& type="system-out" || type="failure"; \
|
||||
if [ $$type = "failure" ]; then \
|
||||
echo "<failure type='buildFailed'>" >> $(testobj)/$*.tmp; \
|
||||
else \
|
||||
echo "<$$type>" >> $(testobj)/$*.tmp; \
|
||||
fi; \
|
||||
echo '<![CDATA[' >> $(testobj)/$*.tmp; \
|
||||
cat $(testobj)/$*.tmp.2 >> $(testobj)/$*.tmp; \
|
||||
echo "]]></$$type>" >> $(testobj)/$*.tmp; \
|
||||
rm -f $(testobj)/$*.tmp.2; \
|
||||
echo "</testcase>" >> $(testobj)/$*.tmp; \
|
||||
if [ $$type != 'failure' ]; then \
|
||||
$(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "run-$*"; \
|
||||
fi
|
||||
|
||||
|
||||
TESTS_BUILD_XML_FILE := $(testobj)/junit-tests-build.xml
|
||||
|
||||
$(TESTS_BUILD_XML_FILE): clean-junit.xml-unit-tests $(addprefix try-,$(alltests))
|
||||
mkdir -p $(dir $@)
|
||||
echo '<?xml version="1.0" encoding="utf-8"?><testsuite>' > $@
|
||||
for tst in $(alltests); do \
|
||||
cat $(testobj)/$$tst.tmp >> $@; \
|
||||
done
|
||||
echo "</testsuite>" >> $@
|
||||
|
||||
junit.xml-unit-tests: $(TESTS_BUILD_XML_FILE)
|
||||
|
||||
clean-junit.xml-unit-tests:
|
||||
rm -f $(TESTS_BUILD_XML_FILE)
|
||||
|
||||
|
||||
# Build a code coverage report by collecting all the gcov files into a single
|
||||
# report. If COV is not set, this might be a user error, and they're trying
|
||||
# to generate a coverage report without first having built and run the code
|
||||
|
|
|
|||
|
|
@ -104,9 +104,10 @@ endif
|
|||
$(MAKE) xcompile=$(COREBOOT_BUILD_DIR)/xcompile $(COREBOOT_BUILD_DIR)/xcompile
|
||||
$(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra124/lp0 BLD=tegra124_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml
|
||||
$(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra210/lp0 BLD=tegra120_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml
|
||||
$(MAKE) unit-tests JUNIT_OUTPUT=y COV=1
|
||||
(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) unit-tests coverage-report JUNIT_OUTPUT=y COV=1)
|
||||
$(MAKE) coverage-report JUNIT_OUTPUT=y COV=1
|
||||
+$(MAKE) junit.xml-unit-tests COV=1
|
||||
+(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) junit.xml-unit-tests COV=1)
|
||||
+(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) coverage-report COV=1)
|
||||
+$(MAKE) coverage-report JUNIT_OUTPUT=y COV=1
|
||||
find . -name 'tests.info' -exec cat {} + >$(COREBOOT_BUILD_DIR)/coverage.info
|
||||
|
||||
test-basic: test-lint test-tools test-abuild test-payloads test-cleanup
|
||||
|
|
|
|||
Loading…
Reference in New Issue