tests: improve code coverage support

Fix the exclusion path for lcov; it should exclude the directory
with source code, not object files.

Use the COV environment variable to
* control whether we build for coverage or not
* select the output directory

Add a separate target for generating the report, so we can get a
report for all of the tests together or just a single test.

Add documentation.

Signed-off-by: Paul Fagerburg <pfagerburg@google.com>
Change-Id: I2bd2bfdedfab291aabeaa968c10b17e9b61c9c0a
Reviewed-on: https://review.coreboot.org/c/coreboot/+/54072
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Jakub Czapiga <jacz@semihalf.com>

Documentation/technotes/2021-05-code-coverage.md

@@ -0,0 +1,56 @@
+# Unit Test Code Coverage
+
+Code coverage for the coreboot unit tests allows us to see what lines of
+code in the coreboot library are covered by unit tests, and allows a test
+author to see where they need to add test cases for additional coverage.
+
+Enable code coverage in your unit test build by setting the environment
+variable `COV` to 1; either `export COV=1` in your shell, or add it to your
+`make` command, e.g. `COV=1 make unit-tests`.
+
+The build output directory is either `build/tests` or `build/coverage`,
+depending on whether `COV=1` is set in the environment.
+
+All of the unit test targets are available with and without `COV=1`
+* `clean-unit-tests`
+* `build-unit-tests`
+* `run-unit-tests`
+* `unit-tests` (which is just `build-unit-tests` followed by `run-unit-tests`)
+
+There are two new `make` targets:
+* `coverage-report` generates a code coverage report from all of the
+GCOV data (`*.gcda` and `*.gcno` files) in the build directory. To view the
+coverage report, open `build/coverage/coverage_reports/index.html` in your web
+browser.
+* `clean-coverage-report` deletes just the coverage report.
+
+The `coverage-report` and `clean-coverage-report` targets automatically set
+`COV=1` if it is not already set in the environment.
+
+
+## Examples
+
+`COV=1 make unit-tests coverage-report` builds all of the unit tests with code
+coverage, runs the unit tests, and generates the code coverage report.
+
+`COV=1 make build-unit-tests` builds all of the unit tests with code coverage.
+
+`COV=1 make run-unit-tests` runs the unit tests, building them with code
+coverage if they are out-of-date.
+
+`COV=1 make coverage-report` creates the code coverage report. This
+target does not explicitly depend on the tests being built and run; it gathers
+the code coverage data from the output directory, which it assumes already
+exists.
+
+`COV=1 make tests/lib/uuid-test coverage-report` builds the uuid test
+with code coverage, runs it, and generates a code coverage report just for
+that test.
+
+As a demonstration that building with and without coverage uses different
+output directories:
+1. `make build-unit-tests` builds unit tests without code coverage into
+`build/tests`.
+2. `COV=1 make clean-unit-tests` cleans `build/coverage`
+3. `make build-unit-tests` doesn't need to build anything in `build/tests`,
+because those files weren't affected by the previous `clean-unit-tests`.

Documentation/tutorial/part3.md

@@ -3,6 +3,8 @@
 ## Introduction
 General thoughts about unit testing coreboot can be found in
 [Unit testing coreboot](../technotes/2020-03-unit-testing-coreboot.md).
+Additionally, [code coverage](../technotes/2021-05-code-coverage.md) support
+is available for unit tests.
 
 This document aims to guide developers through the process of adding and writing
 unit tests for coreboot modules.

Makefile

@@ -124,8 +124,8 @@ ifneq ($(filter help%, $(MAKECMDGOALS)), )
 NOCOMPILE:=1
 UNIT_TEST:=1
 else
-ifneq ($(filter %-test %-tests, $(MAKECMDGOALS)),)
-ifneq ($(filter-out %-test %-tests, $(MAKECMDGOALS)),)
+ifneq ($(filter %-test %-tests %coverage-report, $(MAKECMDGOALS)),)
+ifneq ($(filter-out %-test %-tests %coverage-report, $(MAKECMDGOALS)),)
 $(error Cannot mix unit-tests targets with other targets)
 endif
 UNIT_TEST:=1

tests/Makefile.inc

@@ -1,9 +1,18 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
 testsrc = $(top)/tests
+
+# Place the build output in one of two places depending on COV, so that code
+# built with code coverage never mixes with code built without code coverage.
+ifeq ($(COV),1)
+testobj = $(obj)/coverage
+else
 testobj = $(obj)/tests
+endif
+
 cmockasrc = 3rdparty/cmocka
 cmockaobj = $(objutil)/cmocka
+coverage_dir = coverage_reports
 
 CMOCKA_LIB := $(cmockaobj)/src/libcmocka.so
 
@@ -51,6 +60,12 @@ TEST_LDFLAGS += -Wl,--gc-sections
 TEST_CFLAGS += -fno-pie -fno-pic
 TEST_LDFLAGS += -no-pie
 
+# Enable code coverage if COV=1
+ifeq ($(COV),1)
+TEST_CFLAGS += --coverage
+TEST_LDFLAGS += --coverage
+endif
+
 # Extra attributes for unit tests, declared per test
 attributes:= srcs cflags config mocks stage
 
@@ -99,7 +114,7 @@ $$($(1)-config-file): $(TEST_KCONFIG_AUTOHEADER)
 
 $($(1)-objs): TEST_CFLAGS += -I$$(dir $$($(1)-config-file)) \
 	-D__$$(shell echo $$($(1)-stage) | tr '[:lower:]' '[:upper:]')__
-$($(1)-objs): $(obj)/$(1)/%.o: $$$$*.c $$($(1)-config-file)
+$($(1)-objs): $(testobj)/$(1)/%.o: $$$$*.c $$($(1)-config-file)
 	mkdir -p $$(dir $$@)
 	$(HOSTCC) $(HOSTCFLAGS) $$(TEST_CFLAGS) $($(1)-cflags)  -MMD \
 		-MT $$@ -c $$< -o $$@
@@ -111,10 +126,10 @@ $($(1)-bin): $($(1)-objs) $(CMOCKA_LIB)
 endef
 
 $(foreach test, $(alltests), \
-	$(eval $(test)-objs:=$(addprefix $(obj)/$(test)/, \
+	$(eval $(test)-objs:=$(addprefix $(testobj)/$(test)/, \
 		$(patsubst %.c,%.o,$($(test)-srcs)))))
 $(foreach test, $(alltests), \
-	$(eval $(test)-bin:=$(obj)/$(test)/run))
+	$(eval $(test)-bin:=$(testobj)/$(test)/run))
 $(foreach test, $(alltests), \
 	$(eval $(call TEST_CC_template,$(test))))
 
@@ -168,15 +183,31 @@ $(alltests): $$($$(@)-bin)
 	rm -f $(testobj)/junit-$(subst /,_,$^).xml $(testobj)/$(subst /,_,$^).failed
 	-./$^ || echo failed > $(testobj)/$(subst /,_,$^).failed
 
-.PHONY: coverage-unit-tests
+# Build a code coverage report by collecting all the gcov files into a single
+# report. If COV is not set, this might be a user error, and they're trying
+# to generate a coverage report without first having built and run the code
+# with code coverage. So instead of silently correcting it by adding COV=1,
+# let's flag it to the user so they can be sure they're doing the thing they
+# want to do.
 
-coverage-unit-tests: TEST_CFLAGS += --coverage
-coverage-unit-tests: TEST_LDFLAGS += --coverage
-coverage-unit-tests: clean-unit-tests unit-tests
-	lcov -o $(testobj)/tests.info -c -d $(testobj) --exclude '*/$(testobj)/*'
-	genhtml -q -o build/tests/coverage_rpt -t "coreboot unit tests" \
+.PHONY: coverage-report clean-coverage-report
+
+ifeq ($(COV),1)
+coverage-report:
+	lcov -o $(testobj)/tests.info -c -d $(testobj) --exclude '$(testsrc)/*'
+	genhtml -q -o $(testobj)/$(coverage_dir) -t "coreboot unit tests" \
 	-s $(testobj)/tests.info
 
+clean-coverage-report:
+	rm -Rf $(testobj)/$(coverage_dir)
+else
+coverage-report:
+	COV=1 V=$(V) $(MAKE) coverage-report
+
+clean-coverage-report:
+	COV=1 V=$(V) $(MAKE) clean-coverage-report
+endif
+
 unit-tests: build-unit-tests run-unit-tests
 
 build-unit-tests: $(test-bins)
@@ -195,7 +226,7 @@ run-unit-tests: $(alltests)
 	fi
 
 $(addprefix clean-,$(alltests)): clean-%:
-	rm -rf $(obj)/$*
+	rm -rf $(testobj)/$*
 
 clean-unit-tests:
 	rm -rf $(testobj)
@@ -208,11 +239,12 @@ list-unit-tests:
 
 help-unit-tests help::
 	@echo  '*** coreboot unit-tests targets ***'
+	@echo  '  Use "COV=1 make [target]" to enable code coverage for unit tests'
 	@echo  '  unit-tests            - Run all unit-tests from tests/'
 	@echo  '  clean-unit-tests      - Remove unit-tests build artifacts'
 	@echo  '  list-unit-tests       - List all unit-tests'
 	@echo  '  <unit-test>           - Build and run single unit-test'
 	@echo  '  clean-<unit-test>     - Remove single unit-test build artifacts'
-	@echo  '  coverage-unit-tests   - Build unit tests for code coverage and'
-	@echo  '                            generate a code coverage report'
+	@echo  '  coverage-report       - Generate a code coverage report'
+	@echo  '  clean-coverage-report - Remove the code coverage report'
 	@echo