romcc: Rewrite the test system
Differences: - The test logic is now only implemented in one place (pending the deletion of the old parts), whereas it previously was implemented both as make rules and as a pair of shell scripts. - Tests don't need to be registered anymore. Just adding a new file with the correct name is enough to have it tested. - The code is hopefully more readable and maintainable. - The new test script supports colors (if the standard output is a terminal and --nocolor was not passed on the command line). Things to do in follow-up patches: - Remove the old test code - Test or remove fail_test*.c, hello_world*.c and raminit_test*.c - Fix regressions that have built up over the years, while making sure not to introduce new ones - Makefile integration - Jenkins integration There are tests in the makefile that specify -fno-always-inline, but this option doesn't exist anymore, so I didn't port them over. Change-Id: Idd6b89368c1e36555cb880c37bbe07035c938cd7 Signed-off-by: Jonathan Neuschäfer <j.neuschaefer@gmx.net> Reviewed-on: https://review.coreboot.org/14291 Reviewed-by: Martin Roth <martinroth@google.com> Tested-by: build bot (Jenkins)
This commit is contained in:
parent
3cfbc4a608
commit
e69d6c2e7b
|
@ -96,8 +96,7 @@ util/nvramtool/.dependencies
|
||||||
util/nvramtool/nvramtool
|
util/nvramtool/nvramtool
|
||||||
util/optionlist/Options.wiki
|
util/optionlist/Options.wiki
|
||||||
util/romcc/romcc
|
util/romcc/romcc
|
||||||
util/romcc/tests/fail_test*.S
|
util/romcc/build
|
||||||
util/romcc/tests/*.S-O2-mmmx
|
|
||||||
util/runfw/googlesnow
|
util/runfw/googlesnow
|
||||||
util/superiotool/superiotool
|
util/superiotool/superiotool
|
||||||
util/vgabios/testbios
|
util/vgabios/testbios
|
||||||
|
|
|
@ -0,0 +1,232 @@
|
||||||
|
#!/bin/sh
|
||||||
|
#
|
||||||
|
# This file is part of the coreboot project.
|
||||||
|
#
|
||||||
|
# Copyright 2016 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation; version 2 of the License.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
|
||||||
|
|
||||||
|
# These tests are currently known to be broken and should not result in a
|
||||||
|
# build failure:
|
||||||
|
XFAIL_TESTS="
|
||||||
|
simple_test4.c
|
||||||
|
simple_test6.c
|
||||||
|
simple_test25.c
|
||||||
|
simple_test26.c
|
||||||
|
simple_test46.c
|
||||||
|
simple_test47.c
|
||||||
|
simple_test54.c
|
||||||
|
simple_test72.c
|
||||||
|
simple_test73.c
|
||||||
|
linux_test2.c
|
||||||
|
linux_test5.c
|
||||||
|
linux_test10.c
|
||||||
|
linux_test11.c
|
||||||
|
linux_test12.c
|
||||||
|
"
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
BASEDIR="$(dirname "$0")"
|
||||||
|
BUILDDIR="$BASEDIR/build"
|
||||||
|
LOGDIR="$BUILDDIR/logs"
|
||||||
|
mkdir -p "$BUILDDIR"
|
||||||
|
mkdir -p "$LOGDIR"
|
||||||
|
|
||||||
|
usage () {
|
||||||
|
echo "Usage: test.sh [--nocolor] CLASS"
|
||||||
|
echo ""
|
||||||
|
echo "CLASS selects a group of tests to run. It must be one of the following:"
|
||||||
|
echo " all - all tests"
|
||||||
|
echo " simple - simple tests"
|
||||||
|
echo " linux - linux programs whose output is checked against a reference"
|
||||||
|
echo ""
|
||||||
|
echo "--nocolor disables colors."
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
COLORS=1
|
||||||
|
if [ "$1" = "--nocolor" ]; then
|
||||||
|
shift
|
||||||
|
COLORS=0
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
if [ -t 1 -a "$COLORS" -eq 1 ]; then
|
||||||
|
red() { printf "\033[1;31m%s\033[0m" "$*"; }
|
||||||
|
green() { printf "\033[1;32m%s\033[0m" "$*"; }
|
||||||
|
blue() { printf "\033[1;34m%s\033[0m" "$*"; }
|
||||||
|
else
|
||||||
|
red() { printf "%s" "$*"; }
|
||||||
|
green() { printf "%s" "$*"; }
|
||||||
|
blue() { printf "%s" "$*"; }
|
||||||
|
fi
|
||||||
|
|
||||||
|
init_stats() {
|
||||||
|
NUM_TOTAL=0 # Number of tests that were run
|
||||||
|
NUM_FAIL=0 # Number of tests that failed unexpectedly
|
||||||
|
NUM_BROKEN=0 # Number of tests that failed expectedly
|
||||||
|
NUM_PASS=0 # Number of tests that passed expectedly
|
||||||
|
NUM_FIXED=0 # Number of tests that passed unexpectedly
|
||||||
|
}
|
||||||
|
|
||||||
|
get_romcc() {
|
||||||
|
ROMCC="$BASEDIR/romcc"
|
||||||
|
if [ ! -f "$ROMCC" ]; then
|
||||||
|
echo "romcc not found! Please run \"make romcc\"."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
init_testing() {
|
||||||
|
init_stats
|
||||||
|
get_romcc
|
||||||
|
}
|
||||||
|
|
||||||
|
show_stats() {
|
||||||
|
printf "passed: %s\t(%s newly fixed)\n" $NUM_PASS $NUM_FIXED
|
||||||
|
printf "failed: %s\t(%s known broken)\n" $NUM_FAIL $NUM_BROKEN
|
||||||
|
printf "total: %s\n" $NUM_TOTAL
|
||||||
|
}
|
||||||
|
|
||||||
|
is_xfail() {
|
||||||
|
local t
|
||||||
|
for t in $XFAIL_TESTS; do
|
||||||
|
if [ "$t" = "$1" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
pass() {
|
||||||
|
NUM_TOTAL=$((NUM_TOTAL + 1))
|
||||||
|
NUM_PASS=$((NUM_PASS + 1))
|
||||||
|
|
||||||
|
green "passed"
|
||||||
|
if is_xfail "$(basename "$1")"; then
|
||||||
|
blue " (fixed)"
|
||||||
|
NUM_FIXED=$((NUM_FIXED + 1))
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
NUM_TOTAL=$((NUM_TOTAL + 1))
|
||||||
|
NUM_FAIL=$((NUM_FAIL + 1))
|
||||||
|
|
||||||
|
red "failed"
|
||||||
|
if is_xfail "$(basename "$1")"; then
|
||||||
|
blue " (known broken)"
|
||||||
|
NUM_BROKEN=$((NUM_BROKEN + 1))
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
run_simple_test() {
|
||||||
|
# TODO: "timeout" is not POSIX compliant. Use something that is.
|
||||||
|
timeout 60 "$ROMCC" $1 "$2" -o "$BUILDDIR/dummy.S"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_simple_tests() {
|
||||||
|
echo "Running simple tests..."
|
||||||
|
|
||||||
|
local t
|
||||||
|
for t in $(find "$BASEDIR/tests" -name 'simple_test*.c'); do
|
||||||
|
printf "%s" "$(basename "$t")"
|
||||||
|
|
||||||
|
local result=pass
|
||||||
|
local logfile="$LOGDIR/$(basename "$t").log"
|
||||||
|
rm "$logfile" >/dev/null 2>&1
|
||||||
|
for opt in "" "-O" "-O2" "-mmmx" "-msse" "-mmmx -msse" \
|
||||||
|
"-O -mmmx" "-O -msse" "-O -mmmx -msse" \
|
||||||
|
"-O2 -mmmx" "-O2 -msse" "-O2 -mmmx -msse"; do
|
||||||
|
if run_simple_test "$opt" "$t" \
|
||||||
|
>> "$logfile" 2>&1; then
|
||||||
|
printf .
|
||||||
|
else
|
||||||
|
result=fail
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
printf " "
|
||||||
|
$result "$t"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
run_linux_test() {
|
||||||
|
local base="$(basename "$1")"
|
||||||
|
|
||||||
|
# TODO: "timeout" is not POSIX compliant. Use something that is.
|
||||||
|
|
||||||
|
timeout 60 "$ROMCC" "$1" -o "$BUILDDIR/$base.S" || return 1
|
||||||
|
as --32 "$BUILDDIR/$base.S" -o "$BUILDDIR/$base.o" || return 1
|
||||||
|
ld -m elf_i386 -T "$BASEDIR/tests/ldscript.ld" \
|
||||||
|
"$BUILDDIR/$base.o" -o "$BUILDDIR/$base.elf" || return 1
|
||||||
|
timeout 60 "$BUILDDIR/$base.elf" > "$BUILDDIR/$base.out" || return 1
|
||||||
|
|
||||||
|
diff -u "$BASEDIR/results/${base%.c}.out" "$BUILDDIR/$base.out"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_linux_tests() {
|
||||||
|
echo "Running linux tests..."
|
||||||
|
|
||||||
|
local t
|
||||||
|
for t in $(find "$BASEDIR/tests" -name 'linux_test*.c'); do
|
||||||
|
printf "%s... " "$(basename "$t")"
|
||||||
|
|
||||||
|
local logfile="$LOGDIR/$(basename "$t").log"
|
||||||
|
if run_linux_test "$t" > "$logfile" 2>&1; then
|
||||||
|
pass "$t"
|
||||||
|
else
|
||||||
|
fail "$t"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if [ $# -ne 1 ]; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
|
||||||
|
CLASS="$1"
|
||||||
|
|
||||||
|
case "$CLASS" in
|
||||||
|
all)
|
||||||
|
init_testing
|
||||||
|
run_simple_tests
|
||||||
|
run_linux_tests
|
||||||
|
show_stats
|
||||||
|
;;
|
||||||
|
simple)
|
||||||
|
init_testing
|
||||||
|
run_simple_tests
|
||||||
|
show_stats
|
||||||
|
;;
|
||||||
|
linux)
|
||||||
|
init_testing
|
||||||
|
run_linux_tests
|
||||||
|
show_stats
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Invalid test class $CLASS"
|
||||||
|
echo
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
if [ $NUM_FAIL -ne $NUM_BROKEN ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
Loading…
Reference in New Issue