summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJonathan Neuschäfer <j.neuschaefer@gmx.net>2016-04-13 02:47:24 +0200
committerMartin Roth <martinroth@google.com>2016-04-13 17:37:28 +0200
commite69d6c2e7bef9e8821423cb476d1a727b73ca1a4 (patch)
treed9dc64ab8f0d709c36bb84f119b234a43fb6bc62
parent3cfbc4a60873691d7ee21dda388c268b811782dd (diff)
downloadcoreboot-e69d6c2e7bef9e8821423cb476d1a727b73ca1a4.tar.xz
romcc: Rewrite the test system
Differences: - The test logic is now only implemented in one place (pending the deletion of the old parts), whereas it previously was implemented both as make rules and as a pair of shell scripts. - Tests don't need to be registered anymore. Just adding a new file with the correct name is enough to have it tested. - The code is hopefully more readable and maintainable. - The new test script supports colors (if the standard output is a terminal and --nocolor was not passed on the command line). Things to do in follow-up patches: - Remove the old test code - Test or remove fail_test*.c, hello_world*.c and raminit_test*.c - Fix regressions that have built up over the years, while making sure not to introduce new ones - Makefile integration - Jenkins integration There are tests in the makefile that specify -fno-always-inline, but this option doesn't exist anymore, so I didn't port them over. Change-Id: Idd6b89368c1e36555cb880c37bbe07035c938cd7 Signed-off-by: Jonathan Neuschäfer <j.neuschaefer@gmx.net> Reviewed-on: https://review.coreboot.org/14291 Reviewed-by: Martin Roth <martinroth@google.com> Tested-by: build bot (Jenkins)
-rw-r--r--.gitignore3
-rwxr-xr-xutil/romcc/test.sh232
2 files changed, 233 insertions, 2 deletions
diff --git a/.gitignore b/.gitignore
index 912aa85eb2..07a48358a3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -96,8 +96,7 @@ util/nvramtool/.dependencies
util/nvramtool/nvramtool
util/optionlist/Options.wiki
util/romcc/romcc
-util/romcc/tests/fail_test*.S
-util/romcc/tests/*.S-O2-mmmx
+util/romcc/build
util/runfw/googlesnow
util/superiotool/superiotool
util/vgabios/testbios
diff --git a/util/romcc/test.sh b/util/romcc/test.sh
new file mode 100755
index 0000000000..8aa7c53a79
--- /dev/null
+++ b/util/romcc/test.sh
@@ -0,0 +1,232 @@
+#!/bin/sh
+#
+# This file is part of the coreboot project.
+#
+# Copyright 2016 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+
+# These tests are currently known to be broken and should not result in a
+# build failure:
+XFAIL_TESTS="
+ simple_test4.c
+ simple_test6.c
+ simple_test25.c
+ simple_test26.c
+ simple_test46.c
+ simple_test47.c
+ simple_test54.c
+ simple_test72.c
+ simple_test73.c
+ linux_test2.c
+ linux_test5.c
+ linux_test10.c
+ linux_test11.c
+ linux_test12.c
+"
+
+# ------------------------------------------------------------------- #
+
+BASEDIR="$(dirname "$0")"
+BUILDDIR="$BASEDIR/build"
+LOGDIR="$BUILDDIR/logs"
+mkdir -p "$BUILDDIR"
+mkdir -p "$LOGDIR"
+
+usage () {
+ echo "Usage: test.sh [--nocolor] CLASS"
+ echo ""
+ echo "CLASS selects a group of tests to run. It must be one of the following:"
+ echo " all - all tests"
+ echo " simple - simple tests"
+ echo " linux - linux programs whose output is checked against a reference"
+ echo ""
+ echo "--nocolor disables colors."
+ exit 1
+}
+
+COLORS=1
+if [ "$1" = "--nocolor" ]; then
+ shift
+ COLORS=0
+fi
+
+
+if [ -t 1 -a "$COLORS" -eq 1 ]; then
+ red() { printf "\033[1;31m%s\033[0m" "$*"; }
+ green() { printf "\033[1;32m%s\033[0m" "$*"; }
+ blue() { printf "\033[1;34m%s\033[0m" "$*"; }
+else
+ red() { printf "%s" "$*"; }
+ green() { printf "%s" "$*"; }
+ blue() { printf "%s" "$*"; }
+fi
+
+init_stats() {
+ NUM_TOTAL=0 # Number of tests that were run
+ NUM_FAIL=0 # Number of tests that failed unexpectedly
+ NUM_BROKEN=0 # Number of tests that failed expectedly
+ NUM_PASS=0 # Number of tests that passed expectedly
+ NUM_FIXED=0 # Number of tests that passed unexpectedly
+}
+
+get_romcc() {
+ ROMCC="$BASEDIR/romcc"
+ if [ ! -f "$ROMCC" ]; then
+ echo "romcc not found! Please run \"make romcc\"."
+ exit 1
+ fi
+}
+
+init_testing() {
+ init_stats
+ get_romcc
+}
+
+show_stats() {
+ printf "passed: %s\t(%s newly fixed)\n" $NUM_PASS $NUM_FIXED
+ printf "failed: %s\t(%s known broken)\n" $NUM_FAIL $NUM_BROKEN
+ printf "total: %s\n" $NUM_TOTAL
+}
+
+is_xfail() {
+ local t
+ for t in $XFAIL_TESTS; do
+ if [ "$t" = "$1" ]; then
+ return 0
+ fi
+ done
+ return 1
+}
+
+pass() {
+ NUM_TOTAL=$((NUM_TOTAL + 1))
+ NUM_PASS=$((NUM_PASS + 1))
+
+ green "passed"
+ if is_xfail "$(basename "$1")"; then
+ blue " (fixed)"
+ NUM_FIXED=$((NUM_FIXED + 1))
+ fi
+ echo
+}
+
+fail() {
+ NUM_TOTAL=$((NUM_TOTAL + 1))
+ NUM_FAIL=$((NUM_FAIL + 1))
+
+ red "failed"
+ if is_xfail "$(basename "$1")"; then
+ blue " (known broken)"
+ NUM_BROKEN=$((NUM_BROKEN + 1))
+ fi
+ echo
+}
+
+run_simple_test() {
+ # TODO: "timeout" is not POSIX compliant. Use something that is.
+ timeout 60 "$ROMCC" $1 "$2" -o "$BUILDDIR/dummy.S"
+}
+
+run_simple_tests() {
+ echo "Running simple tests..."
+
+ local t
+ for t in $(find "$BASEDIR/tests" -name 'simple_test*.c'); do
+ printf "%s" "$(basename "$t")"
+
+ local result=pass
+ local logfile="$LOGDIR/$(basename "$t").log"
+ rm "$logfile" >/dev/null 2>&1
+ for opt in "" "-O" "-O2" "-mmmx" "-msse" "-mmmx -msse" \
+ "-O -mmmx" "-O -msse" "-O -mmmx -msse" \
+ "-O2 -mmmx" "-O2 -msse" "-O2 -mmmx -msse"; do
+ if run_simple_test "$opt" "$t" \
+ >> "$logfile" 2>&1; then
+ printf .
+ else
+ result=fail
+ break
+ fi
+ done
+ printf " "
+ $result "$t"
+ done
+
+ echo
+}
+
+run_linux_test() {
+ local base="$(basename "$1")"
+
+ # TODO: "timeout" is not POSIX compliant. Use something that is.
+
+ timeout 60 "$ROMCC" "$1" -o "$BUILDDIR/$base.S" || return 1
+ as --32 "$BUILDDIR/$base.S" -o "$BUILDDIR/$base.o" || return 1
+ ld -m elf_i386 -T "$BASEDIR/tests/ldscript.ld" \
+ "$BUILDDIR/$base.o" -o "$BUILDDIR/$base.elf" || return 1
+ timeout 60 "$BUILDDIR/$base.elf" > "$BUILDDIR/$base.out" || return 1
+
+ diff -u "$BASEDIR/results/${base%.c}.out" "$BUILDDIR/$base.out"
+}
+
+run_linux_tests() {
+ echo "Running linux tests..."
+
+ local t
+ for t in $(find "$BASEDIR/tests" -name 'linux_test*.c'); do
+ printf "%s... " "$(basename "$t")"
+
+ local logfile="$LOGDIR/$(basename "$t").log"
+ if run_linux_test "$t" > "$logfile" 2>&1; then
+ pass "$t"
+ else
+ fail "$t"
+ fi
+ done
+
+ echo
+}
+
+
+if [ $# -ne 1 ]; then
+ usage
+fi
+
+CLASS="$1"
+
+case "$CLASS" in
+ all)
+ init_testing
+ run_simple_tests
+ run_linux_tests
+ show_stats
+ ;;
+ simple)
+ init_testing
+ run_simple_tests
+ show_stats
+ ;;
+ linux)
+ init_testing
+ run_linux_tests
+ show_stats
+ ;;
+ *)
+ echo "Invalid test class $CLASS"
+ echo
+ usage
+ ;;
+esac
+
+if [ $NUM_FAIL -ne $NUM_BROKEN ]; then
+ exit 1
+fi