diff options
Diffstat (limited to 'tests/SConscript')
| -rw-r--r-- | tests/SConscript | 169 |
1 files changed, 103 insertions, 66 deletions
diff --git a/tests/SConscript b/tests/SConscript index 62c4d0508..38e9ae9d2 100644 --- a/tests/SConscript +++ b/tests/SConscript @@ -29,7 +29,7 @@ # Authors: Steve Reinhardt # Kevin Lim -import os +import os, signal import sys import glob from SCons.Script.SConscript import SConsEnvironment @@ -44,57 +44,115 @@ env.Tests = {} def contents(node): return file(str(node)).read() -def check_test(target, source, env): +# functions to parse return value from scons Execute()... not the same +# as wait() etc., so python built-in os funcs don't work. +def signaled(status): + return (status & 0x80) != 0; + +def signum(status): + return (status & 0x7f); + +# List of signals that indicate that we should retry the test rather +# than consider it failed. +retry_signals = (signal.SIGTERM, signal.SIGKILL, signal.SIGINT, + signal.SIGQUIT, signal.SIGHUP) + +# regular expressions of lines to ignore when diffing outputs +output_ignore_regexes = ( + '^command line:', # for stdout file + '^M5 compiled ', # for stderr file + '^M5 started ', # for stderr file + '^M5 executing on ', # for stderr file + '^Simulation complete at', # for stderr file + '^Listening for', # for stderr file + 'listening for remote gdb', # for stderr file + ) + +output_ignore_args = ' '.join(["-I '"+s+"'" for s in output_ignore_regexes]) + +output_ignore_args += ' --exclude=stats.txt --exclude=outdiff' + +def run_test(target, source, env): """Check output from running test. Targets are as follows: - target[0] : outdiff - target[1] : statsdiff - target[2] : status + target[0] : status + + Sources are: + source[0] : M5 binary + source[1] : tests/run.py script + source[2] : reference stats file """ # make sure target files are all gone for t in target: if os.path.exists(t.abspath): - Execute(Delete(t.abspath)) - # Run diff on output & ref directories to find differences. - # Exclude m5stats.txt since we will use diff-out on that. - Execute(env.subst('diff -ubr ${SOURCES[0].dir} ${SOURCES[1].dir} ' + - '-I "^command line:" ' + # for stdout file - '-I "^M5 compiled " ' + # for stderr file - '-I "^M5 started " ' + # for stderr file - '-I "^M5 executing on " ' + # for stderr file - '-I "^Simulation complete at" ' + # for stderr file - '-I "^Listening for" ' + # for stderr file - '-I "listening for remote gdb" ' + # for stderr file - '--exclude=m5stats.txt --exclude=SCCS ' + - '--exclude=${TARGETS[0].file} ' + - '> ${TARGETS[0]}', target=target, source=source), None) - print "===== Output differences =====" - print contents(target[0]) - # Run diff-out on m5stats.txt file - status = Execute(env.subst('$DIFFOUT $SOURCES > ${TARGETS[1]}', - target=target, source=source), - strfunction=None) - print "===== Statistics differences =====" - print contents(target[1]) - # Generate status file contents based on exit status of diff-out + env.Execute(Delete(t.abspath)) + + tgt_dir = os.path.dirname(str(target[0])) + + # Base command for running test. We mess around with indirectly + # referring to files via SOURCES and TARGETS so that scons can mess + # with paths all it wants to and we still get the right files. + cmd = '${SOURCES[0]} -d %s -re ${SOURCES[1]} %s' % (tgt_dir, tgt_dir) + + # Prefix test run with batch job submission command if appropriate. + # Batch command also supports timeout arg (in seconds, not minutes). + timeout = 15 * 60 # used to be a param, probably should be again + if env['BATCH']: + cmd = '%s -t %d %s' % (env['BATCH_CMD'], timeout, cmd) + + status = env.Execute(env.subst(cmd, target=target, source=source)) + if status == 0: + # M5 terminated normally. + # Run diff on output & ref directories to find differences. + # Exclude the stats file since we will use diff-out on that. + outdiff = os.path.join(tgt_dir, 'outdiff') + diffcmd = 'diff -ubr %s ${SOURCES[2].dir} %s > %s' \ + % (output_ignore_args, tgt_dir, outdiff) + env.Execute(env.subst(diffcmd, target=target, source=source)) + print "===== Output differences =====" + print contents(outdiff) + # Run diff-out on stats.txt file + statsdiff = os.path.join(tgt_dir, 'statsdiff') + diffcmd = '$DIFFOUT ${SOURCES[2]} %s > %s' \ + % (os.path.join(tgt_dir, 'stats.txt'), statsdiff) + diffcmd = env.subst(diffcmd, target=target, source=source) + status = env.Execute(diffcmd, strfunction=None) + print "===== Statistics differences =====" + print contents(statsdiff) + + else: # m5 exit status != 0 + # M5 did not terminate properly, so no need to check the output + if signaled(status): + print 'M5 terminated with signal', signum(status) + if signum(status) in retry_signals: + # Consider the test incomplete; don't create a 'status' output. + # Hand the return status to scons and let scons decide what + # to do about it (typically terminate unless run with -k). + return status + else: + print 'M5 exited with non-zero status', status + # complete but failed execution (call to exit() with non-zero + # status, SIGABORT due to assertion failure, etc.)... fall through + # and generate FAILED status as if output comparison had failed + + # Generate status file contents based on exit status of m5 or diff-out if status == 0: status_str = "passed." else: status_str = "FAILED!" - f = file(str(target[2]), 'w') - print >>f, env.subst('${TARGETS[2].dir}', target=target, source=source), \ - status_str + f = file(str(target[0]), 'w') + print >>f, tgt_dir, status_str f.close() # done return 0 -def check_test_string(target, source, env): - return env.subst("Comparing outputs in ${TARGETS[0].dir}.", +def run_test_string(target, source, env): + return env.subst("Running test in ${TARGETS[0].dir}.", target=target, source=source) -testAction = env.Action(check_test, check_test_string) +testAction = env.Action(run_test, run_test_string) def print_test(target, source, env): print '***** ' + contents(source[0]) @@ -111,14 +169,14 @@ Note: The following file(s) will not be copied. New non-standard inputs and are ignored. ''' # - reference files always needed -needed_files = set(['stdout', 'stderr', 'm5stats.txt', 'config.ini']) +needed_files = set(['simout', 'simerr', 'stats.txt', 'config.ini']) # - source files we always want to ignore known_ignores = set(['status', 'outdiff', 'statsdiff']) def update_test(target, source, env): """Update reference test outputs. - Target is phony. First two sources are the ref & new m5stats.txt + Target is phony. First two sources are the ref & new stats.txt file files, respectively. We actually copy everything in the respective directories except the status & diff output files. @@ -147,7 +205,7 @@ def update_test(target, source, env): print " Creating new file", f copyAction = Copy(os.path.join(dest_dir, f), os.path.join(src_dir, f)) copyAction.strfunction = None - Execute(copyAction) + env.Execute(copyAction) return 0 def update_test_string(target, source, env): @@ -170,35 +228,12 @@ def test_builder(env, ref_dir): def tgt(f): return os.path.join(tgt_dir, f) - ref_stats = os.path.join(ref_dir, 'm5stats.txt') - new_stats = tgt('m5stats.txt') + ref_stats = os.path.join(ref_dir, 'stats.txt') + new_stats = tgt('stats.txt') status_file = tgt('status') - # Base command for running test. We mess around with indirectly - # referring to files via SOURCES and TARGETS so that scons can - # mess with paths all it wants to and we still get the right - # files. - base_cmd = '${SOURCES[0]} -d $TARGET.dir ${SOURCES[1]} %s' % tgt_dir - # stdout and stderr files - cmd_stdout = '${TARGETS[0]}' - cmd_stderr = '${TARGETS[1]}' - - # Prefix test run with batch job submission command if appropriate. - # Output redirection is also different for batch runs. - # Batch command also supports timeout arg (in seconds, not minutes). - timeout = 15 # used to be a param, probably should be again - if env['BATCH']: - cmd = [env['BATCH_CMD'], '-t', str(timeout * 60), - '-o', cmd_stdout, '-e', cmd_stderr, base_cmd] - else: - cmd = [base_cmd, '>', cmd_stdout, '2>', cmd_stderr] - - env.Command([tgt('stdout'), tgt('stderr'), new_stats], - [env.M5Binary, 'run.py'], ' '.join(cmd)) - - # order of targets is important... see check_test - env.Command([tgt('outdiff'), tgt('statsdiff'), status_file], - [ref_stats, new_stats], + env.Command([status_file], + [env.M5Binary, 'run.py', ref_stats], testAction) # phony target to echo status @@ -221,13 +256,15 @@ if env['FULL_SYSTEM']: 'tsunami-simple-timing', 'tsunami-simple-atomic-dual', 'tsunami-simple-timing-dual', - 'twosys-tsunami-simple-atomic'] + 'twosys-tsunami-simple-atomic', + 'tsunami-o3', 'tsunami-o3-dual'] if env['TARGET_ISA'] == 'sparc': configs += ['t1000-simple-atomic', 't1000-simple-timing'] else: - configs += ['simple-atomic', 'simple-timing', 'o3-timing', 'memtest'] + configs += ['simple-atomic', 'simple-timing', 'o3-timing', 'memtest', + 'simple-atomic-mp', 'simple-timing-mp'] cwd = os.getcwd() os.chdir(str(Dir('.').srcdir)) |