summaryrefslogtreecommitdiff
path: root/tests/SConscript
blob: 32691c6395dd8d3a10c4735ccd0f23bc26b2271f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
# -*- mode:python -*-

# Copyright (c) 2004-2006 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Steve Reinhardt
#          Kevin Lim

import os, signal
import sys, time
import glob
from SCons.Script.SConscript import SConsEnvironment

Import('env')

env['DIFFOUT'] = File('diff-out')

# get the termcap from the environment
termcap = env['TERMCAP']

# Dict that accumulates lists of tests by category (quick, medium, long)
env.Tests = {}

def contents(node):
    return file(str(node)).read()

# functions to parse return value from scons Execute()... not the same
# as wait() etc., so python built-in os funcs don't work.
def signaled(status):
    return (status & 0x80) != 0;

def signum(status):
    return (status & 0x7f);

# List of signals that indicate that we should retry the test rather
# than consider it failed.
retry_signals = (signal.SIGTERM, signal.SIGKILL, signal.SIGINT,
                 signal.SIGQUIT, signal.SIGHUP)

# regular expressions of lines to ignore when diffing outputs
output_ignore_regexes = (
    '^command line:',		# for stdout file
    '^gem5 compiled ',		# for stderr file
    '^gem5 started ',		# for stderr file
    '^gem5 executing on ',	# for stderr file
    '^Simulation complete at',	# for stderr file
    '^Listening for',		# for stderr file
    'listening for remote gdb',	# for stderr file
    )

output_ignore_args = ' '.join(["-I '"+s+"'" for s in output_ignore_regexes])

output_ignore_args += ' --exclude=stats.txt --exclude=outdiff'

def run_test(target, source, env):
    """Check output from running test.

    Targets are as follows:
    target[0] : status

    Sources are:
    source[0] : gem5 binary
    source[1] : tests/run.py script
    source[2] : reference stats file

    """
    # make sure target files are all gone
    for t in target:
        if os.path.exists(t.abspath):
            env.Execute(Delete(t.abspath))

    tgt_dir = os.path.dirname(str(target[0]))

    # Base command for running test.  We mess around with indirectly
    # referring to files via SOURCES and TARGETS so that scons can mess
    # with paths all it wants to and we still get the right files.
    cmd = '${SOURCES[0]} -d %s -re ${SOURCES[1]} %s' % (tgt_dir, tgt_dir)

    # Prefix test run with batch job submission command if appropriate.
    # Batch command also supports timeout arg (in seconds, not minutes).
    timeout = 15 * 60 # used to be a param, probably should be again
    if env['BATCH']:
        cmd = '%s -t %d %s' % (env['BATCH_CMD'], timeout, cmd)
    # The slowest regression (bzip2) requires ~4 hours;
    # 5 hours was chosen to be conservative.
    elif env['TIMEOUT']:
        cmd = 'timeout --foreground 5h %s' % cmd

    # Create a default value for the status string, changed as needed
    # based on the status.
    status_str = "passed."

    pre_exec_time = time.time()
    status = env.Execute(env.subst(cmd, target=target, source=source))
    if status == 0:
        # gem5 terminated normally.
        # Run diff on output & ref directories to find differences.
        # Exclude the stats file since we will use diff-out on that.

        # NFS file systems can be annoying and not have updated yet
        # wait until we see the file modified
        statsdiff = os.path.join(tgt_dir, 'statsdiff')
        m_time = 0
        nap = 0
        while m_time < pre_exec_time and nap < 10:
            try:
                m_time = os.stat(statsdiff).st_mtime
            except OSError:
                pass
            time.sleep(1)
            nap += 1

        outdiff = os.path.join(tgt_dir, 'outdiff')
        # tack 'true' on the end so scons doesn't report diff's
        # non-zero exit code as a build error
        diffcmd = 'diff -ubrs %s ${SOURCES[2].dir} %s > %s; true' \
                  % (output_ignore_args, tgt_dir, outdiff)
        env.Execute(env.subst(diffcmd, target=target, source=source))
        print "===== Output differences ====="
        print contents(outdiff)
        # Run diff-out on stats.txt file
        diffcmd = '$DIFFOUT ${SOURCES[2]} %s > %s' \
                  % (os.path.join(tgt_dir, 'stats.txt'), statsdiff)
        diffcmd = env.subst(diffcmd, target=target, source=source)
        diff_status = env.Execute(diffcmd, strfunction=None)
        # If there is a difference, change the status string to say so
        if diff_status != 0:
            status_str = "CHANGED!"
        print "===== Statistics differences ====="
        print contents(statsdiff)

    else: # gem5 exit status != 0
        # Consider it a failed test unless the exit status is 2
        status_str = "FAILED!"
        # gem5 did not terminate properly, so no need to check the output
        if env['TIMEOUT'] and status == 124:
            status_str = "TIMED-OUT!"
        elif signaled(status):
            print 'gem5 terminated with signal', signum(status)
            if signum(status) in retry_signals:
                # Consider the test incomplete; don't create a 'status' output.
                # Hand the return status to scons and let scons decide what
                # to do about it (typically terminate unless run with -k).
                return status
        elif status == 2:
            # The test was skipped, change the status string to say so
            status_str = "skipped."
        else:
            print 'gem5 exited with non-zero status', status
        # complete but failed execution (call to exit() with non-zero
        # status, SIGABORT due to assertion failure, etc.)... fall through
        # and generate FAILED status as if output comparison had failed

    # Generate status file contents based on exit status of gem5 and diff-out
    f = file(str(target[0]), 'w')
    print >>f, tgt_dir, status_str
    f.close()
    # done
    return 0

def run_test_string(target, source, env):
    return env.subst("Running test in ${TARGETS[0].dir}.",
                     target=target, source=source)

testAction = env.Action(run_test, run_test_string)

def print_test(target, source, env):
    # print the status with colours to make it easier to see what
    # passed and what failed
    line = contents(source[0])

    # split the line to words and get the last one
    words = line.split()
    status = words[-1]

    # if the test failed make it red, if it passed make it green, and
    # skip the punctuation
    if status == "FAILED!" or status == "TIMED-OUT!":
        status = termcap.Red + status[:-1] + termcap.Normal + status[-1]
    elif status == "CHANGED!":
        status = termcap.Yellow + status[:-1] + termcap.Normal + status[-1]
    elif status == "passed.":
        status = termcap.Green + status[:-1] + termcap.Normal + status[-1]
    elif status == "skipped.":
        status = termcap.Cyan + status[:-1] + termcap.Normal + status[-1]

    # put it back in the list and join with space
    words[-1] = status
    line = " ".join(words)

    print '***** ' + line
    return 0

printAction = env.Action(print_test, strfunction = None)

# Static vars for update_test:
# - long-winded message about ignored sources
ignore_msg = '''
Note: The following file(s) will not be copied.  New non-standard
      output files must be copied manually once before --update-ref will
      recognize them as outputs.  Otherwise they are assumed to be
      inputs and are ignored.
'''
# - reference files always needed
needed_files = set(['simout', 'simerr', 'stats.txt', 'config.ini'])
# - source files we always want to ignore
known_ignores = set(['status', 'outdiff', 'statsdiff'])

def update_test(target, source, env):
    """Update reference test outputs.

    Target is phony.  First two sources are the ref & new stats.txt file
    files, respectively.  We actually copy everything in the
    respective directories except the status & diff output files.

    """
    dest_dir = str(source[0].get_dir())
    src_dir = str(source[1].get_dir())
    dest_files = set(os.listdir(dest_dir))
    src_files = set(os.listdir(src_dir))
    # Copy all of the required files plus any existing dest files.
    wanted_files = needed_files | dest_files
    missing_files = wanted_files - src_files
    if len(missing_files) > 0:
        print "  WARNING: the following file(s) are missing " \
              "and will not be updated:"
        print "    ", " ,".join(missing_files)
    copy_files = wanted_files - missing_files
    warn_ignored_files = (src_files - copy_files) - known_ignores
    if len(warn_ignored_files) > 0:
        print ignore_msg,
        print "       ", ", ".join(warn_ignored_files)
    for f in copy_files:
        if f in dest_files:
            print "  Replacing file", f
            dest_files.remove(f)
        else:
            print "  Creating new file", f
        copyAction = Copy(os.path.join(dest_dir, f), os.path.join(src_dir, f))
        copyAction.strfunction = None
        env.Execute(copyAction)
    return 0

def update_test_string(target, source, env):
    return env.subst("Updating ${SOURCES[0].dir} from ${SOURCES[1].dir}",
                     target=target, source=source)

updateAction = env.Action(update_test, update_test_string)

def test_builder(env, ref_dir):
    """Define a test."""

    path = list(ref_dir.split('/'))

    # target path (where test output goes) consists of category, mode,
    # name, isa, opsys, and config (skips the 'ref' component)
    assert(path.pop(-4) == 'ref')
    tgt_dir = os.path.join(*path[-6:])

    # local closure for prepending target path to filename
    def tgt(f):
        return os.path.join(tgt_dir, f)

    ref_stats = os.path.join(ref_dir, 'stats.txt')
    new_stats = tgt('stats.txt')
    status_file = tgt('status')

    env.Command([status_file, new_stats],
                [env.M5Binary, 'run.py', ref_stats],
                testAction)

    # phony target to echo status
    if GetOption('update_ref'):
        p = env.Command(tgt('_update'),
                        [ref_stats, new_stats, status_file],
                        updateAction)
    else:
        p = env.Command(tgt('_print'), [status_file], printAction)

    env.AlwaysBuild(p)


# Figure out applicable configs based on build type
configs = []
if env['TARGET_ISA'] == 'alpha':
    configs += ['tsunami-simple-atomic',
                'tsunami-simple-timing',
                'tsunami-simple-atomic-dual',
                'tsunami-simple-timing-dual',
                'twosys-tsunami-simple-atomic',
                'tsunami-o3', 'tsunami-o3-dual',
                'tsunami-minor', 'tsunami-minor-dual',
                'tsunami-switcheroo-full']
if env['TARGET_ISA'] == 'sparc':
    configs += ['t1000-simple-atomic',
                't1000-simple-timing']
if env['TARGET_ISA'] == 'arm':
    configs += ['simple-atomic-dummychecker',
                'o3-timing-checker',
                'realview-simple-atomic',
                'realview-simple-atomic-dual',
                'realview-simple-timing',
                'realview-simple-timing-dual',
                'realview-o3',
                'realview-o3-checker',
                'realview-o3-dual',
                'realview-minor',
                'realview-minor-dual',
                'realview-switcheroo-atomic',
                'realview-switcheroo-timing',
                'realview-switcheroo-o3',
                'realview-switcheroo-full',
                'realview64-simple-atomic',
                'realview64-simple-atomic-dual',
                'realview64-simple-timing',
                'realview64-simple-timing-dual',
                'realview64-o3',
                'realview64-o3-checker',
                'realview64-o3-dual',
                'realview64-minor',
                'realview64-minor-dual',
                'realview64-switcheroo-atomic',
                'realview64-switcheroo-timing',
                'realview64-switcheroo-o3',
                'realview64-switcheroo-full']
if env['TARGET_ISA'] == 'x86':
    configs += ['pc-simple-atomic',
                'pc-simple-timing',
                'pc-o3-timing',
                'pc-switcheroo-full']

configs += ['simple-atomic', 'simple-atomic-mp',
            'simple-timing', 'simple-timing-mp',
            'minor-timing', 'minor-timing-mp',
            'o3-timing', 'o3-timing-mp',
            'rubytest', 'memtest', 'memtest-filter',
            'tgen-simple-mem', 'tgen-dram-ctrl']

if env['PROTOCOL'] != 'None':
    if env['PROTOCOL'] == 'MI_example':
        configs += [c + "-ruby" for c in configs]
    else:
        configs = [c + "-ruby-" + env['PROTOCOL'] for c in configs]

src = Dir('.').srcdir
for config in configs:
    dirs = src.glob('*/*/*/ref/%s/*/%s' % (env['TARGET_ISA'], config))
    for d in dirs:
        d = str(d)
        if not os.path.exists(os.path.join(d, 'skip')):
            test_builder(env, d)