summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorGabe Black <gabeblack@google.com>2018-07-27 03:32:22 -0700
committerGabe Black <gabeblack@google.com>2018-09-11 21:51:26 +0000
commit9cfe743a2262d53fdf3637f0424faa66533ca8e4 (patch)
tree55e074cea83b11bf858d74736a7de30cddb48ead /src
parent80da3c3ff1d6944a1dac0640a1a1d008ec891700 (diff)
downloadgem5-9cfe743a2262d53fdf3637f0424faa66533ca8e4.tar.xz
systemc: Teach verify.py to diff files when checking test results.
Currently it just diffs the stdout and ignores other reference files. It also doesn't filter out noise in the diffs from non test related simulator messages. These include startup messages, messages when the simulator finishes executing, and some non-standard warnings, etc. Change-Id: Idcb19edd893cd8818423c2c5ebb6cbfb278baffa Reviewed-on: https://gem5-review.googlesource.com/12054 Reviewed-by: Gabe Black <gabeblack@google.com> Maintainer: Gabe Black <gabeblack@google.com>
Diffstat (limited to 'src')
-rwxr-xr-xsrc/systemc/tests/verify.py137
1 files changed, 106 insertions, 31 deletions
diff --git a/src/systemc/tests/verify.py b/src/systemc/tests/verify.py
index 0c47ce752..c8167d6c7 100755
--- a/src/systemc/tests/verify.py
+++ b/src/systemc/tests/verify.py
@@ -30,12 +30,15 @@
from __future__ import print_function
import argparse
+import collections
+import difflib
import functools
import inspect
import itertools
import json
import multiprocessing.pool
import os
+import re
import subprocess
import sys
@@ -60,9 +63,14 @@ class Test(object):
self.target = target
self.suffix = suffix
self.build_dir = build_dir
+ self.props = {}
for key, val in props.iteritems():
- setattr(self, key, val)
+ self.set_prop(key, val)
+
+ def set_prop(self, key, val):
+ setattr(self, key, val)
+ self.props[key] = val
def dir(self):
return os.path.join(self.build_dir, tests_rel_path, self.path)
@@ -142,6 +150,7 @@ class RunPhase(TestPhaseBase):
test.full_path(),
'-red', test.m5out_dir(),
'--listener-mode=off',
+ '--quiet',
config_path
])
# Ensure the output directory exists.
@@ -165,6 +174,55 @@ class RunPhase(TestPhaseBase):
tp.close()
tp.join()
+class Checker(object):
+ def __init__(self, ref, test, tag):
+ self.ref = ref
+ self.test = test
+ self.tag = tag
+
+ def check(self):
+ with open(self.text) as test_f, open(self.ref) as ref_f:
+ return test_f.read() == ref_f.read()
+
+class LogChecker(Checker):
+ def merge_filts(*filts):
+ filts = map(lambda f: '(' + f + ')', filts)
+ filts = '|'.join(filts)
+ return re.compile(filts, flags=re.MULTILINE)
+
+ ref_filt = merge_filts(
+ r'^\nInfo: /OSCI/SystemC: Simulation stopped by user.\n',
+ r'^SystemC Simulation\n'
+ )
+ test_filt = merge_filts(
+ r'^Global frequency set at \d* ticks per second\n'
+ )
+
+ def __init__(self, ref, test, tag, out_dir):
+ super(LogChecker, self).__init__(ref, test, tag)
+ self.out_dir = out_dir
+
+ def apply_filters(self, data, filts):
+ re.sub(filt, '', data)
+
+ def check(self):
+ test_file = os.path.basename(self.test)
+ ref_file = os.path.basename(self.ref)
+ with open(self.test) as test_f, open(self.ref) as ref_f:
+ test = re.sub(self.test_filt, '', test_f.read())
+ ref = re.sub(self.ref_filt, '', ref_f.read())
+ if test != ref:
+ diff_file = '.'.join([ref_file, 'diff'])
+ diff_path = os.path.join(self.out_dir, diff_file)
+ with open(diff_path, 'w') as diff_f:
+ for line in difflib.unified_diff(
+ ref.splitlines(True), test.splitlines(True),
+ fromfile=ref_file,
+ tofile=test_file):
+ diff_f.write(line)
+ return False
+ return True
+
class VerifyPhase(TestPhaseBase):
name = 'verify'
number = 3
@@ -176,7 +234,8 @@ class VerifyPhase(TestPhaseBase):
def passed(self, test):
self._passed.append(test)
- def failed(self, test, cause):
+ def failed(self, test, cause, note=''):
+ test.set_prop('note', note)
self._failed.setdefault(cause, []).append(test)
def print_status(self):
@@ -187,44 +246,36 @@ class VerifyPhase(TestPhaseBase):
passed=total_passed, failed=total_failed))
def write_result_file(self, path):
- passed = map(lambda t: t.path, self._passed)
- passed.sort()
- failed = {
- cause: map(lambda t: t.path, tests) for
+ results = {
+ 'passed': map(lambda t: t.props, self._passed),
+ 'failed': {
+ cause: map(lambda t: t.props, tests) for
cause, tests in self._failed.iteritems()
+ }
}
- for tests in failed.values():
- tests.sort()
- results = { 'passed': passed, 'failed': failed }
with open(path, 'w') as rf:
json.dump(results, rf)
def print_results(self):
- passed = map(lambda t: t.path, self._passed)
- passed.sort()
- failed = {
- cause: map(lambda t: t.path, tests) for
- cause, tests in self._failed.iteritems()
- }
- for tests in failed.values():
- tests.sort()
-
print()
print('Passed:')
- map(lambda t: print(' ', t), passed)
+ for path in sorted(list([ t.path for t in self._passed ])):
+ print(' ', path)
print()
print('Failed:')
- categories = failed.items()
- categories.sort()
- def cat_str((cause, tests)):
- heading = ' ' + cause.capitalize() + ':\n'
- test_lines = [' ' + test + '\n'for test in tests]
- return heading + ''.join(test_lines)
- blocks = map(cat_str, categories)
+ causes = []
+ for cause, tests in sorted(self._failed.items()):
+ block = ' ' + cause.capitalize() + ':\n'
+ for test in sorted(tests, key=lambda t: t.path):
+ block += ' ' + test.path
+ if test.note:
+ block += ' - ' + test.note
+ block += '\n'
+ causes.append(block)
- print('\n'.join(blocks))
+ print('\n'.join(causes))
def run(self, tests):
parser = argparse.ArgumentParser()
@@ -252,12 +303,36 @@ class VerifyPhase(TestPhaseBase):
with open(test.returncode_file()) as rc:
returncode = int(rc.read())
- if returncode == 0:
- self.passed(test)
- elif returncode == 124:
+ if returncode == 124:
self.failed(test, 'time out')
- else:
+ continue
+ elif returncode != 0:
self.failed(test, 'abort')
+ continue
+
+ out_dir = test.m5out_dir()
+
+ Diff = collections.namedtuple(
+ 'Diff', 'ref, test, tag, ref_filter')
+
+ diffs = []
+
+ log_file = '.'.join([test.name, 'log'])
+ log_path = os.path.join(test.golden_dir(), log_file)
+ simout_path = os.path.join(out_dir, 'simout')
+ if not os.path.exists(simout_path):
+ self.failed(test, 'no log output')
+ if os.path.exists(log_path):
+ diffs.append(LogChecker(
+ log_path, simout_path, log_file, out_dir))
+
+ failed_diffs = filter(lambda d: not d.check(), diffs)
+ if failed_diffs:
+ tags = map(lambda d: d.tag, failed_diffs)
+ self.failed(test, 'failed diffs', ' '.join(tags))
+ continue
+
+ self.passed(test)
if args.print_results:
self.print_results()