From 9cfe743a2262d53fdf3637f0424faa66533ca8e4 Mon Sep 17 00:00:00 2001 From: Gabe Black Date: Fri, 27 Jul 2018 03:32:22 -0700 Subject: systemc: Teach verify.py to diff files when checking test results. Currently it just diffs the stdout and ignores other reference files. It also doesn't filter out noise in the diffs from non test related simulator messages. These include startup messages, messages when the simulator finishes executing, and some non-standard warnings, etc. Change-Id: Idcb19edd893cd8818423c2c5ebb6cbfb278baffa Reviewed-on: https://gem5-review.googlesource.com/12054 Reviewed-by: Gabe Black Maintainer: Gabe Black --- src/systemc/tests/verify.py | 137 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 106 insertions(+), 31 deletions(-) (limited to 'src/systemc/tests') diff --git a/src/systemc/tests/verify.py b/src/systemc/tests/verify.py index 0c47ce752..c8167d6c7 100755 --- a/src/systemc/tests/verify.py +++ b/src/systemc/tests/verify.py @@ -30,12 +30,15 @@ from __future__ import print_function import argparse +import collections +import difflib import functools import inspect import itertools import json import multiprocessing.pool import os +import re import subprocess import sys @@ -60,9 +63,14 @@ class Test(object): self.target = target self.suffix = suffix self.build_dir = build_dir + self.props = {} for key, val in props.iteritems(): - setattr(self, key, val) + self.set_prop(key, val) + + def set_prop(self, key, val): + setattr(self, key, val) + self.props[key] = val def dir(self): return os.path.join(self.build_dir, tests_rel_path, self.path) @@ -142,6 +150,7 @@ class RunPhase(TestPhaseBase): test.full_path(), '-red', test.m5out_dir(), '--listener-mode=off', + '--quiet', config_path ]) # Ensure the output directory exists. @@ -165,6 +174,55 @@ class RunPhase(TestPhaseBase): tp.close() tp.join() +class Checker(object): + def __init__(self, ref, test, tag): + self.ref = ref + self.test = test + self.tag = tag + + def check(self): + with open(self.text) as test_f, open(self.ref) as ref_f: + return test_f.read() == ref_f.read() + +class LogChecker(Checker): + def merge_filts(*filts): + filts = map(lambda f: '(' + f + ')', filts) + filts = '|'.join(filts) + return re.compile(filts, flags=re.MULTILINE) + + ref_filt = merge_filts( + r'^\nInfo: /OSCI/SystemC: Simulation stopped by user.\n', + r'^SystemC Simulation\n' + ) + test_filt = merge_filts( + r'^Global frequency set at \d* ticks per second\n' + ) + + def __init__(self, ref, test, tag, out_dir): + super(LogChecker, self).__init__(ref, test, tag) + self.out_dir = out_dir + + def apply_filters(self, data, filts): + re.sub(filt, '', data) + + def check(self): + test_file = os.path.basename(self.test) + ref_file = os.path.basename(self.ref) + with open(self.test) as test_f, open(self.ref) as ref_f: + test = re.sub(self.test_filt, '', test_f.read()) + ref = re.sub(self.ref_filt, '', ref_f.read()) + if test != ref: + diff_file = '.'.join([ref_file, 'diff']) + diff_path = os.path.join(self.out_dir, diff_file) + with open(diff_path, 'w') as diff_f: + for line in difflib.unified_diff( + ref.splitlines(True), test.splitlines(True), + fromfile=ref_file, + tofile=test_file): + diff_f.write(line) + return False + return True + class VerifyPhase(TestPhaseBase): name = 'verify' number = 3 @@ -176,7 +234,8 @@ class VerifyPhase(TestPhaseBase): def passed(self, test): self._passed.append(test) - def failed(self, test, cause): + def failed(self, test, cause, note=''): + test.set_prop('note', note) self._failed.setdefault(cause, []).append(test) def print_status(self): @@ -187,44 +246,36 @@ class VerifyPhase(TestPhaseBase): passed=total_passed, failed=total_failed)) def write_result_file(self, path): - passed = map(lambda t: t.path, self._passed) - passed.sort() - failed = { - cause: map(lambda t: t.path, tests) for + results = { + 'passed': map(lambda t: t.props, self._passed), + 'failed': { + cause: map(lambda t: t.props, tests) for cause, tests in self._failed.iteritems() + } } - for tests in failed.values(): - tests.sort() - results = { 'passed': passed, 'failed': failed } with open(path, 'w') as rf: json.dump(results, rf) def print_results(self): - passed = map(lambda t: t.path, self._passed) - passed.sort() - failed = { - cause: map(lambda t: t.path, tests) for - cause, tests in self._failed.iteritems() - } - for tests in failed.values(): - tests.sort() - print() print('Passed:') - map(lambda t: print(' ', t), passed) + for path in sorted(list([ t.path for t in self._passed ])): + print(' ', path) print() print('Failed:') - categories = failed.items() - categories.sort() - def cat_str((cause, tests)): - heading = ' ' + cause.capitalize() + ':\n' - test_lines = [' ' + test + '\n'for test in tests] - return heading + ''.join(test_lines) - blocks = map(cat_str, categories) + causes = [] + for cause, tests in sorted(self._failed.items()): + block = ' ' + cause.capitalize() + ':\n' + for test in sorted(tests, key=lambda t: t.path): + block += ' ' + test.path + if test.note: + block += ' - ' + test.note + block += '\n' + causes.append(block) - print('\n'.join(blocks)) + print('\n'.join(causes)) def run(self, tests): parser = argparse.ArgumentParser() @@ -252,12 +303,36 @@ class VerifyPhase(TestPhaseBase): with open(test.returncode_file()) as rc: returncode = int(rc.read()) - if returncode == 0: - self.passed(test) - elif returncode == 124: + if returncode == 124: self.failed(test, 'time out') - else: + continue + elif returncode != 0: self.failed(test, 'abort') + continue + + out_dir = test.m5out_dir() + + Diff = collections.namedtuple( + 'Diff', 'ref, test, tag, ref_filter') + + diffs = [] + + log_file = '.'.join([test.name, 'log']) + log_path = os.path.join(test.golden_dir(), log_file) + simout_path = os.path.join(out_dir, 'simout') + if not os.path.exists(simout_path): + self.failed(test, 'no log output') + if os.path.exists(log_path): + diffs.append(LogChecker( + log_path, simout_path, log_file, out_dir)) + + failed_diffs = filter(lambda d: not d.check(), diffs) + if failed_diffs: + tags = map(lambda d: d.tag, failed_diffs) + self.failed(test, 'failed diffs', ' '.join(tags)) + continue + + self.passed(test) if args.print_results: self.print_results() -- cgit v1.2.3