diff options
Diffstat (limited to 'tests/testing')
| -rw-r--r-- | tests/testing/__init__.py | 38 | ||||
| -rwxr-xr-x | tests/testing/helpers.py | 132 | ||||
| -rw-r--r-- | tests/testing/results.py | 271 | ||||
| -rw-r--r-- | tests/testing/tests.py | 343 | ||||
| -rw-r--r-- | tests/testing/units.py | 290 |
5 files changed, 1074 insertions, 0 deletions
diff --git a/tests/testing/__init__.py b/tests/testing/__init__.py new file mode 100644 index 000000000..e7c83da10 --- /dev/null +++ b/tests/testing/__init__.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg diff --git a/tests/testing/helpers.py b/tests/testing/helpers.py new file mode 100755 index 000000000..dcc48904c --- /dev/null +++ b/tests/testing/helpers.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +import subprocess +from threading import Timer +import time + +class CallTimeoutException(Exception): + """Exception that indicates that a process call timed out""" + + def __init__(self, status, stdout, stderr): + self.status = status + self.stdout = stdout + self.stderr = stderr + +class ProcessHelper(subprocess.Popen): + """Helper class to run child processes. + + This class wraps a subprocess.Popen class and adds support for + using it in a with block. When the process goes out of scope, it's + automatically terminated. + + with ProcessHelper(["/bin/ls"], stdout=subprocess.PIPE) as p: + return p.call() + """ + def __init__(self, *args, **kwargs): + super(ProcessHelper, self).__init__(*args, **kwargs) + + def _terminate_nicely(self, timeout=5): + def on_timeout(): + self.kill() + + if self.returncode is not None: + return self.returncode + + timer = Timer(timeout, on_timeout) + self.terminate() + status = self.wait() + timer.cancel() + + return status + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.returncode is None: + self._terminate_nicely() + + def call(self, timeout=0): + self._timeout = False + def on_timeout(): + self._timeout = True + self._terminate_nicely() + + status, stdout, stderr = None, None, None + timer = Timer(timeout, on_timeout) + if timeout: + timer.start() + + stdout, stderr = self.communicate() + status = self.wait() + + timer.cancel() + + if self._timeout: + self._terminate_nicely() + raise CallTimeoutException(self.returncode, stdout, stderr) + else: + return status, stdout, stderr + +if __name__ == "__main__": + # Run internal self tests to ensure that the helpers are working + # properly. The expected output when running this script is + # "SUCCESS!". + + cmd_foo = [ "/bin/echo", "-n", "foo" ] + cmd_sleep = [ "/bin/sleep", "10" ] + + # Test that things don't break if the process hasn't been started + with ProcessHelper(cmd_foo) as p: + pass + + with ProcessHelper(cmd_foo, stdout=subprocess.PIPE) as p: + status, stdout, stderr = p.call() + assert stdout == "foo" + assert status == 0 + + try: + with ProcessHelper(cmd_sleep) as p: + status, stdout, stderr = p.call(timeout=1) + assert False, "Timeout not triggered" + except CallTimeoutException: + pass + + print "SUCCESS!" diff --git a/tests/testing/results.py b/tests/testing/results.py new file mode 100644 index 000000000..0c46c9665 --- /dev/null +++ b/tests/testing/results.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +from abc import ABCMeta, abstractmethod +import inspect +import pickle +import string +import sys + +import xml.etree.cElementTree as ET + +class UnitResult(object): + """Results of a single test unit. + + A test result can be one of: + - STATE_OK: Test ran successfully. + - STATE_SKIPPED: The test was skipped. + - STATE_ERROR: The test failed to run. + - STATE_FAILED: Test ran, but failed. + + The difference between STATE_ERROR and STATE_FAILED is very + subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed + to start or crashed, while STATE_FAILED would mean that a test + failed (e.g., statistics mismatch). + + """ + + STATE_OK = 0 + STATE_SKIPPED = 1 + STATE_ERROR = 2 + STATE_FAILURE = 3 + + state_names = { + STATE_OK : "OK", + STATE_SKIPPED : "SKIPPED", + STATE_ERROR : "ERROR", + STATE_FAILURE : "FAILURE", + } + + def __init__(self, name, state, message="", stderr="", stdout="", + runtime=0.0): + self.name = name + self.state = state + self.message = message + self.stdout = stdout + self.stderr = stderr + self.runtime = runtime + + def skipped(self): + return self.state == UnitResult.STATE_SKIPPED + + def success(self): + return self.state == UnitResult.STATE_OK + + def state_name(self): + return UnitResult.state_names[self.state] + + def __nonzero__(self): + return self.success() or self.skipped() + + def __str__(self): + state_name = self.state_name() + + status = "%s: %s" % (state_name, self.message) if self.message else \ + state_name + + return "%s: %s" % (self.name, status) + +class TestResult(object): + """Results for from a single test consisting of one or more units.""" + + def __init__(self, name, results=[]): + self.name = name + self.results = results + + def success(self): + return all([ r.success() for r in self.results]) + + def skipped(self): + return all([ r.skipped() for r in self.results]) + + def failed(self): + return any([ not r for r in self.results]) + + def runtime(self): + return sum([ r.runtime for r in self.results ]) + + def __nonzero__(self): + return all([r for r in self.results]) + +class ResultFormatter(object): + __metaclass__ = ABCMeta + + def __init__(self, fout=sys.stdout, verbose=False): + self.verbose = verbose + self.fout = fout + + @abstractmethod + def dump_suites(self, suites): + pass + +class Pickle(ResultFormatter): + """Save test results as a binary using Python's pickle + functionality. + + """ + + def __init__(self, **kwargs): + super(Pickle, self).__init__(**kwargs) + + def dump_suites(self, suites): + pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) + +class Text(ResultFormatter): + """Output test results as text.""" + + def __init__(self, **kwargs): + super(Text, self).__init__(**kwargs) + + def dump_suites(self, suites): + fout = self.fout + for suite in suites: + print >> fout, "--- %s ---" % suite.name + + for t in suite.results: + print >> fout, "*** %s" % t + + if t and not self.verbose: + continue + + if t.message: + print >> fout, t.message + + if t.stderr: + print >> fout, t.stderr + if t.stdout: + print >> fout, t.stdout + +class TextSummary(ResultFormatter): + """Output test results as a text summary""" + + def __init__(self, **kwargs): + super(TextSummary, self).__init__(**kwargs) + + def dump_suites(self, suites): + fout = self.fout + for suite in suites: + status = "SKIPPED" if suite.skipped() else \ + ("OK" if suite else "FAILED") + print >> fout, "%s: %s" % (suite.name, status) + +class JUnit(ResultFormatter): + """Output test results as JUnit XML""" + + def __init__(self, translate_names=True, **kwargs): + super(JUnit, self).__init__(**kwargs) + + if translate_names: + self.name_table = string.maketrans( + "/.", + ".-", + ) + else: + self.name_table = string.maketrans("", "") + + def convert_unit(self, x_suite, test): + x_test = ET.SubElement(x_suite, "testcase", + name=test.name, + time="%f" % test.runtime) + + x_state = None + if test.state == UnitResult.STATE_OK: + pass + elif test.state == UnitResult.STATE_SKIPPED: + x_state = ET.SubElement(x_test, "skipped") + elif test.state == UnitResult.STATE_FAILURE: + x_state = ET.SubElement(x_test, "failure") + elif test.state == UnitResult.STATE_ERROR: + x_state = ET.SubElement(x_test, "error") + else: + assert False, "Unknown test state" + + if x_state is not None: + if test.message: + x_state.set("message", test.message) + + msg = [] + if test.stderr: + msg.append("*** Standard Errror: ***") + msg.append(test.stderr) + if test.stdout: + msg.append("*** Standard Out: ***") + msg.append(test.stdout) + + x_state.text = "\n".join(msg) + + return x_test + + def convert_suite(self, x_suites, suite): + x_suite = ET.SubElement(x_suites, "testsuite", + name=suite.name.translate(self.name_table), + time="%f" % suite.runtime()) + errors = 0 + failures = 0 + skipped = 0 + + for test in suite.results: + if test.state != UnitResult.STATE_OK: + if test.state == UnitResult.STATE_SKIPPED: + skipped += 1 + elif test.state == UnitResult.STATE_ERROR: + errors += 1 + elif test.state == UnitResult.STATE_FAILURE: + failures += 1 + + x_test = self.convert_unit(x_suite, test) + + x_suite.set("errors", str(errors)) + x_suite.set("failures", str(failures)) + x_suite.set("skipped", str(skipped)) + x_suite.set("tests", str(len(suite.results))) + + return x_suite + + def convert_suites(self, suites): + x_root = ET.Element("testsuites") + + for suite in suites: + self.convert_suite(x_root, suite) + + return x_root + + def dump_suites(self, suites): + et = ET.ElementTree(self.convert_suites(suites)) + et.write(self.fout, encoding="UTF-8") diff --git a/tests/testing/tests.py b/tests/testing/tests.py new file mode 100644 index 000000000..4c467f25c --- /dev/null +++ b/tests/testing/tests.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +from abc import ABCMeta, abstractmethod +import os +from collections import namedtuple +from units import * +from results import TestResult +import shutil + +_test_base = os.path.join(os.path.dirname(__file__), "..") + +ClassicConfig = namedtuple("ClassicConfig", ( + "category", + "mode", + "workload", + "isa", + "os", + "config", +)) + +# There are currently two "classes" of test +# configurations. Architecture-specific ones and generic ones +# (typically SE mode tests). In both cases, the configuration name +# matches a file in tests/configs/ that will be picked up by the test +# runner (run.py). +# +# Architecture specific configurations are listed in the arch_configs +# dictionary. This is indexed by a (cpu architecture, gpu +# architecture) tuple. GPU architecture is optional and may be None. +# +# Generic configurations are listed in the generic_configs tuple. +# +# When discovering available test cases, this script look uses the +# test list as a list of /candidate/ configurations. A configuration +# is only used if a test has a reference output for that +# configuration. In addition to the base configurations from +# arch_configs and generic_configs, a Ruby configuration may be +# appended to the base name (this is probed /in addition/ to the +# original name. See get_tests() for details. +# +arch_configs = { + ("alpha", None) : ( + 'tsunami-simple-atomic', + 'tsunami-simple-timing', + 'tsunami-simple-atomic-dual', + 'tsunami-simple-timing-dual', + 'twosys-tsunami-simple-atomic', + 'tsunami-o3', 'tsunami-o3-dual', + 'tsunami-minor', 'tsunami-minor-dual', + 'tsunami-switcheroo-full', + ), + + ("arm", None) : ( + 'simple-atomic-dummychecker', + 'o3-timing-checker', + 'realview-simple-atomic', + 'realview-simple-atomic-dual', + 'realview-simple-atomic-checkpoint', + 'realview-simple-timing', + 'realview-simple-timing-dual', + 'realview-o3', + 'realview-o3-checker', + 'realview-o3-dual', + 'realview-minor', + 'realview-minor-dual', + 'realview-switcheroo-atomic', + 'realview-switcheroo-timing', + 'realview-switcheroo-o3', + 'realview-switcheroo-full', + 'realview64-simple-atomic', + 'realview64-simple-atomic-checkpoint', + 'realview64-simple-atomic-dual', + 'realview64-simple-timing', + 'realview64-simple-timing-dual', + 'realview64-o3', + 'realview64-o3-checker', + 'realview64-o3-dual', + 'realview64-minor', + 'realview64-minor-dual', + 'realview64-switcheroo-atomic', + 'realview64-switcheroo-timing', + 'realview64-switcheroo-o3', + 'realview64-switcheroo-full', + ), + + ("sparc", None) : ( + 't1000-simple-atomic', + 't1000-simple-x86', + ), + + ("timing", None) : ( + 'pc-simple-atomic', + 'pc-simple-timing', + 'pc-o3-timing', + 'pc-switcheroo-full', + ), + + ("x86", "hsail") : ( + 'gpu', + ), +} + +generic_configs = ( + 'simple-atomic', + 'simple-atomic-mp', + 'simple-timing', + 'simple-timing-mp', + + 'minor-timing', + 'minor-timing-mp', + + 'o3-timing', + 'o3-timing-mt', + 'o3-timing-mp', + + 'rubytest', + 'memcheck', + 'memtest', + 'memtest-filter', + 'tgen-simple-mem', + 'tgen-dram-ctrl', + + 'learning-gem5-p1-simple', + 'learning-gem5-p1-two-level', +) + +all_categories = ("quick", "long") +all_modes = ("fs", "se") + +class Test(object): + """Test case base class. + + Test cases consists of one or more test units that are run in two + phases. A run phase (units produced by run_units() and a verify + phase (units from verify_units()). The verify phase is skipped if + the run phase fails. + + """ + + __metaclass__ = ABCMeta + + def __init__(self, name): + self.test_name = name + + @abstractmethod + def ref_files(self): + """Get a list of reference files used by this test case""" + pass + + @abstractmethod + def run_units(self): + """Units (typically RunGem5 instances) that describe the run phase of + this test. + + """ + pass + + @abstractmethod + def verify_units(self): + """Verify the output from the run phase (see run_units()).""" + pass + + @abstractmethod + def update_ref(self): + """Update reference files with files from a test run""" + pass + + def run(self): + """Run this test case and return a list of results""" + + run_results = [ u.run() for u in self.run_units() ] + run_ok = all([not r.skipped() and r for r in run_results ]) + + verify_results = [ + u.run() if run_ok else u.skip() + for u in self.verify_units() + ] + + return TestResult(self.test_name, run_results + verify_results) + + def __str__(self): + return self.test_name + +class ClassicTest(Test): + diff_ignore_files = [ + # Stat files use a special stat differ, so don't include them + # here. + "stats.txt", + ] + + def __init__(self, gem5, output_dir, config_tuple, + timeout=None, + skip=False, skip_diff_out=False, skip_diff_stat=False): + + super(ClassicTest, self).__init__("/".join(config_tuple)) + + ct = config_tuple + + self.gem5 = os.path.abspath(gem5) + self.script = os.path.join(_test_base, "run.py") + self.config_tuple = ct + self.timeout = timeout + + self.output_dir = output_dir + self.ref_dir = os.path.join(_test_base, + ct.category, ct.mode, ct.workload, + "ref", ct.isa, ct.os, ct.config) + self.skip_run = skip + self.skip_diff_out = skip or skip_diff_out + self.skip_diff_stat = skip or skip_diff_stat + + def ref_files(self): + ref_dir = os.path.abspath(self.ref_dir) + for root, dirs, files in os.walk(ref_dir, topdown=False): + for f in files: + fpath = os.path.join(root[len(ref_dir) + 1:], f) + if fpath not in ClassicTest.diff_ignore_files: + yield fpath + + def run_units(self): + args = [ + self.script, + "/".join(self.config_tuple), + ] + + return [ + RunGem5(self.gem5, args, + ref_dir=self.ref_dir, test_dir=self.output_dir, + skip=self.skip_run), + ] + + def verify_units(self): + return [ + DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir, + skip=self.skip_diff_stat) + ] + [ + DiffOutFile(f, + ref_dir=self.ref_dir, test_dir=self.output_dir, + skip=self.skip_diff_out) + for f in self.ref_files() + if f not in ClassicTest.diff_ignore_files + ] + + def update_ref(self): + for fname in self.ref_files(): + shutil.copy( + os.path.join(self.output_dir, fname), + os.path.join(self.ref_dir, fname)) + +def parse_test_filter(test_filter): + wildcards = ("", "*") + + _filter = list(test_filter.split("/")) + if len(_filter) > 3: + raise RuntimeError("Illegal test filter string") + _filter += [ "", ] * (3 - len(_filter)) + + isa, cat, mode = _filter + + if isa in wildcards: + raise RuntimeError("No ISA specified") + + cat = all_categories if cat in wildcards else (cat, ) + mode = all_modes if mode in wildcards else (mode, ) + + return isa, cat, mode + +def get_tests(isa, + categories=all_categories, modes=all_modes, + ruby_protocol=None, gpu_isa=None): + + # Generate a list of candidate configs + configs = list(arch_configs.get((isa, gpu_isa), [])) + + if (isa, gpu_isa) == ("x86", "hsail"): + if ruby_protocol == "GPU_RfO": + configs += ['gpu-randomtest'] + else: + configs += generic_configs + + if ruby_protocol == 'MI_example': + configs += [ "%s-ruby" % (c, ) for c in configs ] + elif ruby_protocol is not None: + configs += [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ] + + # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/ + for conf_script in configs: + for cat in categories: + for mode in modes: + mode_dir = os.path.join(_test_base, cat, mode) + if not os.path.exists(mode_dir): + continue + + for workload in os.listdir(mode_dir): + isa_dir = os.path.join(mode_dir, workload, "ref", isa) + if not os.path.isdir(isa_dir): + continue + + for _os in os.listdir(isa_dir): + test_dir = os.path.join(isa_dir, _os, conf_script) + if not os.path.exists(test_dir) or \ + os.path.exists(os.path.join(test_dir, "skip")): + continue + + yield ClassicConfig(cat, mode, workload, isa, _os, + conf_script) diff --git a/tests/testing/units.py b/tests/testing/units.py new file mode 100644 index 000000000..6214c8f14 --- /dev/null +++ b/tests/testing/units.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +from abc import ABCMeta, abstractmethod +from datetime import datetime +import difflib +import functools +import os +import re +import subprocess +import sys +import traceback + +from results import UnitResult +from helpers import * + +_test_base = os.path.join(os.path.dirname(__file__), "..") + +class TestUnit(object): + """Base class for all test units. + + A test unit is a part of a larger test case. Test cases usually + contain two types of units, run units (run gem5) and verify units + (diff output files). All unit implementations inherit from this + class. + + A unit implementation overrides the _run() method. The test runner + calls the run() method, which wraps _run() to protect against + exceptions. + + """ + + __metaclass__ = ABCMeta + + def __init__(self, name, ref_dir, test_dir, skip=False): + self.name = name + self.ref_dir = ref_dir + self.test_dir = test_dir + self.force_skip = skip + self.start_time = None + self.stop_time = None + + def result(self, state, **kwargs): + if self.start_time is not None and "runtime" not in kwargs: + self.stop_time = datetime.utcnow() + delta = self.stop_time - self.start_time + kwargs["runtime"] = delta.total_seconds() + + return UnitResult(self.name, state, **kwargs) + + def ok(self, **kwargs): + return self.result(UnitResult.STATE_OK, **kwargs) + + def skip(self, **kwargs): + return self.result(UnitResult.STATE_SKIPPED, **kwargs) + + def error(self, message, **kwargs): + return self.result(UnitResult.STATE_ERROR, message=message, **kwargs) + + def failure(self, message, **kwargs): + return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs) + + def ref_file(self, fname): + return os.path.join(self.ref_dir, fname) + + def out_file(self, fname): + return os.path.join(self.test_dir, fname) + + def _read_output(self, fname, default=""): + try: + with open(self.out_file(fname), "r") as f: + return f.read() + except IOError: + return default + + def run(self): + self.start_time = datetime.utcnow() + try: + if self.force_skip: + return self.skip() + else: + return self._run() + except: + return self.error("Python exception:\n%s" % traceback.format_exc()) + + @abstractmethod + def _run(self): + pass + +class RunGem5(TestUnit): + """Test unit representing a gem5 run. + + Possible failure modes: + - gem5 failed to run -> STATE_ERROR + - timeout -> STATE_ERROR + - non-zero exit code -> STATE_ERROR + + Possible non-failure results: + - exit code == 0 -> STATE_OK + - exit code == 2 -> STATE_SKIPPED + """ + + def __init__(self, gem5, gem5_args, timeout=0, **kwargs): + super(RunGem5, self).__init__("gem5", **kwargs) + self.gem5 = gem5 + self.args = gem5_args + self.timeout = timeout + + def _run(self): + gem5_cmd = [ + self.gem5, + "-d", self.test_dir, + "-re", + ] + self.args + + try: + with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout) + except CallTimeoutException as te: + return self.error("Timeout", stdout=te.stdout, stderr=te.stderr) + except OSError as ose: + return self.error("Failed to launch gem5: %s" % ose) + + stderr = "\n".join([ + "*** gem5 stderr ***", + gem5_stderr, + "", + "*** m5out/simerr ***", + self._read_output("simerr"), + ]) + + stdout = "\n".join([ + "*** gem5 stdout ***", + gem5_stdout, + "", + "*** m5out/simout ***", + self._read_output("simout"), + ]) + + # Signal + if status < 0: + return self.error("gem5 terminated by signal %i" % (-status, ), + stdout=stdout, stderr=stderr) + elif status == 2: + return self.skip(stdout=stdout, stderr=stderr) + elif status > 0: + return self.error("gem5 exited with non-zero status: %i" % status, + stdout=stdout, stderr=stderr) + else: + return self.ok(stdout=stdout, stderr=stderr) + +class DiffOutFile(TestUnit): + """Test unit comparing and output file and a reference file.""" + + # regular expressions of lines to ignore when diffing outputs + diff_ignore_regexes = { + "simout" : [ + re.compile('^Redirecting (stdout|stderr) to'), + re.compile('^gem5 compiled '), + re.compile('^gem5 started '), + re.compile('^gem5 executing on '), + re.compile('^command line:'), + re.compile("^Couldn't import dot_parser,"), + re.compile("^info: kernel located at:"), + re.compile("^Couldn't unlink "), + re.compile("^Using GPU kernel code file\(s\) "), + ], + "simerr" : [ + #re.compile('^Simulation complete at'), + ], + "config.ini" : [ + re.compile("^(executable|readfile|kernel|image_file)="), + re.compile("^(cwd|input|codefile)="), + ], + "config.json" : [ + re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''), + re.compile(r'''^\s*"(cwd|input|codefile)":'''), + ], + } + + def __init__(self, fname, **kwargs): + super(DiffOutFile, self).__init__("diff[%s]" % fname, + **kwargs) + + self.fname = fname + self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple()) + + def _filter_file(self, fname): + def match_line(l): + for r in self.line_filters: + if r.match(l): + return True + return False + + with open(fname, "r") as f: + for l in f: + if not match_line(l): + yield l + + + def _run(self): + fname = self.fname + ref = self.ref_file(fname) + out = self.out_file(fname) + + if not os.path.exists(ref): + return self.error("%s doesn't exist in reference directory" \ + % fname) + + if not os.path.exists(out): + return self.error("%s doesn't exist in output directory" % fname) + + diff = difflib.unified_diff( + tuple(self._filter_file(ref)), + tuple(self._filter_file(out)), + fromfile="ref/%s" % fname, tofile="out/%s" % fname) + + diff = list(diff) + if diff: + return self.error("ref/%s and out/%s differ" % (fname, fname), + stderr="".join(diff)) + else: + return self.ok(stdout="-- ref/%s and out/%s are identical --" \ + % (fname, fname)) + +class DiffStatFile(TestUnit): + """Test unit comparing two gem5 stat files.""" + + def __init__(self, **kwargs): + super(DiffStatFile, self).__init__("stat_diff", **kwargs) + + self.stat_diff = os.path.join(_test_base, "diff-out") + + def _run(self): + stats = "stats.txt" + + cmd = [ + self.stat_diff, + self.ref_file(stats), self.out_file(stats), + ] + with ProcessHelper(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + status, stdout, stderr = p.call() + + if status == 0: + return self.ok(stdout=stdout, stderr=stderr) + if status == 1: + return self.failure("Statistics mismatch", + stdout=stdout, stderr=stderr) + else: + return self.error("diff-out returned an error: %i" % status, + stdout=stdout, stderr=stderr) |