diff options
| -rw-r--r-- | tests/testing/__init__.py | 38 | ||||
| -rwxr-xr-x | tests/testing/helpers.py | 132 | ||||
| -rw-r--r-- | tests/testing/results.py | 271 | ||||
| -rw-r--r-- | tests/testing/tests.py | 343 | ||||
| -rw-r--r-- | tests/testing/units.py | 290 | ||||
| -rwxr-xr-x | tests/tests.py | 287 |
6 files changed, 1361 insertions, 0 deletions
diff --git a/tests/testing/__init__.py b/tests/testing/__init__.py new file mode 100644 index 000000000..e7c83da10 --- /dev/null +++ b/tests/testing/__init__.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg diff --git a/tests/testing/helpers.py b/tests/testing/helpers.py new file mode 100755 index 000000000..dcc48904c --- /dev/null +++ b/tests/testing/helpers.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +import subprocess +from threading import Timer +import time + +class CallTimeoutException(Exception): + """Exception that indicates that a process call timed out""" + + def __init__(self, status, stdout, stderr): + self.status = status + self.stdout = stdout + self.stderr = stderr + +class ProcessHelper(subprocess.Popen): + """Helper class to run child processes. + + This class wraps a subprocess.Popen class and adds support for + using it in a with block. When the process goes out of scope, it's + automatically terminated. + + with ProcessHelper(["/bin/ls"], stdout=subprocess.PIPE) as p: + return p.call() + """ + def __init__(self, *args, **kwargs): + super(ProcessHelper, self).__init__(*args, **kwargs) + + def _terminate_nicely(self, timeout=5): + def on_timeout(): + self.kill() + + if self.returncode is not None: + return self.returncode + + timer = Timer(timeout, on_timeout) + self.terminate() + status = self.wait() + timer.cancel() + + return status + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.returncode is None: + self._terminate_nicely() + + def call(self, timeout=0): + self._timeout = False + def on_timeout(): + self._timeout = True + self._terminate_nicely() + + status, stdout, stderr = None, None, None + timer = Timer(timeout, on_timeout) + if timeout: + timer.start() + + stdout, stderr = self.communicate() + status = self.wait() + + timer.cancel() + + if self._timeout: + self._terminate_nicely() + raise CallTimeoutException(self.returncode, stdout, stderr) + else: + return status, stdout, stderr + +if __name__ == "__main__": + # Run internal self tests to ensure that the helpers are working + # properly. The expected output when running this script is + # "SUCCESS!". + + cmd_foo = [ "/bin/echo", "-n", "foo" ] + cmd_sleep = [ "/bin/sleep", "10" ] + + # Test that things don't break if the process hasn't been started + with ProcessHelper(cmd_foo) as p: + pass + + with ProcessHelper(cmd_foo, stdout=subprocess.PIPE) as p: + status, stdout, stderr = p.call() + assert stdout == "foo" + assert status == 0 + + try: + with ProcessHelper(cmd_sleep) as p: + status, stdout, stderr = p.call(timeout=1) + assert False, "Timeout not triggered" + except CallTimeoutException: + pass + + print "SUCCESS!" diff --git a/tests/testing/results.py b/tests/testing/results.py new file mode 100644 index 000000000..0c46c9665 --- /dev/null +++ b/tests/testing/results.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +from abc import ABCMeta, abstractmethod +import inspect +import pickle +import string +import sys + +import xml.etree.cElementTree as ET + +class UnitResult(object): + """Results of a single test unit. + + A test result can be one of: + - STATE_OK: Test ran successfully. + - STATE_SKIPPED: The test was skipped. + - STATE_ERROR: The test failed to run. + - STATE_FAILED: Test ran, but failed. + + The difference between STATE_ERROR and STATE_FAILED is very + subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed + to start or crashed, while STATE_FAILED would mean that a test + failed (e.g., statistics mismatch). + + """ + + STATE_OK = 0 + STATE_SKIPPED = 1 + STATE_ERROR = 2 + STATE_FAILURE = 3 + + state_names = { + STATE_OK : "OK", + STATE_SKIPPED : "SKIPPED", + STATE_ERROR : "ERROR", + STATE_FAILURE : "FAILURE", + } + + def __init__(self, name, state, message="", stderr="", stdout="", + runtime=0.0): + self.name = name + self.state = state + self.message = message + self.stdout = stdout + self.stderr = stderr + self.runtime = runtime + + def skipped(self): + return self.state == UnitResult.STATE_SKIPPED + + def success(self): + return self.state == UnitResult.STATE_OK + + def state_name(self): + return UnitResult.state_names[self.state] + + def __nonzero__(self): + return self.success() or self.skipped() + + def __str__(self): + state_name = self.state_name() + + status = "%s: %s" % (state_name, self.message) if self.message else \ + state_name + + return "%s: %s" % (self.name, status) + +class TestResult(object): + """Results for from a single test consisting of one or more units.""" + + def __init__(self, name, results=[]): + self.name = name + self.results = results + + def success(self): + return all([ r.success() for r in self.results]) + + def skipped(self): + return all([ r.skipped() for r in self.results]) + + def failed(self): + return any([ not r for r in self.results]) + + def runtime(self): + return sum([ r.runtime for r in self.results ]) + + def __nonzero__(self): + return all([r for r in self.results]) + +class ResultFormatter(object): + __metaclass__ = ABCMeta + + def __init__(self, fout=sys.stdout, verbose=False): + self.verbose = verbose + self.fout = fout + + @abstractmethod + def dump_suites(self, suites): + pass + +class Pickle(ResultFormatter): + """Save test results as a binary using Python's pickle + functionality. + + """ + + def __init__(self, **kwargs): + super(Pickle, self).__init__(**kwargs) + + def dump_suites(self, suites): + pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) + +class Text(ResultFormatter): + """Output test results as text.""" + + def __init__(self, **kwargs): + super(Text, self).__init__(**kwargs) + + def dump_suites(self, suites): + fout = self.fout + for suite in suites: + print >> fout, "--- %s ---" % suite.name + + for t in suite.results: + print >> fout, "*** %s" % t + + if t and not self.verbose: + continue + + if t.message: + print >> fout, t.message + + if t.stderr: + print >> fout, t.stderr + if t.stdout: + print >> fout, t.stdout + +class TextSummary(ResultFormatter): + """Output test results as a text summary""" + + def __init__(self, **kwargs): + super(TextSummary, self).__init__(**kwargs) + + def dump_suites(self, suites): + fout = self.fout + for suite in suites: + status = "SKIPPED" if suite.skipped() else \ + ("OK" if suite else "FAILED") + print >> fout, "%s: %s" % (suite.name, status) + +class JUnit(ResultFormatter): + """Output test results as JUnit XML""" + + def __init__(self, translate_names=True, **kwargs): + super(JUnit, self).__init__(**kwargs) + + if translate_names: + self.name_table = string.maketrans( + "/.", + ".-", + ) + else: + self.name_table = string.maketrans("", "") + + def convert_unit(self, x_suite, test): + x_test = ET.SubElement(x_suite, "testcase", + name=test.name, + time="%f" % test.runtime) + + x_state = None + if test.state == UnitResult.STATE_OK: + pass + elif test.state == UnitResult.STATE_SKIPPED: + x_state = ET.SubElement(x_test, "skipped") + elif test.state == UnitResult.STATE_FAILURE: + x_state = ET.SubElement(x_test, "failure") + elif test.state == UnitResult.STATE_ERROR: + x_state = ET.SubElement(x_test, "error") + else: + assert False, "Unknown test state" + + if x_state is not None: + if test.message: + x_state.set("message", test.message) + + msg = [] + if test.stderr: + msg.append("*** Standard Errror: ***") + msg.append(test.stderr) + if test.stdout: + msg.append("*** Standard Out: ***") + msg.append(test.stdout) + + x_state.text = "\n".join(msg) + + return x_test + + def convert_suite(self, x_suites, suite): + x_suite = ET.SubElement(x_suites, "testsuite", + name=suite.name.translate(self.name_table), + time="%f" % suite.runtime()) + errors = 0 + failures = 0 + skipped = 0 + + for test in suite.results: + if test.state != UnitResult.STATE_OK: + if test.state == UnitResult.STATE_SKIPPED: + skipped += 1 + elif test.state == UnitResult.STATE_ERROR: + errors += 1 + elif test.state == UnitResult.STATE_FAILURE: + failures += 1 + + x_test = self.convert_unit(x_suite, test) + + x_suite.set("errors", str(errors)) + x_suite.set("failures", str(failures)) + x_suite.set("skipped", str(skipped)) + x_suite.set("tests", str(len(suite.results))) + + return x_suite + + def convert_suites(self, suites): + x_root = ET.Element("testsuites") + + for suite in suites: + self.convert_suite(x_root, suite) + + return x_root + + def dump_suites(self, suites): + et = ET.ElementTree(self.convert_suites(suites)) + et.write(self.fout, encoding="UTF-8") diff --git a/tests/testing/tests.py b/tests/testing/tests.py new file mode 100644 index 000000000..4c467f25c --- /dev/null +++ b/tests/testing/tests.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +from abc import ABCMeta, abstractmethod +import os +from collections import namedtuple +from units import * +from results import TestResult +import shutil + +_test_base = os.path.join(os.path.dirname(__file__), "..") + +ClassicConfig = namedtuple("ClassicConfig", ( + "category", + "mode", + "workload", + "isa", + "os", + "config", +)) + +# There are currently two "classes" of test +# configurations. Architecture-specific ones and generic ones +# (typically SE mode tests). In both cases, the configuration name +# matches a file in tests/configs/ that will be picked up by the test +# runner (run.py). +# +# Architecture specific configurations are listed in the arch_configs +# dictionary. This is indexed by a (cpu architecture, gpu +# architecture) tuple. GPU architecture is optional and may be None. +# +# Generic configurations are listed in the generic_configs tuple. +# +# When discovering available test cases, this script look uses the +# test list as a list of /candidate/ configurations. A configuration +# is only used if a test has a reference output for that +# configuration. In addition to the base configurations from +# arch_configs and generic_configs, a Ruby configuration may be +# appended to the base name (this is probed /in addition/ to the +# original name. See get_tests() for details. +# +arch_configs = { + ("alpha", None) : ( + 'tsunami-simple-atomic', + 'tsunami-simple-timing', + 'tsunami-simple-atomic-dual', + 'tsunami-simple-timing-dual', + 'twosys-tsunami-simple-atomic', + 'tsunami-o3', 'tsunami-o3-dual', + 'tsunami-minor', 'tsunami-minor-dual', + 'tsunami-switcheroo-full', + ), + + ("arm", None) : ( + 'simple-atomic-dummychecker', + 'o3-timing-checker', + 'realview-simple-atomic', + 'realview-simple-atomic-dual', + 'realview-simple-atomic-checkpoint', + 'realview-simple-timing', + 'realview-simple-timing-dual', + 'realview-o3', + 'realview-o3-checker', + 'realview-o3-dual', + 'realview-minor', + 'realview-minor-dual', + 'realview-switcheroo-atomic', + 'realview-switcheroo-timing', + 'realview-switcheroo-o3', + 'realview-switcheroo-full', + 'realview64-simple-atomic', + 'realview64-simple-atomic-checkpoint', + 'realview64-simple-atomic-dual', + 'realview64-simple-timing', + 'realview64-simple-timing-dual', + 'realview64-o3', + 'realview64-o3-checker', + 'realview64-o3-dual', + 'realview64-minor', + 'realview64-minor-dual', + 'realview64-switcheroo-atomic', + 'realview64-switcheroo-timing', + 'realview64-switcheroo-o3', + 'realview64-switcheroo-full', + ), + + ("sparc", None) : ( + 't1000-simple-atomic', + 't1000-simple-x86', + ), + + ("timing", None) : ( + 'pc-simple-atomic', + 'pc-simple-timing', + 'pc-o3-timing', + 'pc-switcheroo-full', + ), + + ("x86", "hsail") : ( + 'gpu', + ), +} + +generic_configs = ( + 'simple-atomic', + 'simple-atomic-mp', + 'simple-timing', + 'simple-timing-mp', + + 'minor-timing', + 'minor-timing-mp', + + 'o3-timing', + 'o3-timing-mt', + 'o3-timing-mp', + + 'rubytest', + 'memcheck', + 'memtest', + 'memtest-filter', + 'tgen-simple-mem', + 'tgen-dram-ctrl', + + 'learning-gem5-p1-simple', + 'learning-gem5-p1-two-level', +) + +all_categories = ("quick", "long") +all_modes = ("fs", "se") + +class Test(object): + """Test case base class. + + Test cases consists of one or more test units that are run in two + phases. A run phase (units produced by run_units() and a verify + phase (units from verify_units()). The verify phase is skipped if + the run phase fails. + + """ + + __metaclass__ = ABCMeta + + def __init__(self, name): + self.test_name = name + + @abstractmethod + def ref_files(self): + """Get a list of reference files used by this test case""" + pass + + @abstractmethod + def run_units(self): + """Units (typically RunGem5 instances) that describe the run phase of + this test. + + """ + pass + + @abstractmethod + def verify_units(self): + """Verify the output from the run phase (see run_units()).""" + pass + + @abstractmethod + def update_ref(self): + """Update reference files with files from a test run""" + pass + + def run(self): + """Run this test case and return a list of results""" + + run_results = [ u.run() for u in self.run_units() ] + run_ok = all([not r.skipped() and r for r in run_results ]) + + verify_results = [ + u.run() if run_ok else u.skip() + for u in self.verify_units() + ] + + return TestResult(self.test_name, run_results + verify_results) + + def __str__(self): + return self.test_name + +class ClassicTest(Test): + diff_ignore_files = [ + # Stat files use a special stat differ, so don't include them + # here. + "stats.txt", + ] + + def __init__(self, gem5, output_dir, config_tuple, + timeout=None, + skip=False, skip_diff_out=False, skip_diff_stat=False): + + super(ClassicTest, self).__init__("/".join(config_tuple)) + + ct = config_tuple + + self.gem5 = os.path.abspath(gem5) + self.script = os.path.join(_test_base, "run.py") + self.config_tuple = ct + self.timeout = timeout + + self.output_dir = output_dir + self.ref_dir = os.path.join(_test_base, + ct.category, ct.mode, ct.workload, + "ref", ct.isa, ct.os, ct.config) + self.skip_run = skip + self.skip_diff_out = skip or skip_diff_out + self.skip_diff_stat = skip or skip_diff_stat + + def ref_files(self): + ref_dir = os.path.abspath(self.ref_dir) + for root, dirs, files in os.walk(ref_dir, topdown=False): + for f in files: + fpath = os.path.join(root[len(ref_dir) + 1:], f) + if fpath not in ClassicTest.diff_ignore_files: + yield fpath + + def run_units(self): + args = [ + self.script, + "/".join(self.config_tuple), + ] + + return [ + RunGem5(self.gem5, args, + ref_dir=self.ref_dir, test_dir=self.output_dir, + skip=self.skip_run), + ] + + def verify_units(self): + return [ + DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir, + skip=self.skip_diff_stat) + ] + [ + DiffOutFile(f, + ref_dir=self.ref_dir, test_dir=self.output_dir, + skip=self.skip_diff_out) + for f in self.ref_files() + if f not in ClassicTest.diff_ignore_files + ] + + def update_ref(self): + for fname in self.ref_files(): + shutil.copy( + os.path.join(self.output_dir, fname), + os.path.join(self.ref_dir, fname)) + +def parse_test_filter(test_filter): + wildcards = ("", "*") + + _filter = list(test_filter.split("/")) + if len(_filter) > 3: + raise RuntimeError("Illegal test filter string") + _filter += [ "", ] * (3 - len(_filter)) + + isa, cat, mode = _filter + + if isa in wildcards: + raise RuntimeError("No ISA specified") + + cat = all_categories if cat in wildcards else (cat, ) + mode = all_modes if mode in wildcards else (mode, ) + + return isa, cat, mode + +def get_tests(isa, + categories=all_categories, modes=all_modes, + ruby_protocol=None, gpu_isa=None): + + # Generate a list of candidate configs + configs = list(arch_configs.get((isa, gpu_isa), [])) + + if (isa, gpu_isa) == ("x86", "hsail"): + if ruby_protocol == "GPU_RfO": + configs += ['gpu-randomtest'] + else: + configs += generic_configs + + if ruby_protocol == 'MI_example': + configs += [ "%s-ruby" % (c, ) for c in configs ] + elif ruby_protocol is not None: + configs += [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ] + + # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/ + for conf_script in configs: + for cat in categories: + for mode in modes: + mode_dir = os.path.join(_test_base, cat, mode) + if not os.path.exists(mode_dir): + continue + + for workload in os.listdir(mode_dir): + isa_dir = os.path.join(mode_dir, workload, "ref", isa) + if not os.path.isdir(isa_dir): + continue + + for _os in os.listdir(isa_dir): + test_dir = os.path.join(isa_dir, _os, conf_script) + if not os.path.exists(test_dir) or \ + os.path.exists(os.path.join(test_dir, "skip")): + continue + + yield ClassicConfig(cat, mode, workload, isa, _os, + conf_script) diff --git a/tests/testing/units.py b/tests/testing/units.py new file mode 100644 index 000000000..6214c8f14 --- /dev/null +++ b/tests/testing/units.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +from abc import ABCMeta, abstractmethod +from datetime import datetime +import difflib +import functools +import os +import re +import subprocess +import sys +import traceback + +from results import UnitResult +from helpers import * + +_test_base = os.path.join(os.path.dirname(__file__), "..") + +class TestUnit(object): + """Base class for all test units. + + A test unit is a part of a larger test case. Test cases usually + contain two types of units, run units (run gem5) and verify units + (diff output files). All unit implementations inherit from this + class. + + A unit implementation overrides the _run() method. The test runner + calls the run() method, which wraps _run() to protect against + exceptions. + + """ + + __metaclass__ = ABCMeta + + def __init__(self, name, ref_dir, test_dir, skip=False): + self.name = name + self.ref_dir = ref_dir + self.test_dir = test_dir + self.force_skip = skip + self.start_time = None + self.stop_time = None + + def result(self, state, **kwargs): + if self.start_time is not None and "runtime" not in kwargs: + self.stop_time = datetime.utcnow() + delta = self.stop_time - self.start_time + kwargs["runtime"] = delta.total_seconds() + + return UnitResult(self.name, state, **kwargs) + + def ok(self, **kwargs): + return self.result(UnitResult.STATE_OK, **kwargs) + + def skip(self, **kwargs): + return self.result(UnitResult.STATE_SKIPPED, **kwargs) + + def error(self, message, **kwargs): + return self.result(UnitResult.STATE_ERROR, message=message, **kwargs) + + def failure(self, message, **kwargs): + return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs) + + def ref_file(self, fname): + return os.path.join(self.ref_dir, fname) + + def out_file(self, fname): + return os.path.join(self.test_dir, fname) + + def _read_output(self, fname, default=""): + try: + with open(self.out_file(fname), "r") as f: + return f.read() + except IOError: + return default + + def run(self): + self.start_time = datetime.utcnow() + try: + if self.force_skip: + return self.skip() + else: + return self._run() + except: + return self.error("Python exception:\n%s" % traceback.format_exc()) + + @abstractmethod + def _run(self): + pass + +class RunGem5(TestUnit): + """Test unit representing a gem5 run. + + Possible failure modes: + - gem5 failed to run -> STATE_ERROR + - timeout -> STATE_ERROR + - non-zero exit code -> STATE_ERROR + + Possible non-failure results: + - exit code == 0 -> STATE_OK + - exit code == 2 -> STATE_SKIPPED + """ + + def __init__(self, gem5, gem5_args, timeout=0, **kwargs): + super(RunGem5, self).__init__("gem5", **kwargs) + self.gem5 = gem5 + self.args = gem5_args + self.timeout = timeout + + def _run(self): + gem5_cmd = [ + self.gem5, + "-d", self.test_dir, + "-re", + ] + self.args + + try: + with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout) + except CallTimeoutException as te: + return self.error("Timeout", stdout=te.stdout, stderr=te.stderr) + except OSError as ose: + return self.error("Failed to launch gem5: %s" % ose) + + stderr = "\n".join([ + "*** gem5 stderr ***", + gem5_stderr, + "", + "*** m5out/simerr ***", + self._read_output("simerr"), + ]) + + stdout = "\n".join([ + "*** gem5 stdout ***", + gem5_stdout, + "", + "*** m5out/simout ***", + self._read_output("simout"), + ]) + + # Signal + if status < 0: + return self.error("gem5 terminated by signal %i" % (-status, ), + stdout=stdout, stderr=stderr) + elif status == 2: + return self.skip(stdout=stdout, stderr=stderr) + elif status > 0: + return self.error("gem5 exited with non-zero status: %i" % status, + stdout=stdout, stderr=stderr) + else: + return self.ok(stdout=stdout, stderr=stderr) + +class DiffOutFile(TestUnit): + """Test unit comparing and output file and a reference file.""" + + # regular expressions of lines to ignore when diffing outputs + diff_ignore_regexes = { + "simout" : [ + re.compile('^Redirecting (stdout|stderr) to'), + re.compile('^gem5 compiled '), + re.compile('^gem5 started '), + re.compile('^gem5 executing on '), + re.compile('^command line:'), + re.compile("^Couldn't import dot_parser,"), + re.compile("^info: kernel located at:"), + re.compile("^Couldn't unlink "), + re.compile("^Using GPU kernel code file\(s\) "), + ], + "simerr" : [ + #re.compile('^Simulation complete at'), + ], + "config.ini" : [ + re.compile("^(executable|readfile|kernel|image_file)="), + re.compile("^(cwd|input|codefile)="), + ], + "config.json" : [ + re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''), + re.compile(r'''^\s*"(cwd|input|codefile)":'''), + ], + } + + def __init__(self, fname, **kwargs): + super(DiffOutFile, self).__init__("diff[%s]" % fname, + **kwargs) + + self.fname = fname + self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple()) + + def _filter_file(self, fname): + def match_line(l): + for r in self.line_filters: + if r.match(l): + return True + return False + + with open(fname, "r") as f: + for l in f: + if not match_line(l): + yield l + + + def _run(self): + fname = self.fname + ref = self.ref_file(fname) + out = self.out_file(fname) + + if not os.path.exists(ref): + return self.error("%s doesn't exist in reference directory" \ + % fname) + + if not os.path.exists(out): + return self.error("%s doesn't exist in output directory" % fname) + + diff = difflib.unified_diff( + tuple(self._filter_file(ref)), + tuple(self._filter_file(out)), + fromfile="ref/%s" % fname, tofile="out/%s" % fname) + + diff = list(diff) + if diff: + return self.error("ref/%s and out/%s differ" % (fname, fname), + stderr="".join(diff)) + else: + return self.ok(stdout="-- ref/%s and out/%s are identical --" \ + % (fname, fname)) + +class DiffStatFile(TestUnit): + """Test unit comparing two gem5 stat files.""" + + def __init__(self, **kwargs): + super(DiffStatFile, self).__init__("stat_diff", **kwargs) + + self.stat_diff = os.path.join(_test_base, "diff-out") + + def _run(self): + stats = "stats.txt" + + cmd = [ + self.stat_diff, + self.ref_file(stats), self.out_file(stats), + ] + with ProcessHelper(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + status, stdout, stderr = p.call() + + if status == 0: + return self.ok(stdout=stdout, stderr=stderr) + if status == 1: + return self.failure("Statistics mismatch", + stdout=stdout, stderr=stderr) + else: + return self.error("diff-out returned an error: %i" % status, + stdout=stdout, stderr=stderr) diff --git a/tests/tests.py b/tests/tests.py new file mode 100755 index 000000000..05d68881e --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +import argparse +import sys +import os +import pickle + +from testing.tests import * +import testing.results + +class ParagraphHelpFormatter(argparse.HelpFormatter): + def _fill_text(self, text, width, indent): + return "\n\n".join([ + super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \ + for p in text.split("\n\n") ]) + +formatters = { + "junit" : testing.results.JUnit, + "text" : testing.results.Text, + "summary" : testing.results.TextSummary, + "pickle" : testing.results.Pickle, +} + + +def _add_format_args(parser): + parser.add_argument("--format", choices=formatters, default="text", + help="Output format") + + parser.add_argument("--no-junit-xlate-names", action="store_true", + help="Don't translate test names to " \ + "package-like names") + + parser.add_argument("--output", "-o", + type=argparse.FileType('w'), default=sys.stdout, + help="Test result output file") + + +def _create_formatter(args): + formatter = formatters[args.format] + kwargs = { + "fout" : args.output, + "verbose" : args.verbose + } + + if issubclass(formatter, testing.results.JUnit): + kwargs.update({ + "translate_names" : not args.no_junit_xlate_names, + }) + + return formatter(**kwargs) + + +def _list_tests_args(subparsers): + parser = subparsers.add_parser( + "list", + formatter_class=ParagraphHelpFormatter, + help="List available tests", + description="List available tests", + epilog=""" + Generate a list of available tests using a list filter. + + The filter is a string consisting of the target ISA optionally + followed by the test category and mode separated by + slashes. The test names emitted by this command can be fed + into the run command. + + For example, to list all quick arm tests, run the following: + tests.py list arm/quick + + Non-mandatory parts of the filter string (anything other than + the ISA) can be left out or replaced with the wildcard + character. For example, all full-system tests can be listed + with this command: tests.py list arm/*/fs""") + + parser.add_argument("--ruby-protocol", type=str, default=None, + help="Ruby protocol") + + parser.add_argument("--gpu-isa", type=str, default=None, + help="GPU ISA") + + parser.add_argument("list_filter", metavar="ISA[/category/mode]", + action="append", type=str, + help="List available test cases") + +def _list_tests(args): + for isa, categories, modes in \ + ( parse_test_filter(f) for f in args.list_filter ): + + for test in get_tests(isa, categories=categories, modes=modes, + ruby_protocol=args.ruby_protocol, + gpu_isa=args.gpu_isa): + print "/".join(test) + sys.exit(0) + +def _run_tests_args(subparsers): + parser = subparsers.add_parser( + "run", + formatter_class=ParagraphHelpFormatter, + help='Run one or more tests', + description="Run one or more tests.", + epilog=""" + Run one or more tests described by a gem5 test tuple. + + The test tuple consists of a test category (quick or long), a + test mode (fs or se), a workload name, an isa, an operating + system, and a config name separate by slashes. For example: + quick/se/00.hello/arm/linux/simple-timing + + Available tests can be listed using the 'list' sub-command + (e.g., "tests.py list arm/quick" or one of the scons test list + targets (e.g., "scons build/ARM/tests/opt/quick.list"). + + The test results can be stored in multiple different output + formats. See the help for the show command for more details + about output formatting.""") + + parser.add_argument("gem5", type=str, + help="gem5 binary") + + parser.add_argument("test", type=str, nargs="*", + help="List of tests to execute") + + parser.add_argument("--directory", "-d", + type=str, default="m5tests", + help="Test work directory") + + parser.add_argument("--timeout", "-t", + type=int, default="0", metavar="MINUTES", + help="Timeout, 0 to disable") + + parser.add_argument("--skip-diff-out", action="store_true", + help="Skip output diffing stage") + + parser.add_argument("--skip-diff-stat", action="store_true", + help="Skip stat diffing stage") + + _add_format_args(parser) + +def _run_tests(args): + formatter = _create_formatter(args) + + out_base = os.path.abspath(args.directory) + if not os.path.exists(out_base): + os.mkdir(out_base) + tests = [] + for test_name in args.test: + config = ClassicConfig(*test_name.split("/")) + out_dir = os.path.join(out_base, "/".join(config)) + tests.append( + ClassicTest(args.gem5, out_dir, config, + timeout=args.timeout, + skip_diff_stat=args.skip_diff_stat, + skip_diff_out=args.skip_diff_out)) + + all_results = [] + print "Running %i tests" % len(tests) + for testno, test in enumerate(tests): + print "%i: Running '%s'..." % (testno, test) + + all_results.append(test.run()) + + formatter.dump_suites(all_results) + +def _show_args(subparsers): + parser = subparsers.add_parser( + "show", + formatter_class=ParagraphHelpFormatter, + help='Display pickled test results', + description='Display pickled test results', + epilog=""" + Reformat the pickled output from one or more test runs. This + command is typically used with the output from a single test + run, but it can also be used to merge the outputs from + multiple runs. + + The 'text' format is a verbose output format that provides + information about individual test units and the output from + failed tests. It's mainly useful for debugging test failures. + + The 'summary' format provides outputs the results of one test + per line with the test's overall status (OK, SKIPPED, or + FAILED). + + The 'junit' format is primarily intended for use with CI + systems. It provides an XML representation of test + status. Similar to the text format, it includes detailed + information about test failures. Since many JUnit parser make + assume that test names look like Java packet strings, the + JUnit formatter automatically to something the looks like a + Java class path ('.'->'-', '/'->'.'). + + The 'pickle' format stores the raw results in a format that + can be reformatted using this command. It's typically used + with the show command to merge multiple test results into one + pickle file.""") + + _add_format_args(parser) + + parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", + help="Pickled test results") + +def _show(args): + formatter = _create_formatter(args) + suites = sum([ pickle.load(f) for f in args.result ], []) + formatter.dump_suites(suites) + +_commands = { + "list" : (_list_tests, _list_tests_args), + "run" : (_run_tests, _run_tests_args), + "show" : (_show, _show_args), +} + +def main(): + parser = argparse.ArgumentParser( + formatter_class=ParagraphHelpFormatter, + description="""gem5 testing multi tool.""", + epilog=""" + This tool provides an interface to gem5's test framework that + doesn't depend on gem5's build system. It supports test + listing, running, and output formatting. + + The list sub-command (e.g., "test.py list arm/quick") produces + a list of tests tuples that can be used by the run command + (e.g., "tests.py run gem5.opt + quick/se/00.hello/arm/linux/simple-timing"). + + The run command supports several output formats. One of them, + pickle, contains the raw output from the tests and can be + re-formatted using the show command (e.g., "tests.py show + --format summary *.pickle"). Such pickle files are also + generated by the build system when scons is used to run + regressions. + + See the usage strings for the individual sub-commands for + details.""") + + parser.add_argument("--verbose", action="store_true", + help="Produce more verbose output") + + subparsers = parser.add_subparsers(dest="command") + + for key, (impl, cmd_parser) in _commands.items(): + cmd_parser(subparsers) + + args = parser.parse_args() + impl, cmd_parser = _commands[args.command] + impl(args) + +if __name__ == "__main__": + main() |