From 07ce662bd212246e20d85de1e4f3d537565449d1 Mon Sep 17 00:00:00 2001 From: Sean Wilson Date: Thu, 3 Aug 2017 11:28:49 -0500 Subject: tests,ext: Add a new testing library proposal The new test library is split into two parts: The framework which resides in ext/, and the gem5 helping components in /tests/gem5. Change-Id: Ib4f3ae8d7eb96a7306335a3e739b7e8041aa99b9 Signed-off-by: Sean Wilson Reviewed-on: https://gem5-review.googlesource.com/4421 Reviewed-by: Giacomo Travaglini Maintainer: Jason Lowe-Power --- tests/.gitignore | 1 + tests/gem5/__init__.py | 33 ++++++ tests/gem5/fixture.py | 258 ++++++++++++++++++++++++++++++++++++++++++++ tests/gem5/suite.py | 163 ++++++++++++++++++++++++++++ tests/gem5/verifier.py | 205 +++++++++++++++++++++++++++++++++++ tests/legacy-configs/run.py | 121 +++++++++++++++++++++ tests/main.py | 24 +++++ 7 files changed, 805 insertions(+) create mode 100644 tests/.gitignore create mode 100644 tests/gem5/__init__.py create mode 100644 tests/gem5/fixture.py create mode 100644 tests/gem5/suite.py create mode 100644 tests/gem5/verifier.py create mode 100644 tests/legacy-configs/run.py create mode 100755 tests/main.py (limited to 'tests') diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 000000000..63c0e1ba6 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +.testing-results diff --git a/tests/gem5/__init__.py b/tests/gem5/__init__.py new file mode 100644 index 000000000..ac21eba9e --- /dev/null +++ b/tests/gem5/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2017 Mark D. Hill and David A. Wood +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Sean Wilson + +import suite +import fixture + +from suite import * +from fixture import * diff --git a/tests/gem5/fixture.py b/tests/gem5/fixture.py new file mode 100644 index 000000000..a50d73cff --- /dev/null +++ b/tests/gem5/fixture.py @@ -0,0 +1,258 @@ +# Copyright (c) 2017 Mark D. Hill and David A. Wood +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Sean Wilson + +import os +import tempfile +import shutil + +from testlib.fixture import Fixture, globalfixture +from testlib.config import config, constants +from testlib.helper import log_call, cacheresult, joinpath, absdirpath +import testlib.log as log + + +class VariableFixture(Fixture): + def __init__(self, value=None, name=None): + super(VariableFixture, self).__init__(name=name) + self.value = value + + +class TempdirFixture(Fixture): + def __init__(self): + self.path = None + super(TempdirFixture, self).__init__( + name=constants.tempdir_fixture_name) + + def setup(self, testitem): + self.path = tempfile.mkdtemp(prefix='gem5out') + + def teardown(self, testitem): + if self.path is not None: + shutil.rmtree(self.path) + + +class SConsFixture(Fixture): + ''' + Fixture will wait until all SCons targets are collected and tests are + about to be ran, then will invocate a single instance of SCons for all + targets. + + :param directory: The directory which scons will -C (cd) into before + executing. If None is provided, will choose the config base_dir. + ''' + def __init__(self, directory=None, target_class=None): + self.directory = directory if directory else config.base_dir + self.target_class = target_class if target_class else SConsTarget + self.threads = config.threads + self.targets = set() + super(SConsFixture, self).__init__() + + def setup(self, testitem): + if config.skip_build: + return + + command = [ + 'scons', '-C', self.directory, + '-j', str(self.threads), + '--ignore-style' + ] + + if not self.targets: + log.test_log.warn( + 'No SCons targets specified, this will' + ' build the default all target.\n' + 'This is likely unintended, and you' + ' may wish to kill testlib and reconfigure.') + else: + log.test_log.message( + 'Building the following targets.' + ' This may take a while.') + log.test_log.message('%s' % (', '.join(self.targets))) + log.test_log.message( + "You may want to run with only a single ISA" + "(--isa=), use --skip-build, or use 'rerun'.") + + + + command.extend(self.targets) + log_call(log.test_log, command) + + +class SConsTarget(Fixture): + # The singleton scons fixture we'll use for all targets. + default_scons_invocation = None + + def __init__(self, target, build_dir=None, invocation=None): + ''' + Represents a target to be built by an 'invocation' of scons. + + :param target: The target known to scons. + + :param build_dir: The 'build' directory path which will be prepended + to the target name. + + :param invocation: Represents an invocation of scons which we will + automatically attach this target to. If None provided, uses the + main 'scons' invocation. + ''' + + if build_dir is None: + build_dir = config.build_dir + self.target = os.path.join(build_dir, target) + super(SConsTarget, self).__init__(name=target) + + if invocation is None: + if self.default_scons_invocation is None: + SConsTarget.default_scons_invocation = SConsFixture() + globalfixture(SConsTarget.default_scons_invocation) + + invocation = self.default_scons_invocation + self.invocation = invocation + + def schedule_finalized(self, schedule): + self.invocation.targets.add(self.target) + return Fixture.schedule_finalized(self, schedule) + +class Gem5Fixture(SConsTarget): + def __init__(self, isa, variant): + target = joinpath(isa.upper(), 'gem5.%s' % variant) + super(Gem5Fixture, self).__init__(target) + + self.name = constants.gem5_binary_fixture_name + self.path = self.target + self.isa = isa + self.variant = variant + + +class MakeFixture(Fixture): + def __init__(self, directory, *args, **kwargs): + name = 'make -C %s' % directory + super(MakeFixture, self).__init__(build_once=True, lazy_init=False, + name=name, + *args, **kwargs) + self.targets = [] + self.directory = directory + + def setup(self): + super(MakeFixture, self).setup() + targets = set(self.required_by) + command = ['make', '-C', self.directory] + command.extend([target.target for target in targets]) + log_call(command) + + +class MakeTarget(Fixture): + def __init__(self, target, make_fixture=None, *args, **kwargs): + ''' + :param make_fixture: The make invocation we will be attached to. + Since we don't have a single global instance of make in gem5 like we do + scons we need to know what invocation to attach to. If none given, + creates its own. + ''' + super(MakeTarget, self).__init__(name=target, *args, **kwargs) + self.target = self.name + + if make_fixture is None: + make_fixture = MakeFixture( + absdirpath(target), + lazy_init=True, + build_once=False) + + self.make_fixture = make_fixture + + # Add our self to the required targets of the main MakeFixture + self.require(self.make_fixture) + + def setup(self, testitem): + super(MakeTarget, self).setup() + self.make_fixture.setup() + return self + +class TestProgram(MakeTarget): + def __init__(self, program, isa, os, recompile=False): + make_dir = joinpath('test-progs', program) + make_fixture = MakeFixture(make_dir) + target = joinpath('bin', isa, os, program) + super(TestProgram, self).__init__(target, make_fixture) + self.path = joinpath(make_dir, target) + self.recompile = recompile + + def setup(self, testitem): + # Check if the program exists if it does then only compile if + # recompile was given. + if self.recompile: + super(MakeTarget, self).setup() + elif not os.path.exists(self.path): + super(MakeTarget, self).setup() + +class DownloadedProgram(Fixture): + """ Like TestProgram, but checks the version in the gem5 binary repository + and downloads an updated version if it is needed. + """ + urlbase = "http://gem5.org/dist/current/" + + def __init__(self, path, program, **kwargs): + super(DownloadedProgram, self).__init__("download-" + program, + build_once=True, **kwargs) + + self.program_dir = joinpath('test-progs', path) + self.path = joinpath(self.program_dir, program) + + self.url = self.urlbase + self.path + + def _download(self): + import urllib + log.test_log.debug("Downloading " + self.url + " to " + self.path) + if not os.path.exists(self.program_dir): + os.makedirs(self.program_dir) + urllib.urlretrieve(self.url, self.path) + + def _getremotetime(self): + import urllib2, datetime, time + import _strptime # Needed for python threading bug + + u = urllib2.urlopen(self.url) + return time.mktime(datetime.datetime.strptime( \ + u.info().getheaders("Last-Modified")[0], + "%a, %d %b %Y %X GMT").timetuple()) + + def setup(self, testitem): + import urllib2 + # Check to see if there is a file downloaded + if not os.path.exists(self.path): + self._download() + else: + try: + t = self._getremotetime() + except urllib2.URLError: + # Problem checking the server, use the old files. + log.debug("Could not contact server. Binaries may be old.") + return + # If the server version is more recent, download it + if t > os.path.getmtime(self.path): + self._download() diff --git a/tests/gem5/suite.py b/tests/gem5/suite.py new file mode 100644 index 000000000..47cf421e1 --- /dev/null +++ b/tests/gem5/suite.py @@ -0,0 +1,163 @@ +# Copyright (c) 2017 Mark D. Hill and David A. Wood +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Sean Wilson + +import os +import copy +import subprocess + +from testlib.test import TestFunction +from testlib.suite import TestSuite +from testlib.helper import log_call +from testlib.config import constants, config +from fixture import TempdirFixture, Gem5Fixture, VariableFixture +import verifier + +def gem5_verify_config(name, + config, + config_args, + verifiers, + gem5_args=tuple(), + fixtures=[], + valid_isas=constants.supported_isas, + valid_variants=constants.supported_variants, + length=constants.supported_lengths[0]): + ''' + Helper class to generate common gem5 tests using verifiers. + + The generated TestSuite will run gem5 with the provided config and + config_args. After that it will run any provided verifiers to verify + details about the gem5 run. + + .. seealso:: For the verifiers see :mod:`testlib.gem5.verifier` + + :param name: Name of the test. + :param config: The config to give gem5. + :param config_args: A list of arguments to pass to the given config. + + :param verifiers: An iterable with Verifier instances which will be placed + into a suite that will be ran after a gem5 run. + + :param gem5_args: An iterable with arguments to give to gem5. (Arguments + that would normally go before the config path.) + + :param valid_isas: An iterable with the isas that this test can be ran + for. If None given, will run for all supported_isas. + + :param valid_variants: An iterable with the variant levels that + this test can be ran for. (E.g. opt, debug) + ''' + fixtures = list(fixtures) + testsuites = [] + for opt in valid_variants: + for isa in valid_isas: + + # Create a tempdir fixture to be shared throughout the test. + tempdir = TempdirFixture() + gem5_returncode = VariableFixture( + name=constants.gem5_returncode_fixture_name) + + # Common name of this generated testcase. + _name = '{given_name}-{isa}-{opt}'.format( + given_name=name, + isa=isa, + opt=opt) + + # Create the running of gem5 subtest. + # NOTE: We specifically create this test before our verifiers so + # this is listed first. + tests = [] + gem5_execution = TestFunction( + _create_test_run_gem5(config, config_args, gem5_args), + name=_name) + tests.append(gem5_execution) + + # Create copies of the verifier subtests for this isa and + # variant. + for verifier in verifiers: + tests.append(verifier.instantiate_test(_name)) + + # Add the isa and variant to tags list. + tags = [isa, opt, length] + + # Create the gem5 target for the specific architecture and + # variant. + _fixtures = copy.copy(fixtures) + _fixtures.append(Gem5Fixture(isa, opt)) + _fixtures.append(tempdir) + _fixtures.append(gem5_returncode) + + # Finally construct the self contained TestSuite out of our + # tests. + testsuites.append(TestSuite( + name=_name, + fixtures=_fixtures, + tags=tags, + tests=tests)) + return testsuites + +def _create_test_run_gem5(config, config_args, gem5_args): + def test_run_gem5(params): + ''' + Simple \'test\' which runs gem5 and saves the result into a tempdir. + + NOTE: Requires fixtures: tempdir, gem5 + ''' + fixtures = params.fixtures + + if gem5_args is None: + _gem5_args = tuple() + elif isinstance(gem5_args, str): + # If just a single str, place it in an iterable + _gem5_args = (gem5_args,) + else: + _gem5_args = gem5_args + + # FIXME/TODO: I don't like the idea of having to modify this test run + # or always collect results even if not using a verifier. There should + # be some configuration in here that only gathers certain results for + # certain verifiers. + # + # I.E. Only the returncode verifier will use the gem5_returncode + # fixture, but we always require it even if that verifier isn't being + # ran. + returncode = fixtures[constants.gem5_returncode_fixture_name] + tempdir = fixtures[constants.tempdir_fixture_name].path + gem5 = fixtures[constants.gem5_binary_fixture_name].path + command = [ + gem5, + '-d', # Set redirect dir to tempdir. + tempdir, + '-re',# TODO: Change to const. Redirect stdout and stderr + ] + command.extend(_gem5_args) + command.append(config) + # Config_args should set up the program args. + command.extend(config_args) + returncode.value = log_call(params.log, command) + + return test_run_gem5 diff --git a/tests/gem5/verifier.py b/tests/gem5/verifier.py new file mode 100644 index 000000000..ab1265cfa --- /dev/null +++ b/tests/gem5/verifier.py @@ -0,0 +1,205 @@ +# Copyright (c) 2017 Mark D. Hill and David A. Wood +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Sean Wilson + +''' +Built in test cases that verify particular details about a gem5 run. +''' +import re + +from testlib import test +from testlib.config import constants +from testlib.helper import joinpath, diff_out_file + +class Verifier(object): + def __init__(self, fixtures=tuple()): + self.fixtures = fixtures + + def _test(self, *args, **kwargs): + # Use a callback wrapper to make stack + # traces easier to understand. + self.test(*args, **kwargs) + + def instantiate_test(self, name_pfx): + name = '-'.join([name_pfx, self.__class__.__name__]) + return test.TestFunction(self._test, + name=name, fixtures=self.fixtures) + + def failed(self, fixtures): + ''' + Called if this verifier fails to cleanup (or not) as needed. + ''' + try: + fixtures[constants.tempdir_fixture_name].skip_cleanup() + except KeyError: + pass # No need to do anything if the tempdir fixture doesn't exist + + +class MatchGoldStandard(Verifier): + ''' + Compares a standard output to the test output and passes if they match, + fails if they do not. + ''' + def __init__(self, standard_filename, ignore_regex=None, + test_filename='simout'): + ''' + :param standard_filename: The path of the standard file to compare + output to. + + :param ignore_regex: A string, compiled regex, or iterable containing + either which will be ignored in 'standard' and test output files when + diffing. + ''' + super(MatchGoldStandard, self).__init__() + self.standard_filename = standard_filename + self.test_filename = test_filename + + self.ignore_regex = _iterable_regex(ignore_regex) + + def test(self, params): + # We need a tempdir fixture from our parent verifier suite. + fixtures = params.fixtures + # Get the file from the tempdir of the test. + tempdir = fixtures[constants.tempdir_fixture_name].path + self.test_filename = joinpath(tempdir, self.test_filename) + + diff = diff_out_file(self.standard_filename, + self.test_filename, + ignore_regexes=self.ignore_regex, + logger=params.log) + if diff is not None: + self.failed(fixtures) + test.fail('Stdout did not match:\n%s\nSee %s for full results' + % (diff, tempdir)) + + def _generic_instance_warning(self, kwargs): + ''' + Method for helper classes to tell users to use this more generic class + if they are going to manually override the test_filename param. + ''' + if 'test_filename' in kwargs: + raise ValueError('If you are setting test_filename use the more' + ' generic %s' + ' instead' % MatchGoldStandard.__name__) + +class DerivedGoldStandard(MatchGoldStandard): + __ignore_regex_sentinel = object() + _file = None + _default_ignore_regex = [] + + def __init__(self, standard_filename, + ignore_regex=__ignore_regex_sentinel, **kwargs): + + if ignore_regex == self.__ignore_regex_sentinel: + ignore_regex = self._default_ignore_regex + + self._generic_instance_warning(kwargs) + + super(DerivedGoldStandard, self).__init__( + standard_filename, + test_filename=self._file, + ignore_regex=ignore_regex, + **kwargs) + +class MatchStdout(DerivedGoldStandard): + _file = constants.gem5_simulation_stdout + _default_ignore_regex = [ + re.compile('^Redirecting (stdout|stderr) to'), + re.compile('^gem5 compiled '), + re.compile('^gem5 started '), + re.compile('^gem5 executing on '), + re.compile('^command line:'), + re.compile("^Couldn't import dot_parser,"), + re.compile("^info: kernel located at:"), + re.compile("^Couldn't unlink "), + re.compile("^Using GPU kernel code file\(s\) "), + ] + +class MatchStdoutNoPerf(MatchStdout): + _file = constants.gem5_simulation_stdout + _default_ignore_regex = MatchStdout._default_ignore_regex + [ + re.compile('^Exiting @ tick'), + ] + +class MatchStderr(DerivedGoldStandard): + _file = constants.gem5_simulation_stderr + _default_ignore_regex = [] + +class MatchStats(DerivedGoldStandard): + # TODO: Likely will want to change this verifier since we have the weird + # perl script right now. A simple diff probably isn't going to work. + _file = constants.gem5_simulation_stats + _default_ignore_regex = [] + +class MatchConfigINI(DerivedGoldStandard): + _file = constants.gem5_simulation_config_ini + _default_ignore_regex = ( + re.compile("^(executable|readfile|kernel|image_file)="), + re.compile("^(cwd|input|codefile)="), + ) + +class MatchConfigJSON(DerivedGoldStandard): + _file = constants.gem5_simulation_config_json + _default_ignore_regex = ( + re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''), + re.compile(r'''^\s*"(cwd|input|codefile)":'''), + ) + +class MatchRegex(Verifier): + def __init__(self, regex, match_stderr=True, match_stdout=True): + super(MatchRegex, self).__init__() + self.regex = _iterable_regex(regex) + self.match_stderr = match_stderr + self.match_stdout = match_stdout + + def test(self, params): + fixtures = params.fixtures + # Get the file from the tempdir of the test. + tempdir = fixtures[constants.tempdir_fixture_name].path + + def parse_file(fname): + with open(fname, 'r') as file_: + for line in file_: + for regex in self.regex: + if re.match(regex, line): + return True + if self.match_stdout: + if parse_file(joinpath(tempdir, + constants.gem5_simulation_stdout)): + return # Success + if self.match_stderr: + if parse_file(joinpath(tempdir, + constants.gem5_simulation_stderr)): + return # Success + self.failed(fixtures) + test.fail('Could not match regex.') + +_re_type = type(re.compile('')) +def _iterable_regex(regex): + if isinstance(regex, _re_type) or isinstance(regex, str): + regex = (regex,) + return regex diff --git a/tests/legacy-configs/run.py b/tests/legacy-configs/run.py new file mode 100644 index 000000000..3eac49095 --- /dev/null +++ b/tests/legacy-configs/run.py @@ -0,0 +1,121 @@ +# Copyright (c) 2017 Mark D. Hill and David A. Wood +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Sean Wilson + +''' +New version of the run.py script. For this, all dependencies should be +handled outside of the script. + +.. warning:: This script is NOT the recommended way to handle configurations + for new tests. This exists for legacy support only. New Tests should + either use configs from the normal gem5 configs or create their own for + a test. +''' +import argparse +import sys +import os +from os.path import abspath, join as joinpath, dirname + +import m5 + +# Add the normal gem5 config path to system path. +# This requirement should be removed if possible from all legacy scripts, but +# I've left it here for now. +sys.path.insert(0, abspath(joinpath(dirname(__file__), '../../configs'))) + +# set default maxtick... script can override +# -1 means run forever +maxtick = m5.MaxTick + +def run_test(root): + """Default run_test implementations. Scripts can override it.""" + + # instantiate configuration + m5.instantiate() + + # simulate until program terminates + exit_event = m5.simulate(maxtick) + print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause() + +test_progs = os.environ.get('M5_TEST_PROGS', '/dist/m5/regression/test-progs') + +# Since we're in batch mode, dont allow tcp socket connections +m5.disableAllListeners() + +parser = argparse.ArgumentParser() +parser.add_argument('--cmd', + action='store', + type=str, + help='Command to pass to the test system') +parser.add_argument('--executable', + action='store', + type=str, + help='Executable to pass to the test system') +parser.add_argument('--config', + action='append', + type=str, + help='A config file to initialize the system with.'\ + + ' If more than one given, loads them in order given.') +args = parser.parse_args() + +executable = args.executable + +for config in args.config: + execfile(config) + +# Initialize all CPUs in a system +def initCPUs(sys): + def initCPU(cpu): + # We might actually have a MemTest object or something similar + # here that just pretends to be a CPU. + try: + cpu.createThreads() + except: + pass + + # The CPU attribute doesn't exist in some cases, e.g. the Ruby testers. + if not hasattr(sys, "cpu"): + return + + # The CPU can either be a list of CPUs or a single object. + if isinstance(sys.cpu, list): + [ initCPU(cpu) for cpu in sys.cpu ] + else: + initCPU(sys.cpu) + +# TODO: Might want to automatically place the cmd and executable on the +# cpu[0].workload, although I think most legacy configs do this automatically +# or somewhere in their `test.py` config. + + +# We might be creating a single system or a dual system. Try +# initializing the CPUs in all known system attributes. +for sysattr in [ "system", "testsys", "drivesys" ]: + if hasattr(root, sysattr): + initCPUs(getattr(root, sysattr)) + +run_test(root) diff --git a/tests/main.py b/tests/main.py new file mode 100755 index 000000000..c8dc9b961 --- /dev/null +++ b/tests/main.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python2 +''' +The main source for testlib. Ties together the default test runners and +loaders. + +Discovers and runs all tests from a given root directory. +''' +from __future__ import print_function + +import sys +import os + +base_dir = os.path.dirname(os.path.abspath(__name__)) +ext_path = os.path.join(base_dir, os.pardir, 'ext') + +sys.path.insert(0, base_dir) +sys.path.insert(0, ext_path) + +import testlib.main as testlib +import testlib.config as config +import testlib.helper as helper + +config.basedir = helper.absdirpath(__file__) +testlib() -- cgit v1.2.3