tests,ext: Add a new testing library proposal

The new test library is split into two parts: The framework which resides
in ext/, and the gem5 helping components in /tests/gem5.

Change-Id: Ib4f3ae8d7eb96a7306335a3e739b7e8041aa99b9
Signed-off-by: Sean Wilson <spwilson2@wisc.edu>
Reviewed-on: https://gem5-review.googlesource.com/4421
Reviewed-by: Giacomo Travaglini <giacomo.travaglini@arm.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>

7 files changed, 805 insertions, 0 deletions

diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 000000000..63c0e1ba6
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1 @@
+.testing-results
diff --git a/tests/gem5/__init__.py b/tests/gem5/__init__.py
new file mode 100644
index 000000000..ac21eba9e
--- /dev/null
+++ b/tests/gem5/__init__.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import suite
+import fixture
+
+from suite import *
+from fixture import *
diff --git a/tests/gem5/fixture.py b/tests/gem5/fixture.py
new file mode 100644
index 000000000..a50d73cff
--- /dev/null
+++ b/tests/gem5/fixture.py
@@ -0,0 +1,258 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import os
+import tempfile
+import shutil
+
+from testlib.fixture import Fixture, globalfixture
+from testlib.config import config, constants
+from testlib.helper import log_call, cacheresult, joinpath, absdirpath
+import testlib.log as log
+
+
+class VariableFixture(Fixture):
+    def __init__(self, value=None, name=None):
+        super(VariableFixture, self).__init__(name=name)
+        self.value = value
+
+
+class TempdirFixture(Fixture):
+    def __init__(self):
+        self.path = None
+        super(TempdirFixture, self).__init__(
+                name=constants.tempdir_fixture_name)
+
+    def setup(self, testitem):
+        self.path = tempfile.mkdtemp(prefix='gem5out')
+
+    def teardown(self, testitem):
+        if self.path is not None:
+            shutil.rmtree(self.path)
+
+
+class SConsFixture(Fixture):
+    '''
+    Fixture will wait until all SCons targets are collected and tests are
+    about to be ran, then will invocate a single instance of SCons for all
+    targets.
+
+    :param directory: The directory which scons will -C (cd) into before
+        executing. If None is provided, will choose the config base_dir.
+    '''
+    def __init__(self, directory=None, target_class=None):
+        self.directory = directory if directory else config.base_dir
+        self.target_class = target_class if target_class else SConsTarget
+        self.threads = config.threads
+        self.targets = set()
+        super(SConsFixture, self).__init__()
+
+    def setup(self, testitem):
+        if config.skip_build:
+            return
+
+        command = [
+            'scons', '-C', self.directory,
+            '-j', str(self.threads),
+            '--ignore-style'
+        ]
+
+        if not self.targets:
+            log.test_log.warn(
+                'No SCons targets specified, this will'
+                ' build the default all target.\n'
+                'This is likely unintended, and you'
+                ' may wish to kill testlib and reconfigure.')
+        else:
+            log.test_log.message(
+                    'Building the following targets.'
+                    ' This may take a while.')
+            log.test_log.message('%s' % (', '.join(self.targets)))
+            log.test_log.message(
+                    "You may want to run with only a single ISA"
+                    "(--isa=), use --skip-build, or use 'rerun'.")
+
+
+
+        command.extend(self.targets)
+        log_call(log.test_log, command)
+
+
+class SConsTarget(Fixture):
+    # The singleton scons fixture we'll use for all targets.
+    default_scons_invocation = None
+
+    def __init__(self, target, build_dir=None, invocation=None):
+        '''
+        Represents a target to be built by an 'invocation' of scons.
+
+        :param target: The target known to scons.
+
+        :param build_dir: The 'build' directory path which will be prepended
+            to the target name.
+
+        :param invocation: Represents an invocation of scons which we will
+            automatically attach this target to. If None provided, uses the
+            main 'scons' invocation.
+        '''
+
+        if build_dir is None:
+            build_dir = config.build_dir
+        self.target = os.path.join(build_dir, target)
+        super(SConsTarget, self).__init__(name=target)
+
+        if invocation is None:
+            if self.default_scons_invocation is None:
+                SConsTarget.default_scons_invocation = SConsFixture()
+                globalfixture(SConsTarget.default_scons_invocation)
+
+            invocation = self.default_scons_invocation
+        self.invocation = invocation
+
+    def schedule_finalized(self, schedule):
+        self.invocation.targets.add(self.target)
+        return Fixture.schedule_finalized(self, schedule)
+
+class Gem5Fixture(SConsTarget):
+    def __init__(self, isa, variant):
+        target = joinpath(isa.upper(), 'gem5.%s' % variant)
+        super(Gem5Fixture, self).__init__(target)
+
+        self.name = constants.gem5_binary_fixture_name
+        self.path = self.target
+        self.isa = isa
+        self.variant = variant
+
+
+class MakeFixture(Fixture):
+    def __init__(self, directory, *args, **kwargs):
+        name = 'make -C %s' % directory
+        super(MakeFixture, self).__init__(build_once=True, lazy_init=False,
+                                          name=name,
+                                          *args, **kwargs)
+        self.targets = []
+        self.directory = directory
+
+    def setup(self):
+        super(MakeFixture, self).setup()
+        targets = set(self.required_by)
+        command = ['make', '-C', self.directory]
+        command.extend([target.target for target in targets])
+        log_call(command)
+
+
+class MakeTarget(Fixture):
+    def __init__(self, target, make_fixture=None, *args, **kwargs):
+        '''
+        :param make_fixture: The make invocation we will be attached to.
+        Since we don't have a single global instance of make in gem5 like we do
+        scons we need to know what invocation to attach to. If none given,
+        creates its own.
+        '''
+        super(MakeTarget, self).__init__(name=target, *args, **kwargs)
+        self.target = self.name
+
+        if make_fixture is None:
+            make_fixture = MakeFixture(
+                    absdirpath(target),
+                    lazy_init=True,
+                    build_once=False)
+
+        self.make_fixture = make_fixture
+
+        # Add our self to the required targets of the main MakeFixture
+        self.require(self.make_fixture)
+
+    def setup(self, testitem):
+        super(MakeTarget, self).setup()
+        self.make_fixture.setup()
+        return self
+
+class TestProgram(MakeTarget):
+    def __init__(self, program, isa, os, recompile=False):
+        make_dir = joinpath('test-progs', program)
+        make_fixture = MakeFixture(make_dir)
+        target = joinpath('bin', isa, os, program)
+        super(TestProgram, self).__init__(target, make_fixture)
+        self.path = joinpath(make_dir, target)
+        self.recompile = recompile
+
+    def setup(self, testitem):
+        # Check if the program exists if it does then only compile if
+        # recompile was given.
+        if self.recompile:
+            super(MakeTarget, self).setup()
+        elif not os.path.exists(self.path):
+            super(MakeTarget, self).setup()
+
+class DownloadedProgram(Fixture):
+    """ Like TestProgram, but checks the version in the gem5 binary repository
+        and downloads an updated version if it is needed.
+    """
+    urlbase = "http://gem5.org/dist/current/"
+
+    def __init__(self, path, program, **kwargs):
+        super(DownloadedProgram, self).__init__("download-" + program,
+                                                build_once=True, **kwargs)
+
+        self.program_dir = joinpath('test-progs', path)
+        self.path = joinpath(self.program_dir, program)
+
+        self.url = self.urlbase + self.path
+
+    def _download(self):
+        import urllib
+        log.test_log.debug("Downloading " + self.url + " to " + self.path)
+        if not os.path.exists(self.program_dir):
+            os.makedirs(self.program_dir)
+        urllib.urlretrieve(self.url, self.path)
+
+    def _getremotetime(self):
+        import  urllib2, datetime, time
+        import _strptime # Needed for python threading bug
+
+        u = urllib2.urlopen(self.url)
+        return time.mktime(datetime.datetime.strptime( \
+                    u.info().getheaders("Last-Modified")[0],
+                    "%a, %d %b %Y %X GMT").timetuple())
+
+    def setup(self, testitem):
+        import urllib2
+        # Check to see if there is a file downloaded
+        if not os.path.exists(self.path):
+            self._download()
+        else:
+            try:
+                t = self._getremotetime()
+            except urllib2.URLError:
+                # Problem checking the server, use the old files.
+                log.debug("Could not contact server. Binaries may be old.")
+                return
+            # If the server version is more recent, download it
+            if t > os.path.getmtime(self.path):
+                self._download()
diff --git a/tests/gem5/suite.py b/tests/gem5/suite.py
new file mode 100644
index 000000000..47cf421e1
--- /dev/null
+++ b/tests/gem5/suite.py
@@ -0,0 +1,163 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import os
+import copy
+import subprocess
+
+from testlib.test import TestFunction
+from testlib.suite import TestSuite
+from testlib.helper import log_call
+from testlib.config import constants, config
+from fixture import TempdirFixture, Gem5Fixture, VariableFixture
+import verifier
+
+def gem5_verify_config(name,
+                       config,
+                       config_args,
+                       verifiers,
+                       gem5_args=tuple(),
+                       fixtures=[],
+                       valid_isas=constants.supported_isas,
+                       valid_variants=constants.supported_variants,
+                       length=constants.supported_lengths[0]):
+    '''
+    Helper class to generate common gem5 tests using verifiers.
+
+    The generated TestSuite will run gem5 with the provided config and
+    config_args. After that it will run any provided verifiers to verify
+    details about the gem5 run.
+
+    .. seealso::  For the verifiers see :mod:`testlib.gem5.verifier`
+
+    :param name: Name of the test.
+    :param config: The config to give gem5.
+    :param config_args: A list of arguments to pass to the given config.
+
+    :param verifiers: An iterable with Verifier instances which will be placed
+        into a suite that will be ran after a gem5 run.
+
+    :param gem5_args: An iterable with arguments to give to gem5. (Arguments
+        that would normally go before the config path.)
+
+    :param valid_isas: An iterable with the isas that this test can be ran
+        for. If None given, will run for all supported_isas.
+
+    :param valid_variants: An iterable with the variant levels that
+        this test can be ran for. (E.g. opt, debug)
+    '''
+    fixtures = list(fixtures)
+    testsuites = []
+    for opt in valid_variants:
+        for isa in valid_isas:
+
+            # Create a tempdir fixture to be shared throughout the test.
+            tempdir = TempdirFixture()
+            gem5_returncode = VariableFixture(
+                    name=constants.gem5_returncode_fixture_name)
+
+            # Common name of this generated testcase.
+            _name = '{given_name}-{isa}-{opt}'.format(
+                    given_name=name,
+                    isa=isa,
+                    opt=opt)
+
+            # Create the running of gem5 subtest.
+            # NOTE: We specifically create this test before our verifiers so
+            # this is listed first.
+            tests = []
+            gem5_execution = TestFunction(
+                    _create_test_run_gem5(config, config_args, gem5_args),
+                    name=_name)
+            tests.append(gem5_execution)
+
+            # Create copies of the verifier subtests for this isa and
+            # variant.
+            for verifier in verifiers:
+                tests.append(verifier.instantiate_test(_name))
+
+            # Add the isa and variant to tags list.
+            tags = [isa, opt, length]
+
+            # Create the gem5 target for the specific architecture and
+            # variant.
+            _fixtures = copy.copy(fixtures)
+            _fixtures.append(Gem5Fixture(isa, opt))
+            _fixtures.append(tempdir)
+            _fixtures.append(gem5_returncode)
+
+            # Finally construct the self contained TestSuite out of our
+            # tests.
+            testsuites.append(TestSuite(
+                name=_name,
+                fixtures=_fixtures,
+                tags=tags,
+                tests=tests))
+    return testsuites
+
+def _create_test_run_gem5(config, config_args, gem5_args):
+    def test_run_gem5(params):
+        '''
+        Simple \'test\' which runs gem5 and saves the result into a tempdir.
+
+        NOTE: Requires fixtures: tempdir, gem5
+        '''
+        fixtures = params.fixtures
+
+        if gem5_args is None:
+            _gem5_args = tuple()
+        elif isinstance(gem5_args, str):
+            # If just a single str, place it in an iterable
+            _gem5_args = (gem5_args,)
+        else:
+            _gem5_args = gem5_args
+
+        # FIXME/TODO: I don't like the idea of having to modify this test run
+        # or always collect results even if not using a verifier. There should
+        # be some configuration in here that only gathers certain results for
+        # certain verifiers.
+        #
+        # I.E. Only the returncode verifier will use the gem5_returncode
+        # fixture, but we always require it even if that verifier isn't being
+        # ran.
+        returncode = fixtures[constants.gem5_returncode_fixture_name]
+        tempdir = fixtures[constants.tempdir_fixture_name].path
+        gem5 = fixtures[constants.gem5_binary_fixture_name].path
+        command = [
+            gem5,
+            '-d',  # Set redirect dir to tempdir.
+            tempdir,
+            '-re',# TODO: Change to const. Redirect stdout and stderr
+        ]
+        command.extend(_gem5_args)
+        command.append(config)
+        # Config_args should set up the program args.
+        command.extend(config_args)
+        returncode.value = log_call(params.log, command)
+
+    return test_run_gem5
diff --git a/tests/gem5/verifier.py b/tests/gem5/verifier.py
new file mode 100644
index 000000000..ab1265cfa
--- /dev/null
+++ b/tests/gem5/verifier.py
@@ -0,0 +1,205 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+Built in test cases that verify particular details about a gem5 run.
+'''
+import re
+
+from testlib import test
+from testlib.config import constants
+from testlib.helper import joinpath, diff_out_file
+
+class Verifier(object):
+    def __init__(self, fixtures=tuple()):
+        self.fixtures = fixtures
+
+    def _test(self, *args, **kwargs):
+        # Use a callback wrapper to make stack
+        # traces easier to understand.
+        self.test(*args, **kwargs)
+
+    def instantiate_test(self, name_pfx):
+        name = '-'.join([name_pfx, self.__class__.__name__])
+        return test.TestFunction(self._test,
+                name=name, fixtures=self.fixtures)
+
+    def failed(self, fixtures):
+        '''
+        Called if this verifier fails to cleanup (or not) as needed.
+        '''
+        try:
+            fixtures[constants.tempdir_fixture_name].skip_cleanup()
+        except KeyError:
+            pass # No need to do anything if the tempdir fixture doesn't exist
+
+
+class MatchGoldStandard(Verifier):
+    '''
+    Compares a standard output to the test output and passes if they match,
+    fails if they do not.
+    '''
+    def __init__(self, standard_filename, ignore_regex=None,
+                 test_filename='simout'):
+        '''
+        :param standard_filename: The path of the standard file to compare
+        output to.
+
+        :param ignore_regex: A string, compiled regex, or iterable containing
+        either which will be ignored in 'standard' and test output files when
+        diffing.
+        '''
+        super(MatchGoldStandard, self).__init__()
+        self.standard_filename = standard_filename
+        self.test_filename = test_filename
+
+        self.ignore_regex = _iterable_regex(ignore_regex)
+
+    def test(self, params):
+        # We need a tempdir fixture from our parent verifier suite.
+        fixtures = params.fixtures
+        # Get the file from the tempdir of the test.
+        tempdir = fixtures[constants.tempdir_fixture_name].path
+        self.test_filename = joinpath(tempdir, self.test_filename)
+
+        diff = diff_out_file(self.standard_filename,
+                            self.test_filename,
+                            ignore_regexes=self.ignore_regex,
+                            logger=params.log)
+        if diff is not None:
+            self.failed(fixtures)
+            test.fail('Stdout did not match:\n%s\nSee %s for full results'
+                      % (diff, tempdir))
+
+    def _generic_instance_warning(self, kwargs):
+        '''
+        Method for helper classes to tell users to use this more generic class
+        if they are going to manually override the test_filename param.
+        '''
+        if 'test_filename' in kwargs:
+            raise ValueError('If you are setting test_filename use the more'
+                             ' generic %s'
+                             ' instead' % MatchGoldStandard.__name__)
+
+class DerivedGoldStandard(MatchGoldStandard):
+    __ignore_regex_sentinel = object()
+    _file = None
+    _default_ignore_regex = []
+
+    def __init__(self, standard_filename,
+                 ignore_regex=__ignore_regex_sentinel, **kwargs):
+
+        if ignore_regex == self.__ignore_regex_sentinel:
+            ignore_regex = self._default_ignore_regex
+
+        self._generic_instance_warning(kwargs)
+
+        super(DerivedGoldStandard, self).__init__(
+            standard_filename,
+            test_filename=self._file,
+            ignore_regex=ignore_regex,
+            **kwargs)
+
+class MatchStdout(DerivedGoldStandard):
+    _file = constants.gem5_simulation_stdout
+    _default_ignore_regex = [
+            re.compile('^Redirecting (stdout|stderr) to'),
+            re.compile('^gem5 compiled '),
+            re.compile('^gem5 started '),
+            re.compile('^gem5 executing on '),
+            re.compile('^command line:'),
+            re.compile("^Couldn't import dot_parser,"),
+            re.compile("^info: kernel located at:"),
+            re.compile("^Couldn't unlink "),
+            re.compile("^Using GPU kernel code file\(s\) "),
+        ]
+
+class MatchStdoutNoPerf(MatchStdout):
+    _file = constants.gem5_simulation_stdout
+    _default_ignore_regex = MatchStdout._default_ignore_regex + [
+            re.compile('^Exiting @ tick'),
+        ]
+
+class MatchStderr(DerivedGoldStandard):
+    _file = constants.gem5_simulation_stderr
+    _default_ignore_regex = []
+
+class MatchStats(DerivedGoldStandard):
+    # TODO: Likely will want to change this verifier since we have the weird
+    # perl script right now. A simple diff probably isn't going to work.
+    _file = constants.gem5_simulation_stats
+    _default_ignore_regex = []
+
+class MatchConfigINI(DerivedGoldStandard):
+    _file = constants.gem5_simulation_config_ini
+    _default_ignore_regex = (
+            re.compile("^(executable|readfile|kernel|image_file)="),
+            re.compile("^(cwd|input|codefile)="),
+            )
+
+class MatchConfigJSON(DerivedGoldStandard):
+    _file = constants.gem5_simulation_config_json
+    _default_ignore_regex = (
+            re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
+            re.compile(r'''^\s*"(cwd|input|codefile)":'''),
+            )
+
+class MatchRegex(Verifier):
+    def __init__(self, regex, match_stderr=True, match_stdout=True):
+        super(MatchRegex, self).__init__()
+        self.regex = _iterable_regex(regex)
+        self.match_stderr = match_stderr
+        self.match_stdout = match_stdout
+
+    def test(self, params):
+        fixtures = params.fixtures
+        # Get the file from the tempdir of the test.
+        tempdir = fixtures[constants.tempdir_fixture_name].path
+
+        def parse_file(fname):
+            with open(fname, 'r') as file_:
+                for line in file_:
+                    for regex in self.regex:
+                        if re.match(regex, line):
+                            return True
+        if self.match_stdout:
+            if parse_file(joinpath(tempdir,
+                                   constants.gem5_simulation_stdout)):
+                return # Success
+        if self.match_stderr:
+            if parse_file(joinpath(tempdir,
+                                   constants.gem5_simulation_stderr)):
+                return # Success
+        self.failed(fixtures)
+        test.fail('Could not match regex.')
+
+_re_type = type(re.compile(''))
+def _iterable_regex(regex):
+    if isinstance(regex, _re_type) or isinstance(regex, str):
+        regex = (regex,)
+    return regex
diff --git a/tests/legacy-configs/run.py b/tests/legacy-configs/run.py
new file mode 100644
index 000000000..3eac49095
--- /dev/null
+++ b/tests/legacy-configs/run.py
@@ -0,0 +1,121 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+New version of the run.py script. For this, all dependencies should be
+handled outside of the script.
+
+.. warning:: This script is NOT the recommended way to handle configurations
+    for new tests. This exists for legacy support only. New Tests should
+    either use configs from the normal gem5 configs or create their own for
+    a test.
+'''
+import argparse
+import sys
+import os
+from os.path import abspath, join as joinpath, dirname
+
+import m5
+
+# Add the normal gem5 config path to system path.
+# This requirement should be removed if possible from all legacy scripts, but
+# I've left it here for now.
+sys.path.insert(0, abspath(joinpath(dirname(__file__), '../../configs')))
+
+# set default maxtick... script can override
+# -1 means run forever
+maxtick = m5.MaxTick
+
+def run_test(root):
+    """Default run_test implementations. Scripts can override it."""
+
+    # instantiate configuration
+    m5.instantiate()
+
+    # simulate until program terminates
+    exit_event = m5.simulate(maxtick)
+    print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause()
+
+test_progs = os.environ.get('M5_TEST_PROGS', '/dist/m5/regression/test-progs')
+
+# Since we're in batch mode, dont allow tcp socket connections
+m5.disableAllListeners()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--cmd',
+                    action='store',
+                    type=str,
+                    help='Command to pass to the test system')
+parser.add_argument('--executable',
+                    action='store',
+                    type=str,
+                    help='Executable to pass to the test system')
+parser.add_argument('--config',
+                    action='append',
+                    type=str,
+                    help='A config file to initialize the system with.'\
+                    + ' If more than one given, loads them in order given.')
+args = parser.parse_args()
+
+executable = args.executable
+
+for config in args.config:
+    execfile(config)
+
+# Initialize all CPUs in a system
+def initCPUs(sys):
+    def initCPU(cpu):
+        # We might actually have a MemTest object or something similar
+        # here that just pretends to be a CPU.
+        try:
+            cpu.createThreads()
+        except:
+            pass
+
+    # The CPU attribute doesn't exist in some cases, e.g. the Ruby testers.
+    if not hasattr(sys, "cpu"):
+        return
+
+    # The CPU can either be a list of CPUs or a single object.
+    if isinstance(sys.cpu, list):
+        [ initCPU(cpu) for cpu in sys.cpu ]
+    else:
+        initCPU(sys.cpu)
+
+# TODO: Might want to automatically place the cmd and executable on the
+# cpu[0].workload, although I think most legacy configs do this automatically
+# or somewhere in their `test.py` config.
+
+
+# We might be creating a single system or a dual system. Try
+# initializing the CPUs in all known system attributes.
+for sysattr in [ "system", "testsys", "drivesys" ]:
+    if hasattr(root, sysattr):
+        initCPUs(getattr(root, sysattr))
+
+run_test(root)
diff --git a/tests/main.py b/tests/main.py
new file mode 100755
index 000000000..c8dc9b961
--- /dev/null
+++ b/tests/main.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python2
+'''
+The main source for testlib. Ties together the default test runners and
+loaders.
+
+Discovers and runs all tests from a given root directory.
+'''
+from __future__ import print_function
+
+import sys
+import os
+
+base_dir = os.path.dirname(os.path.abspath(__name__))
+ext_path = os.path.join(base_dir, os.pardir, 'ext')
+
+sys.path.insert(0, base_dir)
+sys.path.insert(0, ext_path)
+
+import testlib.main as testlib
+import testlib.config as config
+import testlib.helper as helper
+
+config.basedir = helper.absdirpath(__file__)
+testlib()