diff options
Diffstat (limited to 'tests/testing/tests.py')
-rw-r--r-- | tests/testing/tests.py | 343 |
1 files changed, 343 insertions, 0 deletions
diff --git a/tests/testing/tests.py b/tests/testing/tests.py new file mode 100644 index 000000000..4c467f25c --- /dev/null +++ b/tests/testing/tests.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +from abc import ABCMeta, abstractmethod +import os +from collections import namedtuple +from units import * +from results import TestResult +import shutil + +_test_base = os.path.join(os.path.dirname(__file__), "..") + +ClassicConfig = namedtuple("ClassicConfig", ( + "category", + "mode", + "workload", + "isa", + "os", + "config", +)) + +# There are currently two "classes" of test +# configurations. Architecture-specific ones and generic ones +# (typically SE mode tests). In both cases, the configuration name +# matches a file in tests/configs/ that will be picked up by the test +# runner (run.py). +# +# Architecture specific configurations are listed in the arch_configs +# dictionary. This is indexed by a (cpu architecture, gpu +# architecture) tuple. GPU architecture is optional and may be None. +# +# Generic configurations are listed in the generic_configs tuple. +# +# When discovering available test cases, this script look uses the +# test list as a list of /candidate/ configurations. A configuration +# is only used if a test has a reference output for that +# configuration. In addition to the base configurations from +# arch_configs and generic_configs, a Ruby configuration may be +# appended to the base name (this is probed /in addition/ to the +# original name. See get_tests() for details. +# +arch_configs = { + ("alpha", None) : ( + 'tsunami-simple-atomic', + 'tsunami-simple-timing', + 'tsunami-simple-atomic-dual', + 'tsunami-simple-timing-dual', + 'twosys-tsunami-simple-atomic', + 'tsunami-o3', 'tsunami-o3-dual', + 'tsunami-minor', 'tsunami-minor-dual', + 'tsunami-switcheroo-full', + ), + + ("arm", None) : ( + 'simple-atomic-dummychecker', + 'o3-timing-checker', + 'realview-simple-atomic', + 'realview-simple-atomic-dual', + 'realview-simple-atomic-checkpoint', + 'realview-simple-timing', + 'realview-simple-timing-dual', + 'realview-o3', + 'realview-o3-checker', + 'realview-o3-dual', + 'realview-minor', + 'realview-minor-dual', + 'realview-switcheroo-atomic', + 'realview-switcheroo-timing', + 'realview-switcheroo-o3', + 'realview-switcheroo-full', + 'realview64-simple-atomic', + 'realview64-simple-atomic-checkpoint', + 'realview64-simple-atomic-dual', + 'realview64-simple-timing', + 'realview64-simple-timing-dual', + 'realview64-o3', + 'realview64-o3-checker', + 'realview64-o3-dual', + 'realview64-minor', + 'realview64-minor-dual', + 'realview64-switcheroo-atomic', + 'realview64-switcheroo-timing', + 'realview64-switcheroo-o3', + 'realview64-switcheroo-full', + ), + + ("sparc", None) : ( + 't1000-simple-atomic', + 't1000-simple-x86', + ), + + ("timing", None) : ( + 'pc-simple-atomic', + 'pc-simple-timing', + 'pc-o3-timing', + 'pc-switcheroo-full', + ), + + ("x86", "hsail") : ( + 'gpu', + ), +} + +generic_configs = ( + 'simple-atomic', + 'simple-atomic-mp', + 'simple-timing', + 'simple-timing-mp', + + 'minor-timing', + 'minor-timing-mp', + + 'o3-timing', + 'o3-timing-mt', + 'o3-timing-mp', + + 'rubytest', + 'memcheck', + 'memtest', + 'memtest-filter', + 'tgen-simple-mem', + 'tgen-dram-ctrl', + + 'learning-gem5-p1-simple', + 'learning-gem5-p1-two-level', +) + +all_categories = ("quick", "long") +all_modes = ("fs", "se") + +class Test(object): + """Test case base class. + + Test cases consists of one or more test units that are run in two + phases. A run phase (units produced by run_units() and a verify + phase (units from verify_units()). The verify phase is skipped if + the run phase fails. + + """ + + __metaclass__ = ABCMeta + + def __init__(self, name): + self.test_name = name + + @abstractmethod + def ref_files(self): + """Get a list of reference files used by this test case""" + pass + + @abstractmethod + def run_units(self): + """Units (typically RunGem5 instances) that describe the run phase of + this test. + + """ + pass + + @abstractmethod + def verify_units(self): + """Verify the output from the run phase (see run_units()).""" + pass + + @abstractmethod + def update_ref(self): + """Update reference files with files from a test run""" + pass + + def run(self): + """Run this test case and return a list of results""" + + run_results = [ u.run() for u in self.run_units() ] + run_ok = all([not r.skipped() and r for r in run_results ]) + + verify_results = [ + u.run() if run_ok else u.skip() + for u in self.verify_units() + ] + + return TestResult(self.test_name, run_results + verify_results) + + def __str__(self): + return self.test_name + +class ClassicTest(Test): + diff_ignore_files = [ + # Stat files use a special stat differ, so don't include them + # here. + "stats.txt", + ] + + def __init__(self, gem5, output_dir, config_tuple, + timeout=None, + skip=False, skip_diff_out=False, skip_diff_stat=False): + + super(ClassicTest, self).__init__("/".join(config_tuple)) + + ct = config_tuple + + self.gem5 = os.path.abspath(gem5) + self.script = os.path.join(_test_base, "run.py") + self.config_tuple = ct + self.timeout = timeout + + self.output_dir = output_dir + self.ref_dir = os.path.join(_test_base, + ct.category, ct.mode, ct.workload, + "ref", ct.isa, ct.os, ct.config) + self.skip_run = skip + self.skip_diff_out = skip or skip_diff_out + self.skip_diff_stat = skip or skip_diff_stat + + def ref_files(self): + ref_dir = os.path.abspath(self.ref_dir) + for root, dirs, files in os.walk(ref_dir, topdown=False): + for f in files: + fpath = os.path.join(root[len(ref_dir) + 1:], f) + if fpath not in ClassicTest.diff_ignore_files: + yield fpath + + def run_units(self): + args = [ + self.script, + "/".join(self.config_tuple), + ] + + return [ + RunGem5(self.gem5, args, + ref_dir=self.ref_dir, test_dir=self.output_dir, + skip=self.skip_run), + ] + + def verify_units(self): + return [ + DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir, + skip=self.skip_diff_stat) + ] + [ + DiffOutFile(f, + ref_dir=self.ref_dir, test_dir=self.output_dir, + skip=self.skip_diff_out) + for f in self.ref_files() + if f not in ClassicTest.diff_ignore_files + ] + + def update_ref(self): + for fname in self.ref_files(): + shutil.copy( + os.path.join(self.output_dir, fname), + os.path.join(self.ref_dir, fname)) + +def parse_test_filter(test_filter): + wildcards = ("", "*") + + _filter = list(test_filter.split("/")) + if len(_filter) > 3: + raise RuntimeError("Illegal test filter string") + _filter += [ "", ] * (3 - len(_filter)) + + isa, cat, mode = _filter + + if isa in wildcards: + raise RuntimeError("No ISA specified") + + cat = all_categories if cat in wildcards else (cat, ) + mode = all_modes if mode in wildcards else (mode, ) + + return isa, cat, mode + +def get_tests(isa, + categories=all_categories, modes=all_modes, + ruby_protocol=None, gpu_isa=None): + + # Generate a list of candidate configs + configs = list(arch_configs.get((isa, gpu_isa), [])) + + if (isa, gpu_isa) == ("x86", "hsail"): + if ruby_protocol == "GPU_RfO": + configs += ['gpu-randomtest'] + else: + configs += generic_configs + + if ruby_protocol == 'MI_example': + configs += [ "%s-ruby" % (c, ) for c in configs ] + elif ruby_protocol is not None: + configs += [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ] + + # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/ + for conf_script in configs: + for cat in categories: + for mode in modes: + mode_dir = os.path.join(_test_base, cat, mode) + if not os.path.exists(mode_dir): + continue + + for workload in os.listdir(mode_dir): + isa_dir = os.path.join(mode_dir, workload, "ref", isa) + if not os.path.isdir(isa_dir): + continue + + for _os in os.listdir(isa_dir): + test_dir = os.path.join(isa_dir, _os, conf_script) + if not os.path.exists(test_dir) or \ + os.path.exists(os.path.join(test_dir, "skip")): + continue + + yield ClassicConfig(cat, mode, workload, isa, _os, + conf_script) |