diff options
Diffstat (limited to 'tests/tests.py')
| -rwxr-xr-x | tests/tests.py | 287 |
1 files changed, 287 insertions, 0 deletions
diff --git a/tests/tests.py b/tests/tests.py new file mode 100755 index 000000000..05d68881e --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +import argparse +import sys +import os +import pickle + +from testing.tests import * +import testing.results + +class ParagraphHelpFormatter(argparse.HelpFormatter): + def _fill_text(self, text, width, indent): + return "\n\n".join([ + super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \ + for p in text.split("\n\n") ]) + +formatters = { + "junit" : testing.results.JUnit, + "text" : testing.results.Text, + "summary" : testing.results.TextSummary, + "pickle" : testing.results.Pickle, +} + + +def _add_format_args(parser): + parser.add_argument("--format", choices=formatters, default="text", + help="Output format") + + parser.add_argument("--no-junit-xlate-names", action="store_true", + help="Don't translate test names to " \ + "package-like names") + + parser.add_argument("--output", "-o", + type=argparse.FileType('w'), default=sys.stdout, + help="Test result output file") + + +def _create_formatter(args): + formatter = formatters[args.format] + kwargs = { + "fout" : args.output, + "verbose" : args.verbose + } + + if issubclass(formatter, testing.results.JUnit): + kwargs.update({ + "translate_names" : not args.no_junit_xlate_names, + }) + + return formatter(**kwargs) + + +def _list_tests_args(subparsers): + parser = subparsers.add_parser( + "list", + formatter_class=ParagraphHelpFormatter, + help="List available tests", + description="List available tests", + epilog=""" + Generate a list of available tests using a list filter. + + The filter is a string consisting of the target ISA optionally + followed by the test category and mode separated by + slashes. The test names emitted by this command can be fed + into the run command. + + For example, to list all quick arm tests, run the following: + tests.py list arm/quick + + Non-mandatory parts of the filter string (anything other than + the ISA) can be left out or replaced with the wildcard + character. For example, all full-system tests can be listed + with this command: tests.py list arm/*/fs""") + + parser.add_argument("--ruby-protocol", type=str, default=None, + help="Ruby protocol") + + parser.add_argument("--gpu-isa", type=str, default=None, + help="GPU ISA") + + parser.add_argument("list_filter", metavar="ISA[/category/mode]", + action="append", type=str, + help="List available test cases") + +def _list_tests(args): + for isa, categories, modes in \ + ( parse_test_filter(f) for f in args.list_filter ): + + for test in get_tests(isa, categories=categories, modes=modes, + ruby_protocol=args.ruby_protocol, + gpu_isa=args.gpu_isa): + print "/".join(test) + sys.exit(0) + +def _run_tests_args(subparsers): + parser = subparsers.add_parser( + "run", + formatter_class=ParagraphHelpFormatter, + help='Run one or more tests', + description="Run one or more tests.", + epilog=""" + Run one or more tests described by a gem5 test tuple. + + The test tuple consists of a test category (quick or long), a + test mode (fs or se), a workload name, an isa, an operating + system, and a config name separate by slashes. For example: + quick/se/00.hello/arm/linux/simple-timing + + Available tests can be listed using the 'list' sub-command + (e.g., "tests.py list arm/quick" or one of the scons test list + targets (e.g., "scons build/ARM/tests/opt/quick.list"). + + The test results can be stored in multiple different output + formats. See the help for the show command for more details + about output formatting.""") + + parser.add_argument("gem5", type=str, + help="gem5 binary") + + parser.add_argument("test", type=str, nargs="*", + help="List of tests to execute") + + parser.add_argument("--directory", "-d", + type=str, default="m5tests", + help="Test work directory") + + parser.add_argument("--timeout", "-t", + type=int, default="0", metavar="MINUTES", + help="Timeout, 0 to disable") + + parser.add_argument("--skip-diff-out", action="store_true", + help="Skip output diffing stage") + + parser.add_argument("--skip-diff-stat", action="store_true", + help="Skip stat diffing stage") + + _add_format_args(parser) + +def _run_tests(args): + formatter = _create_formatter(args) + + out_base = os.path.abspath(args.directory) + if not os.path.exists(out_base): + os.mkdir(out_base) + tests = [] + for test_name in args.test: + config = ClassicConfig(*test_name.split("/")) + out_dir = os.path.join(out_base, "/".join(config)) + tests.append( + ClassicTest(args.gem5, out_dir, config, + timeout=args.timeout, + skip_diff_stat=args.skip_diff_stat, + skip_diff_out=args.skip_diff_out)) + + all_results = [] + print "Running %i tests" % len(tests) + for testno, test in enumerate(tests): + print "%i: Running '%s'..." % (testno, test) + + all_results.append(test.run()) + + formatter.dump_suites(all_results) + +def _show_args(subparsers): + parser = subparsers.add_parser( + "show", + formatter_class=ParagraphHelpFormatter, + help='Display pickled test results', + description='Display pickled test results', + epilog=""" + Reformat the pickled output from one or more test runs. This + command is typically used with the output from a single test + run, but it can also be used to merge the outputs from + multiple runs. + + The 'text' format is a verbose output format that provides + information about individual test units and the output from + failed tests. It's mainly useful for debugging test failures. + + The 'summary' format provides outputs the results of one test + per line with the test's overall status (OK, SKIPPED, or + FAILED). + + The 'junit' format is primarily intended for use with CI + systems. It provides an XML representation of test + status. Similar to the text format, it includes detailed + information about test failures. Since many JUnit parser make + assume that test names look like Java packet strings, the + JUnit formatter automatically to something the looks like a + Java class path ('.'->'-', '/'->'.'). + + The 'pickle' format stores the raw results in a format that + can be reformatted using this command. It's typically used + with the show command to merge multiple test results into one + pickle file.""") + + _add_format_args(parser) + + parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", + help="Pickled test results") + +def _show(args): + formatter = _create_formatter(args) + suites = sum([ pickle.load(f) for f in args.result ], []) + formatter.dump_suites(suites) + +_commands = { + "list" : (_list_tests, _list_tests_args), + "run" : (_run_tests, _run_tests_args), + "show" : (_show, _show_args), +} + +def main(): + parser = argparse.ArgumentParser( + formatter_class=ParagraphHelpFormatter, + description="""gem5 testing multi tool.""", + epilog=""" + This tool provides an interface to gem5's test framework that + doesn't depend on gem5's build system. It supports test + listing, running, and output formatting. + + The list sub-command (e.g., "test.py list arm/quick") produces + a list of tests tuples that can be used by the run command + (e.g., "tests.py run gem5.opt + quick/se/00.hello/arm/linux/simple-timing"). + + The run command supports several output formats. One of them, + pickle, contains the raw output from the tests and can be + re-formatted using the show command (e.g., "tests.py show + --format summary *.pickle"). Such pickle files are also + generated by the build system when scons is used to run + regressions. + + See the usage strings for the individual sub-commands for + details.""") + + parser.add_argument("--verbose", action="store_true", + help="Produce more verbose output") + + subparsers = parser.add_subparsers(dest="command") + + for key, (impl, cmd_parser) in _commands.items(): + cmd_parser(subparsers) + + args = parser.parse_args() + impl, cmd_parser = _commands[args.command] + impl(args) + +if __name__ == "__main__": + main() |