diff options
Diffstat (limited to 'tests/testing/results.py')
| -rw-r--r-- | tests/testing/results.py | 271 |
1 files changed, 271 insertions, 0 deletions
diff --git a/tests/testing/results.py b/tests/testing/results.py new file mode 100644 index 000000000..0c46c9665 --- /dev/null +++ b/tests/testing/results.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Andreas Sandberg + +from abc import ABCMeta, abstractmethod +import inspect +import pickle +import string +import sys + +import xml.etree.cElementTree as ET + +class UnitResult(object): + """Results of a single test unit. + + A test result can be one of: + - STATE_OK: Test ran successfully. + - STATE_SKIPPED: The test was skipped. + - STATE_ERROR: The test failed to run. + - STATE_FAILED: Test ran, but failed. + + The difference between STATE_ERROR and STATE_FAILED is very + subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed + to start or crashed, while STATE_FAILED would mean that a test + failed (e.g., statistics mismatch). + + """ + + STATE_OK = 0 + STATE_SKIPPED = 1 + STATE_ERROR = 2 + STATE_FAILURE = 3 + + state_names = { + STATE_OK : "OK", + STATE_SKIPPED : "SKIPPED", + STATE_ERROR : "ERROR", + STATE_FAILURE : "FAILURE", + } + + def __init__(self, name, state, message="", stderr="", stdout="", + runtime=0.0): + self.name = name + self.state = state + self.message = message + self.stdout = stdout + self.stderr = stderr + self.runtime = runtime + + def skipped(self): + return self.state == UnitResult.STATE_SKIPPED + + def success(self): + return self.state == UnitResult.STATE_OK + + def state_name(self): + return UnitResult.state_names[self.state] + + def __nonzero__(self): + return self.success() or self.skipped() + + def __str__(self): + state_name = self.state_name() + + status = "%s: %s" % (state_name, self.message) if self.message else \ + state_name + + return "%s: %s" % (self.name, status) + +class TestResult(object): + """Results for from a single test consisting of one or more units.""" + + def __init__(self, name, results=[]): + self.name = name + self.results = results + + def success(self): + return all([ r.success() for r in self.results]) + + def skipped(self): + return all([ r.skipped() for r in self.results]) + + def failed(self): + return any([ not r for r in self.results]) + + def runtime(self): + return sum([ r.runtime for r in self.results ]) + + def __nonzero__(self): + return all([r for r in self.results]) + +class ResultFormatter(object): + __metaclass__ = ABCMeta + + def __init__(self, fout=sys.stdout, verbose=False): + self.verbose = verbose + self.fout = fout + + @abstractmethod + def dump_suites(self, suites): + pass + +class Pickle(ResultFormatter): + """Save test results as a binary using Python's pickle + functionality. + + """ + + def __init__(self, **kwargs): + super(Pickle, self).__init__(**kwargs) + + def dump_suites(self, suites): + pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) + +class Text(ResultFormatter): + """Output test results as text.""" + + def __init__(self, **kwargs): + super(Text, self).__init__(**kwargs) + + def dump_suites(self, suites): + fout = self.fout + for suite in suites: + print >> fout, "--- %s ---" % suite.name + + for t in suite.results: + print >> fout, "*** %s" % t + + if t and not self.verbose: + continue + + if t.message: + print >> fout, t.message + + if t.stderr: + print >> fout, t.stderr + if t.stdout: + print >> fout, t.stdout + +class TextSummary(ResultFormatter): + """Output test results as a text summary""" + + def __init__(self, **kwargs): + super(TextSummary, self).__init__(**kwargs) + + def dump_suites(self, suites): + fout = self.fout + for suite in suites: + status = "SKIPPED" if suite.skipped() else \ + ("OK" if suite else "FAILED") + print >> fout, "%s: %s" % (suite.name, status) + +class JUnit(ResultFormatter): + """Output test results as JUnit XML""" + + def __init__(self, translate_names=True, **kwargs): + super(JUnit, self).__init__(**kwargs) + + if translate_names: + self.name_table = string.maketrans( + "/.", + ".-", + ) + else: + self.name_table = string.maketrans("", "") + + def convert_unit(self, x_suite, test): + x_test = ET.SubElement(x_suite, "testcase", + name=test.name, + time="%f" % test.runtime) + + x_state = None + if test.state == UnitResult.STATE_OK: + pass + elif test.state == UnitResult.STATE_SKIPPED: + x_state = ET.SubElement(x_test, "skipped") + elif test.state == UnitResult.STATE_FAILURE: + x_state = ET.SubElement(x_test, "failure") + elif test.state == UnitResult.STATE_ERROR: + x_state = ET.SubElement(x_test, "error") + else: + assert False, "Unknown test state" + + if x_state is not None: + if test.message: + x_state.set("message", test.message) + + msg = [] + if test.stderr: + msg.append("*** Standard Errror: ***") + msg.append(test.stderr) + if test.stdout: + msg.append("*** Standard Out: ***") + msg.append(test.stdout) + + x_state.text = "\n".join(msg) + + return x_test + + def convert_suite(self, x_suites, suite): + x_suite = ET.SubElement(x_suites, "testsuite", + name=suite.name.translate(self.name_table), + time="%f" % suite.runtime()) + errors = 0 + failures = 0 + skipped = 0 + + for test in suite.results: + if test.state != UnitResult.STATE_OK: + if test.state == UnitResult.STATE_SKIPPED: + skipped += 1 + elif test.state == UnitResult.STATE_ERROR: + errors += 1 + elif test.state == UnitResult.STATE_FAILURE: + failures += 1 + + x_test = self.convert_unit(x_suite, test) + + x_suite.set("errors", str(errors)) + x_suite.set("failures", str(failures)) + x_suite.set("skipped", str(skipped)) + x_suite.set("tests", str(len(suite.results))) + + return x_suite + + def convert_suites(self, suites): + x_root = ET.Element("testsuites") + + for suite in suites: + self.convert_suite(x_root, suite) + + return x_root + + def dump_suites(self, suites): + et = ET.ElementTree(self.convert_suites(suites)) + et.write(self.fout, encoding="UTF-8") |