#!/usr/bin/env python2 # # Copyright (c) 2016 ARM Limited # All rights reserved # # The license below extends only to copyright in the software and shall # not be construed as granting a license to any other intellectual # property including but not limited to intellectual property relating # to a hardware implementation of the functionality of the software # licensed hereunder. You may use the software subject to the license # terms below provided that you ensure that this notice is replicated # unmodified and in its entirety in all distributions of the software, # modified or unmodified, in source code or in binary form. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer; # redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution; # neither the name of the copyright holders nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Authors: Andreas Sandberg from __future__ import print_function from abc import ABCMeta, abstractmethod import inspect import pickle import string import sys import xml.etree.cElementTree as ET class UnitResult(object): """Results of a single test unit. A test result can be one of: - STATE_OK: Test ran successfully. - STATE_SKIPPED: The test was skipped. - STATE_ERROR: The test failed to run. - STATE_FAILED: Test ran, but failed. The difference between STATE_ERROR and STATE_FAILED is very subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed to start or crashed, while STATE_FAILED would mean that a test failed (e.g., statistics mismatch). """ STATE_OK = 0 STATE_SKIPPED = 1 STATE_ERROR = 2 STATE_FAILURE = 3 state_names = { STATE_OK : "OK", STATE_SKIPPED : "SKIPPED", STATE_ERROR : "ERROR", STATE_FAILURE : "FAILURE", } def __init__(self, name, state, message="", stderr="", stdout="", runtime=0.0): self.name = name self.state = state self.message = message self.stdout = stdout self.stderr = stderr self.runtime = runtime def skipped(self): return self.state == UnitResult.STATE_SKIPPED def success(self): return self.state == UnitResult.STATE_OK def state_name(self): return UnitResult.state_names[self.state] def __nonzero__(self): return self.success() or self.skipped() def __str__(self): state_name = self.state_name() status = "%s: %s" % (state_name, self.message) if self.message else \ state_name return "%s: %s" % (self.name, status) class TestResult(object): """Results for from a single test consisting of one or more units.""" def __init__(self, name, run_results=[], verify_results=[]): self.name = name self.results = run_results + verify_results self.run_results = run_results self.verify_results = verify_results def success(self): return self.success_run() and self.success_verify() def success_run(self): return all([ r.success() for r in self.run_results ]) def success_verify(self): return all([ r.success() for r in self.verify_results ]) def failed(self): return self.failed_run() or self.failed_verify() def failed_run(self): return any([ not r for r in self.run_results ]) def failed_verify(self): return any([ not r for r in self.verify_results ]) def skipped(self): return all([ r.skipped() for r in self.run_results ]) def changed(self): return self.success_run() and self.failed_verify() def runtime(self): return sum([ r.runtime for r in self.results ]) def __nonzero__(self): return all([ r for r in self.results ]) class ResultFormatter(object): __metaclass__ = ABCMeta def __init__(self, fout=sys.stdout, verbose=False): self.verbose = verbose self.fout = fout @abstractmethod def dump_suites(self, suites): pass class Pickle(ResultFormatter): """Save test results as a binary using Python's pickle functionality. """ def __init__(self, **kwargs): super(Pickle, self).__init__(**kwargs) def dump_suites(self, suites): pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) class Text(ResultFormatter): """Output test results as text.""" def __init__(self, **kwargs): super(Text, self).__init__(**kwargs) def dump_suites(self, suites): fout = self.fout for suite in suites: print("--- %s ---" % suite.name, file=fout) for t in suite.results: print("*** %s" % t, file=fout) if t and not self.verbose: continue if t.message: print(t.message, file=fout) if t.stderr: print(t.stderr, file=fout) if t.stdout: print(t.stdout, file=fout) class TextSummary(ResultFormatter): """Output test results as a text summary""" def __init__(self, **kwargs): super(TextSummary, self).__init__(**kwargs) def test_status(self, suite): if suite.skipped(): return "SKIPPED" elif suite.changed(): return "CHANGED" elif suite: return "OK" else: return "FAILED" def dump_suites(self, suites): fout = self.fout for suite in suites: status = self.test_status(suite) print("%s: %s" % (suite.name, status), file=fout) class JUnit(ResultFormatter): """Output test results as JUnit XML""" def __init__(self, translate_names=True, **kwargs): super(JUnit, self).__init__(**kwargs) if translate_names: self.name_table = string.maketrans( "/.", ".-", ) else: self.name_table = string.maketrans("", "") def convert_unit(self, x_suite, test): x_test = ET.SubElement(x_suite, "testcase", name=test.name, time="%f" % test.runtime) x_state = None if test.state == UnitResult.STATE_OK: pass elif test.state == UnitResult.STATE_SKIPPED: x_state = ET.SubElement(x_test, "skipped") elif test.state == UnitResult.STATE_FAILURE: x_state = ET.SubElement(x_test, "failure") elif test.state == UnitResult.STATE_ERROR: x_state = ET.SubElement(x_test, "error") else: assert False, "Unknown test state" if x_state is not None: if test.message: x_state.set("message", test.message) msg = [] if test.stderr: msg.append("*** Standard Errror: ***") msg.append(test.stderr) if test.stdout: msg.append("*** Standard Out: ***") msg.append(test.stdout) x_state.text = "\n".join(msg) return x_test def convert_suite(self, x_suites, suite): x_suite = ET.SubElement(x_suites, "testsuite", name=suite.name.translate(self.name_table), time="%f" % suite.runtime()) errors = 0 failures = 0 skipped = 0 for test in suite.results: if test.state != UnitResult.STATE_OK: if test.state == UnitResult.STATE_SKIPPED: skipped += 1 elif test.state == UnitResult.STATE_ERROR: errors += 1 elif test.state == UnitResult.STATE_FAILURE: failures += 1 x_test = self.convert_unit(x_suite, test) x_suite.set("errors", str(errors)) x_suite.set("failures", str(failures)) x_suite.set("skipped", str(skipped)) x_suite.set("tests", str(len(suite.results))) return x_suite def convert_suites(self, suites): x_root = ET.Element("testsuites") for suite in suites: self.convert_suite(x_root, suite) return x_root def dump_suites(self, suites): et = ET.ElementTree(self.convert_suites(suites)) et.write(self.fout, encoding="UTF-8")