diff options
Diffstat (limited to 'ext/testlib/result.py')
-rw-r--r-- | ext/testlib/result.py | 303 |
1 files changed, 303 insertions, 0 deletions
diff --git a/ext/testlib/result.py b/ext/testlib/result.py new file mode 100644 index 000000000..8e3f38d25 --- /dev/null +++ b/ext/testlib/result.py @@ -0,0 +1,303 @@ +# Copyright (c) 2017 Mark D. Hill and David A. Wood +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Sean Wilson + +import os +import pickle +import xml.sax.saxutils + +from config import config +import helper +import state +import log + +def _create_uid_index(iterable): + index = {} + for item in iterable: + assert item.uid not in index + index[item.uid] = item + return index + + +class _CommonMetadataMixin: + @property + def name(self): + return self._metadata.name + @property + def uid(self): + return self._metadata.uid + @property + def result(self): + return self._metadata.result + @result.setter + def result(self, result): + self._metadata.result = result + + @property + def unsucessful(self): + return self._metadata.result.value != state.Result.Passed + + +class InternalTestResult(object, _CommonMetadataMixin): + def __init__(self, obj, suite, directory): + self._metadata = obj.metadata + self.suite = suite + + self.stderr = os.path.join( + InternalSavedResults.output_path(self.uid, suite.uid), + 'stderr' + ) + self.stdout = os.path.join( + InternalSavedResults.output_path(self.uid, suite.uid), + 'stdout' + ) + + +class InternalSuiteResult(object, _CommonMetadataMixin): + def __init__(self, obj, directory): + self._metadata = obj.metadata + self.directory = directory + self._wrap_tests(obj) + + def _wrap_tests(self, obj): + self._tests = [InternalTestResult(test, self, self.directory) + for test in obj] + self._tests_index = _create_uid_index(self._tests) + + def get_test(self, uid): + return self._tests_index[uid] + + def __iter__(self): + return iter(self._tests) + + def get_test_result(self, uid): + return self.get_test(uid) + + def aggregate_test_results(self): + results = {} + for test in self: + helper.append_dictlist(results, test.result.value, test) + return results + + +class InternalLibraryResults(object, _CommonMetadataMixin): + def __init__(self, obj, directory): + self.directory = directory + self._metadata = obj.metadata + self._wrap_suites(obj) + + def __iter__(self): + return iter(self._suites) + + def _wrap_suites(self, obj): + self._suites = [InternalSuiteResult(suite, self.directory) + for suite in obj] + self._suites_index = _create_uid_index(self._suites) + + def add_suite(self, suite): + if suite.uid in self._suites: + raise ValueError('Cannot have duplicate suite UIDs.') + self._suites[suite.uid] = suite + + def get_suite_result(self, suite_uid): + return self._suites_index[suite_uid] + + def get_test_result(self, test_uid, suite_uid): + return self.get_suite_result(suite_uid).get_test_result(test_uid) + + def aggregate_test_results(self): + results = {} + for suite in self._suites: + for test in suite: + helper.append_dictlist(results, test.result.value, test) + return results + +class InternalSavedResults: + @staticmethod + def output_path(test_uid, suite_uid, base=None): + ''' + Return the path which results for a specific test case should be + stored. + ''' + if base is None: + base = config.result_path + return os.path.join( + base, + str(suite_uid).replace(os.path.sep, '-'), + str(test_uid).replace(os.path.sep, '-')) + + @staticmethod + def save(results, path, protocol=pickle.HIGHEST_PROTOCOL): + if not os.path.exists(os.path.dirname(path)): + try: + os.makedirs(os.path.dirname(path)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + + with open(path, 'w') as f: + pickle.dump(results, f, protocol) + + @staticmethod + def load(path): + with open(path, 'r') as f: + return pickle.load(f) + + +class XMLElement(object): + def write(self, file_): + self.begin(file_) + self.end(file_) + + def begin(self, file_): + file_.write('<') + file_.write(self.name) + for attr in self.attributes: + file_.write(' ') + attr.write(file_) + file_.write('>') + + self.body(file_) + + def body(self, file_): + for elem in self.elements: + file_.write('\n') + elem.write(file_) + file_.write('\n') + + def end(self, file_): + file_.write('</%s>' % self.name) + +class XMLAttribute(object): + def __init__(self, name, value): + self.name = name + self.value = value + + def write(self, file_): + file_.write('%s=%s' % (self.name, + xml.sax.saxutils.quoteattr(self.value))) + + +class JUnitTestSuites(XMLElement): + name = 'testsuites' + result_map = { + state.Result.Errored: 'errors', + state.Result.Failed: 'failures', + state.Result.Passed: 'tests' + } + + def __init__(self, internal_results): + results = internal_results.aggregate_test_results() + + self.attributes = [] + for result, tests in results.items(): + self.attributes.append(self.result_attribute(result, + str(len(tests)))) + + self.elements = [] + for suite in internal_results: + self.elements.append(JUnitTestSuite(suite)) + + def result_attribute(self, result, count): + return XMLAttribute(self.result_map[result], count) + +class JUnitTestSuite(JUnitTestSuites): + name = 'testsuite' + result_map = { + state.Result.Errored: 'errors', + state.Result.Failed: 'failures', + state.Result.Passed: 'tests', + state.Result.Skipped: 'skipped' + } + + def __init__(self, suite_result): + results = suite_result.aggregate_test_results() + + self.attributes = [ + XMLAttribute('name', suite_result.name) + ] + for result, tests in results.items(): + self.attributes.append(self.result_attribute(result, + str(len(tests)))) + + self.elements = [] + for test in suite_result: + self.elements.append(JUnitTestCase(test)) + + def result_attribute(self, result, count): + return XMLAttribute(self.result_map[result], count) + +class JUnitTestCase(XMLElement): + name = 'testcase' + def __init__(self, test_result): + self.attributes = [ + XMLAttribute('name', test_result.name), + # TODO JUnit expects class of test.. add as test metadata. + XMLAttribute('classname', str(test_result.uid)), + XMLAttribute('status', str(test_result.result)), + ] + + # TODO JUnit expects a message for the reason a test was + # skipped or errored, save this with the test metadata. + # http://llg.cubic.org/docs/junit/ + self.elements = [ + LargeFileElement('system-err', test_result.stderr), + LargeFileElement('system-out', test_result.stdout), + ] + +class LargeFileElement(XMLElement): + def __init__(self, name, filename): + self.name = name + self.filename = filename + self.attributes = [] + + def body(self, file_): + try: + with open(self.filename, 'r') as f: + for line in f: + file_.write(xml.sax.saxutils.escape(line)) + except IOError: + # TODO Better error logic, this is sometimes O.K. + # if there was no stdout/stderr captured for the test + # + # TODO If that was the case, the file should still be made and it + # should just be empty instead of not existing. + pass + + + +class JUnitSavedResults: + @staticmethod + def save(results, path): + ''' + Compile the internal results into JUnit format writting it to the + given file. + ''' + results = JUnitTestSuites(results) + with open(path, 'w') as f: + results.write(f) + |