diff options
Diffstat (limited to 'ext/testlib/loader.py')
| -rw-r--r-- | ext/testlib/loader.py | 302 |
1 files changed, 302 insertions, 0 deletions
diff --git a/ext/testlib/loader.py b/ext/testlib/loader.py new file mode 100644 index 000000000..e788c33a9 --- /dev/null +++ b/ext/testlib/loader.py @@ -0,0 +1,302 @@ +# Copyright (c) 2017 Mark D. Hill and David A. Wood +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Sean Wilson + +''' +Contains the :class:`Loader` which is responsible for discovering and loading +tests. + +Loading typically follows the following stages. + +1. Recurse down a given directory looking for tests which match a given regex. + + The default regex used will match any python file (ending in .py) that has + a name starting or ending in test(s). If there are any additional + components of the name they must be connected with '-' or '_'. Lastly, + file names that begin with '.' will be ignored. + + The following names would match: + + - `tests.py` + - `test.py` + - `test-this.py` + - `tests-that.py` + - `these-test.py` + + These would not match: + + - `.test.py` - 'hidden' files are ignored. + - `test` - Must end in '.py' + - `test-.py` - Needs a character after the hypen. + - `testthis.py` - Needs a hypen or underscore to separate 'test' and 'this' + + +2. With all files discovered execute each file gathering its test items we + care about collecting. (`TestCase`, `TestSuite` and `Fixture` objects.) + +As a final note, :class:`TestCase` instances which are not put into +a :class:`TestSuite` by the test writer will be placed into +a :class:`TestSuite` named after the module. + +.. seealso:: :func:`load_file` +''' + +import os +import re +import sys +import traceback + +import config +import log +import suite as suite_mod +import test as test_mod +import fixture as fixture_mod +import wrappers +import uid + +class DuplicateTestItemException(Exception): + ''' + Exception indicates multiple test items with the same UID + were discovered. + ''' + pass + + +# Match filenames that either begin or end with 'test' or tests and use +# - or _ to separate additional name components. +default_filepath_regex = re.compile( + r'(((.+[_])?tests?)|(tests?([-_].+)?))\.py$') + +def default_filepath_filter(filepath): + '''The default filter applied to filepaths to marks as test sources.''' + filepath = os.path.basename(filepath) + if default_filepath_regex.match(filepath): + # Make sure doesn't start with . + return not filepath.startswith('.') + return False + +def path_as_modulename(filepath): + '''Return the given filepath as a module name.''' + # Remove the file extention (.py) + return os.path.splitext(os.path.basename(filepath))[0] + +def path_as_suitename(filepath): + return os.path.split(os.path.dirname(os.path.abspath((filepath))))[-1] + +def _assert_files_in_same_dir(files): + if __debug__: + if files: + directory = os.path.dirname(files[0]) + for f in files: + assert os.path.dirname(f) == directory + +class Loader(object): + ''' + Class for discovering tests. + + Discovered :class:`TestCase` and :class:`TestSuite` objects are wrapped by + :class:`LoadedTest` and :class:`LoadedSuite` objects respectively. + These objects provided additional methods and metadata about the loaded + objects and are the internal representation used by testlib. + + To simply discover and load all tests using the default filter create an + instance and `load_root`. + + >>> import os + >>> tl = Loader() + >>> tl.load_root(os.getcwd()) + + .. note:: If tests are not contained in a TestSuite, they will + automatically be placed into one for the module. + + .. warn:: This class is extremely thread-unsafe. + It modifies the sys path and global config. + Use with care. + ''' + def __init__(self): + self.suites = [] + self.suite_uids = {} + self.filepath_filter = default_filepath_filter + + # filepath -> Successful | Failed to load + self._files = {} + + @property + def schedule(self): + return wrappers.LoadedLibrary(self.suites, fixture_mod.global_fixtures) + + def load_schedule_for_suites(self, *uids): + files = {uid.UID.uid_to_path(id_) for id_ in uids} + for file_ in files: + self.load_file(file_) + + return wrappers.LoadedLibrary( + [self.suite_uids[id_] for id_ in uids], + fixture_mod.global_fixtures) + + def _verify_no_duplicate_suites(self, new_suites): + new_suite_uids = self.suite_uids.copy() + for suite in new_suites: + if suite.uid in new_suite_uids: + raise DuplicateTestItemException( + "More than one suite with UID '%s' was defined" %\ + suite.uid) + new_suite_uids[suite.uid] = suite + + def _verify_no_duplicate_tests_in_suites(self, new_suites): + for suite in new_suites: + test_uids = set() + for test in suite: + if test.uid in test_uids: + raise DuplicateTestItemException( + "More than one test with UID '%s' was defined" + " in suite '%s'" + % (test.uid, suite.uid)) + test_uids.add(test.uid) + + def load_root(self, root): + ''' + Load files from the given root directory which match + `self.filepath_filter`. + ''' + if __debug__: + self._loaded_a_file = True + + for directory in self._discover_files(root): + if directory: + _assert_files_in_same_dir(directory) + for f in directory: + self.load_file(f) + + def load_dir(self, directory): + for dir_ in self._discover_files(directory): + _assert_files_in_same_dir(dir_) + for f in dir_: + self.load_file(f) + + def load_file(self, path): + path = os.path.abspath(path) + + if path in self._files: + if not self._files[path]: + raise Exception('Attempted to load a file which already' + ' failed to load') + else: + log.test_log.debug('Tried to reload: %s' % path) + return + + # Create a custom dictionary for the loaded module. + newdict = { + '__builtins__':__builtins__, + '__name__': path_as_modulename(path), + '__file__': path, + } + + # Add the file's containing directory to the system path. So it can do + # relative imports naturally. + old_path = sys.path[:] + sys.path.insert(0, os.path.dirname(path)) + cwd = os.getcwd() + os.chdir(os.path.dirname(path)) + config.config.file_under_load = path + + new_tests = test_mod.TestCase.collector.create() + new_suites = suite_mod.TestSuite.collector.create() + new_fixtures = fixture_mod.Fixture.collector.create() + + def cleanup(): + config.config.file_under_load = None + sys.path[:] = old_path + os.chdir(cwd) + test_mod.TestCase.collector.remove(new_tests) + suite_mod.TestSuite.collector.remove(new_suites) + fixture_mod.Fixture.collector.remove(new_fixtures) + + try: + execfile(path, newdict, newdict) + except Exception as e: + log.test_log.debug(traceback.format_exc()) + log.test_log.warn( + 'Exception thrown while loading "%s"\n' + 'Ignoring all tests in this file.' + % (path)) + cleanup() + return + + # Create a module test suite for those not contained in a suite. + orphan_tests = set(new_tests) + for suite in new_suites: + for test in suite: + # Remove the test if it wasn't already removed. + # (Suites may contain copies of tests.) + if test in orphan_tests: + orphan_tests.remove(test) + if orphan_tests: + orphan_tests = sorted(orphan_tests, key=new_tests.index) + # FIXME Use the config based default to group all uncollected + # tests. + # NOTE: This is automatically collected (we still have the + # collector active.) + suite_mod.TestSuite(tests=orphan_tests, + name=path_as_suitename(path)) + + try: + loaded_suites = [wrappers.LoadedSuite(suite, path) + for suite in new_suites] + + self._verify_no_duplicate_suites(loaded_suites) + self._verify_no_duplicate_tests_in_suites(loaded_suites) + except Exception as e: + log.test_log.warn('%s\n' + 'Exception thrown while loading "%s"\n' + 'Ignoring all tests in this file.' + % (traceback.format_exc(), path)) + else: + log.test_log.info('Discovered %d tests and %d suites in %s' + '' % (len(new_tests), len(loaded_suites), path)) + + self.suites.extend(loaded_suites) + self.suite_uids.update({suite.uid: suite + for suite in loaded_suites}) + cleanup() + + def _discover_files(self, root): + ''' + Recurse down from the given root directory returning a list of + directories which contain a list of files matching + `self.filepath_filter`. + ''' + # Will probably want to order this traversal. + for root, dirnames, filenames in os.walk(root): + dirnames.sort() + if filenames: + filenames.sort() + filepaths = [os.path.join(root, filename) \ + for filename in filenames] + filepaths = filter(self.filepath_filter, filepaths) + if filepaths: + yield filepaths |