1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
|
# Copyright (c) 2017 Mark D. Hill and David A. Wood
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Sean Wilson
'''
Contains the :class:`Loader` which is responsible for discovering and loading
tests.
Loading typically follows the following stages.
1. Recurse down a given directory looking for tests which match a given regex.
The default regex used will match any python file (ending in .py) that has
a name starting or ending in test(s). If there are any additional
components of the name they must be connected with '-' or '_'. Lastly,
file names that begin with '.' will be ignored.
The following names would match:
- `tests.py`
- `test.py`
- `test-this.py`
- `tests-that.py`
- `these-test.py`
These would not match:
- `.test.py` - 'hidden' files are ignored.
- `test` - Must end in '.py'
- `test-.py` - Needs a character after the hypen.
- `testthis.py` - Needs a hypen or underscore to separate 'test' and 'this'
2. With all files discovered execute each file gathering its test items we
care about collecting. (`TestCase`, `TestSuite` and `Fixture` objects.)
As a final note, :class:`TestCase` instances which are not put into
a :class:`TestSuite` by the test writer will be placed into
a :class:`TestSuite` named after the module.
.. seealso:: :func:`load_file`
'''
import os
import re
import sys
import traceback
import config
import log
import suite as suite_mod
import test as test_mod
import fixture as fixture_mod
import wrappers
import uid
class DuplicateTestItemException(Exception):
'''
Exception indicates multiple test items with the same UID
were discovered.
'''
pass
# Match filenames that either begin or end with 'test' or tests and use
# - or _ to separate additional name components.
default_filepath_regex = re.compile(
r'(((.+[_])?tests?)|(tests?([-_].+)?))\.py$')
def default_filepath_filter(filepath):
'''The default filter applied to filepaths to marks as test sources.'''
filepath = os.path.basename(filepath)
if default_filepath_regex.match(filepath):
# Make sure doesn't start with .
return not filepath.startswith('.')
return False
def path_as_modulename(filepath):
'''Return the given filepath as a module name.'''
# Remove the file extention (.py)
return os.path.splitext(os.path.basename(filepath))[0]
def path_as_suitename(filepath):
return os.path.split(os.path.dirname(os.path.abspath((filepath))))[-1]
def _assert_files_in_same_dir(files):
if __debug__:
if files:
directory = os.path.dirname(files[0])
for f in files:
assert os.path.dirname(f) == directory
class Loader(object):
'''
Class for discovering tests.
Discovered :class:`TestCase` and :class:`TestSuite` objects are wrapped by
:class:`LoadedTest` and :class:`LoadedSuite` objects respectively.
These objects provided additional methods and metadata about the loaded
objects and are the internal representation used by testlib.
To simply discover and load all tests using the default filter create an
instance and `load_root`.
>>> import os
>>> tl = Loader()
>>> tl.load_root(os.getcwd())
.. note:: If tests are not contained in a TestSuite, they will
automatically be placed into one for the module.
.. warn:: This class is extremely thread-unsafe.
It modifies the sys path and global config.
Use with care.
'''
def __init__(self):
self.suites = []
self.suite_uids = {}
self.filepath_filter = default_filepath_filter
# filepath -> Successful | Failed to load
self._files = {}
@property
def schedule(self):
return wrappers.LoadedLibrary(self.suites)
def load_schedule_for_suites(self, *uids):
files = {uid.UID.uid_to_path(id_) for id_ in uids}
for file_ in files:
self.load_file(file_)
return wrappers.LoadedLibrary(
[self.suite_uids[id_] for id_ in uids])
def _verify_no_duplicate_suites(self, new_suites):
new_suite_uids = self.suite_uids.copy()
for suite in new_suites:
if suite.uid in new_suite_uids:
raise DuplicateTestItemException(
"More than one suite with UID '%s' was defined" %\
suite.uid)
new_suite_uids[suite.uid] = suite
def _verify_no_duplicate_tests_in_suites(self, new_suites):
for suite in new_suites:
test_uids = set()
for test in suite:
if test.uid in test_uids:
raise DuplicateTestItemException(
"More than one test with UID '%s' was defined"
" in suite '%s'"
% (test.uid, suite.uid))
test_uids.add(test.uid)
def load_root(self, root):
'''
Load files from the given root directory which match
`self.filepath_filter`.
'''
if __debug__:
self._loaded_a_file = True
for directory in self._discover_files(root):
if directory:
_assert_files_in_same_dir(directory)
for f in directory:
self.load_file(f)
def load_dir(self, directory):
for dir_ in self._discover_files(directory):
_assert_files_in_same_dir(dir_)
for f in dir_:
self.load_file(f)
def load_file(self, path):
path = os.path.abspath(path)
if path in self._files:
if not self._files[path]:
raise Exception('Attempted to load a file which already'
' failed to load')
else:
log.test_log.debug('Tried to reload: %s' % path)
return
# Create a custom dictionary for the loaded module.
newdict = {
'__builtins__':__builtins__,
'__name__': path_as_modulename(path),
'__file__': path,
}
# Add the file's containing directory to the system path. So it can do
# relative imports naturally.
old_path = sys.path[:]
sys.path.insert(0, os.path.dirname(path))
cwd = os.getcwd()
os.chdir(os.path.dirname(path))
config.config.file_under_load = path
new_tests = test_mod.TestCase.collector.create()
new_suites = suite_mod.TestSuite.collector.create()
new_fixtures = fixture_mod.Fixture.collector.create()
def cleanup():
config.config.file_under_load = None
sys.path[:] = old_path
os.chdir(cwd)
test_mod.TestCase.collector.remove(new_tests)
suite_mod.TestSuite.collector.remove(new_suites)
fixture_mod.Fixture.collector.remove(new_fixtures)
try:
execfile(path, newdict, newdict)
except Exception as e:
log.test_log.debug(traceback.format_exc())
log.test_log.warn(
'Exception thrown while loading "%s"\n'
'Ignoring all tests in this file.'
% (path))
cleanup()
return
# Create a module test suite for those not contained in a suite.
orphan_tests = set(new_tests)
for suite in new_suites:
for test in suite:
# Remove the test if it wasn't already removed.
# (Suites may contain copies of tests.)
if test in orphan_tests:
orphan_tests.remove(test)
if orphan_tests:
orphan_tests = sorted(orphan_tests, key=new_tests.index)
# FIXME Use the config based default to group all uncollected
# tests.
# NOTE: This is automatically collected (we still have the
# collector active.)
suite_mod.TestSuite(tests=orphan_tests,
name=path_as_suitename(path))
try:
loaded_suites = [wrappers.LoadedSuite(suite, path)
for suite in new_suites]
self._verify_no_duplicate_suites(loaded_suites)
self._verify_no_duplicate_tests_in_suites(loaded_suites)
except Exception as e:
log.test_log.warn('%s\n'
'Exception thrown while loading "%s"\n'
'Ignoring all tests in this file.'
% (traceback.format_exc(), path))
else:
log.test_log.info('Discovered %d tests and %d suites in %s'
'' % (len(new_tests), len(loaded_suites), path))
self.suites.extend(loaded_suites)
self.suite_uids.update({suite.uid: suite
for suite in loaded_suites})
cleanup()
def _discover_files(self, root):
'''
Recurse down from the given root directory returning a list of
directories which contain a list of files matching
`self.filepath_filter`.
'''
# Will probably want to order this traversal.
for root, dirnames, filenames in os.walk(root):
dirnames.sort()
if filenames:
filenames.sort()
filepaths = [os.path.join(root, filename) \
for filename in filenames]
filepaths = filter(self.filepath_filter, filepaths)
if filepaths:
yield filepaths
|