1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
|
# Copyright (c) 2017 Mark D. Hill and David A. Wood
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Sean Wilson
import os
import itertools
import config
import fixture as fixture_mod
import handlers
import loader as loader_mod
import log
import query
import result
import runner
import terminal
import uid
def entry_message():
log.test_log.message("Running the new gem5 testing script.")
log.test_log.message("For more information see TESTING.md.")
log.test_log.message("To see details as the testing scripts are"
" running, use the option"
" -v, -vv, or -vvv")
class RunLogHandler():
def __init__(self):
term_handler = handlers.TerminalHandler(
verbosity=config.config.verbose+log.LogLevel.Info
)
summary_handler = handlers.SummaryHandler()
self.mp_handler = handlers.MultiprocessingHandlerWrapper(
summary_handler, term_handler)
self.mp_handler.async_process()
log.test_log.log_obj.add_handler(self.mp_handler)
entry_message()
def schedule_finalized(self, test_schedule):
# Create the result handler object.
self.result_handler = handlers.ResultHandler(
test_schedule, config.config.result_path)
self.mp_handler.add_handler(self.result_handler)
def finish_testing(self):
self.result_handler.close()
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
return False
def close(self):
self.mp_handler.close()
def unsuccessful(self):
'''
Performs an or reduce on all of the results.
Returns true if at least one test is unsuccessful, false when all tests
pass
'''
return self.result_handler.unsuccessful()
def get_config_tags():
return getattr(config.config,
config.StorePositionalTagsAction.position_kword)
def filter_with_config_tags(loaded_library):
tags = get_config_tags()
final_tags = []
regex_fmt = '^%s$'
cfg = config.config
def _append_inc_tag_filter(name):
if hasattr(cfg, name):
tag_opts = getattr(cfg, name)
for tag in tag_opts:
final_tags.append(config.TagRegex(True, regex_fmt % tag))
def _append_rem_tag_filter(name):
if hasattr(cfg, name):
tag_opts = getattr(cfg, name)
for tag in cfg.constants.supported_tags[name]:
if tag not in tag_opts:
final_tags.append(config.TagRegex(False, regex_fmt % tag))
# Append additional tags for the isa, length, and variant options.
# They apply last (they take priority)
special_tags = (
cfg.constants.isa_tag_type,
cfg.constants.length_tag_type,
cfg.constants.variant_tag_type
)
for tagname in special_tags:
_append_inc_tag_filter(tagname)
for tagname in special_tags:
_append_rem_tag_filter(tagname)
if tags is None:
tags = tuple()
filters = list(itertools.chain(tags, final_tags))
string = 'Filtering suites with tags as follows:\n'
filter_string = '\t\n'.join((str(f) for f in filters))
log.test_log.trace(string + filter_string)
return filter_with_tags(loaded_library, filters)
def filter_with_tags(loaded_library, filters):
'''
Filter logic supports two filter types:
--include-tags <regex>
--exclude-tags <regex>
The logic maintains a `set` of test suites.
If the regex provided with the `--include-tags` flag matches a tag of a
suite, that suite will added to the set.
If the regex provided with the `--exclude-tags` flag matches a tag of a
suite, that suite will removed to the set.
Suites can be added and removed multiple times.
First Flag Special Case Logic:
If include is the first flag, start with an empty set of suites.
If exclude is the first flag, start with the set of all collected suites.
Let's trace out the set as we go through the flags to clarify::
# Say our collection of suites looks like this: set(suite_ARM64,
# suite_X86, suite_Other).
#
# Additionally, we've passed the flags in the following order:
# --include-tags "ARM64" --exclude-tags ".*" --include-tags "X86"
# Process --include-tags "ARM64"
set(suite_ARM64) # Suite begins empty, but adds the ARM64 suite
# Process --exclude-tags ".*"
set() # Removed all suites which have tags
# Process --include-tags "X86"
set(suite_X86)
'''
if not filters:
return
query_runner = query.QueryRunner(loaded_library)
tags = query_runner.tags()
if not filters[0].include:
suites = set(query_runner.suites())
else:
suites = set()
def exclude(excludes):
return suites - excludes
def include(includes):
return suites | includes
for tag_regex in filters:
matched_tags = (tag for tag in tags if tag_regex.regex.search(tag))
for tag in matched_tags:
matched_suites = set(query_runner.suites_with_tag(tag))
suites = include(matched_suites) if tag_regex.include \
else exclude(matched_suites)
# Set the library's suites to only those which where accepted by our filter
loaded_library.suites = [suite for suite in loaded_library.suites
if suite in suites]
# TODO Add results command for listing previous results.
def load_tests():
'''
Create a TestLoader and load tests for the directory given by the config.
'''
testloader = loader_mod.Loader()
log.test_log.message(terminal.separator())
log.test_log.message('Loading Tests', bold=True)
testloader.load_root(config.config.directory)
return testloader
def do_list():
term_handler = handlers.TerminalHandler(
verbosity=config.config.verbose+log.LogLevel.Info,
machine_only=config.config.quiet
)
log.test_log.log_obj.add_handler(term_handler)
entry_message()
test_schedule = load_tests().schedule
filter_with_config_tags(test_schedule)
qrunner = query.QueryRunner(test_schedule)
if config.config.suites:
qrunner.list_suites()
elif config.config.tests:
qrunner.list_tests()
elif config.config.all_tags:
qrunner.list_tags()
else:
qrunner.list_suites()
qrunner.list_tests()
qrunner.list_tags()
return 0
def run_schedule(test_schedule, log_handler):
'''
Test Phases
-----------
* Test Collection
* Fixture Parameterization
* Global Fixture Setup
* Iteratevely run suites:
* Suite Fixture Setup
* Iteratively run tests:
* Test Fixture Setup
* Run Test
* Test Fixture Teardown
* Suite Fixture Teardown
* Global Fixture Teardown
'''
log_handler.schedule_finalized(test_schedule)
log.test_log.message(terminal.separator())
log.test_log.message('Running Tests from {} suites'
.format(len(test_schedule.suites)), bold=True)
log.test_log.message("Results will be stored in {}".format(
config.config.result_path))
log.test_log.message(terminal.separator())
# Build global fixtures and exectute scheduled test suites.
if config.config.test_threads > 1:
library_runner = runner.LibraryParallelRunner(test_schedule)
library_runner.set_threads(config.config.test_threads)
else:
library_runner = runner.LibraryRunner(test_schedule)
library_runner.run()
failed = log_handler.unsuccessful()
log_handler.finish_testing()
return 1 if failed else 0
def do_run():
# Initialize early parts of the log.
with RunLogHandler() as log_handler:
if config.config.uid:
uid_ = uid.UID.from_uid(config.config.uid)
if isinstance(uid_, uid.TestUID):
log.test_log.error('Unable to run a standalone test.\n'
'Gem5 expects test suites to be the smallest unit '
' of test.\n\n'
'Pass a SuiteUID instead.')
return
test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_)
if get_config_tags():
log.test_log.warn(
"The '--uid' flag was supplied,"
" '--include-tags' and '--exclude-tags' will be ignored."
)
else:
test_schedule = load_tests().schedule
# Filter tests based on tags
filter_with_config_tags(test_schedule)
# Execute the tests
return run_schedule(test_schedule, log_handler)
def do_rerun():
# Init early parts of log
with RunLogHandler() as log_handler:
# Load previous results
results = result.InternalSavedResults.load(
os.path.join(config.config.result_path,
config.constants.pickle_filename))
rerun_suites = (suite.uid for suite in results if suite.unsuccessful)
# Use loader to load suites
loader = loader_mod.Loader()
test_schedule = loader.load_schedule_for_suites(*rerun_suites)
# Execute the tests
return run_schedule(test_schedule, log_handler)
def main():
'''
Main entrypoint for the testlib test library.
Returns 0 on success and 1 otherwise so it can be used as a return code
for scripts.
'''
config.initialize_config()
# 'do' the given command.
result = globals()['do_'+config.config.command]()
log.test_log.close()
return result
|