summaryrefslogtreecommitdiff
path: root/testing/tools/test_runner.py
diff options
context:
space:
mode:
Diffstat (limited to 'testing/tools/test_runner.py')
-rw-r--r--testing/tools/test_runner.py73
1 files changed, 54 insertions, 19 deletions
diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py
index 6cd3a6c05a..97a2513385 100644
--- a/testing/tools/test_runner.py
+++ b/testing/tools/test_runner.py
@@ -39,6 +39,7 @@ def TestOneFileParallel(this, test_case):
class TestRunner:
def __init__(self, dirname):
self.test_dir = dirname
+ self.enforce_expected_images = False
# GenerateAndTest returns a tuple <success, outputfiles> where
# success is a boolean indicating whether the tests passed comparison
@@ -62,8 +63,8 @@ class TestRunner:
raised_exception = self.Generate(source_dir, input_filename, input_root,
pdf_path)
- if raised_exception != None:
- print "FAILURE: " + input_filename + "; " + str(raised_exception)
+ if raised_exception is not None:
+ print 'FAILURE: %s; %s' % (input_filename, raised_exception)
return False, []
results = []
@@ -72,23 +73,29 @@ class TestRunner:
else:
raised_exception, results = self.TestPixel(input_root, pdf_path)
- if raised_exception != None:
- print "FAILURE: " + input_filename + "; " + str(raised_exception)
+ if raised_exception is not None:
+ print 'FAILURE: %s; %s' % (input_filename, raised_exception)
return False, results
- if len(actual_images):
+ if actual_images:
if self.image_differ.HasDifferences(input_filename, source_dir,
self.working_dir):
return False, results
+ else:
+ if (self.enforce_expected_images
+ and not self.test_suppressor.IsPixelDiffSuppressed(input_filename)):
+ print 'FAILURE: %s; Missing expected images' % input_filename
+ return False, results
+
return True, results
def Generate(self, source_dir, input_filename, input_root, pdf_path):
original_path = os.path.join(source_dir, input_filename)
input_path = os.path.join(source_dir, input_root + '.in')
- input_event_path = os.path.join(source_dir, input_root + ".evt")
+ input_event_path = os.path.join(source_dir, input_root + '.evt')
if os.path.exists(input_event_path):
- output_event_path = os.path.splitext(pdf_path)[0] + ".evt"
+ output_event_path = os.path.splitext(pdf_path)[0] + '.evt'
shutil.copyfile(input_event_path, output_event_path)
if not os.path.exists(input_path):
@@ -131,6 +138,7 @@ class TestRunner:
self.gold_results.AddTestResult(test_name, md5_hash, img_path)
if self.test_suppressor.IsResultSuppressed(input_filename):
+ self.result_suppressed_cases.append(input_filename)
if success:
self.surprises.append(input_path)
else:
@@ -178,7 +186,7 @@ class TestRunner:
self.pdfium_test_path = finder.ExecutablePath('pdfium_test')
if not os.path.exists(self.pdfium_test_path):
print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path
- print "Use --build-dir to specify its location."
+ print 'Use --build-dir to specify its location.'
return 1
self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir))
@@ -192,45 +200,49 @@ class TestRunner:
walk_from_dir = finder.TestingDir(test_dir);
- test_cases = []
+ self.test_cases = []
+ self.execution_suppressed_cases = []
input_file_re = re.compile('^[a-zA-Z0-9_.]+[.](in|pdf)$')
- if len(args):
+ if args:
for file_name in args:
- file_name.replace(".pdf", ".in")
+ file_name.replace('.pdf', '.in')
input_path = os.path.join(walk_from_dir, file_name)
if not os.path.isfile(input_path):
print "Can't find test file '%s'" % file_name
return 1
- test_cases.append((os.path.basename(input_path),
+ self.test_cases.append((os.path.basename(input_path),
os.path.dirname(input_path)))
else:
for file_dir, _, filename_list in os.walk(walk_from_dir):
for input_filename in filename_list:
if input_file_re.match(input_filename):
input_path = os.path.join(file_dir, input_filename)
- if not self.test_suppressor.IsExecutionSuppressed(input_path):
+ if self.test_suppressor.IsExecutionSuppressed(input_path):
+ self.execution_suppressed_cases.append(input_path)
+ else:
if os.path.isfile(input_path):
- test_cases.append((input_filename, file_dir))
+ self.test_cases.append((input_filename, file_dir))
self.failures = []
self.surprises = []
+ self.result_suppressed_cases = []
# Collect Gold results if an output directory was named.
self.gold_results = None
if options.gold_output_dir:
- self.gold_results = gold.GoldResults("pdfium",
+ self.gold_results = gold.GoldResults('pdfium',
options.gold_output_dir,
options.gold_properties,
options.gold_key,
options.gold_ignore_hashes)
- if options.num_workers > 1 and len(test_cases) > 1:
+ if options.num_workers > 1 and len(self.test_cases) > 1:
try:
pool = multiprocessing.Pool(options.num_workers)
worker_func = functools.partial(TestOneFileParallel, self)
- worker_results = pool.imap(worker_func, test_cases)
+ worker_results = pool.imap(worker_func, self.test_cases)
for worker_result in worker_results:
result, input_filename, source_dir = worker_result
input_path = os.path.join(source_dir, input_filename)
@@ -243,7 +255,7 @@ class TestRunner:
pool.close()
pool.join()
else:
- for test_case in test_cases:
+ for test_case in self.test_cases:
input_filename, input_file_dir = test_case
result = self.GenerateAndTest(input_filename, input_file_dir)
self.HandleResult(input_filename,
@@ -256,7 +268,7 @@ class TestRunner:
self.surprises.sort()
print '\n\nUnexpected Successes:'
for surprise in self.surprises:
- print surprise;
+ print surprise
if self.failures:
self.failures.sort()
@@ -264,6 +276,29 @@ class TestRunner:
for failure in self.failures:
print failure
+ self._PrintSummary()
+
+ if self.failures:
if not options.ignore_errors:
return 1
+
return 0
+
+ def _PrintSummary(self):
+ number_test_cases = len(self.test_cases)
+ number_failures = len(self.failures)
+ number_suppressed = len(self.result_suppressed_cases)
+ number_successes = number_test_cases - number_failures - number_suppressed
+ number_surprises = len(self.surprises)
+ print
+ print 'Test cases executed: %d' % number_test_cases
+ print ' Successes: %d' % number_successes
+ print ' Suppressed: %d' % number_suppressed
+ print ' Surprises: %d' % number_surprises
+ print ' Failures: %d' % number_failures
+ print
+ print 'Test cases not executed: %d' % len(self.execution_suppressed_cases)
+
+ def SetEnforceExpectedImages(self, new_value):
+ """Set whether to enforce that each test case provide an expected image."""
+ self.enforce_expected_images = new_value