diff options
| -rw-r--r-- | testing/SUPPRESSIONS_PIXEL | 33 | ||||
| -rwxr-xr-x | testing/tools/run_corpus_tests.py | 1 | ||||
| -rwxr-xr-x | testing/tools/run_pixel_tests.py | 1 | ||||
| -rwxr-xr-x | testing/tools/suppressor.py | 16 | ||||
| -rw-r--r-- | testing/tools/test_runner.py | 73 |
5 files changed, 102 insertions, 22 deletions
diff --git a/testing/SUPPRESSIONS_PIXEL b/testing/SUPPRESSIONS_PIXEL new file mode 100644 index 0000000000..6bd0511199 --- /dev/null +++ b/testing/SUPPRESSIONS_PIXEL @@ -0,0 +1,33 @@ +# Copyright 2017 The PDFium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# List of tests to have pixel diff tests skipped, one per line. +# There are four space-separated columns per line +# Each column (except column 0) can contain a comma-separated list of values. +# +# Column 0: test file name +# Column 1: platform: *, win, mac, linux +# Column 2: v8 support: *, nov8, v8 +# Column 3: xfa support: *, noxfa, xfa +# +# All columns on a line on a line must match, but filenames may be repeated +# on subsequent lines to suppress more cases. Within each column, any one of +# the comma-separated values must match in order for the colum to "match". +# The filenames and keywords are case-sensitive. +# +# Try to keep the file alphabetized within each category of test. + +# +# Corpus tests +# +FRC_3.5_CF_Strf_stmf_DefaultCryptFilter.pdf * * * +FRC_3.5_EncryptMetadata_T.pdf * * * +FRC_3.5_Encrypt_is_damage.pdf * * * +FRC_3.5_Filter_PubSec_SubFilter_s5.pdf * * * +FRC_3.5_Filter_PubSec_Sub_SubFilter_s4.pdf * * * +Line_Stroke.pdf.pdf * * * +MouseEvents.pdf * * * +Oneof.pdf * * * +bug_651304.pdf * * * +outline.pdf * * * diff --git a/testing/tools/run_corpus_tests.py b/testing/tools/run_corpus_tests.py index 4942406579..5932685f32 100755 --- a/testing/tools/run_corpus_tests.py +++ b/testing/tools/run_corpus_tests.py @@ -9,6 +9,7 @@ import test_runner def main(): runner = test_runner.TestRunner('corpus') + runner.SetEnforceExpectedImages(True) return runner.Run() if __name__ == '__main__': diff --git a/testing/tools/run_pixel_tests.py b/testing/tools/run_pixel_tests.py index aad39c5600..1d61966c2e 100755 --- a/testing/tools/run_pixel_tests.py +++ b/testing/tools/run_pixel_tests.py @@ -9,6 +9,7 @@ import test_runner def main(): runner = test_runner.TestRunner('pixel') + runner.SetEnforceExpectedImages(True) return runner.Run() if __name__ == '__main__': diff --git a/testing/tools/suppressor.py b/testing/tools/suppressor.py index 3b2872df95..be03e21516 100755 --- a/testing/tools/suppressor.py +++ b/testing/tools/suppressor.py @@ -12,11 +12,15 @@ class Suppressor: feature_vector = feature_string.strip().split(",") self.has_v8 = "V8" in feature_vector self.has_xfa = "XFA" in feature_vector + self.suppression_set = self._LoadSuppressedSet('SUPPRESSIONS', finder) + self.pixel_suppression_set = self._LoadSuppressedSet('SUPPRESSIONS_PIXEL', + finder) + + def _LoadSuppressedSet(self, suppressions_filename, finder): v8_option = "v8" if self.has_v8 else "nov8" xfa_option = "xfa" if self.has_xfa else "noxfa" - - with open(os.path.join(finder.TestingDir(), 'SUPPRESSIONS')) as f: - self.suppression_set = set(self._FilterSuppressions( + with open(os.path.join(finder.TestingDir(), suppressions_filename)) as f: + return set(self._FilterSuppressions( common.os_name(), v8_option, xfa_option, self._ExtractSuppressions(f))) def _ExtractSuppressions(self, f): @@ -47,3 +51,9 @@ class Suppressor: print "%s execution is suppressed" % input_filepath return True return False + + def IsPixelDiffSuppressed(self, input_filename): + if input_filename in self.pixel_suppression_set: + print "%s is pixel suppressed" % input_filename + return True + return False diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py index 6cd3a6c05a..97a2513385 100644 --- a/testing/tools/test_runner.py +++ b/testing/tools/test_runner.py @@ -39,6 +39,7 @@ def TestOneFileParallel(this, test_case): class TestRunner: def __init__(self, dirname): self.test_dir = dirname + self.enforce_expected_images = False # GenerateAndTest returns a tuple <success, outputfiles> where # success is a boolean indicating whether the tests passed comparison @@ -62,8 +63,8 @@ class TestRunner: raised_exception = self.Generate(source_dir, input_filename, input_root, pdf_path) - if raised_exception != None: - print "FAILURE: " + input_filename + "; " + str(raised_exception) + if raised_exception is not None: + print 'FAILURE: %s; %s' % (input_filename, raised_exception) return False, [] results = [] @@ -72,23 +73,29 @@ class TestRunner: else: raised_exception, results = self.TestPixel(input_root, pdf_path) - if raised_exception != None: - print "FAILURE: " + input_filename + "; " + str(raised_exception) + if raised_exception is not None: + print 'FAILURE: %s; %s' % (input_filename, raised_exception) return False, results - if len(actual_images): + if actual_images: if self.image_differ.HasDifferences(input_filename, source_dir, self.working_dir): return False, results + else: + if (self.enforce_expected_images + and not self.test_suppressor.IsPixelDiffSuppressed(input_filename)): + print 'FAILURE: %s; Missing expected images' % input_filename + return False, results + return True, results def Generate(self, source_dir, input_filename, input_root, pdf_path): original_path = os.path.join(source_dir, input_filename) input_path = os.path.join(source_dir, input_root + '.in') - input_event_path = os.path.join(source_dir, input_root + ".evt") + input_event_path = os.path.join(source_dir, input_root + '.evt') if os.path.exists(input_event_path): - output_event_path = os.path.splitext(pdf_path)[0] + ".evt" + output_event_path = os.path.splitext(pdf_path)[0] + '.evt' shutil.copyfile(input_event_path, output_event_path) if not os.path.exists(input_path): @@ -131,6 +138,7 @@ class TestRunner: self.gold_results.AddTestResult(test_name, md5_hash, img_path) if self.test_suppressor.IsResultSuppressed(input_filename): + self.result_suppressed_cases.append(input_filename) if success: self.surprises.append(input_path) else: @@ -178,7 +186,7 @@ class TestRunner: self.pdfium_test_path = finder.ExecutablePath('pdfium_test') if not os.path.exists(self.pdfium_test_path): print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path - print "Use --build-dir to specify its location." + print 'Use --build-dir to specify its location.' return 1 self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir)) @@ -192,45 +200,49 @@ class TestRunner: walk_from_dir = finder.TestingDir(test_dir); - test_cases = [] + self.test_cases = [] + self.execution_suppressed_cases = [] input_file_re = re.compile('^[a-zA-Z0-9_.]+[.](in|pdf)$') - if len(args): + if args: for file_name in args: - file_name.replace(".pdf", ".in") + file_name.replace('.pdf', '.in') input_path = os.path.join(walk_from_dir, file_name) if not os.path.isfile(input_path): print "Can't find test file '%s'" % file_name return 1 - test_cases.append((os.path.basename(input_path), + self.test_cases.append((os.path.basename(input_path), os.path.dirname(input_path))) else: for file_dir, _, filename_list in os.walk(walk_from_dir): for input_filename in filename_list: if input_file_re.match(input_filename): input_path = os.path.join(file_dir, input_filename) - if not self.test_suppressor.IsExecutionSuppressed(input_path): + if self.test_suppressor.IsExecutionSuppressed(input_path): + self.execution_suppressed_cases.append(input_path) + else: if os.path.isfile(input_path): - test_cases.append((input_filename, file_dir)) + self.test_cases.append((input_filename, file_dir)) self.failures = [] self.surprises = [] + self.result_suppressed_cases = [] # Collect Gold results if an output directory was named. self.gold_results = None if options.gold_output_dir: - self.gold_results = gold.GoldResults("pdfium", + self.gold_results = gold.GoldResults('pdfium', options.gold_output_dir, options.gold_properties, options.gold_key, options.gold_ignore_hashes) - if options.num_workers > 1 and len(test_cases) > 1: + if options.num_workers > 1 and len(self.test_cases) > 1: try: pool = multiprocessing.Pool(options.num_workers) worker_func = functools.partial(TestOneFileParallel, self) - worker_results = pool.imap(worker_func, test_cases) + worker_results = pool.imap(worker_func, self.test_cases) for worker_result in worker_results: result, input_filename, source_dir = worker_result input_path = os.path.join(source_dir, input_filename) @@ -243,7 +255,7 @@ class TestRunner: pool.close() pool.join() else: - for test_case in test_cases: + for test_case in self.test_cases: input_filename, input_file_dir = test_case result = self.GenerateAndTest(input_filename, input_file_dir) self.HandleResult(input_filename, @@ -256,7 +268,7 @@ class TestRunner: self.surprises.sort() print '\n\nUnexpected Successes:' for surprise in self.surprises: - print surprise; + print surprise if self.failures: self.failures.sort() @@ -264,6 +276,29 @@ class TestRunner: for failure in self.failures: print failure + self._PrintSummary() + + if self.failures: if not options.ignore_errors: return 1 + return 0 + + def _PrintSummary(self): + number_test_cases = len(self.test_cases) + number_failures = len(self.failures) + number_suppressed = len(self.result_suppressed_cases) + number_successes = number_test_cases - number_failures - number_suppressed + number_surprises = len(self.surprises) + print + print 'Test cases executed: %d' % number_test_cases + print ' Successes: %d' % number_successes + print ' Suppressed: %d' % number_suppressed + print ' Surprises: %d' % number_surprises + print ' Failures: %d' % number_failures + print + print 'Test cases not executed: %d' % len(self.execution_suppressed_cases) + + def SetEnforceExpectedImages(self, new_value): + """Set whether to enforce that each test case provide an expected image.""" + self.enforce_expected_images = new_value |