From 3218fffe6f92b62d05ab5c2d1c78285a0a0f0647 Mon Sep 17 00:00:00 2001 From: Tom Sepez Date: Fri, 12 Jun 2015 09:11:00 -0700 Subject: Corpus tests check for unexpected successes. Update run_corpus_tests.py to always run all test cases, and just ignore the results that are supposed to fail. R=thestig@chromium.org Review URL: https://codereview.chromium.org/1178393002. --- testing/tools/run_corpus_tests.py | 17 +++++++++++++---- testing/tools/suppressor.py | 4 ++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/testing/tools/run_corpus_tests.py b/testing/tools/run_corpus_tests.py index 1876581e40..dbc8f35a91 100755 --- a/testing/tools/run_corpus_tests.py +++ b/testing/tools/run_corpus_tests.py @@ -55,6 +55,7 @@ def main(): # test files are under .../pdfium/testing/corpus. failures = [] + surprises = [] walk_from_dir = finder.TestingDir('corpus'); input_file_re = re.compile('^[a-zA-Z0-9_.]+[.]pdf$') for source_dir, _, filename_list in os.walk(walk_from_dir): @@ -62,11 +63,19 @@ def main(): if input_file_re.match(input_filename): input_path = os.path.join(source_dir, input_filename) if os.path.isfile(input_path): + result = test_one_file(input_filename, source_dir, working_dir, + pdfium_test_path, image_differ) if test_suppressor.IsSuppressed(input_filename): - continue - if not test_one_file(input_filename, source_dir, working_dir, - pdfium_test_path, image_differ): - failures.append(input_path) + if result: + surprises.append(input_path) + else: + if not result: + failures.append(input_path) + + if surprises: + print '\n\nUnexpected Successes:' + for surprise in surprises: + print surprise; if failures: print '\n\nSummary of Failures:' diff --git a/testing/tools/suppressor.py b/testing/tools/suppressor.py index fff9bc8381..2bb31171ab 100755 --- a/testing/tools/suppressor.py +++ b/testing/tools/suppressor.py @@ -24,11 +24,11 @@ class Suppressor: def IsSuppressed(self, input_filename): if input_filename in self.suppression_list: - print ("Not running %s, found in %s file" % + print ("%s is suppressed, found in %s file" % (input_filename, self.SUPPRESSIONS_FILENAME)) return True if input_filename in self.platform_suppression_list: - print ("Not running %s, found in %s file" % + print ("%s is suppressed, found in %s file" % (input_filename, self.PLATFORM_SUPPRESSIONS_FILENAME)) return True return False -- cgit v1.2.3