From 3218fffe6f92b62d05ab5c2d1c78285a0a0f0647 Mon Sep 17 00:00:00 2001
From: Tom Sepez <tsepez@chromium.org>
Date: Fri, 12 Jun 2015 09:11:00 -0700
Subject: Corpus tests check for unexpected successes.

Update run_corpus_tests.py to always run all test cases, and just
ignore the results that are supposed to fail.

R=thestig@chromium.org

Review URL: https://codereview.chromium.org/1178393002.
---
 testing/tools/run_corpus_tests.py | 17 +++++++++++++----
 testing/tools/suppressor.py       |  4 ++--
 2 files changed, 15 insertions(+), 6 deletions(-)

(limited to 'testing')

diff --git a/testing/tools/run_corpus_tests.py b/testing/tools/run_corpus_tests.py
index 1876581e40..dbc8f35a91 100755
--- a/testing/tools/run_corpus_tests.py
+++ b/testing/tools/run_corpus_tests.py
@@ -55,6 +55,7 @@ def main():
 
   # test files are under .../pdfium/testing/corpus.
   failures = []
+  surprises = []
   walk_from_dir = finder.TestingDir('corpus');
   input_file_re = re.compile('^[a-zA-Z0-9_.]+[.]pdf$')
   for source_dir, _, filename_list in os.walk(walk_from_dir):
@@ -62,11 +63,19 @@ def main():
       if input_file_re.match(input_filename):
          input_path = os.path.join(source_dir, input_filename)
          if os.path.isfile(input_path):
+           result = test_one_file(input_filename, source_dir, working_dir,
+                                  pdfium_test_path, image_differ)
            if test_suppressor.IsSuppressed(input_filename):
-             continue
-         if not test_one_file(input_filename, source_dir, working_dir,
-                              pdfium_test_path, image_differ):
-             failures.append(input_path)
+             if result:
+               surprises.append(input_path)
+           else:
+             if not result:
+               failures.append(input_path)
+
+  if surprises:
+    print '\n\nUnexpected Successes:'
+    for surprise in surprises:
+      print surprise;
 
   if failures:
     print '\n\nSummary of Failures:'
diff --git a/testing/tools/suppressor.py b/testing/tools/suppressor.py
index fff9bc8381..2bb31171ab 100755
--- a/testing/tools/suppressor.py
+++ b/testing/tools/suppressor.py
@@ -24,11 +24,11 @@ class Suppressor:
 
   def IsSuppressed(self, input_filename):
     if input_filename in self.suppression_list:
-      print ("Not running %s, found in %s file" %
+      print ("%s is suppressed, found in %s file" %
              (input_filename, self.SUPPRESSIONS_FILENAME))
       return True
     if input_filename in self.platform_suppression_list:
-      print ("Not running %s, found in %s file" %
+      print ("%s is suppressed, found in %s file" %
              (input_filename, self.PLATFORM_SUPPRESSIONS_FILENAME))
       return True
     return False
-- 
cgit v1.2.3