Fail pixel tests when they do not provide the expected image.

Also showing stats on number of failures, successes, suppressions,
etc.

Change-Id: Id12faf504ea8d6b1c343d7a8b412d03dc787ca3a
Reviewed-on: https://pdfium-review.googlesource.com/6834
Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>
Reviewed-by: dsinclair <dsinclair@chromium.org>

5 files changed, 102 insertions, 22 deletions

diff --git a/testing/SUPPRESSIONS_PIXEL b/testing/SUPPRESSIONS_PIXEL
new file mode 100644
index 0000000000..6bd0511199
--- /dev/null
+++ b/testing/SUPPRESSIONS_PIXEL
@@ -0,0 +1,33 @@
+# Copyright 2017 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# List of tests to have pixel diff tests skipped, one per line.
+# There are four space-separated columns per line
+# Each column (except column 0) can contain a comma-separated list of values.
+#
+# Column 0: test file name
+# Column 1: platform: *, win, mac, linux
+# Column 2: v8 support: *, nov8, v8
+# Column 3: xfa support: *, noxfa, xfa
+#
+# All columns on a line on a line must match, but filenames may be repeated
+# on subsequent lines to suppress more cases.  Within each column, any one of
+# the comma-separated values must match in order for the colum to "match".
+# The filenames and keywords are case-sensitive.
+#
+# Try to keep the file alphabetized within each category of test.
+
+#
+# Corpus tests
+#
+FRC_3.5_CF_Strf_stmf_DefaultCryptFilter.pdf * * *
+FRC_3.5_EncryptMetadata_T.pdf * * *
+FRC_3.5_Encrypt_is_damage.pdf * * *
+FRC_3.5_Filter_PubSec_SubFilter_s5.pdf * * *
+FRC_3.5_Filter_PubSec_Sub_SubFilter_s4.pdf * * *
+Line_Stroke.pdf.pdf * * *
+MouseEvents.pdf * * *
+Oneof.pdf * * *
+bug_651304.pdf * * *
+outline.pdf * * *
diff --git a/testing/tools/run_corpus_tests.py b/testing/tools/run_corpus_tests.py
index 4942406579..5932685f32 100755
--- a/testing/tools/run_corpus_tests.py
+++ b/testing/tools/run_corpus_tests.py
@@ -9,6 +9,7 @@ import test_runner
 
 def main():
   runner = test_runner.TestRunner('corpus')
+  runner.SetEnforceExpectedImages(True)
   return runner.Run()
 
 if __name__ == '__main__':
diff --git a/testing/tools/run_pixel_tests.py b/testing/tools/run_pixel_tests.py
index aad39c5600..1d61966c2e 100755
--- a/testing/tools/run_pixel_tests.py
+++ b/testing/tools/run_pixel_tests.py
@@ -9,6 +9,7 @@ import test_runner
 
 def main():
   runner = test_runner.TestRunner('pixel')
+  runner.SetEnforceExpectedImages(True)
   return runner.Run()
 
 if __name__ == '__main__':
diff --git a/testing/tools/suppressor.py b/testing/tools/suppressor.py
index 3b2872df95..be03e21516 100755
--- a/testing/tools/suppressor.py
+++ b/testing/tools/suppressor.py
@@ -12,11 +12,15 @@ class Suppressor:
     feature_vector = feature_string.strip().split(",")
     self.has_v8 = "V8" in feature_vector
     self.has_xfa = "XFA" in feature_vector
+    self.suppression_set = self._LoadSuppressedSet('SUPPRESSIONS', finder)
+    self.pixel_suppression_set = self._LoadSuppressedSet('SUPPRESSIONS_PIXEL',
+                                                         finder)
+
+  def _LoadSuppressedSet(self, suppressions_filename, finder):
     v8_option = "v8" if self.has_v8 else "nov8"
     xfa_option = "xfa" if self.has_xfa else "noxfa"
-
-    with open(os.path.join(finder.TestingDir(), 'SUPPRESSIONS')) as f:
-      self.suppression_set = set(self._FilterSuppressions(
+    with open(os.path.join(finder.TestingDir(), suppressions_filename)) as f:
+      return set(self._FilterSuppressions(
         common.os_name(), v8_option, xfa_option, self._ExtractSuppressions(f)))
 
   def _ExtractSuppressions(self, f):
@@ -47,3 +51,9 @@ class Suppressor:
       print "%s execution is suppressed" % input_filepath
       return True
     return False
+
+  def IsPixelDiffSuppressed(self, input_filename):
+    if input_filename in self.pixel_suppression_set:
+      print "%s is pixel suppressed" % input_filename
+      return True
+    return False
diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py
index 6cd3a6c05a..97a2513385 100644
--- a/testing/tools/test_runner.py
+++ b/testing/tools/test_runner.py
@@ -39,6 +39,7 @@ def TestOneFileParallel(this, test_case):
 class TestRunner:
   def __init__(self, dirname):
     self.test_dir = dirname
+    self.enforce_expected_images = False
 
   # GenerateAndTest returns a tuple <success, outputfiles> where
   # success is a boolean indicating whether the tests passed comparison
@@ -62,8 +63,8 @@ class TestRunner:
     raised_exception = self.Generate(source_dir, input_filename, input_root,
                                      pdf_path)
 
-    if raised_exception != None:
-      print "FAILURE: " + input_filename + "; " + str(raised_exception)
+    if raised_exception is not None:
+      print 'FAILURE: %s; %s' % (input_filename, raised_exception)
       return False, []
 
     results = []
@@ -72,23 +73,29 @@ class TestRunner:
     else:
       raised_exception, results = self.TestPixel(input_root, pdf_path)
 
-    if raised_exception != None:
-      print "FAILURE: " + input_filename + "; " + str(raised_exception)
+    if raised_exception is not None:
+      print 'FAILURE: %s; %s' % (input_filename, raised_exception)
       return False, results
 
-    if len(actual_images):
+    if actual_images:
       if self.image_differ.HasDifferences(input_filename, source_dir,
                                           self.working_dir):
         return False, results
+    else:
+      if (self.enforce_expected_images
+          and not self.test_suppressor.IsPixelDiffSuppressed(input_filename)):
+        print 'FAILURE: %s; Missing expected images' % input_filename
+        return False, results
+
     return True, results
 
   def Generate(self, source_dir, input_filename, input_root, pdf_path):
     original_path = os.path.join(source_dir, input_filename)
     input_path = os.path.join(source_dir, input_root + '.in')
 
-    input_event_path = os.path.join(source_dir, input_root + ".evt")
+    input_event_path = os.path.join(source_dir, input_root + '.evt')
     if os.path.exists(input_event_path):
-      output_event_path = os.path.splitext(pdf_path)[0] + ".evt"
+      output_event_path = os.path.splitext(pdf_path)[0] + '.evt'
       shutil.copyfile(input_event_path, output_event_path)
 
     if not os.path.exists(input_path):
@@ -131,6 +138,7 @@ class TestRunner:
           self.gold_results.AddTestResult(test_name, md5_hash, img_path)
 
     if self.test_suppressor.IsResultSuppressed(input_filename):
+      self.result_suppressed_cases.append(input_filename)
       if success:
         self.surprises.append(input_path)
     else:
@@ -178,7 +186,7 @@ class TestRunner:
     self.pdfium_test_path = finder.ExecutablePath('pdfium_test')
     if not os.path.exists(self.pdfium_test_path):
       print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path
-      print "Use --build-dir to specify its location."
+      print 'Use --build-dir to specify its location.'
       return 1
 
     self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir))
@@ -192,45 +200,49 @@ class TestRunner:
 
     walk_from_dir = finder.TestingDir(test_dir);
 
-    test_cases = []
+    self.test_cases = []
+    self.execution_suppressed_cases = []
     input_file_re = re.compile('^[a-zA-Z0-9_.]+[.](in|pdf)$')
-    if len(args):
+    if args:
       for file_name in args:
-        file_name.replace(".pdf", ".in")
+        file_name.replace('.pdf', '.in')
         input_path = os.path.join(walk_from_dir, file_name)
         if not os.path.isfile(input_path):
           print "Can't find test file '%s'" % file_name
           return 1
 
-        test_cases.append((os.path.basename(input_path),
+        self.test_cases.append((os.path.basename(input_path),
                            os.path.dirname(input_path)))
     else:
       for file_dir, _, filename_list in os.walk(walk_from_dir):
         for input_filename in filename_list:
           if input_file_re.match(input_filename):
             input_path = os.path.join(file_dir, input_filename)
-            if not self.test_suppressor.IsExecutionSuppressed(input_path):
+            if self.test_suppressor.IsExecutionSuppressed(input_path):
+              self.execution_suppressed_cases.append(input_path)
+            else:
               if os.path.isfile(input_path):
-                test_cases.append((input_filename, file_dir))
+                self.test_cases.append((input_filename, file_dir))
 
     self.failures = []
     self.surprises = []
+    self.result_suppressed_cases = []
 
     # Collect Gold results if an output directory was named.
     self.gold_results = None
     if options.gold_output_dir:
-      self.gold_results = gold.GoldResults("pdfium",
+      self.gold_results = gold.GoldResults('pdfium',
                                            options.gold_output_dir,
                                            options.gold_properties,
                                            options.gold_key,
                                            options.gold_ignore_hashes)
 
-    if options.num_workers > 1 and len(test_cases) > 1:
+    if options.num_workers > 1 and len(self.test_cases) > 1:
       try:
         pool = multiprocessing.Pool(options.num_workers)
         worker_func = functools.partial(TestOneFileParallel, self)
 
-        worker_results = pool.imap(worker_func, test_cases)
+        worker_results = pool.imap(worker_func, self.test_cases)
         for worker_result in worker_results:
           result, input_filename, source_dir = worker_result
           input_path = os.path.join(source_dir, input_filename)
@@ -243,7 +255,7 @@ class TestRunner:
         pool.close()
         pool.join()
     else:
-      for test_case in test_cases:
+      for test_case in self.test_cases:
         input_filename, input_file_dir = test_case
         result = self.GenerateAndTest(input_filename, input_file_dir)
         self.HandleResult(input_filename,
@@ -256,7 +268,7 @@ class TestRunner:
       self.surprises.sort()
       print '\n\nUnexpected Successes:'
       for surprise in self.surprises:
-        print surprise;
+        print surprise
 
     if self.failures:
       self.failures.sort()
@@ -264,6 +276,29 @@ class TestRunner:
       for failure in self.failures:
         print failure
 
+    self._PrintSummary()
+
+    if self.failures:
       if not options.ignore_errors:
         return 1
+
     return 0
+
+  def _PrintSummary(self):
+    number_test_cases = len(self.test_cases)
+    number_failures = len(self.failures)
+    number_suppressed = len(self.result_suppressed_cases)
+    number_successes = number_test_cases - number_failures - number_suppressed
+    number_surprises = len(self.surprises)
+    print
+    print 'Test cases executed: %d' % number_test_cases
+    print '  Successes: %d' % number_successes
+    print '  Suppressed: %d' % number_suppressed
+    print '    Surprises: %d' % number_surprises
+    print '  Failures: %d' % number_failures
+    print
+    print 'Test cases not executed: %d' % len(self.execution_suppressed_cases)
+
+  def SetEnforceExpectedImages(self, new_value):
+    """Set whether to enforce that each test case provide an expected image."""
+    self.enforce_expected_images = new_value