Re-Reland Combined test runner.

Original Commit Message:

This CL revives the old CL to combine the test runners [1] which was reverted
due to failing font_size tests. I've deleted the font_size.pdf as it is not
needed and for testing. Pixel tests are either .in or .pdf files, not both.

Original description:
This CL takes the three test runners (corpus, javascript, pixel) and combines
the code into a single test_runner file. Each of the individual runners still
exists and calls the test runner with their data directory.

With this change, the pixel and javascript test will now run in parallel if
multiple processors are available.

1-https://codereview.chromium.org/1430623006/

BUG=

Review URL: https://codereview.chromium.org/1888673003

6 files changed, 259 insertions, 427 deletions

diff --git a/testing/resources/pixel/font_size.pdf b/testing/resources/pixel/font_size.pdf
deleted file mode 100644
index 5a42260ce9..0000000000
--- a/testing/resources/pixel/font_size.pdf
+++ /dev/null
@@ -1,72 +0,0 @@
-%PDF-1.7
-% ò¤ô
-1 0 obj <<
-  /Type /Catalog
-  /Pages 2 0 R
->>
-2 0 obj <<
-  /Type /Pages
-  /MediaBox [ 0 0 100 400 ]
-  /Count 1
-  /Kids [ 3 0 R ]
->>
-endobj
-3 0 obj <<
-  /Type /Page
-  /Parent 2 0 R
-  /Resources <<
-    /Font <<
-      /F1 4 0 R
-    >>
-  >>
-  /Contents 6 0 R
->>
-endobj
-4 0 obj <<
-  /Type /Font
-  /Subtype /Type1
-  /BaseFont /Times-Roman
->>
-endobj
-6 0 obj <<
->>
-stream
-BT
-20 0 Td
-0 20 Td /F1 0 Tf (Size 0) Tj
-0 20 Td /F1 1 Tf (Size 1) Tj
-0 20 Td /F1 2 Tf (Size 2) Tj
-0 20 Td /F1 3 Tf (Size 3) Tj
-0 20 Td /F1 4 Tf (Size 4) Tj
-0 20 Td /F1 5 Tf (Size 5) Tj
-0 20 Td /F1 6 Tf (Size 6) Tj
-0 20 Td /F1 7 Tf (Size 7) Tj
-0 20 Td /F1 8 Tf (Size 8) Tj
-0 20 Td /F1 9 Tf (Size 9) Tj
-0 20 Td /F1 10 Tf (Size 10) Tj
-0 20 Td /F1 11 Tf (Size 11) Tj
-0 20 Td /F1 12 Tf (Size 12) Tj
-0 20 Td /F1 13 Tf (Size 13) Tj
-0 20 Td /F1 14 Tf (Size 14) Tj
-0 20 Td /F1 15 Tf (Size 15) Tj
-0 20 Td /F1 16 Tf (Size 16) Tj
-0 20 Td /F1 17 Tf (Size 17) Tj
-0 20 Td /F1 18 Tf (Size 18) Tj
-ET
-endstream
-endobj
-xref
-0 7
-0000000000 65535 f 
-0000000015 00000 n 
-0000000061 00000 n 
-0000000154 00000 n 
-0000000280 00000 n 
-0000000000 65535 f 
-0000000358 00000 n 
-trailer <<
-  /Root 1 0 R
->>
-startxref
-979
-%%EOF
diff --git a/testing/tools/common.py b/testing/tools/common.py
index 6e9de7c82c..ce292bf41f 100755
--- a/testing/tools/common.py
+++ b/testing/tools/common.py
@@ -18,6 +18,13 @@ def os_name():
   raise Exception('Confused, can not determine OS, aborting.')
 
 
+def RunCommandToFile(cmd, file):
+  try:
+    subprocess.check_call(cmd, stdout=file)
+    return None
+  except subprocess.CalledProcessError as e:
+    return e
+
 def RunCommand(cmd, redirect_output=False):
   try:
     if redirect_output:
@@ -28,6 +35,7 @@ def RunCommand(cmd, redirect_output=False):
   except subprocess.CalledProcessError as e:
     return e
 
+
 # Adjust Dr. Memory wrapper to have separate log directory for each test
 # for better error reporting.
 def DrMemoryWrapper(wrapper, pdf_name):
diff --git a/testing/tools/run_corpus_tests.py b/testing/tools/run_corpus_tests.py
index 82c3d2aefe..22b5d734af 100755
--- a/testing/tools/run_corpus_tests.py
+++ b/testing/tools/run_corpus_tests.py
@@ -3,175 +3,12 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import cStringIO
-import functools
-import multiprocessing
-import optparse
-import os
-import re
-import shutil
-import subprocess
 import sys
-
-import common
-import pngdiffer
-import suppressor
-
-class KeyboardInterruptError(Exception): pass
-
-# Nomenclature:
-#   x_root - "x"
-#   x_filename - "x.ext"
-#   x_path - "path/to/a/b/c/x.ext"
-#   c_dir - "path/to/a/b/c"
-
-def test_one_file(input_filename, source_dir, working_dir,
-                  pdfium_test_path, image_differ, drmem_wrapper,
-                  redirect_output=False):
-  input_path = os.path.join(source_dir, input_filename)
-  pdf_path = os.path.join(working_dir, input_filename)
-  # Remove any existing generated images from previous runs.
-  actual_images = image_differ.GetActualFiles(
-      input_filename, source_dir, working_dir)
-  for image in actual_images:
-    if os.path.exists(image):
-      os.remove(image)
-
-  shutil.copyfile(input_path, pdf_path)
-  sys.stdout.flush()
-  # add Dr. Memory wrapper if exist
-  # remove .pdf suffix
-  cmd_to_run = common.DrMemoryWrapper(drmem_wrapper,
-                                      os.path.splitext(input_filename)[0])
-  cmd_to_run.extend([pdfium_test_path, '--png', pdf_path])
-  # run test
-  error = common.RunCommand(cmd_to_run, redirect_output)
-  if error:
-    print "FAILURE: " + input_filename + "; " + str(error)
-    return False
-  return not image_differ.HasDifferences(input_filename, source_dir,
-                                         working_dir, redirect_output)
-
-
-def test_one_file_parallel(working_dir, pdfium_test_path, image_differ,
-                           test_case):
-  """Wrapper function to call test_one_file() and redirect output to stdout."""
-  try:
-    old_stdout = sys.stdout
-    old_stderr = sys.stderr
-    sys.stdout = cStringIO.StringIO()
-    sys.stderr = sys.stdout
-    input_filename, source_dir = test_case
-    result = test_one_file(input_filename, source_dir, working_dir,
-                           pdfium_test_path, image_differ, "", True);
-    output = sys.stdout
-    sys.stdout = old_stdout
-    sys.stderr = old_stderr
-    return (result, output.getvalue(), input_filename, source_dir)
-  except KeyboardInterrupt:
-    raise KeyboardInterruptError()
-
-
-def handle_result(test_suppressor, input_filename, input_path, result,
-                  surprises, failures):
-  if test_suppressor.IsResultSuppressed(input_filename):
-    if result:
-      surprises.append(input_path)
-  else:
-    if not result:
-      failures.append(input_path)
-
+import test_runner
 
 def main():
-  parser = optparse.OptionParser()
-  parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
-                    help='relative path from the base source directory')
-  parser.add_option('-j', default=multiprocessing.cpu_count(),
-                    dest='num_workers', type='int',
-                    help='run NUM_WORKERS jobs in parallel')
-  parser.add_option('--wrapper', default='', dest="wrapper",
-                    help='Dr. Memory wrapper for running test under Dr. Memory')
-  options, args = parser.parse_args()
-  finder = common.DirectoryFinder(options.build_dir)
-  pdfium_test_path = finder.ExecutablePath('pdfium_test')
-  if not os.path.exists(pdfium_test_path):
-    print "FAILURE: Can't find test executable '%s'" % pdfium_test_path
-    print "Use --build-dir to specify its location."
-    return 1
-  working_dir = finder.WorkingDir(os.path.join('testing', 'corpus'))
-  if not os.path.exists(working_dir):
-    os.makedirs(working_dir)
-
-  feature_string = subprocess.check_output([pdfium_test_path, '--show-config'])
-  test_suppressor = suppressor.Suppressor(finder, feature_string)
-  image_differ = pngdiffer.PNGDiffer(finder)
-
-  # test files are under .../pdfium/testing/corpus.
-  failures = []
-  surprises = []
-  walk_from_dir = finder.TestingDir('corpus');
-  input_file_re = re.compile('^[a-zA-Z0-9_.]+[.]pdf$')
-  test_cases = []
-
-  if len(args):
-    for file_name in args:
-      input_path = os.path.join(walk_from_dir, file_name)
-      if not os.path.isfile(input_path):
-        print "Can't find test file '%s'" % file_name
-        return 1
-
-      test_cases.append((os.path.basename(input_path),
-                         os.path.dirname(input_path)))
-  else:
-    for source_dir, _, filename_list in os.walk(walk_from_dir):
-      for input_filename in filename_list:
-        if input_file_re.match(input_filename):
-          input_path = os.path.join(source_dir, input_filename)
-          if not test_suppressor.IsExecutionSuppressed(input_path):
-            if os.path.isfile(input_path):
-              test_cases.append((input_filename, source_dir))
-
-  if options.num_workers > 1 and len(test_cases) > 1:
-    try:
-      pool = multiprocessing.Pool(options.num_workers)
-      worker_func = functools.partial(test_one_file_parallel, working_dir,
-                                      pdfium_test_path, image_differ)
-      worker_results = pool.imap(worker_func, test_cases)
-      for worker_result in worker_results:
-        result, output, input_filename, source_dir = worker_result
-        input_path = os.path.join(source_dir, input_filename)
-        sys.stdout.write(output)
-        handle_result(test_suppressor, input_filename, input_path, result,
-                      surprises, failures)
-      pool.close()
-    except KeyboardInterrupt:
-      pool.terminate()
-    finally:
-      pool.join()
-  else:
-    for test_case in test_cases:
-      input_filename, source_dir = test_case
-      result = test_one_file(input_filename, source_dir, working_dir,
-                             pdfium_test_path, image_differ,
-                             options.wrapper)
-      handle_result(test_suppressor, input_filename, input_path, result,
-                    surprises, failures)
-
-  if surprises:
-    surprises.sort()
-    print '\n\nUnexpected Successes:'
-    for surprise in surprises:
-      print surprise;
-
-  if failures:
-    failures.sort()
-    print '\n\nSummary of Failures:'
-    for failure in failures:
-      print failure
-    return 1
-
-  return 0
-
+  runner = test_runner.TestRunner('corpus')
+  return runner.Run()
 
 if __name__ == '__main__':
   sys.exit(main())
diff --git a/testing/tools/run_javascript_tests.py b/testing/tools/run_javascript_tests.py
index 4e066a1c97..76d2379dbb 100755
--- a/testing/tools/run_javascript_tests.py
+++ b/testing/tools/run_javascript_tests.py
@@ -3,100 +3,13 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import optparse
-import os
-import re
-import subprocess
 import sys
 
-import common
-
-# Nomenclature:
-#   x_root - "x"
-#   x_filename - "x.ext"
-#   x_path - "path/to/a/b/c/x.ext"
-#   c_dir - "path/to/a/b/c"
-
-def generate_and_test(input_filename, source_dir, working_dir,
-                      fixup_path, pdfium_test_path, text_diff_path,
-                      drmem_wrapper):
-  input_root, _ = os.path.splitext(input_filename)
-  input_path = os.path.join(source_dir, input_root + '.in')
-  pdf_path = os.path.join(working_dir, input_root + '.pdf')
-  txt_path = os.path.join(working_dir, input_root + '.txt')
-  expected_path = os.path.join(source_dir, input_root + '_expected.txt')
-  try:
-    sys.stdout.flush()
-    subprocess.check_call(
-        [sys.executable, fixup_path, '--output-dir=' + working_dir, input_path])
-    with open(txt_path, 'w') as outfile:
-      # add Dr. Memory wrapper if exist
-      cmd_to_run = common.DrMemoryWrapper(drmem_wrapper, input_root)
-      cmd_to_run.extend([pdfium_test_path, pdf_path])
-      # run test
-      subprocess.check_call(cmd_to_run, stdout=outfile)
-    subprocess.check_call(
-        [sys.executable, text_diff_path, expected_path, txt_path])
-  except subprocess.CalledProcessError as e:
-    print "FAILURE: " + input_filename + "; " + str(e)
-    return False
-  return True
+import test_runner
 
 def main():
-  parser = optparse.OptionParser()
-  parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
-                    help='relative path from the base source directory')
-  parser.add_option('-j', default=1,
-                    dest='num_workers', type='int',
-                    help='run NUM_WORKERS jobs in parallel (currently ignored)')
-  parser.add_option('--wrapper', default='', dest="wrapper",
-                    help='Dr. Memory wrapper for running test under Dr. Memory')
-  options, args = parser.parse_args()
-
-  finder = common.DirectoryFinder(options.build_dir)
-  fixup_path = finder.ScriptPath('fixup_pdf_template.py')
-  text_diff_path = finder.ScriptPath('text_diff.py')
-  source_dir = finder.TestingDir(os.path.join('resources', 'javascript'))
-  pdfium_test_path = finder.ExecutablePath('pdfium_test')
-  if not os.path.exists(pdfium_test_path):
-    print "FAILURE: Can't find test executable '%s'" % pdfium_test_path
-    print "Use --build-dir to specify its location."
-    return 1
-  working_dir = finder.WorkingDir(os.path.join('testing', 'javascript'))
-  if not os.path.exists(working_dir):
-    os.makedirs(working_dir)
-
-  feature_string = subprocess.check_output([pdfium_test_path, '--show-config'])
-  if "V8" not in feature_string.strip().split(","):
-    print "V8 not enabled, skipping."
-    return 0
-
-  input_files = []
-  if len(args):
-    for file_name in args:
-      input_files.append(file_name.replace(".pdf", ".in"))
-  else:
-    input_files = os.listdir(source_dir)
-
-  failures = []
-  input_file_re = re.compile('^[a-zA-Z0-9_.]+[.]in$')
-  for input_filename in input_files:
-    if input_file_re.match(input_filename):
-      input_path = os.path.join(source_dir, input_filename)
-      if os.path.isfile(input_path):
-        if not generate_and_test(input_filename, source_dir, working_dir,
-                                 fixup_path, pdfium_test_path, text_diff_path,
-                                 options.wrapper):
-          failures.append(input_path)
-
-  if failures:
-    failures.sort()
-    print '\n\nSummary of Failures:'
-    for failure in failures:
-      print failure
-    return 1
-  return 0
-
+  runner = test_runner.TestRunner('javascript')
+  return runner.Run()
 
 if __name__ == '__main__':
   sys.exit(main())
diff --git a/testing/tools/run_pixel_tests.py b/testing/tools/run_pixel_tests.py
index 1595ef7e45..f48a57d5a4 100755
--- a/testing/tools/run_pixel_tests.py
+++ b/testing/tools/run_pixel_tests.py
@@ -3,108 +3,12 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import optparse
-import os
-import re
-import subprocess
 import sys
-
-import common
-import pngdiffer
-import suppressor
-
-# Nomenclature:
-#   x_root - "x"
-#   x_filename - "x.ext"
-#   x_path - "path/to/a/b/c/x.ext"
-#   c_dir - "path/to/a/b/c"
-
-def generate_and_test(input_filename, source_dir, working_dir,
-                      fixup_path, pdfium_test_path, image_differ,
-                      drmem_wrapper):
-  input_root, _ = os.path.splitext(input_filename)
-  input_path = os.path.join(source_dir, input_root + '.in')
-  pdf_path = os.path.join(working_dir, input_root + '.pdf')
-
-  # Remove any existing generated images from previous runs.
-  actual_images = image_differ.GetActualFiles(
-      input_filename, source_dir, working_dir)
-  for image in actual_images:
-    if os.path.exists(image):
-      os.remove(image)
-
-  try:
-    sys.stdout.flush()
-    subprocess.check_call(
-        [sys.executable, fixup_path, '--output-dir=' + working_dir, input_path])
-    # add Dr. Memory wrapper if exist
-    cmd_to_run = common.DrMemoryWrapper(drmem_wrapper, input_root)
-    cmd_to_run.extend([pdfium_test_path, '--png', pdf_path])
-    # run test
-    subprocess.check_call(cmd_to_run)
-  except subprocess.CalledProcessError as e:
-    print "FAILURE: " + input_filename + "; " + str(e)
-    return False
-  if image_differ.HasDifferences(input_filename, source_dir, working_dir):
-    print "FAILURE: " + input_filename
-    return False
-  return True
-
+import test_runner
 
 def main():
-  parser = optparse.OptionParser()
-  parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
-                    help='relative path from the base source directory')
-  parser.add_option('-j', default=1,
-                    dest='num_workers', type='int',
-                    help='run NUM_WORKERS jobs in parallel (currently ignored)')
-  parser.add_option('--wrapper', default='', dest="wrapper",
-                    help='Dr. Memory wrapper for running test under Dr. Memory')
-  options, args = parser.parse_args()
-  finder = common.DirectoryFinder(options.build_dir)
-  fixup_path = finder.ScriptPath('fixup_pdf_template.py')
-  source_dir = finder.TestingDir(os.path.join('resources', 'pixel'))
-  pdfium_test_path = finder.ExecutablePath('pdfium_test')
-  if not os.path.exists(pdfium_test_path):
-    print "FAILURE: Can't find test executable '%s'" % pdfium_test_path
-    print "Use --build-dir to specify its location."
-    return 1
-  working_dir = finder.WorkingDir(os.path.join('testing', 'pixel'))
-  if not os.path.exists(working_dir):
-    os.makedirs(working_dir)
-
-  feature_string = subprocess.check_output([pdfium_test_path, '--show-config'])
-  test_suppressor = suppressor.Suppressor(finder, feature_string)
-  image_differ = pngdiffer.PNGDiffer(finder)
-
-  input_files = []
-  if len(args):
-    for file_name in args:
-      input_files.append(file_name.replace(".pdf", ".in"))
-  else:
-    input_files = os.listdir(source_dir)
-
-  failures = []
-  input_file_re = re.compile('^[a-zA-Z0-9_.]+[.]in$')
-  for input_filename in input_files:
-    if input_file_re.match(input_filename):
-      input_path = os.path.join(source_dir, input_filename)
-      if os.path.isfile(input_path):
-        if test_suppressor.IsResultSuppressed(input_filename):
-          continue
-        if not generate_and_test(input_filename, source_dir, working_dir,
-                                 fixup_path, pdfium_test_path, image_differ,
-                                 options.wrapper):
-          failures.append(input_path)
-
-  if failures:
-    failures.sort()
-    print '\n\nSummary of Failures:'
-    for failure in failures:
-      print failure
-    return 1
-  return 0
-
+  runner = test_runner.TestRunner('pixel')
+  return runner.Run()
 
 if __name__ == '__main__':
   sys.exit(main())
diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py
new file mode 100644
index 0000000000..698428ebc2
--- /dev/null
+++ b/testing/tools/test_runner.py
@@ -0,0 +1,242 @@
+#!/usr/bin/env python
+# Copyright 2015 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import cStringIO
+import functools
+import multiprocessing
+import optparse
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+import common
+import pngdiffer
+import suppressor
+
+class KeyboardInterruptError(Exception): pass
+
+# Nomenclature:
+#   x_root - "x"
+#   x_filename - "x.ext"
+#   x_path - "path/to/a/b/c/x.ext"
+#   c_dir - "path/to/a/b/c"
+
+def TestOneFileParallel(this, test_case):
+  """Wrapper to call GenerateAndTest() and redirect output to stdout."""
+  try:
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = cStringIO.StringIO()
+    sys.stderr = sys.stdout
+
+    input_filename, source_dir = test_case
+    result = this.GenerateAndTest(input_filename, source_dir, True);
+
+    output = sys.stdout
+    sys.stdout = old_stdout
+    sys.stderr = old_stderr
+    return (result, output.getvalue(), input_filename, source_dir)
+  except KeyboardInterrupt:
+    raise KeyboardInterruptError()
+
+class TestRunner:
+  def __init__(self, dirname):
+    self.test_dir = dirname
+
+  def GenerateAndTest(self, input_filename, source_dir, redirect_output=False):
+    input_root, _ = os.path.splitext(input_filename)
+    expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt')
+
+    pdf_path = os.path.join(self.working_dir, input_root + '.pdf')
+
+    # Remove any existing generated images from previous runs.
+    actual_images = self.image_differ.GetActualFiles(input_filename, source_dir,
+                                                     self.working_dir)
+    for image in actual_images:
+      if os.path.exists(image):
+        os.remove(image)
+
+    sys.stdout.flush()
+
+    raised_exception = self.Generate(source_dir, input_filename, input_root,
+                                     pdf_path, redirect_output)
+
+    if raised_exception != None:
+      print "FAILURE: " + input_filename + "; " + str(raised_exception)
+      return False
+
+    if os.path.exists(expected_txt_path):
+      raised_exception = self.TestText(input_root, expected_txt_path, pdf_path,
+                                       redirect_output)
+    else:
+      raised_exception = self.TestPixel(pdf_path, redirect_output)
+
+    if raised_exception != None:
+      print "FAILURE: " + input_filename + "; " + str(raised_exception)
+      return False
+
+    if len(actual_images):
+      if self.image_differ.HasDifferences(input_filename, source_dir,
+                                          self.working_dir, redirect_output):
+        return False
+
+    return True
+
+
+  def Generate(self, source_dir, input_filename, input_root, pdf_path,
+               redirect_output):
+    original_path = os.path.join(source_dir, input_filename)
+    input_path = os.path.join(source_dir, input_root + '.in')
+    txt_path = os.path.join(self.working_dir, input_root + '.txt')
+
+    if not os.path.exists(input_path):
+      if os.path.exists(original_path):
+        shutil.copyfile(original_path, pdf_path)
+      return None
+
+    sys.stdout.flush()
+    subprocess.check_call(
+        [sys.executable, self.fixup_path, '--output-dir=' + self.working_dir,
+            input_path])
+
+    with open(txt_path, 'w') as outfile:
+      # Add Dr. Memory wrapper if exists.
+      cmd = common.DrMemoryWrapper(self.drmem_wrapper, input_root)
+      cmd.extend([self.pdfium_test_path, pdf_path])
+      ret = common.RunCommandToFile(cmd, outfile)
+    return ret
+
+  def TestText(self, input_root, expected_txt_path, pdf_path, redirect_output):
+    txt_path = os.path.join(self.working_dir, input_root + '.txt')
+    with open(txt_path, 'w') as outfile:
+      subprocess.check_call([self.pdfium_test_path, pdf_path], stdout=outfile)
+
+    cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path]
+    return common.RunCommand(cmd, redirect_output)
+
+
+  def TestPixel(self, pdf_path, redirect_output):
+    return common.RunCommand(
+        [self.pdfium_test_path, '--png', pdf_path], redirect_output)
+
+
+  def HandleResult(self, input_filename, input_path, result):
+    if self.test_suppressor.IsResultSuppressed(input_filename):
+      if result:
+        self.surprises.append(input_path)
+    else:
+      if not result:
+        self.failures.append(input_path)
+
+
+  def Run(self):
+    parser = optparse.OptionParser()
+    parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
+                      help='relative path from the base source directory')
+    parser.add_option('-j', default=multiprocessing.cpu_count(),
+                      dest='num_workers', type='int',
+                      help='run NUM_WORKERS jobs in parallel')
+    parser.add_option('--wrapper', default='', dest="wrapper",
+                      help='wrapper for running test under Dr. Memory')
+    options, args = parser.parse_args()
+
+    finder = common.DirectoryFinder(options.build_dir)
+    self.fixup_path = finder.ScriptPath('fixup_pdf_template.py')
+    self.text_diff_path = finder.ScriptPath('text_diff.py')
+
+    self.drmem_wrapper = options.wrapper
+
+    self.source_dir = finder.TestingDir()
+
+    if self.test_dir != 'corpus':
+      test_dir = finder.TestingDir(os.path.join('resources', self.test_dir))
+    else:
+      test_dir = finder.TestingDir(self.test_dir)
+
+    self.pdfium_test_path = finder.ExecutablePath('pdfium_test')
+    if not os.path.exists(self.pdfium_test_path):
+      print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path
+      print "Use --build-dir to specify its location."
+      return 1
+
+    self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir))
+    if not os.path.exists(self.working_dir):
+      os.makedirs(self.working_dir)
+
+    self.feature_string = subprocess.check_output([self.pdfium_test_path,
+                                                  '--show-config'])
+    self.test_suppressor = suppressor.Suppressor(finder, self.feature_string)
+    self.image_differ = pngdiffer.PNGDiffer(finder)
+
+    walk_from_dir = finder.TestingDir(test_dir);
+
+    test_cases = []
+    input_file_re = re.compile('^[a-zA-Z0-9_.]+[.](in|pdf)$')
+
+    if len(args):
+      for file_name in args:
+        file_name.replace(".pdf", ".in")
+        input_path = os.path.join(walk_from_dir, file_name)
+        if not os.path.isfile(input_path):
+          print "Can't find test file '%s'" % file_name
+          return 1
+
+        test_cases.append((os.path.basename(input_path),
+                           os.path.dirname(input_path)))
+    else:
+      for file_dir, _, filename_list in os.walk(walk_from_dir):
+        for input_filename in filename_list:
+          if input_file_re.match(input_filename):
+            input_path = os.path.join(file_dir, input_filename)
+            if not self.test_suppressor.IsExecutionSuppressed(input_path):
+              if os.path.isfile(input_path):
+                test_cases.append((input_filename, file_dir))
+
+    self.failures = []
+    self.surprises = []
+
+    if options.num_workers > 1 and not len(args):
+      try:
+        pool = multiprocessing.Pool(options.num_workers)
+        worker_func = functools.partial(TestOneFileParallel, self)
+
+        worker_results = pool.imap(worker_func, test_cases)
+        for worker_result in worker_results:
+          result, output, input_filename, source_dir = worker_result
+          input_path = os.path.join(source_dir, input_filename)
+          sys.stdout.write(output)
+
+          self.HandleResult(input_filename, input_path, result)
+      except KeyboardInterrupt:
+        pool.terminate()
+      finally:
+        pool.close()
+        pool.join()
+    else:
+      for test_case in test_cases:
+        input_filename, input_file_dir = test_case
+        result = self.GenerateAndTest(input_filename, input_file_dir)
+        self.HandleResult(input_filename,
+                          os.path.join(input_file_dir, input_filename), result)
+
+    if self.surprises:
+      self.surprises.sort()
+      print '\n\nUnexpected Successes:'
+      for surprise in self.surprises:
+        print surprise;
+
+    if self.failures:
+      self.failures.sort()
+      print '\n\nSummary of Failures:'
+      for failure in self.failures:
+        print failure
+      return 1
+
+    return 0
+
+
+