Enable Dr. Memory to run javascript, pixel, and corpus testschromium/2575chromium/2574

- add DrMemoryWrapper in common.py to adjust Dr. Memory wrapper
- add --wrapper option to run_*_tests.py for Dr. Mempry wrapper
- update run_*_tests.py to handle Dr. Memory wrapper
- add TestPDFiumTest in pdfium_tests.py to support run_*_tests.py
- remove ValgrindTool in valgrind_tests.py

R=thestig@chromium.org
BUG=pdfium:238

Review URL: https://codereview.chromium.org/1464453003 .

4 files changed, 69 insertions, 12 deletions

diff --git a/testing/tools/common.py b/testing/tools/common.py
index d45404b4d4..6e9de7c82c 100755
--- a/testing/tools/common.py
+++ b/testing/tools/common.py
@@ -3,6 +3,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import glob
 import os
 import subprocess
 import sys
@@ -27,6 +28,38 @@ def RunCommand(cmd, redirect_output=False):
   except subprocess.CalledProcessError as e:
     return e
 
+# Adjust Dr. Memory wrapper to have separate log directory for each test
+# for better error reporting.
+def DrMemoryWrapper(wrapper, pdf_name):
+  if not wrapper:
+    return []
+  # convert string to list
+  cmd_to_run = wrapper.split()
+
+  # Do nothing if using default log directory.
+  if cmd_to_run.count("-logdir") == 0:
+    return cmd_to_run
+  # Usually, we pass "-logdir" "foo\bar\spam path" args to Dr. Memory.
+  # To group reports per test, we want to put the reports for each test into a
+  # separate directory. This code can be simplified when we have
+  # https://github.com/DynamoRIO/drmemory/issues/684 fixed.
+  logdir_idx = cmd_to_run.index("-logdir")
+  old_logdir = cmd_to_run[logdir_idx + 1]
+  wrapper_pid = str(os.getpid())
+
+  # We are using the same pid of the same python process, so append the number
+  # of entries in the logdir at the end of wrapper_pid to avoid conflict.
+  wrapper_pid += "_%d" % len(glob.glob(old_logdir + "\\*"))
+
+  cmd_to_run[logdir_idx + 1] += "\\testcase.%s.logs" % wrapper_pid
+  os.makedirs(cmd_to_run[logdir_idx + 1])
+
+  f = open(old_logdir + "\\testcase.%s.name" % wrapper_pid, "w")
+  print >>f, pdf_name + ".pdf"
+  f.close()
+
+  return cmd_to_run
+
 
 class DirectoryFinder:
   '''A class for finding directories and paths under either a standalone
diff --git a/testing/tools/run_corpus_tests.py b/testing/tools/run_corpus_tests.py
index 29f23b504a..e2b950c363 100755
--- a/testing/tools/run_corpus_tests.py
+++ b/testing/tools/run_corpus_tests.py
@@ -26,10 +26,10 @@ class KeyboardInterruptError(Exception): pass
 #   c_dir - "path/to/a/b/c"
 
 def test_one_file(input_filename, source_dir, working_dir,
-                  pdfium_test_path, image_differ, redirect_output=False):
+                  pdfium_test_path, image_differ, drmem_wrapper,
+                  redirect_output=False):
   input_path = os.path.join(source_dir, input_filename)
   pdf_path = os.path.join(working_dir, input_filename)
-
   # Remove any existing generated images from previous runs.
   actual_images = image_differ.GetActualFiles(
       input_filename, source_dir, working_dir)
@@ -39,8 +39,13 @@ def test_one_file(input_filename, source_dir, working_dir,
 
   shutil.copyfile(input_path, pdf_path)
   sys.stdout.flush()
-  error = common.RunCommand([pdfium_test_path, '--png', pdf_path],
-                            redirect_output)
+  # add Dr. Memory wrapper if exist
+  # remove .pdf suffix
+  cmd_to_run = common.DrMemoryWrapper(drmem_wrapper,
+                                      os.path.splitext(input_filename)[0])
+  cmd_to_run.extend([pdfium_test_path, '--png', pdf_path])
+  # run test
+  error = common.RunCommand(cmd_to_run, redirect_output)
   if error:
     print "FAILURE: " + input_filename + "; " + str(error)
     return False
@@ -58,7 +63,7 @@ def test_one_file_parallel(working_dir, pdfium_test_path, image_differ,
     sys.stderr = sys.stdout
     input_filename, source_dir = test_case
     result = test_one_file(input_filename, source_dir, working_dir,
-                           pdfium_test_path, image_differ, True);
+                           pdfium_test_path, image_differ, "", True);
     output = sys.stdout
     sys.stdout = old_stdout
     sys.stderr = old_stderr
@@ -84,6 +89,8 @@ def main():
   parser.add_option('-j', default=multiprocessing.cpu_count(),
                     dest='num_workers', type='int',
                     help='run NUM_WORKERS jobs in parallel')
+  parser.add_option('--wrapper', default='', dest="wrapper",
+                    help='Dr. Memory wrapper for running test under Dr. Memory')
   options, args = parser.parse_args()
   finder = common.DirectoryFinder(options.build_dir)
   pdfium_test_path = finder.ExecutablePath('pdfium_test')
@@ -143,7 +150,8 @@ def main():
     for test_case in test_cases:
       input_filename, source_dir = test_case
       result = test_one_file(input_filename, source_dir, working_dir,
-                             pdfium_test_path, image_differ)
+                             pdfium_test_path, image_differ,
+                             options.wrapper)
       handle_result(test_suppressor, input_filename, input_path, result,
                     surprises, failures)
 
diff --git a/testing/tools/run_javascript_tests.py b/testing/tools/run_javascript_tests.py
index e2fdc66918..0c2401774e 100755
--- a/testing/tools/run_javascript_tests.py
+++ b/testing/tools/run_javascript_tests.py
@@ -18,7 +18,8 @@ import common
 #   c_dir - "path/to/a/b/c"
 
 def generate_and_test(input_filename, source_dir, working_dir,
-                      fixup_path, pdfium_test_path, text_diff_path):
+                      fixup_path, pdfium_test_path, text_diff_path,
+                      drmem_wrapper):
   input_root, _ = os.path.splitext(input_filename)
   input_path = os.path.join(source_dir, input_root + '.in')
   pdf_path = os.path.join(working_dir, input_root + '.pdf')
@@ -29,7 +30,11 @@ def generate_and_test(input_filename, source_dir, working_dir,
     subprocess.check_call(
         [sys.executable, fixup_path, '--output-dir=' + working_dir, input_path])
     with open(txt_path, 'w') as outfile:
-      subprocess.check_call([pdfium_test_path, pdf_path], stdout=outfile)
+      # add Dr. Memory wrapper if exist
+      cmd_to_run = common.DrMemoryWrapper(drmem_wrapper, input_root)
+      cmd_to_run.extend([pdfium_test_path, pdf_path])
+      # run test
+      subprocess.check_call(cmd_to_run, stdout=outfile)
     subprocess.check_call(
         [sys.executable, text_diff_path, expected_path, txt_path])
   except subprocess.CalledProcessError as e:
@@ -41,6 +46,8 @@ def main():
   parser = optparse.OptionParser()
   parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
                     help='relative path from the base source directory')
+  parser.add_option('--wrapper', default='', dest="wrapper",
+                    help='Dr. Memory wrapper for running test under Dr. Memory')
   options, args = parser.parse_args()
 
   finder = common.DirectoryFinder(options.build_dir)
@@ -70,7 +77,8 @@ def main():
       input_path = os.path.join(source_dir, input_filename)
       if os.path.isfile(input_path):
         if not generate_and_test(input_filename, source_dir, working_dir,
-                                 fixup_path, pdfium_test_path, text_diff_path):
+                                 fixup_path, pdfium_test_path, text_diff_path,
+                                 options.wrapper):
           failures.append(input_path)
 
   if failures:
diff --git a/testing/tools/run_pixel_tests.py b/testing/tools/run_pixel_tests.py
index b167923b86..8d838402f9 100755
--- a/testing/tools/run_pixel_tests.py
+++ b/testing/tools/run_pixel_tests.py
@@ -20,7 +20,8 @@ import suppressor
 #   c_dir - "path/to/a/b/c"
 
 def generate_and_test(input_filename, source_dir, working_dir,
-                      fixup_path, pdfium_test_path, image_differ):
+                      fixup_path, pdfium_test_path, image_differ,
+                      drmem_wrapper):
   input_root, _ = os.path.splitext(input_filename)
   input_path = os.path.join(source_dir, input_root + '.in')
   pdf_path = os.path.join(working_dir, input_root + '.pdf')
@@ -36,7 +37,11 @@ def generate_and_test(input_filename, source_dir, working_dir,
     sys.stdout.flush()
     subprocess.check_call(
         [sys.executable, fixup_path, '--output-dir=' + working_dir, input_path])
-    subprocess.check_call([pdfium_test_path, '--png', pdf_path])
+    # add Dr. Memory wrapper if exist
+    cmd_to_run = common.DrMemoryWrapper(drmem_wrapper, input_root)
+    cmd_to_run.extend([pdfium_test_path, '--png', pdf_path])
+    # run test
+    subprocess.check_call(cmd_to_run)
   except subprocess.CalledProcessError as e:
     print "FAILURE: " + input_filename + "; " + str(e)
     return False
@@ -50,6 +55,8 @@ def main():
   parser = optparse.OptionParser()
   parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
                     help='relative path from the base source directory')
+  parser.add_option('--wrapper', default='', dest="wrapper",
+                    help='Dr. Memory wrapper for running test under Dr. Memory')
   options, args = parser.parse_args()
   finder = common.DirectoryFinder(options.build_dir)
   fixup_path = finder.ScriptPath('fixup_pdf_template.py')
@@ -82,7 +89,8 @@ def main():
         if test_suppressor.IsSuppressed(input_filename):
           continue
         if not generate_and_test(input_filename, source_dir, working_dir,
-                                 fixup_path, pdfium_test_path, image_differ):
+                                 fixup_path, pdfium_test_path, image_differ,
+                                 options.wrapper):
           failures.append(input_path)
 
   if failures: