diff options
| -rwxr-xr-x | testing/tools/common.py | 33 | ||||
| -rwxr-xr-x | testing/tools/run_corpus_tests.py | 20 | ||||
| -rwxr-xr-x | testing/tools/run_javascript_tests.py | 14 | ||||
| -rwxr-xr-x | testing/tools/run_pixel_tests.py | 14 | ||||
| -rw-r--r-- | tools/drmemory/scripts/pdfium_tests.py | 54 | ||||
| -rw-r--r-- | tools/drmemory/scripts/valgrind_test.py | 301 |
6 files changed, 126 insertions, 310 deletions
diff --git a/testing/tools/common.py b/testing/tools/common.py index d45404b4d4..6e9de7c82c 100755 --- a/testing/tools/common.py +++ b/testing/tools/common.py @@ -3,6 +3,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import glob import os import subprocess import sys @@ -27,6 +28,38 @@ def RunCommand(cmd, redirect_output=False): except subprocess.CalledProcessError as e: return e +# Adjust Dr. Memory wrapper to have separate log directory for each test +# for better error reporting. +def DrMemoryWrapper(wrapper, pdf_name): + if not wrapper: + return [] + # convert string to list + cmd_to_run = wrapper.split() + + # Do nothing if using default log directory. + if cmd_to_run.count("-logdir") == 0: + return cmd_to_run + # Usually, we pass "-logdir" "foo\bar\spam path" args to Dr. Memory. + # To group reports per test, we want to put the reports for each test into a + # separate directory. This code can be simplified when we have + # https://github.com/DynamoRIO/drmemory/issues/684 fixed. + logdir_idx = cmd_to_run.index("-logdir") + old_logdir = cmd_to_run[logdir_idx + 1] + wrapper_pid = str(os.getpid()) + + # We are using the same pid of the same python process, so append the number + # of entries in the logdir at the end of wrapper_pid to avoid conflict. + wrapper_pid += "_%d" % len(glob.glob(old_logdir + "\\*")) + + cmd_to_run[logdir_idx + 1] += "\\testcase.%s.logs" % wrapper_pid + os.makedirs(cmd_to_run[logdir_idx + 1]) + + f = open(old_logdir + "\\testcase.%s.name" % wrapper_pid, "w") + print >>f, pdf_name + ".pdf" + f.close() + + return cmd_to_run + class DirectoryFinder: '''A class for finding directories and paths under either a standalone diff --git a/testing/tools/run_corpus_tests.py b/testing/tools/run_corpus_tests.py index 29f23b504a..e2b950c363 100755 --- a/testing/tools/run_corpus_tests.py +++ b/testing/tools/run_corpus_tests.py @@ -26,10 +26,10 @@ class KeyboardInterruptError(Exception): pass # c_dir - "path/to/a/b/c" def test_one_file(input_filename, source_dir, working_dir, - pdfium_test_path, image_differ, redirect_output=False): + pdfium_test_path, image_differ, drmem_wrapper, + redirect_output=False): input_path = os.path.join(source_dir, input_filename) pdf_path = os.path.join(working_dir, input_filename) - # Remove any existing generated images from previous runs. actual_images = image_differ.GetActualFiles( input_filename, source_dir, working_dir) @@ -39,8 +39,13 @@ def test_one_file(input_filename, source_dir, working_dir, shutil.copyfile(input_path, pdf_path) sys.stdout.flush() - error = common.RunCommand([pdfium_test_path, '--png', pdf_path], - redirect_output) + # add Dr. Memory wrapper if exist + # remove .pdf suffix + cmd_to_run = common.DrMemoryWrapper(drmem_wrapper, + os.path.splitext(input_filename)[0]) + cmd_to_run.extend([pdfium_test_path, '--png', pdf_path]) + # run test + error = common.RunCommand(cmd_to_run, redirect_output) if error: print "FAILURE: " + input_filename + "; " + str(error) return False @@ -58,7 +63,7 @@ def test_one_file_parallel(working_dir, pdfium_test_path, image_differ, sys.stderr = sys.stdout input_filename, source_dir = test_case result = test_one_file(input_filename, source_dir, working_dir, - pdfium_test_path, image_differ, True); + pdfium_test_path, image_differ, "", True); output = sys.stdout sys.stdout = old_stdout sys.stderr = old_stderr @@ -84,6 +89,8 @@ def main(): parser.add_option('-j', default=multiprocessing.cpu_count(), dest='num_workers', type='int', help='run NUM_WORKERS jobs in parallel') + parser.add_option('--wrapper', default='', dest="wrapper", + help='Dr. Memory wrapper for running test under Dr. Memory') options, args = parser.parse_args() finder = common.DirectoryFinder(options.build_dir) pdfium_test_path = finder.ExecutablePath('pdfium_test') @@ -143,7 +150,8 @@ def main(): for test_case in test_cases: input_filename, source_dir = test_case result = test_one_file(input_filename, source_dir, working_dir, - pdfium_test_path, image_differ) + pdfium_test_path, image_differ, + options.wrapper) handle_result(test_suppressor, input_filename, input_path, result, surprises, failures) diff --git a/testing/tools/run_javascript_tests.py b/testing/tools/run_javascript_tests.py index e2fdc66918..0c2401774e 100755 --- a/testing/tools/run_javascript_tests.py +++ b/testing/tools/run_javascript_tests.py @@ -18,7 +18,8 @@ import common # c_dir - "path/to/a/b/c" def generate_and_test(input_filename, source_dir, working_dir, - fixup_path, pdfium_test_path, text_diff_path): + fixup_path, pdfium_test_path, text_diff_path, + drmem_wrapper): input_root, _ = os.path.splitext(input_filename) input_path = os.path.join(source_dir, input_root + '.in') pdf_path = os.path.join(working_dir, input_root + '.pdf') @@ -29,7 +30,11 @@ def generate_and_test(input_filename, source_dir, working_dir, subprocess.check_call( [sys.executable, fixup_path, '--output-dir=' + working_dir, input_path]) with open(txt_path, 'w') as outfile: - subprocess.check_call([pdfium_test_path, pdf_path], stdout=outfile) + # add Dr. Memory wrapper if exist + cmd_to_run = common.DrMemoryWrapper(drmem_wrapper, input_root) + cmd_to_run.extend([pdfium_test_path, pdf_path]) + # run test + subprocess.check_call(cmd_to_run, stdout=outfile) subprocess.check_call( [sys.executable, text_diff_path, expected_path, txt_path]) except subprocess.CalledProcessError as e: @@ -41,6 +46,8 @@ def main(): parser = optparse.OptionParser() parser.add_option('--build-dir', default=os.path.join('out', 'Debug'), help='relative path from the base source directory') + parser.add_option('--wrapper', default='', dest="wrapper", + help='Dr. Memory wrapper for running test under Dr. Memory') options, args = parser.parse_args() finder = common.DirectoryFinder(options.build_dir) @@ -70,7 +77,8 @@ def main(): input_path = os.path.join(source_dir, input_filename) if os.path.isfile(input_path): if not generate_and_test(input_filename, source_dir, working_dir, - fixup_path, pdfium_test_path, text_diff_path): + fixup_path, pdfium_test_path, text_diff_path, + options.wrapper): failures.append(input_path) if failures: diff --git a/testing/tools/run_pixel_tests.py b/testing/tools/run_pixel_tests.py index b167923b86..8d838402f9 100755 --- a/testing/tools/run_pixel_tests.py +++ b/testing/tools/run_pixel_tests.py @@ -20,7 +20,8 @@ import suppressor # c_dir - "path/to/a/b/c" def generate_and_test(input_filename, source_dir, working_dir, - fixup_path, pdfium_test_path, image_differ): + fixup_path, pdfium_test_path, image_differ, + drmem_wrapper): input_root, _ = os.path.splitext(input_filename) input_path = os.path.join(source_dir, input_root + '.in') pdf_path = os.path.join(working_dir, input_root + '.pdf') @@ -36,7 +37,11 @@ def generate_and_test(input_filename, source_dir, working_dir, sys.stdout.flush() subprocess.check_call( [sys.executable, fixup_path, '--output-dir=' + working_dir, input_path]) - subprocess.check_call([pdfium_test_path, '--png', pdf_path]) + # add Dr. Memory wrapper if exist + cmd_to_run = common.DrMemoryWrapper(drmem_wrapper, input_root) + cmd_to_run.extend([pdfium_test_path, '--png', pdf_path]) + # run test + subprocess.check_call(cmd_to_run) except subprocess.CalledProcessError as e: print "FAILURE: " + input_filename + "; " + str(e) return False @@ -50,6 +55,8 @@ def main(): parser = optparse.OptionParser() parser.add_option('--build-dir', default=os.path.join('out', 'Debug'), help='relative path from the base source directory') + parser.add_option('--wrapper', default='', dest="wrapper", + help='Dr. Memory wrapper for running test under Dr. Memory') options, args = parser.parse_args() finder = common.DirectoryFinder(options.build_dir) fixup_path = finder.ScriptPath('fixup_pdf_template.py') @@ -82,7 +89,8 @@ def main(): if test_suppressor.IsSuppressed(input_filename): continue if not generate_and_test(input_filename, source_dir, working_dir, - fixup_path, pdfium_test_path, image_differ): + fixup_path, pdfium_test_path, image_differ, + options.wrapper): failures.append(input_path) if failures: diff --git a/tools/drmemory/scripts/pdfium_tests.py b/tools/drmemory/scripts/pdfium_tests.py index 5486b16046..c1733f5592 100644 --- a/tools/drmemory/scripts/pdfium_tests.py +++ b/tools/drmemory/scripts/pdfium_tests.py @@ -50,10 +50,10 @@ class ChromeTests: self._options = options self._args = args - script_dir = path_utils.ScriptDir() - # Compute the top of the tree (the "source dir") from the script dir (where - # this script lives). We assume that the script dir is in tools/valgrind/ - # relative to the top of the tree. + # Compute the top of the tree (the "source dir") from the script dir + # (where this script lives). We assume that the script dir is in + # tools/drmemory/scripts relative to the top of the tree. + script_dir = os.path.dirname(path_utils.ScriptDir()) self._source_dir = os.path.dirname(os.path.dirname(script_dir)) # since this path is used for string matching, make sure it's always # an absolute Unix-style path @@ -269,11 +269,53 @@ class ChromeTests: def TestPDFiumEmbedderTests(self): return self.SimpleTest("pdfium_embeddertests", "pdfium_embeddertests") + def TestPDFiumTest(self, script_name): + # Build the command line in 'cmd'. + # It's going to be roughly + # python valgrind_test.py ... + # but we'll use the --indirect_pdfium_test flag to valgrind_test.py + # to avoid valgrinding python. + + # Start by building the valgrind_test.py commandline. + tool = valgrind_test.CreateTool(self._options.valgrind_tool) + cmd = self._DefaultCommand(tool) + cmd.append("--trace_children") + cmd.append("--indirect_pdfium_test") + cmd.append("--ignore_exit_code") + # Now build script_cmd, the run_corpus_tests commandline. + script = os.path.join(self._source_dir, "testing", "tools", script_name) + script_cmd = ["python", script] + if self._options.build_dir: + script_cmd.extend(["--build-dir", self._options.build_dir]) + # TODO(zhaoqin): it only runs in single process mode now, + # need figure out why it does not work with test_one_file_parallel + # in run_corpus_tests.py. + if script_name == "run_corpus_tests.py": + script_cmd.extend(["-j", "1"]) + # Now run script_cmd with the wrapper in cmd + cmd.append("--") + cmd.extend(script_cmd) + + ret = tool.Run(cmd, "layout", min_runtime_in_seconds=0) + return ret + + def TestPDFiumJavascript(self): + return self.TestPDFiumTest("run_javascript_tests.py") + + def TestPDFiumPixel(self): + return self.TestPDFiumTest("run_pixel_tests.py") + + def TestPDFiumCorpus(self): + return self.TestPDFiumTest("run_corpus_tests.py") + # The known list of tests. _test_list = { - "cmdline" : RunCmdLine, - "pdfium_unittests": TestPDFiumUnitTests, + "cmdline" : RunCmdLine, + "pdfium_corpus": TestPDFiumCorpus, "pdfium_embeddertests": TestPDFiumEmbedderTests, + "pdfium_javascript": TestPDFiumJavascript, + "pdfium_pixel": TestPDFiumPixel, + "pdfium_unittests": TestPDFiumUnitTests, } diff --git a/tools/drmemory/scripts/valgrind_test.py b/tools/drmemory/scripts/valgrind_test.py index bde300241c..d0581f3a8a 100644 --- a/tools/drmemory/scripts/valgrind_test.py +++ b/tools/drmemory/scripts/valgrind_test.py @@ -230,277 +230,6 @@ class BaseTool(object): return self.Main(args, check_sanity, min_runtime_in_seconds) -class ValgrindTool(BaseTool): - """Abstract class for running Valgrind tools. - - Always subclass this and implement ToolSpecificFlags() and - ExtendOptionParser() for tool-specific stuff. - """ - def __init__(self): - super(ValgrindTool, self).__init__() - self.RegisterOptionParserHook(ValgrindTool.ExtendOptionParser) - - def UseXML(self): - # Override if tool prefers nonxml output - return True - - def ExtendOptionParser(self, parser): - parser.add_option("", "--suppressions", default=[], - action="append", - help="path to a valgrind suppression file") - parser.add_option("", "--indirect", action="store_true", - default=False, - help="set BROWSER_WRAPPER rather than " - "running valgrind directly") - parser.add_option("", "--indirect_webkit_layout", action="store_true", - default=False, - help="set --wrapper rather than running Dr. Memory " - "directly.") - parser.add_option("", "--trace_children", action="store_true", - default=False, - help="also trace child processes") - parser.add_option("", "--num-callers", - dest="num_callers", default=30, - help="number of callers to show in stack traces") - parser.add_option("", "--generate_dsym", action="store_true", - default=False, - help="Generate .dSYM file on Mac if needed. Slow!") - - def Setup(self, args): - if not BaseTool.Setup(self, args): - return False - if common.IsMac(): - self.PrepareForTestMac() - return True - - def PrepareForTestMac(self): - """Runs dsymutil if needed. - - Valgrind for Mac OS X requires that debugging information be in a .dSYM - bundle generated by dsymutil. It is not currently able to chase DWARF - data into .o files like gdb does, so executables without .dSYM bundles or - with the Chromium-specific "fake_dsym" bundles generated by - build/mac/strip_save_dsym won't give source file and line number - information in valgrind. - - This function will run dsymutil if the .dSYM bundle is missing or if - it looks like a fake_dsym. A non-fake dsym that already exists is assumed - to be up-to-date. - """ - test_command = self._args[0] - dsym_bundle = self._args[0] + '.dSYM' - dsym_file = os.path.join(dsym_bundle, 'Contents', 'Resources', 'DWARF', - os.path.basename(test_command)) - dsym_info_plist = os.path.join(dsym_bundle, 'Contents', 'Info.plist') - - needs_dsymutil = True - saved_test_command = None - - if os.path.exists(dsym_file) and os.path.exists(dsym_info_plist): - # Look for the special fake_dsym tag in dsym_info_plist. - dsym_info_plist_contents = open(dsym_info_plist).read() - - if not re.search('^\s*<key>fake_dsym</key>$', dsym_info_plist_contents, - re.MULTILINE): - # fake_dsym is not set, this is a real .dSYM bundle produced by - # dsymutil. dsymutil does not need to be run again. - needs_dsymutil = False - else: - # fake_dsym is set. dsym_file is a copy of the original test_command - # before it was stripped. Copy it back to test_command so that - # dsymutil has unstripped input to work with. Move the stripped - # test_command out of the way, it will be restored when this is - # done. - saved_test_command = test_command + '.stripped' - os.rename(test_command, saved_test_command) - shutil.copyfile(dsym_file, test_command) - shutil.copymode(saved_test_command, test_command) - - if needs_dsymutil: - if self._options.generate_dsym: - # Remove the .dSYM bundle if it exists. - shutil.rmtree(dsym_bundle, True) - - dsymutil_command = ['dsymutil', test_command] - - # dsymutil is crazy slow. Ideally we'd have a timeout here, - # but common.RunSubprocess' timeout is only checked - # after each line of output; dsymutil is silent - # until the end, and is then killed, which is silly. - common.RunSubprocess(dsymutil_command) - - if saved_test_command: - os.rename(saved_test_command, test_command) - else: - logging.info("No real .dSYM for test_command. Line numbers will " - "not be shown. Either tell xcode to generate .dSYM " - "file, or use --generate_dsym option to this tool.") - - def ToolCommand(self): - """Get the valgrind command to run.""" - # Note that self._args begins with the exe to be run. - tool_name = self.ToolName() - - # Construct the valgrind command. - if 'CHROME_VALGRIND' in os.environ: - path = os.path.join(os.environ['CHROME_VALGRIND'], "bin", "valgrind") - else: - path = "valgrind" - proc = [path, "--tool=%s" % tool_name] - - proc += ["--num-callers=%i" % int(self._options.num_callers)] - - if self._options.trace_children: - proc += ["--trace-children=yes"] - proc += ["--trace-children-skip='*dbus-daemon*'"] - proc += ["--trace-children-skip='*dbus-launch*'"] - proc += ["--trace-children-skip='*perl*'"] - proc += ["--trace-children-skip='*python*'"] - # This is really Python, but for some reason Valgrind follows it. - proc += ["--trace-children-skip='*lsb_release*'"] - - proc += self.ToolSpecificFlags() - proc += self._tool_flags - - suppression_count = 0 - for suppression_file in self._options.suppressions: - if os.path.exists(suppression_file): - suppression_count += 1 - proc += ["--suppressions=%s" % suppression_file] - - if not suppression_count: - logging.warning("WARNING: NOT USING SUPPRESSIONS!") - - logfilename = self.log_dir + ("/%s." % tool_name) + "%p" - if self.UseXML(): - proc += ["--xml=yes", "--xml-file=" + logfilename] - else: - proc += ["--log-file=" + logfilename] - - # The Valgrind command is constructed. - - # Handle --indirect_webkit_layout separately. - if self._options.indirect_webkit_layout: - # Need to create the wrapper before modifying |proc|. - wrapper = self.CreateBrowserWrapper(proc, webkit=True) - proc = self._args - proc.append("--wrapper") - proc.append(wrapper) - return proc - - if self._options.indirect: - wrapper = self.CreateBrowserWrapper(proc) - os.environ["BROWSER_WRAPPER"] = wrapper - logging.info('export BROWSER_WRAPPER=' + wrapper) - proc = [] - proc += self._args - return proc - - def ToolSpecificFlags(self): - raise NotImplementedError, "This method should be implemented " \ - "in the tool-specific subclass" - - def CreateBrowserWrapper(self, proc, webkit=False): - """The program being run invokes Python or something else that can't stand - to be valgrinded, and also invokes the Chrome browser. In this case, use a - magic wrapper to only valgrind the Chrome browser. Build the wrapper here. - Returns the path to the wrapper. It's up to the caller to use the wrapper - appropriately. - """ - command = " ".join(proc) - # Add the PID of the browser wrapper to the logfile names so we can - # separate log files for different UI tests at the analyze stage. - command = command.replace("%p", "$$.%p") - - (fd, indirect_fname) = tempfile.mkstemp(dir=self.log_dir, - prefix="browser_wrapper.", - text=True) - f = os.fdopen(fd, "w") - f.write('#!/bin/bash\n' - 'echo "Started Valgrind wrapper for this test, PID=$$" >&2\n') - - f.write('DIR=`dirname $0`\n' - 'TESTNAME_FILE=$DIR/testcase.$$.name\n\n') - - if webkit: - # Webkit layout_tests pass the URL as the first line of stdin. - f.write('tee $TESTNAME_FILE | %s "$@"\n' % command) - else: - # Try to get the test case name by looking at the program arguments. - # i.e. Chromium ui_tests used --test-name arg. - # TODO(timurrrr): This doesn't handle "--test-name Test.Name" - # TODO(timurrrr): ui_tests are dead. Where do we use the non-webkit - # wrapper now? browser_tests? What do they do? - f.write('for arg in $@\ndo\n' - ' if [[ "$arg" =~ --test-name=(.*) ]]\n then\n' - ' echo ${BASH_REMATCH[1]} >$TESTNAME_FILE\n' - ' fi\n' - 'done\n\n' - '%s "$@"\n' % command) - - f.close() - os.chmod(indirect_fname, stat.S_IRUSR|stat.S_IXUSR) - return indirect_fname - - def CreateAnalyzer(self): - raise NotImplementedError, "This method should be implemented " \ - "in the tool-specific subclass" - - def GetAnalyzeResults(self, check_sanity=False): - # Glob all the files in the log directory - filenames = glob.glob(self.log_dir + "/" + self.ToolName() + ".*") - - # If we have browser wrapper, the logfiles are named as - # "toolname.wrapper_PID.valgrind_PID". - # Let's extract the list of wrapper_PIDs and name it ppids - ppids = set([int(f.split(".")[-2]) \ - for f in filenames if re.search("\.[0-9]+\.[0-9]+$", f)]) - - analyzer = self.CreateAnalyzer() - if len(ppids) == 0: - # Fast path - no browser wrapper was set. - return analyzer.Report(filenames, None, check_sanity) - - ret = 0 - for ppid in ppids: - testcase_name = None - try: - f = open(self.log_dir + ("/testcase.%d.name" % ppid)) - testcase_name = f.read().strip() - f.close() - wk_layout_prefix="third_party/WebKit/LayoutTests/" - wk_prefix_at = testcase_name.rfind(wk_layout_prefix) - if wk_prefix_at != -1: - testcase_name = testcase_name[wk_prefix_at + len(wk_layout_prefix):] - except IOError: - pass - print "=====================================================" - print " Below is the report for valgrind wrapper PID=%d." % ppid - if testcase_name: - print " It was used while running the `%s` test." % testcase_name - else: - print " You can find the corresponding test" - print " by searching the above log for 'PID=%d'" % ppid - sys.stdout.flush() - - ppid_filenames = [f for f in filenames \ - if re.search("\.%d\.[0-9]+$" % ppid, f)] - # check_sanity won't work with browser wrappers - assert check_sanity == False - ret |= analyzer.Report(ppid_filenames, testcase_name) - print "=====================================================" - sys.stdout.flush() - - if ret != 0: - print "" - print "The Valgrind reports are grouped by test names." - print "Each test has its PID printed in the log when the test was run" - print "and at the beginning of its Valgrind report." - print "Hint: you can search for the reports by Ctrl+F -> `=#`" - sys.stdout.flush() - - return ret - class DrMemory(BaseTool): """Dr.Memory Dynamic memory error detector for Windows. @@ -527,13 +256,9 @@ class DrMemory(BaseTool): help="Monitor python child processes. If off, neither " "python children nor any children of python children " "will be monitored.") - parser.add_option("", "--indirect", action="store_true", + parser.add_option("", "--indirect_pdfium_test", action="store_true", default=False, - help="set BROWSER_WRAPPER rather than " - "running Dr. Memory directly on the harness") - parser.add_option("", "--indirect_webkit_layout", action="store_true", - default=False, - help="set --wrapper rather than running valgrind " + help="set --wrapper rather than running Dr. Memory " "directly.") parser.add_option("", "--use_debug", action="store_true", default=False, dest="use_debug", @@ -677,20 +402,12 @@ class DrMemory(BaseTool): # Dr.Memory requires -- to separate tool flags from the executable name. proc += ["--"] - if self._options.indirect or self._options.indirect_webkit_layout: - wrapper_path = os.path.join(self._source_dir, - "tools", "valgrind", "browser_wrapper_win.py") - wrapper = " ".join(["python", wrapper_path] + proc) - self.CreateBrowserWrapper(wrapper) - logging.info("browser wrapper = " + " ".join(proc)) - if self._options.indirect_webkit_layout: - proc = self._args - # Layout tests want forward slashes. - wrapper = wrapper.replace('\\', '/') - proc += ["--wrapper", wrapper] - return proc - else: - proc = [] + if self._options.indirect_pdfium_test: + wrapper = " ".join(proc) + logging.info("pdfium wrapper = " + wrapper) + proc = self._args + proc += ["--wrapper", wrapper] + return proc # Note that self._args begins with the name of the exe to be run. self._args[0] = common.NormalizeWindowsPath(self._args[0]) @@ -708,7 +425,7 @@ class DrMemory(BaseTool): analyzer = drmemory_analyze.DrMemoryAnalyzer() ret = 0 - if not self._options.indirect and not self._options.indirect_webkit_layout: + if not self._options.indirect_pdfium_test: filenames = glob.glob(self.log_dir + "/*/results.txt") ret = analyzer.Report(filenames, None, check_sanity) |