safetynet_compare.py generates pngs to compare visually

Change-Id: Ief6853cbfdb5dca0c81d76978075a2fd04989f41
Reviewed-on: https://pdfium-review.googlesource.com/8830
Reviewed-by: dsinclair <dsinclair@chromium.org>
Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>

1 files changed, 64 insertions, 0 deletions

diff --git a/testing/tools/safetynet_compare.py b/testing/tools/safetynet_compare.py
index 0cf1aec036..3c5cff86be 100755
--- a/testing/tools/safetynet_compare.py
+++ b/testing/tools/safetynet_compare.py
@@ -7,6 +7,7 @@
 
 import argparse
 import functools
+import glob
 import json
 import multiprocessing
 import os
@@ -24,6 +25,7 @@ from safetynet_conclusions import ComparisonConclusions
 from safetynet_conclusions import PrintConclusionsDictHumanReadable
 from safetynet_conclusions import RATING_IMPROVEMENT
 from safetynet_conclusions import RATING_REGRESSION
+from safetynet_image import ImageComparison
 
 
 def RunSingleTestCaseParallel(this, run_label, build_dir, test_case):
@@ -107,6 +109,15 @@ class CompareRun(object):
 
     self._CleanUp(conclusions)
 
+    if self.args.png_dir:
+      image_comparison = ImageComparison(
+          self.after_build_dir,
+          self.args.png_dir,
+          ('before', 'after'),
+          self.args.num_workers,
+          self.args.png_threshold)
+      image_comparison.Run()
+
     return 0
 
   def _FreezeMeasureScript(self):
@@ -484,11 +495,20 @@ class CompareRun(object):
     if profile_file_path:
       command.append('--output-path=%s' % profile_file_path)
 
+    if self.args.png_dir:
+      command.append('--png')
+
+    if self.args.pages:
+      command.extend(['--pages', self.args.pages])
+
     output = RunCommandPropagateErr(command)
 
     if output is None:
       return None
 
+    if self.args.png_dir:
+      self._MoveImages(test_case, run_label)
+
     # Get the time number as output, making sure it's just a number
     output = output.strip()
     if re.match('^[0-9]+$', output):
@@ -496,6 +516,17 @@ class CompareRun(object):
 
     return None
 
+  def _MoveImages(self, test_case, run_label):
+    png_dir = os.path.join(self.args.png_dir, run_label)
+    if not os.path.exists(png_dir):
+      os.makedirs(png_dir)
+
+    test_case_dir, test_case_filename = os.path.split(test_case)
+    test_case_png_matcher = '%s.*.png' % test_case_filename
+    for output_png in glob.glob(os.path.join(test_case_dir,
+                                             test_case_png_matcher)):
+      shutil.move(output_png, png_dir)
+
   def _GetProfileFilePath(self, run_label, test_case):
     if self.args.output_dir:
       output_filename = ('callgrind.out.%s.%s'
@@ -620,10 +651,22 @@ def main():
                            'the whole test harness. Limiting to only the '
                            'interesting section does not work on Release since '
                            'the delimiters are optimized out')
+  parser.add_argument('--pages',
+                      help='selects some pages to be rendered. Page numbers '
+                           'are 0-based. "--pages A" will render only page A. '
+                           '"--pages A-B" will render pages A to B '
+                           '(inclusive).')
   parser.add_argument('--num-workers', default=multiprocessing.cpu_count(),
                       type=int, help='run NUM_WORKERS jobs in parallel')
   parser.add_argument('--output-dir',
                       help='directory to write the profile data output files')
+  parser.add_argument('--png-dir', default=None,
+                      help='outputs pngs to the specified directory that can '
+                           'be compared with a static html generated. Will '
+                           'affect performance measurements.')
+  parser.add_argument('--png-threshold', default=0.0, type=float,
+                      help='Requires --png-dir. Threshold above which a png '
+                           'is considered to have changed.')
   parser.add_argument('--threshold-significant', default=0.02, type=float,
                       help='variations in performance above this factor are '
                            'considered significant')
@@ -668,10 +711,31 @@ def main():
       PrintErr('"%s" is not a directory' % args.output_dir)
       return 1
 
+  if args.png_dir:
+    args.png_dir = os.path.expanduser(args.png_dir)
+    if not os.path.isdir(args.png_dir):
+      PrintErr('"%s" is not a directory' % args.png_dir)
+      return 1
+
   if args.threshold_significant <= 0.0:
     PrintErr('--threshold-significant should receive a positive float')
     return 1
 
+  if args.png_threshold:
+    if not args.png_dir:
+      PrintErr('--png-threshold requires --png-dir to be specified.')
+      return 1
+
+    if args.png_threshold <= 0.0:
+      PrintErr('--png-threshold should receive a positive float')
+      return 1
+
+  if args.pages:
+    if not re.match(r'^\d+(-\d+)?$', args.pages):
+      PrintErr('Supported formats for --pages are "--pages 7" and '
+               '"--pages 3-6"')
+      return 1
+
   run = CompareRun(args)
   return run.Run()