summaryrefslogtreecommitdiff
path: root/testing/tools/safetynet_compare.py
diff options
context:
space:
mode:
authorHenrique Nakashima <hnakashima@chromium.org>2018-05-09 21:37:42 +0000
committerChromium commit bot <commit-bot@chromium.org>2018-05-09 21:37:42 +0000
commit52489ee8114cf476342231be610e75b324290920 (patch)
tree186112625fbab0dfbfaad1442817b09b806bb54f /testing/tools/safetynet_compare.py
parent81981300c2892598784f8b7096d5143ac04293a7 (diff)
downloadpdfium-52489ee8114cf476342231be610e75b324290920.tar.xz
safetynet_compare.py generates pngs to compare visually
Change-Id: Ief6853cbfdb5dca0c81d76978075a2fd04989f41 Reviewed-on: https://pdfium-review.googlesource.com/8830 Reviewed-by: dsinclair <dsinclair@chromium.org> Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>
Diffstat (limited to 'testing/tools/safetynet_compare.py')
-rwxr-xr-xtesting/tools/safetynet_compare.py64
1 files changed, 64 insertions, 0 deletions
diff --git a/testing/tools/safetynet_compare.py b/testing/tools/safetynet_compare.py
index 0cf1aec036..3c5cff86be 100755
--- a/testing/tools/safetynet_compare.py
+++ b/testing/tools/safetynet_compare.py
@@ -7,6 +7,7 @@
import argparse
import functools
+import glob
import json
import multiprocessing
import os
@@ -24,6 +25,7 @@ from safetynet_conclusions import ComparisonConclusions
from safetynet_conclusions import PrintConclusionsDictHumanReadable
from safetynet_conclusions import RATING_IMPROVEMENT
from safetynet_conclusions import RATING_REGRESSION
+from safetynet_image import ImageComparison
def RunSingleTestCaseParallel(this, run_label, build_dir, test_case):
@@ -107,6 +109,15 @@ class CompareRun(object):
self._CleanUp(conclusions)
+ if self.args.png_dir:
+ image_comparison = ImageComparison(
+ self.after_build_dir,
+ self.args.png_dir,
+ ('before', 'after'),
+ self.args.num_workers,
+ self.args.png_threshold)
+ image_comparison.Run()
+
return 0
def _FreezeMeasureScript(self):
@@ -484,11 +495,20 @@ class CompareRun(object):
if profile_file_path:
command.append('--output-path=%s' % profile_file_path)
+ if self.args.png_dir:
+ command.append('--png')
+
+ if self.args.pages:
+ command.extend(['--pages', self.args.pages])
+
output = RunCommandPropagateErr(command)
if output is None:
return None
+ if self.args.png_dir:
+ self._MoveImages(test_case, run_label)
+
# Get the time number as output, making sure it's just a number
output = output.strip()
if re.match('^[0-9]+$', output):
@@ -496,6 +516,17 @@ class CompareRun(object):
return None
+ def _MoveImages(self, test_case, run_label):
+ png_dir = os.path.join(self.args.png_dir, run_label)
+ if not os.path.exists(png_dir):
+ os.makedirs(png_dir)
+
+ test_case_dir, test_case_filename = os.path.split(test_case)
+ test_case_png_matcher = '%s.*.png' % test_case_filename
+ for output_png in glob.glob(os.path.join(test_case_dir,
+ test_case_png_matcher)):
+ shutil.move(output_png, png_dir)
+
def _GetProfileFilePath(self, run_label, test_case):
if self.args.output_dir:
output_filename = ('callgrind.out.%s.%s'
@@ -620,10 +651,22 @@ def main():
'the whole test harness. Limiting to only the '
'interesting section does not work on Release since '
'the delimiters are optimized out')
+ parser.add_argument('--pages',
+ help='selects some pages to be rendered. Page numbers '
+ 'are 0-based. "--pages A" will render only page A. '
+ '"--pages A-B" will render pages A to B '
+ '(inclusive).')
parser.add_argument('--num-workers', default=multiprocessing.cpu_count(),
type=int, help='run NUM_WORKERS jobs in parallel')
parser.add_argument('--output-dir',
help='directory to write the profile data output files')
+ parser.add_argument('--png-dir', default=None,
+ help='outputs pngs to the specified directory that can '
+ 'be compared with a static html generated. Will '
+ 'affect performance measurements.')
+ parser.add_argument('--png-threshold', default=0.0, type=float,
+ help='Requires --png-dir. Threshold above which a png '
+ 'is considered to have changed.')
parser.add_argument('--threshold-significant', default=0.02, type=float,
help='variations in performance above this factor are '
'considered significant')
@@ -668,10 +711,31 @@ def main():
PrintErr('"%s" is not a directory' % args.output_dir)
return 1
+ if args.png_dir:
+ args.png_dir = os.path.expanduser(args.png_dir)
+ if not os.path.isdir(args.png_dir):
+ PrintErr('"%s" is not a directory' % args.png_dir)
+ return 1
+
if args.threshold_significant <= 0.0:
PrintErr('--threshold-significant should receive a positive float')
return 1
+ if args.png_threshold:
+ if not args.png_dir:
+ PrintErr('--png-threshold requires --png-dir to be specified.')
+ return 1
+
+ if args.png_threshold <= 0.0:
+ PrintErr('--png-threshold should receive a positive float')
+ return 1
+
+ if args.pages:
+ if not re.match(r'^\d+(-\d+)?$', args.pages):
+ PrintErr('Supported formats for --pages are "--pages 7" and '
+ '"--pages 3-6"')
+ return 1
+
run = CompareRun(args)
return run.Run()