diff options
Diffstat (limited to 'testing/tools/safetynet_compare.py')
| -rwxr-xr-x | testing/tools/safetynet_compare.py | 64 |
1 files changed, 64 insertions, 0 deletions
diff --git a/testing/tools/safetynet_compare.py b/testing/tools/safetynet_compare.py index 0cf1aec036..3c5cff86be 100755 --- a/testing/tools/safetynet_compare.py +++ b/testing/tools/safetynet_compare.py @@ -7,6 +7,7 @@ import argparse import functools +import glob import json import multiprocessing import os @@ -24,6 +25,7 @@ from safetynet_conclusions import ComparisonConclusions from safetynet_conclusions import PrintConclusionsDictHumanReadable from safetynet_conclusions import RATING_IMPROVEMENT from safetynet_conclusions import RATING_REGRESSION +from safetynet_image import ImageComparison def RunSingleTestCaseParallel(this, run_label, build_dir, test_case): @@ -107,6 +109,15 @@ class CompareRun(object): self._CleanUp(conclusions) + if self.args.png_dir: + image_comparison = ImageComparison( + self.after_build_dir, + self.args.png_dir, + ('before', 'after'), + self.args.num_workers, + self.args.png_threshold) + image_comparison.Run() + return 0 def _FreezeMeasureScript(self): @@ -484,11 +495,20 @@ class CompareRun(object): if profile_file_path: command.append('--output-path=%s' % profile_file_path) + if self.args.png_dir: + command.append('--png') + + if self.args.pages: + command.extend(['--pages', self.args.pages]) + output = RunCommandPropagateErr(command) if output is None: return None + if self.args.png_dir: + self._MoveImages(test_case, run_label) + # Get the time number as output, making sure it's just a number output = output.strip() if re.match('^[0-9]+$', output): @@ -496,6 +516,17 @@ class CompareRun(object): return None + def _MoveImages(self, test_case, run_label): + png_dir = os.path.join(self.args.png_dir, run_label) + if not os.path.exists(png_dir): + os.makedirs(png_dir) + + test_case_dir, test_case_filename = os.path.split(test_case) + test_case_png_matcher = '%s.*.png' % test_case_filename + for output_png in glob.glob(os.path.join(test_case_dir, + test_case_png_matcher)): + shutil.move(output_png, png_dir) + def _GetProfileFilePath(self, run_label, test_case): if self.args.output_dir: output_filename = ('callgrind.out.%s.%s' @@ -620,10 +651,22 @@ def main(): 'the whole test harness. Limiting to only the ' 'interesting section does not work on Release since ' 'the delimiters are optimized out') + parser.add_argument('--pages', + help='selects some pages to be rendered. Page numbers ' + 'are 0-based. "--pages A" will render only page A. ' + '"--pages A-B" will render pages A to B ' + '(inclusive).') parser.add_argument('--num-workers', default=multiprocessing.cpu_count(), type=int, help='run NUM_WORKERS jobs in parallel') parser.add_argument('--output-dir', help='directory to write the profile data output files') + parser.add_argument('--png-dir', default=None, + help='outputs pngs to the specified directory that can ' + 'be compared with a static html generated. Will ' + 'affect performance measurements.') + parser.add_argument('--png-threshold', default=0.0, type=float, + help='Requires --png-dir. Threshold above which a png ' + 'is considered to have changed.') parser.add_argument('--threshold-significant', default=0.02, type=float, help='variations in performance above this factor are ' 'considered significant') @@ -668,10 +711,31 @@ def main(): PrintErr('"%s" is not a directory' % args.output_dir) return 1 + if args.png_dir: + args.png_dir = os.path.expanduser(args.png_dir) + if not os.path.isdir(args.png_dir): + PrintErr('"%s" is not a directory' % args.png_dir) + return 1 + if args.threshold_significant <= 0.0: PrintErr('--threshold-significant should receive a positive float') return 1 + if args.png_threshold: + if not args.png_dir: + PrintErr('--png-threshold requires --png-dir to be specified.') + return 1 + + if args.png_threshold <= 0.0: + PrintErr('--png-threshold should receive a positive float') + return 1 + + if args.pages: + if not re.match(r'^\d+(-\d+)?$', args.pages): + PrintErr('Supported formats for --pages are "--pages 7" and ' + '"--pages 3-6"') + return 1 + run = CompareRun(args) return run.Run() |