#!/usr/bin/env python # Copyright 2017 The PDFium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Looks for performance regressions on all pushes since the last run. Run this nightly to have a periodical check for performance regressions. Stores the results for each run and last checkpoint in a results directory. """ import argparse import datetime import json import os import sys from common import PrintWithTime from common import RunCommandPropagateErr from githelper import GitHelper from safetynet_conclusions import PrintConclusionsDictHumanReadable class JobContext(object): """Context for a single run, including name and directory paths.""" def __init__(self, args): self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') self.results_dir = args.results_dir self.last_revision_covered_file = os.path.join(self.results_dir, 'last_revision_covered') self.run_output_dir = os.path.join(self.results_dir, 'profiles_%s' % self.datetime) self.run_output_log_file = os.path.join(self.results_dir, '%s.log' % self.datetime) class JobRun(object): """A single run looking for regressions since the last one.""" def __init__(self, args, context): """Constructor. Args: args: Namespace with arguments passed to the script. context: JobContext for this run. """ self.git = GitHelper() self.args = args self.context = context def Run(self): """Searches for regressions. Will only write a checkpoint when first run, and on all subsequent runs a comparison is done against the last checkpoint. Returns: Exit code for the script: 0 if no significant changes are found; 1 if there was an error in the comparison; 3 if there was a regression; 4 if there was an improvement and no regression. """ pdfium_src_dir = os.path.join( os.path.dirname(__file__), os.path.pardir, os.path.pardir) os.chdir(pdfium_src_dir) if not self.git.IsCurrentBranchClean() and not self.args.no_checkout: PrintWithTime('Current branch is not clean, aborting') return 1 branch_to_restore = self.git.GetCurrentBranchName() if not self.args.no_checkout: self.git.FetchOriginMaster() self.git.Checkout('origin/master') # Make sure results dir exists if not os.path.exists(self.context.results_dir): os.makedirs(self.context.results_dir) if not os.path.exists(self.context.last_revision_covered_file): result = self._InitialRun() else: with open(self.context.last_revision_covered_file) as f: last_revision_covered = f.read().strip() result = self._IncrementalRun(last_revision_covered) self.git.Checkout(branch_to_restore) return result def _InitialRun(self): """Initial run, just write a checkpoint. Returns: Exit code for the script. """ current = self.git.GetCurrentBranchHash() PrintWithTime('Initial run, current is %s' % current) self._WriteCheckpoint(current) PrintWithTime('All set up, next runs will be incremental and perform ' 'comparisons') return 0 def _IncrementalRun(self, last_revision_covered): """Incremental run, compare against last checkpoint and update it. Args: last_revision_covered: String with hash for last checkpoint. Returns: Exit code for the script. """ current = self.git.GetCurrentBranchHash() PrintWithTime('Incremental run, current is %s, last is %s' % (current, last_revision_covered)) if not os.path.exists(self.context.run_output_dir): os.makedirs(self.context.run_output_dir) if current == last_revision_covered: PrintWithTime('No changes seen, finishing job') output_info = { 'metadata': self._BuildRunMetadata(last_revision_covered, current, False)} self._WriteRawJson(output_info) return 0 # Run compare cmd = ['testing/tools/safetynet_compare.py', '--this-repo', '--machine-readable', '--branch-before=%s' % last_revision_covered, '--output-dir=%s' % self.context.run_output_dir] cmd.extend(self.args.input_paths) json_output = RunCommandPropagateErr(cmd) if json_output is None: return 1 output_info = json.loads(json_output) run_metadata = self._BuildRunMetadata(last_revision_covered, current, True) output_info.setdefault('metadata', {}).update(run_metadata) self._WriteRawJson(output_info) PrintConclusionsDictHumanReadable(output_info, colored=(not self.args.output_to_log and not self.args.no_color), key='after') status = 0 if output_info['summary']['improvement']: PrintWithTime('Improvement detected.') status = 4 if output_info['summary']['regression']: PrintWithTime('Regression detected.') status = 3 if status == 0: PrintWithTime('Nothing detected.') self._WriteCheckpoint(current) return status def _WriteRawJson(self, output_info): json_output_file = os.path.join(self.context.run_output_dir, 'raw.json') with open(json_output_file, 'w') as f: json.dump(output_info, f) def _BuildRunMetadata(self, revision_before, revision_after, comparison_performed): return { 'datetime': self.context.datetime, 'revision_before': revision_before, 'revision_after': revision_after, 'comparison_performed': comparison_performed, } def _WriteCheckpoint(self, checkpoint): if not self.args.no_checkpoint: with open(self.context.last_revision_covered_file, 'w') as f: f.write(checkpoint + '\n') def main(): parser = argparse.ArgumentParser() parser.add_argument('results_dir', help='where to write the job results') parser.add_argument('input_paths', nargs='+', help='pdf files or directories to search for pdf files ' 'to run as test cases') parser.add_argument('--no-checkout', action='store_true', help='whether to skip checking out origin/master. Use ' 'for script debugging.') parser.add_argument('--no-checkpoint', action='store_true', help='whether to skip writing the new checkpoint. Use ' 'for script debugging.') parser.add_argument('--no-color', action='store_true', help='whether to write output without color escape ' 'codes.') parser.add_argument('--output-to-log', action='store_true', help='whether to write output to a log file') args = parser.parse_args() job_context = JobContext(args) if args.output_to_log: log_file = open(job_context.run_output_log_file, 'w') sys.stdout = log_file sys.stderr = log_file run = JobRun(args, job_context) result = run.Run() if args.output_to_log: log_file.close() return result if __name__ == '__main__': sys.exit(main())