From fdc3acb42a7983053c53d1445c62654d7ef6b3b2 Mon Sep 17 00:00:00 2001 From: Henrique Nakashima Date: Thu, 10 Aug 2017 17:40:11 -0400 Subject: Add safetynet_job.py to run safetynet_compare.py periodically. safetynet.job.py verifies if there were performance changes since its last run. Its state is comprised only of the last checkpoint, and is kept in a directory passed by argument. Results of runs are written to this directory as well. Change-Id: I94e0e176b10fd1d2b18d84f82427f63be107ae17 Reviewed-on: https://pdfium-review.googlesource.com/10370 Commit-Queue: Henrique Nakashima Reviewed-by: Lei Zhang --- testing/tools/githelper.py | 12 +++ testing/tools/safetynet_job.py | 203 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100755 testing/tools/safetynet_job.py (limited to 'testing') diff --git a/testing/tools/githelper.py b/testing/tools/githelper.py index 42cc57d304..021ed4e61e 100644 --- a/testing/tools/githelper.py +++ b/testing/tools/githelper.py @@ -17,6 +17,10 @@ class GitHelper(object): """Checks out a branch.""" subprocess.check_output(['git', 'checkout', branch]) + def FetchOriginMaster(self): + """Fetches new changes on origin/master.""" + subprocess.check_output(['git', 'fetch', 'origin', 'master']) + def StashPush(self): """Stashes uncommitted changes.""" output = subprocess.check_output(['git', 'stash', '--include-untracked']) @@ -37,6 +41,14 @@ class GitHelper(object): return subprocess.check_output( ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + def GetCurrentBranchHash(self): + return subprocess.check_output( + ['git', 'rev-parse', 'HEAD']).strip() + + def IsCurrentBranchClean(self): + output = subprocess.check_output(['git', 'status', '--porcelain']) + return not output + def BranchExists(self, branch_name): """Return whether a branch with the given name exists.""" try: diff --git a/testing/tools/safetynet_job.py b/testing/tools/safetynet_job.py new file mode 100755 index 0000000000..09e6b75097 --- /dev/null +++ b/testing/tools/safetynet_job.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python +# Copyright 2017 The PDFium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Looks for performance regressions on all pushes since the last run. + +Run this nightly to have a periodical check for performance regressions. + +Stores the results for each run and last checkpoint in a results directory. +""" + +import argparse +import datetime +import json +import os +import subprocess +import sys + +from githelper import GitHelper +from safetynet_conclusions import PrintConclusionsDictHumanReadable + + +class JobContext(object): + """Context for a single run, including name and directory paths.""" + + def __init__(self, args): + self.run_name = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + self.results_dir = args.results_dir + self.last_revision_covered_file = os.path.join(self.results_dir, + 'last_revision_covered') + self.run_output_dir = os.path.join(self.results_dir, + 'profiles_%s' % self.run_name) + self.run_output_log_file = os.path.join(self.results_dir, + '%s.log' % self.run_name) + + +class JobRun(object): + """A single run looking for regressions since the last one.""" + + def __init__(self, args, context): + """Constructor. + + Args: + args: Namespace with arguments passed to the script. + context: JobContext for this run. + """ + self.git = GitHelper() + self.args = args + self.context = context + + def Run(self): + """Searches for regressions. + + Will only write a checkpoint when first run, and on all subsequent runs + a comparison is done against the last checkpoint. + + Returns: + Exit code for the script: 0 if no significant changes are found; 1 if + there was an error in the comparison; 2 if there was a regression; 3 if + there was an improvement and no regression. + """ + pdfium_src_dir = os.path.join( + os.path.dirname(__file__), + os.path.pardir, + os.path.pardir) + os.chdir(pdfium_src_dir) + + if not self.git.IsCurrentBranchClean(): + print 'Current branch is not clean, aborting' + return 1 + + branch_to_restore = self.git.GetCurrentBranchName() + + if not self.args.no_fetch: + self.git.FetchOriginMaster() + + self.git.Checkout('origin/master') + + # Make sure results dir exists + if not os.path.exists(self.context.results_dir): + os.makedirs(self.context.results_dir) + + if not os.path.exists(self.context.last_revision_covered_file): + result = self._InitialRun() + else: + with open(self.context.last_revision_covered_file) as f: + last_revision_covered = f.read().strip() + result = self._IncrementalRun(last_revision_covered) + + self.git.Checkout(branch_to_restore) + return result + + def _InitialRun(self): + """Initial run, just write a checkpoint. + + Returns: + Exit code for the script. + """ + current = self.git.GetCurrentBranchHash() + + print 'Initial run, current is %s' % current + + self._WriteCheckpoint(current) + + print 'All set up, next runs will be incremental and perform comparisons' + return 0 + + def _IncrementalRun(self, last_revision_covered): + """Incremental run, compare against last checkpoint and update it. + + Args: + last_revision_covered: String with hash for last checkpoint. + + Returns: + Exit code for the script. + """ + current = self.git.GetCurrentBranchHash() + + print ('Incremental run, current is %s, last is %s' + % (current, last_revision_covered)) + + if current == last_revision_covered: + print 'No changes seen, finishing job' + return 0 + + # Run compare + if not os.path.exists(self.context.run_output_dir): + os.makedirs(self.context.run_output_dir) + cmd = ['testing/tools/safetynet_compare.py', + '--this-repo', + '--machine-readable', + '--branch-before=%s' % last_revision_covered, + '--output-dir=%s' % self.context.run_output_dir] + cmd.extend(self.args.input_paths) + + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + json_output, _ = p.communicate() + + output_info = json.loads(json_output) + PrintConclusionsDictHumanReadable(output_info, + colored=(not self.args.output_to_log), + key='after') + + status = 0 + + if output_info['summary']['improvement']: + print 'Improvement detected.' + status = 3 + + if output_info['summary']['regression']: + print 'Regression detected.' + status = 2 + + if status == 0: + print 'Nothing detected.' + + self._WriteCheckpoint(current) + + return status + + def _WriteCheckpoint(self, checkpoint): + if not self.args.no_checkpoint: + with open(self.context.last_revision_covered_file, 'w') as f: + f.write(checkpoint + '\n') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('results_dir', + help='where to write the job results') + parser.add_argument('input_paths', nargs='+', + help='pdf files or directories to search for pdf files ' + 'to run as test cases') + parser.add_argument('--no-fetch', action='store_true', + help='whether to skip the git fetch. Use for script ' + 'debugging.') + parser.add_argument('--no-checkpoint', action='store_true', + help='whether to skip writing the new checkpoint. Use ' + 'for script debugging.') + parser.add_argument('--output-to-log', action='store_true', + help='whether to write output to a log file') + args = parser.parse_args() + + job_context = JobContext(args) + + if args.output_to_log: + log_file = open(job_context.run_output_log_file, 'w') + sys.stdout = log_file + sys.stderr = log_file + + run = JobRun(args, job_context) + result = run.Run() + + if args.output_to_log: + log_file.close() + + return result + + +if __name__ == '__main__': + sys.exit(main()) + -- cgit v1.2.3