Add safetynet_job.py to run safetynet_compare.py periodically.chromium/3182

safetynet.job.py verifies if there were performance changes since
its last run. Its state is comprised only of the last checkpoint,
and is kept in a directory passed by argument. Results of runs are
written to this directory as well.

Change-Id: I94e0e176b10fd1d2b18d84f82427f63be107ae17
Reviewed-on: https://pdfium-review.googlesource.com/10370
Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>

2 files changed, 215 insertions, 0 deletions

diff --git a/testing/tools/githelper.py b/testing/tools/githelper.py
index 42cc57d304..021ed4e61e 100644
--- a/testing/tools/githelper.py
+++ b/testing/tools/githelper.py
@@ -17,6 +17,10 @@ class GitHelper(object):
     """Checks out a branch."""
     subprocess.check_output(['git', 'checkout', branch])
 
+  def FetchOriginMaster(self):
+    """Fetches new changes on origin/master."""
+    subprocess.check_output(['git', 'fetch', 'origin', 'master'])
+
   def StashPush(self):
     """Stashes uncommitted changes."""
     output = subprocess.check_output(['git', 'stash', '--include-untracked'])
@@ -37,6 +41,14 @@ class GitHelper(object):
     return subprocess.check_output(
         ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
 
+  def GetCurrentBranchHash(self):
+    return subprocess.check_output(
+        ['git', 'rev-parse', 'HEAD']).strip()
+
+  def IsCurrentBranchClean(self):
+    output = subprocess.check_output(['git', 'status', '--porcelain'])
+    return not output
+
   def BranchExists(self, branch_name):
     """Return whether a branch with the given name exists."""
     try:
diff --git a/testing/tools/safetynet_job.py b/testing/tools/safetynet_job.py
new file mode 100755
index 0000000000..09e6b75097
--- /dev/null
+++ b/testing/tools/safetynet_job.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+# Copyright 2017 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Looks for performance regressions on all pushes since the last run.
+
+Run this nightly to have a periodical check for performance regressions.
+
+Stores the results for each run and last checkpoint in a results directory.
+"""
+
+import argparse
+import datetime
+import json
+import os
+import subprocess
+import sys
+
+from githelper import GitHelper
+from safetynet_conclusions import PrintConclusionsDictHumanReadable
+
+
+class JobContext(object):
+  """Context for a single run, including name and directory paths."""
+
+  def __init__(self, args):
+    self.run_name = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
+    self.results_dir = args.results_dir
+    self.last_revision_covered_file = os.path.join(self.results_dir,
+                                                   'last_revision_covered')
+    self.run_output_dir = os.path.join(self.results_dir,
+                                       'profiles_%s' % self.run_name)
+    self.run_output_log_file = os.path.join(self.results_dir,
+                                            '%s.log' % self.run_name)
+
+
+class JobRun(object):
+  """A single run looking for regressions since the last one."""
+
+  def __init__(self, args, context):
+    """Constructor.
+
+    Args:
+      args: Namespace with arguments passed to the script.
+      context: JobContext for this run.
+    """
+    self.git = GitHelper()
+    self.args = args
+    self.context = context
+
+  def Run(self):
+    """Searches for regressions.
+
+    Will only write a checkpoint when first run, and on all subsequent runs
+    a comparison is done against the last checkpoint.
+
+    Returns:
+      Exit code for the script: 0 if no significant changes are found; 1 if
+      there was an error in the comparison; 2 if there was a regression; 3 if
+      there was an improvement and no regression.
+    """
+    pdfium_src_dir = os.path.join(
+        os.path.dirname(__file__),
+        os.path.pardir,
+        os.path.pardir)
+    os.chdir(pdfium_src_dir)
+
+    if not self.git.IsCurrentBranchClean():
+      print 'Current branch is not clean, aborting'
+      return 1
+
+    branch_to_restore = self.git.GetCurrentBranchName()
+
+    if not self.args.no_fetch:
+      self.git.FetchOriginMaster()
+
+    self.git.Checkout('origin/master')
+
+    # Make sure results dir exists
+    if not os.path.exists(self.context.results_dir):
+      os.makedirs(self.context.results_dir)
+
+    if not os.path.exists(self.context.last_revision_covered_file):
+      result = self._InitialRun()
+    else:
+      with open(self.context.last_revision_covered_file) as f:
+        last_revision_covered = f.read().strip()
+      result = self._IncrementalRun(last_revision_covered)
+
+    self.git.Checkout(branch_to_restore)
+    return result
+
+  def _InitialRun(self):
+    """Initial run, just write a checkpoint.
+
+    Returns:
+      Exit code for the script.
+    """
+    current = self.git.GetCurrentBranchHash()
+
+    print 'Initial run, current is %s' % current
+
+    self._WriteCheckpoint(current)
+
+    print 'All set up, next runs will be incremental and perform comparisons'
+    return 0
+
+  def _IncrementalRun(self, last_revision_covered):
+    """Incremental run, compare against last checkpoint and update it.
+
+    Args:
+      last_revision_covered: String with hash for last checkpoint.
+
+    Returns:
+      Exit code for the script.
+    """
+    current = self.git.GetCurrentBranchHash()
+
+    print ('Incremental run, current is %s, last is %s'
+           % (current, last_revision_covered))
+
+    if current == last_revision_covered:
+      print 'No changes seen, finishing job'
+      return 0
+
+    # Run compare
+    if not os.path.exists(self.context.run_output_dir):
+      os.makedirs(self.context.run_output_dir)
+    cmd = ['testing/tools/safetynet_compare.py',
+           '--this-repo',
+           '--machine-readable',
+           '--branch-before=%s' % last_revision_covered,
+           '--output-dir=%s' % self.context.run_output_dir]
+    cmd.extend(self.args.input_paths)
+
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    json_output, _ = p.communicate()
+
+    output_info = json.loads(json_output)
+    PrintConclusionsDictHumanReadable(output_info,
+                                      colored=(not self.args.output_to_log),
+                                      key='after')
+
+    status = 0
+
+    if output_info['summary']['improvement']:
+      print 'Improvement detected.'
+      status = 3
+
+    if output_info['summary']['regression']:
+      print 'Regression detected.'
+      status = 2
+
+    if status == 0:
+      print 'Nothing detected.'
+
+    self._WriteCheckpoint(current)
+
+    return status
+
+  def _WriteCheckpoint(self, checkpoint):
+    if not self.args.no_checkpoint:
+      with open(self.context.last_revision_covered_file, 'w') as f:
+        f.write(checkpoint + '\n')
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('results_dir',
+                      help='where to write the job results')
+  parser.add_argument('input_paths', nargs='+',
+                      help='pdf files or directories to search for pdf files '
+                           'to run as test cases')
+  parser.add_argument('--no-fetch', action='store_true',
+                      help='whether to skip the git fetch. Use for script '
+                      'debugging.')
+  parser.add_argument('--no-checkpoint', action='store_true',
+                      help='whether to skip writing the new checkpoint. Use '
+                           'for script debugging.')
+  parser.add_argument('--output-to-log', action='store_true',
+                      help='whether to write output to a log file')
+  args = parser.parse_args()
+
+  job_context = JobContext(args)
+
+  if args.output_to_log:
+    log_file = open(job_context.run_output_log_file, 'w')
+    sys.stdout = log_file
+    sys.stderr = log_file
+
+  run = JobRun(args, job_context)
+  result = run.Run()
+
+  if args.output_to_log:
+    log_file.close()
+
+  return result
+
+
+if __name__ == '__main__':
+  sys.exit(main())
+