Store raw json with run results when running safetynet_job.py.

Storing the machine-readable data is a good idea in general, in this
case allowing us to create graphs with the historical runs.

The json now also contains:
- a version number in case there are future changes in the format
- the date and time when the run started
- which profiler was used
- which commits were being compared
- whether a comparison was run or not, that is, if the commit hashes
  for before and after differ

Change-Id: I2913beaef30b90669ac4ffe60a656d4b69030588
Reviewed-on: https://pdfium-review.googlesource.com/13370
Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>
Reviewed-by: Nicolás Peña <npm@chromium.org>

3 files changed, 35 insertions, 7 deletions

diff --git a/testing/tools/safetynet_compare.py b/testing/tools/safetynet_compare.py
index d583312bab..ce0505f0a4 100755
--- a/testing/tools/safetynet_compare.py
+++ b/testing/tools/safetynet_compare.py
@@ -101,6 +101,7 @@ class CompareRun(object):
 
     conclusions = self._DrawConclusions(before, after)
     conclusions_dict = conclusions.GetOutputDict()
+    conclusions_dict.setdefault('metadata', {})['profiler'] = self.args.profiler
 
     self._PrintConclusions(conclusions_dict)
 
diff --git a/testing/tools/safetynet_conclusions.py b/testing/tools/safetynet_conclusions.py
index 3d8758764c..fdbc10df14 100644
--- a/testing/tools/safetynet_conclusions.py
+++ b/testing/tools/safetynet_conclusions.py
@@ -124,6 +124,7 @@ class ComparisonConclusions(object):
     Returns:
       A serializable dict with the format illustrated below:
       {
+        "version": 1,
         "params": {
           "threshold": 0.02
         },
@@ -176,6 +177,7 @@ class ComparisonConclusions(object):
       }
     """
     output_dict = {}
+    output_dict['version'] = 1
     output_dict['params'] = {'threshold': self.threshold_significant}
     output_dict['summary'] = self.summary.GetOutputDict()
     output_dict['comparison_by_case'] = {
diff --git a/testing/tools/safetynet_job.py b/testing/tools/safetynet_job.py
index f3ea9c5d9a..7416533222 100755
--- a/testing/tools/safetynet_job.py
+++ b/testing/tools/safetynet_job.py
@@ -14,10 +14,8 @@ import argparse
 import datetime
 import json
 import os
-import subprocess
 import sys
 
-from common import PrintErr
 from common import PrintWithTime
 from common import RunCommandPropagateErr
 from githelper import GitHelper
@@ -28,14 +26,14 @@ class JobContext(object):
   """Context for a single run, including name and directory paths."""
 
   def __init__(self, args):
-    self.run_name = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
+    self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
     self.results_dir = args.results_dir
     self.last_revision_covered_file = os.path.join(self.results_dir,
                                                    'last_revision_covered')
     self.run_output_dir = os.path.join(self.results_dir,
-                                       'profiles_%s' % self.run_name)
+                                       'profiles_%s' % self.datetime)
     self.run_output_log_file = os.path.join(self.results_dir,
-                                            '%s.log' % self.run_name)
+                                            '%s.log' % self.datetime)
 
 
 class JobRun(object):
@@ -123,13 +121,19 @@ class JobRun(object):
     PrintWithTime('Incremental run, current is %s, last is %s'
                   % (current, last_revision_covered))
 
+    if not os.path.exists(self.context.run_output_dir):
+      os.makedirs(self.context.run_output_dir)
+
     if current == last_revision_covered:
       PrintWithTime('No changes seen, finishing job')
+      output_info = {
+          'metadata': self._BuildRunMetadata(last_revision_covered,
+                                             current,
+                                             False)}
+      self._WriteRawJson(output_info)
       return 0
 
     # Run compare
-    if not os.path.exists(self.context.run_output_dir):
-      os.makedirs(self.context.run_output_dir)
     cmd = ['testing/tools/safetynet_compare.py',
            '--this-repo',
            '--machine-readable',
@@ -143,6 +147,13 @@ class JobRun(object):
       return 1
 
     output_info = json.loads(json_output)
+
+    run_metadata = self._BuildRunMetadata(last_revision_covered,
+                                          current,
+                                          True)
+    output_info.setdefault('metadata', {}).update(run_metadata)
+    self._WriteRawJson(output_info)
+
     PrintConclusionsDictHumanReadable(output_info,
                                       colored=(not self.args.output_to_log
                                                and not self.args.no_color),
@@ -165,6 +176,20 @@ class JobRun(object):
 
     return status
 
+  def _WriteRawJson(self, output_info):
+    json_output_file = os.path.join(self.context.run_output_dir, 'raw.json')
+    with open(json_output_file, 'w') as f:
+      json.dump(output_info, f)
+
+  def _BuildRunMetadata(self, revision_before, revision_after,
+                        comparison_performed):
+    return {
+        'datetime': self.context.datetime,
+        'revision_before': revision_before,
+        'revision_after': revision_after,
+        'comparison_performed': comparison_performed,
+    }
+
   def _WriteCheckpoint(self, checkpoint):
     if not self.args.no_checkpoint:
       with open(self.context.last_revision_covered_file, 'w') as f: