summaryrefslogtreecommitdiff
path: root/testing
diff options
context:
space:
mode:
Diffstat (limited to 'testing')
-rwxr-xr-xtesting/tools/safetynet_compare.py1
-rw-r--r--testing/tools/safetynet_conclusions.py2
-rwxr-xr-xtesting/tools/safetynet_job.py39
3 files changed, 35 insertions, 7 deletions
diff --git a/testing/tools/safetynet_compare.py b/testing/tools/safetynet_compare.py
index d583312bab..ce0505f0a4 100755
--- a/testing/tools/safetynet_compare.py
+++ b/testing/tools/safetynet_compare.py
@@ -101,6 +101,7 @@ class CompareRun(object):
conclusions = self._DrawConclusions(before, after)
conclusions_dict = conclusions.GetOutputDict()
+ conclusions_dict.setdefault('metadata', {})['profiler'] = self.args.profiler
self._PrintConclusions(conclusions_dict)
diff --git a/testing/tools/safetynet_conclusions.py b/testing/tools/safetynet_conclusions.py
index 3d8758764c..fdbc10df14 100644
--- a/testing/tools/safetynet_conclusions.py
+++ b/testing/tools/safetynet_conclusions.py
@@ -124,6 +124,7 @@ class ComparisonConclusions(object):
Returns:
A serializable dict with the format illustrated below:
{
+ "version": 1,
"params": {
"threshold": 0.02
},
@@ -176,6 +177,7 @@ class ComparisonConclusions(object):
}
"""
output_dict = {}
+ output_dict['version'] = 1
output_dict['params'] = {'threshold': self.threshold_significant}
output_dict['summary'] = self.summary.GetOutputDict()
output_dict['comparison_by_case'] = {
diff --git a/testing/tools/safetynet_job.py b/testing/tools/safetynet_job.py
index f3ea9c5d9a..7416533222 100755
--- a/testing/tools/safetynet_job.py
+++ b/testing/tools/safetynet_job.py
@@ -14,10 +14,8 @@ import argparse
import datetime
import json
import os
-import subprocess
import sys
-from common import PrintErr
from common import PrintWithTime
from common import RunCommandPropagateErr
from githelper import GitHelper
@@ -28,14 +26,14 @@ class JobContext(object):
"""Context for a single run, including name and directory paths."""
def __init__(self, args):
- self.run_name = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
+ self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
self.results_dir = args.results_dir
self.last_revision_covered_file = os.path.join(self.results_dir,
'last_revision_covered')
self.run_output_dir = os.path.join(self.results_dir,
- 'profiles_%s' % self.run_name)
+ 'profiles_%s' % self.datetime)
self.run_output_log_file = os.path.join(self.results_dir,
- '%s.log' % self.run_name)
+ '%s.log' % self.datetime)
class JobRun(object):
@@ -123,13 +121,19 @@ class JobRun(object):
PrintWithTime('Incremental run, current is %s, last is %s'
% (current, last_revision_covered))
+ if not os.path.exists(self.context.run_output_dir):
+ os.makedirs(self.context.run_output_dir)
+
if current == last_revision_covered:
PrintWithTime('No changes seen, finishing job')
+ output_info = {
+ 'metadata': self._BuildRunMetadata(last_revision_covered,
+ current,
+ False)}
+ self._WriteRawJson(output_info)
return 0
# Run compare
- if not os.path.exists(self.context.run_output_dir):
- os.makedirs(self.context.run_output_dir)
cmd = ['testing/tools/safetynet_compare.py',
'--this-repo',
'--machine-readable',
@@ -143,6 +147,13 @@ class JobRun(object):
return 1
output_info = json.loads(json_output)
+
+ run_metadata = self._BuildRunMetadata(last_revision_covered,
+ current,
+ True)
+ output_info.setdefault('metadata', {}).update(run_metadata)
+ self._WriteRawJson(output_info)
+
PrintConclusionsDictHumanReadable(output_info,
colored=(not self.args.output_to_log
and not self.args.no_color),
@@ -165,6 +176,20 @@ class JobRun(object):
return status
+ def _WriteRawJson(self, output_info):
+ json_output_file = os.path.join(self.context.run_output_dir, 'raw.json')
+ with open(json_output_file, 'w') as f:
+ json.dump(output_info, f)
+
+ def _BuildRunMetadata(self, revision_before, revision_after,
+ comparison_performed):
+ return {
+ 'datetime': self.context.datetime,
+ 'revision_before': revision_before,
+ 'revision_after': revision_after,
+ 'comparison_performed': comparison_performed,
+ }
+
def _WriteCheckpoint(self, checkpoint):
if not self.args.no_checkpoint:
with open(self.context.last_revision_covered_file, 'w') as f: