Add pass/fail Gold summary to outputchromium/3469

- Changes the URL to retrieve baselines which returns the
  baselines based on the issue number (if available)
- Compare against a single baseline (instead of multiple)
- Writes the results of comparing to the baselines to a
  separate output file that can then be parsed by the
  recipe code.

See complementary CLs:

https://skia-review.googlesource.com/c/buildbot/+/131923
https://chromium-review.googlesource.com/c/chromium/tools/build/+/1085853

Change-Id: I90cdd77d0d26e63a5e686f4862fc68dc9c4429d3
Reviewed-on: https://pdfium-review.googlesource.com/33772
Commit-Queue: Stephan Altmueller <stephana@chromium.org>
Reviewed-by: Ryan Harrison <rharrison@chromium.org>

2 files changed, 35 insertions, 34 deletions

diff --git a/testing/tools/gold.py b/testing/tools/gold.py
index 89dfce1b81..445102fef3 100644
--- a/testing/tools/gold.py
+++ b/testing/tools/gold.py
@@ -65,39 +65,31 @@ class GoldBaseline(object):
     Download the baseline json and return a list of the two baselines that
     should be used to match hashes (master and cl#).
     """
-    GOLD_BASELINE_URL = ('https://storage.googleapis.com/skia-infra-gm/'
-                         'hash_files/gold-pdfium-baseline.json')
+    GOLD_BASELINE_URL = 'https://pdfium-gold.skia.org/json/baseline'
+
+    # If we have an issue number add it to the baseline URL
+    cl_number_str = self._properties.get('issue', None)
+    url = GOLD_BASELINE_URL + ('/' + cl_number_str if cl_number_str else '')
+
     try:
-      response = urllib2.urlopen(GOLD_BASELINE_URL, timeout=2)
+      response = urllib2.urlopen(url, timeout=2)
+      c_type = response.headers.get('Content-type', '')
+      if c_type != 'application/json':
+        raise ValueError('Invalid content type. Got %s instead of %s' % (
+          c_type, 'application/json'))
       json_data = response.read()
     except (urllib2.HTTPError, urllib2.URLError) as e:
       print ('Error: Unable to read skia gold json from %s: %s'
-             % (GOLD_BASELINE_URL, e))
+             % (url, e))
       return None
 
     try:
       data = json.loads(json_data)
-    except ValueError:
-      print 'Error: Malformed json read from %s: %s' % (GOLD_BASELINE_URL, e)
-      return None
-
-    try:
-      master_baseline = data['master']
-    except (KeyError, TypeError):
-      print ('Error: "master" key not in json read from %s: %s'
-             % (GOLD_BASELINE_URL, e))
+    except ValueError as e:
+      print 'Error: Malformed json read from %s: %s' % (url, e)
       return None
 
-    cl_number_str = self._properties.get('issue')
-    if cl_number_str is None:
-      return [master_baseline]
-
-    try:
-      cl_baseline = data['changeLists'][cl_number_str]
-    except KeyError:
-      return [master_baseline]
-
-    return [cl_baseline, master_baseline]
+    return data.get('master', {})
 
   # Return values for MatchLocalResult().
   MATCH = 'match'
@@ -124,11 +116,10 @@ class GoldBaseline(object):
       return GoldBaseline.BASELINE_DOWNLOAD_FAILED
 
     found_test_case = False
-    for baseline in self._baselines:
-      if test_name in baseline:
-        found_test_case = True
-        if md5_hash in baseline[test_name]:
-          return GoldBaseline.MATCH
+    if test_name in self._baselines:
+      found_test_case = True
+      if md5_hash in self._baselines[test_name]:
+        return GoldBaseline.MATCH
 
     return (GoldBaseline.MISMATCH if found_test_case
             else GoldBaseline.NO_BASELINE)
@@ -195,6 +186,7 @@ class GoldResults(object):
     self._properties = _ParseKeyValuePairs(propertiesStr)
     self._properties["key"] = _ParseKeyValuePairs(keyStr)
     self._results =  []
+    self._passfail = []
     self._outputDir = outputDir
 
     # make sure the output directory exists and is empty.
@@ -208,7 +200,7 @@ class GoldResults(object):
         hashes=[x.strip() for x in ig_file.readlines() if x.strip()]
         self._ignore_hashes = set(hashes)
 
-  def AddTestResult(self, testName, md5Hash, outputImagePath):
+  def AddTestResult(self, testName, md5Hash, outputImagePath, matchResult):
     # If the hash is in the list of hashes to ignore then we don'try
     # make a copy, but add it to the result.
     imgExt = os.path.splitext(outputImagePath)[1].lstrip(".")
@@ -232,6 +224,8 @@ class GoldResults(object):
       }
     })
 
+    self._passfail.append((testName, matchResult))
+
   def WriteResults(self):
     self._properties.update({
       "results": self._results
@@ -242,6 +236,11 @@ class GoldResults(object):
       json.dump(self._properties, outfile, indent=1)
       outfile.write("\n")
 
+    outputFileName = os.path.join(self._outputDir, "passfail.json")
+    with open(outputFileName, 'wb') as outfile:
+      json.dump(self._passfail, outfile, indent=1)
+      outfile.write("\n")
+
 # Produce example output for manual testing.
 if __name__ == "__main__":
   # Create a test directory with three empty 'image' files.
@@ -261,8 +260,9 @@ if __name__ == "__main__":
   with open(hash_file, 'wb') as f:
     f.write("\n".join(["hash-1","hash-4"]) + "\n")
 
-  gr = GoldResults("pdfium", testDir, propStr, keyStr, hash_file)
-  gr.AddTestResult("test-1", "hash-1", os.path.join(testDir, "image1.png"))
-  gr.AddTestResult("test-2", "hash-2", os.path.join(testDir, "image2.png"))
-  gr.AddTestResult("test-3", "hash-3", os.path.join(testDir, "image3.png"))
+  outputDir = "./output_directory"
+  gr = GoldResults("pdfium", outputDir, propStr, keyStr, hash_file)
+  gr.AddTestResult("test-1", "hash-1", os.path.join(testDir, "image1.png"), GoldBaseline.MATCH)
+  gr.AddTestResult("test-2", "hash-2", os.path.join(testDir, "image2.png"), GoldBaseline.MATCH)
+  gr.AddTestResult("test-3", "hash-3", os.path.join(testDir, "image3.png"), GoldBaseline.MISMATCH)
   gr.WriteResults()
diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py
index 3737969fb2..ecd87ad082 100644
--- a/testing/tools/test_runner.py
+++ b/testing/tools/test_runner.py
@@ -160,6 +160,7 @@ class TestRunner:
         # becomes "example_005.pdf.0".
         test_name = os.path.splitext(os.path.split(img_path)[1])[0]
 
+        matched = "suppressed"
         if not self.test_suppressor.IsResultSuppressed(input_filename):
           matched = self.gold_baseline.MatchLocalResult(test_name, md5_hash)
           if matched == gold.GoldBaseline.MISMATCH:
@@ -168,7 +169,7 @@ class TestRunner:
             print 'No Skia Gold baseline found for test case: %s' % test_name
 
         if self.gold_results:
-          self.gold_results.AddTestResult(test_name, md5_hash, img_path)
+          self.gold_results.AddTestResult(test_name, md5_hash, img_path, matched)
 
     if self.test_suppressor.IsResultSuppressed(input_filename):
       self.result_suppressed_cases.append(input_filename)