summaryrefslogtreecommitdiff
path: root/testing
diff options
context:
space:
mode:
authorStephan Altmueller <stephana@google.com>2018-06-21 14:45:44 +0000
committerChromium commit bot <commit-bot@chromium.org>2018-06-21 14:45:44 +0000
commitbcd66f54de1291ddf8b7b7caca3e7360e14fcc8d (patch)
tree0d49a08a25969c8c6af53755b9919b1c84207f1f /testing
parentd9fb538bc08c4923fbfd3d50c47a61716d679956 (diff)
downloadpdfium-bcd66f54de1291ddf8b7b7caca3e7360e14fcc8d.tar.xz
Add pass/fail Gold summary to outputchromium/3469
- Changes the URL to retrieve baselines which returns the baselines based on the issue number (if available) - Compare against a single baseline (instead of multiple) - Writes the results of comparing to the baselines to a separate output file that can then be parsed by the recipe code. See complementary CLs: https://skia-review.googlesource.com/c/buildbot/+/131923 https://chromium-review.googlesource.com/c/chromium/tools/build/+/1085853 Change-Id: I90cdd77d0d26e63a5e686f4862fc68dc9c4429d3 Reviewed-on: https://pdfium-review.googlesource.com/33772 Commit-Queue: Stephan Altmueller <stephana@chromium.org> Reviewed-by: Ryan Harrison <rharrison@chromium.org>
Diffstat (limited to 'testing')
-rw-r--r--testing/tools/gold.py66
-rw-r--r--testing/tools/test_runner.py3
2 files changed, 35 insertions, 34 deletions
diff --git a/testing/tools/gold.py b/testing/tools/gold.py
index 89dfce1b81..445102fef3 100644
--- a/testing/tools/gold.py
+++ b/testing/tools/gold.py
@@ -65,39 +65,31 @@ class GoldBaseline(object):
Download the baseline json and return a list of the two baselines that
should be used to match hashes (master and cl#).
"""
- GOLD_BASELINE_URL = ('https://storage.googleapis.com/skia-infra-gm/'
- 'hash_files/gold-pdfium-baseline.json')
+ GOLD_BASELINE_URL = 'https://pdfium-gold.skia.org/json/baseline'
+
+ # If we have an issue number add it to the baseline URL
+ cl_number_str = self._properties.get('issue', None)
+ url = GOLD_BASELINE_URL + ('/' + cl_number_str if cl_number_str else '')
+
try:
- response = urllib2.urlopen(GOLD_BASELINE_URL, timeout=2)
+ response = urllib2.urlopen(url, timeout=2)
+ c_type = response.headers.get('Content-type', '')
+ if c_type != 'application/json':
+ raise ValueError('Invalid content type. Got %s instead of %s' % (
+ c_type, 'application/json'))
json_data = response.read()
except (urllib2.HTTPError, urllib2.URLError) as e:
print ('Error: Unable to read skia gold json from %s: %s'
- % (GOLD_BASELINE_URL, e))
+ % (url, e))
return None
try:
data = json.loads(json_data)
- except ValueError:
- print 'Error: Malformed json read from %s: %s' % (GOLD_BASELINE_URL, e)
- return None
-
- try:
- master_baseline = data['master']
- except (KeyError, TypeError):
- print ('Error: "master" key not in json read from %s: %s'
- % (GOLD_BASELINE_URL, e))
+ except ValueError as e:
+ print 'Error: Malformed json read from %s: %s' % (url, e)
return None
- cl_number_str = self._properties.get('issue')
- if cl_number_str is None:
- return [master_baseline]
-
- try:
- cl_baseline = data['changeLists'][cl_number_str]
- except KeyError:
- return [master_baseline]
-
- return [cl_baseline, master_baseline]
+ return data.get('master', {})
# Return values for MatchLocalResult().
MATCH = 'match'
@@ -124,11 +116,10 @@ class GoldBaseline(object):
return GoldBaseline.BASELINE_DOWNLOAD_FAILED
found_test_case = False
- for baseline in self._baselines:
- if test_name in baseline:
- found_test_case = True
- if md5_hash in baseline[test_name]:
- return GoldBaseline.MATCH
+ if test_name in self._baselines:
+ found_test_case = True
+ if md5_hash in self._baselines[test_name]:
+ return GoldBaseline.MATCH
return (GoldBaseline.MISMATCH if found_test_case
else GoldBaseline.NO_BASELINE)
@@ -195,6 +186,7 @@ class GoldResults(object):
self._properties = _ParseKeyValuePairs(propertiesStr)
self._properties["key"] = _ParseKeyValuePairs(keyStr)
self._results = []
+ self._passfail = []
self._outputDir = outputDir
# make sure the output directory exists and is empty.
@@ -208,7 +200,7 @@ class GoldResults(object):
hashes=[x.strip() for x in ig_file.readlines() if x.strip()]
self._ignore_hashes = set(hashes)
- def AddTestResult(self, testName, md5Hash, outputImagePath):
+ def AddTestResult(self, testName, md5Hash, outputImagePath, matchResult):
# If the hash is in the list of hashes to ignore then we don'try
# make a copy, but add it to the result.
imgExt = os.path.splitext(outputImagePath)[1].lstrip(".")
@@ -232,6 +224,8 @@ class GoldResults(object):
}
})
+ self._passfail.append((testName, matchResult))
+
def WriteResults(self):
self._properties.update({
"results": self._results
@@ -242,6 +236,11 @@ class GoldResults(object):
json.dump(self._properties, outfile, indent=1)
outfile.write("\n")
+ outputFileName = os.path.join(self._outputDir, "passfail.json")
+ with open(outputFileName, 'wb') as outfile:
+ json.dump(self._passfail, outfile, indent=1)
+ outfile.write("\n")
+
# Produce example output for manual testing.
if __name__ == "__main__":
# Create a test directory with three empty 'image' files.
@@ -261,8 +260,9 @@ if __name__ == "__main__":
with open(hash_file, 'wb') as f:
f.write("\n".join(["hash-1","hash-4"]) + "\n")
- gr = GoldResults("pdfium", testDir, propStr, keyStr, hash_file)
- gr.AddTestResult("test-1", "hash-1", os.path.join(testDir, "image1.png"))
- gr.AddTestResult("test-2", "hash-2", os.path.join(testDir, "image2.png"))
- gr.AddTestResult("test-3", "hash-3", os.path.join(testDir, "image3.png"))
+ outputDir = "./output_directory"
+ gr = GoldResults("pdfium", outputDir, propStr, keyStr, hash_file)
+ gr.AddTestResult("test-1", "hash-1", os.path.join(testDir, "image1.png"), GoldBaseline.MATCH)
+ gr.AddTestResult("test-2", "hash-2", os.path.join(testDir, "image2.png"), GoldBaseline.MATCH)
+ gr.AddTestResult("test-3", "hash-3", os.path.join(testDir, "image3.png"), GoldBaseline.MISMATCH)
gr.WriteResults()
diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py
index 3737969fb2..ecd87ad082 100644
--- a/testing/tools/test_runner.py
+++ b/testing/tools/test_runner.py
@@ -160,6 +160,7 @@ class TestRunner:
# becomes "example_005.pdf.0".
test_name = os.path.splitext(os.path.split(img_path)[1])[0]
+ matched = "suppressed"
if not self.test_suppressor.IsResultSuppressed(input_filename):
matched = self.gold_baseline.MatchLocalResult(test_name, md5_hash)
if matched == gold.GoldBaseline.MISMATCH:
@@ -168,7 +169,7 @@ class TestRunner:
print 'No Skia Gold baseline found for test case: %s' % test_name
if self.gold_results:
- self.gold_results.AddTestResult(test_name, md5_hash, img_path)
+ self.gold_results.AddTestResult(test_name, md5_hash, img_path, matched)
if self.test_suppressor.IsResultSuppressed(input_filename):
self.result_suppressed_cases.append(input_filename)