diff options
Diffstat (limited to 'testing/tools')
| -rw-r--r-- | testing/tools/gold.py | 136 | ||||
| -rw-r--r-- | testing/tools/test_runner.py | 30 |
2 files changed, 147 insertions, 19 deletions
diff --git a/testing/tools/gold.py b/testing/tools/gold.py index bdae9f4195..a8222378e8 100644 --- a/testing/tools/gold.py +++ b/testing/tools/gold.py @@ -7,6 +7,132 @@ import json import os import shlex import shutil +import urllib2 + + +def _ParseKeyValuePairs(kv_str): + """ + Parses a string of the type 'key1 value1 key2 value2' into a dict. + """ + kv_pairs = shlex.split(kv_str) + if len(kv_pairs) % 2: + raise ValueError('Uneven number of key/value pairs. Got %s' % kv_str) + return { kv_pairs[i]:kv_pairs[i + 1] for i in xrange(0, len(kv_pairs), 2) } + + +# This module downloads a json provided by Skia Gold with the expected baselines +# for each test file. +# +# The expected format for the json is: +# { +# "commit": { +# "author": "John Doe (jdoe@chromium.org)", +# "commit_time": 1510598123, +# "hash": "cee39e6e90c219cc91f2c94a912a06977f4461a0" +# }, +# "master": { +# "abc.pdf.1": { +# "0ec3d86f545052acd7c9a16fde8ca9d4": 1, +# "80455b71673becc9fbc100d6da56ca65": 1, +# "b68e2ecb80090b4502ec89ad1be2322c": 1 +# }, +# "defgh.pdf.0": { +# "01e020cd4cd05c6738e479a46a506044": 1, +# "b68e2ecb80090b4502ec89ad1be2322c": 1 +# } +# }, +# "changeLists": { +# "18499" : { +# "abc.pdf.1": { +# "d5dd649124cf1779152253dc8fb239c5": 1, +# "42a270581930579cdb0f28674972fb1a": 1, +# } +# } +# } +# } +class GoldBaseline(object): + + def __init__(self, properties_str): + """ + properties_str is a string with space separated key/value pairs that + is used to find the cl number for which to baseline + """ + self._properties = _ParseKeyValuePairs(properties_str) + self._baselines = self._LoadSkiaGoldBaselines() + + def _LoadSkiaGoldBaselines(self): + """ + Download the baseline json and return a list of the two baselines that + should be used to match hashes (master and cl#). + """ + GOLD_BASELINE_URL = ('https://storage.googleapis.com/skia-infra-gm/' + 'hash_files/gold-pdfium-baseline.json') + try: + response = urllib2.urlopen(GOLD_BASELINE_URL) + json_data = response.read() + except (urllib2.HTTPError, urllib2.URLError) as e: + print ('Error: Unable to read skia gold json from %s: %s' + % (GOLD_BASELINE_URL, e)) + return None + + try: + data = json.loads(json_data) + except ValueError: + print 'Error: Malformed json read from %s: %s' % (GOLD_BASELINE_URL, e) + return None + + try: + master_baseline = data['master'] + except (KeyError, TypeError): + print ('Error: "master" key not in json read from %s: %s' + % (GOLD_BASELINE_URL, e)) + return None + + cl_number_str = self._properties.get('issue') + if cl_number_str is None: + return [master_baseline] + + try: + cl_baseline = data['changeLists'][cl_number_str] + except KeyError: + return [master_baseline] + + return [cl_baseline, master_baseline] + + # Return values for MatchLocalResult(). + MATCH = 'match' + MISMATCH = 'mismatch' + NO_BASELINE = 'no_baseline' + BASELINE_DOWNLOAD_FAILED = 'baseline_download_failed' + + def MatchLocalResult(self, test_name, md5_hash): + """ + Match a locally generated hash of a test cases rendered image with the + expected hashes downloaded in the baselines json. + + Each baseline is a dict mapping the test case name to a dict with the + expected hashes as keys. Therefore, this list of baselines should be + searched until the test case name is found, then the hash should be matched + with the options in that dict. If the hashes don't match, it should be + considered a failure and we should not continue searching the baseline list. + + Returns MATCH if the md5 provided matches the ones in the baseline json, + MISMATCH if it does not, NO_BASELINE if the test case has no baseline, or + BASELINE_DOWNLOAD_FAILED if the baseline could not be downloaded and parsed. + """ + if self._baselines is None: + return GoldBaseline.BASELINE_DOWNLOAD_FAILED + + found_test_case = False + for baseline in self._baselines: + if test_name in baseline: + found_test_case = True + if md5_hash in baseline[test_name]: + return GoldBaseline.MATCH + + return (GoldBaseline.MISMATCH if found_test_case + else GoldBaseline.NO_BASELINE) + # This module collects and writes output in a format expected by the # Gold baseline tool. Based on meta data provided explicitly and by @@ -66,8 +192,8 @@ class GoldResults(object): that should be ignored. """ self._source_type = source_type - self._properties = self._parseKeyValuePairs(propertiesStr) - self._properties["key"] = self._parseKeyValuePairs(keyStr) + self._properties = _ParseKeyValuePairs(propertiesStr) + self._properties["key"] = _ParseKeyValuePairs(keyStr) self._results = [] self._outputDir = outputDir @@ -106,12 +232,6 @@ class GoldResults(object): } }) - def _parseKeyValuePairs(self, kvStr): - kvPairs = shlex.split(kvStr) - if len(kvPairs) % 2: - raise ValueError("Uneven number of key/value pairs. Got %s" % kvStr) - return { kvPairs[i]:kvPairs[i+1] for i in range(0, len(kvPairs), 2) } - def WriteResults(self): self._properties.update({ "results": self._results diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py index 8921b2f005..e895552bc8 100644 --- a/testing/tools/test_runner.py +++ b/testing/tools/test_runner.py @@ -116,7 +116,6 @@ class TestRunner: [sys.executable, self.fixup_path, '--output-dir=' + self.working_dir, input_path]) - def TestText(self, input_root, expected_txt_path, pdf_path): txt_path = os.path.join(self.working_dir, input_root + '.txt') @@ -127,11 +126,8 @@ class TestRunner: cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path] return common.RunCommand(cmd) - def TestPixel(self, input_root, pdf_path): - cmd_to_run = [self.pdfium_test_path, '--send-events', '--png'] - if self.gold_results: - cmd_to_run.append('--md5') + cmd_to_run = [self.pdfium_test_path, '--send-events', '--png', '--md5'] if self.oneshot_renderer: cmd_to_run.append('--render-oneshot') cmd_to_run.append(pdf_path) @@ -139,11 +135,22 @@ class TestRunner: def HandleResult(self, input_filename, input_path, result): success, image_paths = result - if self.gold_results: - if image_paths: - for img_path, md5_hash in image_paths: - # the output filename (without extension becomes the test name) - test_name = os.path.splitext(os.path.split(img_path)[1])[0] + + if image_paths: + for img_path, md5_hash in image_paths: + # The output filename without image extension becomes the test name. + # For example, "/path/to/.../testing/corpus/example_005.pdf.0.png" + # becomes "example_005.pdf.0". + test_name = os.path.splitext(os.path.split(img_path)[1])[0] + + if not self.test_suppressor.IsResultSuppressed(input_filename): + matched = self.gold_baseline.MatchLocalResult(test_name, md5_hash) + if matched == gold.GoldBaseline.MISMATCH: + print 'Skia Gold hash mismatch for test case: %s' % test_name + elif matched == gold.GoldBaseline.NO_BASELINE: + print 'No Skia Gold baseline found for test case: %s' % test_name + + if self.gold_results: self.gold_results.AddTestResult(test_name, md5_hash, img_path) if self.test_suppressor.IsResultSuppressed(input_filename): @@ -154,7 +161,6 @@ class TestRunner: if not success: self.failures.append(input_path) - def Run(self): parser = optparse.OptionParser() @@ -225,6 +231,8 @@ class TestRunner: self.test_suppressor = suppressor.Suppressor(finder, self.feature_string) self.image_differ = pngdiffer.PNGDiffer(finder) + self.gold_baseline = gold.GoldBaseline(self.options.gold_properties) + walk_from_dir = finder.TestingDir(test_dir); self.test_cases = [] |