summaryrefslogtreecommitdiff
path: root/testing
diff options
context:
space:
mode:
authorTom Sepez <tsepez@chromium.org>2015-06-12 09:11:00 -0700
committerTom Sepez <tsepez@chromium.org>2015-06-12 09:11:00 -0700
commit3218fffe6f92b62d05ab5c2d1c78285a0a0f0647 (patch)
treeba3754048f8544dbc7804cecc119228bd20a70c5 /testing
parentd9e1477c88692ac6eaf8bd20a573a2fdccb1ff41 (diff)
downloadpdfium-3218fffe6f92b62d05ab5c2d1c78285a0a0f0647.tar.xz
Corpus tests check for unexpected successes.
Update run_corpus_tests.py to always run all test cases, and just ignore the results that are supposed to fail. R=thestig@chromium.org Review URL: https://codereview.chromium.org/1178393002.
Diffstat (limited to 'testing')
-rwxr-xr-xtesting/tools/run_corpus_tests.py17
-rwxr-xr-xtesting/tools/suppressor.py4
2 files changed, 15 insertions, 6 deletions
diff --git a/testing/tools/run_corpus_tests.py b/testing/tools/run_corpus_tests.py
index 1876581e40..dbc8f35a91 100755
--- a/testing/tools/run_corpus_tests.py
+++ b/testing/tools/run_corpus_tests.py
@@ -55,6 +55,7 @@ def main():
# test files are under .../pdfium/testing/corpus.
failures = []
+ surprises = []
walk_from_dir = finder.TestingDir('corpus');
input_file_re = re.compile('^[a-zA-Z0-9_.]+[.]pdf$')
for source_dir, _, filename_list in os.walk(walk_from_dir):
@@ -62,11 +63,19 @@ def main():
if input_file_re.match(input_filename):
input_path = os.path.join(source_dir, input_filename)
if os.path.isfile(input_path):
+ result = test_one_file(input_filename, source_dir, working_dir,
+ pdfium_test_path, image_differ)
if test_suppressor.IsSuppressed(input_filename):
- continue
- if not test_one_file(input_filename, source_dir, working_dir,
- pdfium_test_path, image_differ):
- failures.append(input_path)
+ if result:
+ surprises.append(input_path)
+ else:
+ if not result:
+ failures.append(input_path)
+
+ if surprises:
+ print '\n\nUnexpected Successes:'
+ for surprise in surprises:
+ print surprise;
if failures:
print '\n\nSummary of Failures:'
diff --git a/testing/tools/suppressor.py b/testing/tools/suppressor.py
index fff9bc8381..2bb31171ab 100755
--- a/testing/tools/suppressor.py
+++ b/testing/tools/suppressor.py
@@ -24,11 +24,11 @@ class Suppressor:
def IsSuppressed(self, input_filename):
if input_filename in self.suppression_list:
- print ("Not running %s, found in %s file" %
+ print ("%s is suppressed, found in %s file" %
(input_filename, self.SUPPRESSIONS_FILENAME))
return True
if input_filename in self.platform_suppression_list:
- print ("Not running %s, found in %s file" %
+ print ("%s is suppressed, found in %s file" %
(input_filename, self.PLATFORM_SUPPRESSIONS_FILENAME))
return True
return False