tests: Split test results into running and verification

The test base class already assumes that test cases consists of a run
stage and a verification stage. Reflect this in the results class to
make it possible to detect cases where a run was successful, but
didn't verify.

Change-Id: I31ef393e496671221c5408aca41649cd8dda74ca
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-by: Curtis Dunham <curtis.dunham@arm.com>

2 files changed, 26 insertions, 10 deletions

diff --git a/tests/testing/results.py b/tests/testing/results.py
index 9432a0f10..1df14bc4f 100644
--- a/tests/testing/results.py
+++ b/tests/testing/results.py
@@ -105,27 +105,41 @@ class UnitResult(object):
 class TestResult(object):
     """Results for from a single test consisting of one or more units."""
 
-    def __init__(self, name, results=[]):
+    def __init__(self, name, run_results=[], verify_results=[]):
         self.name = name
-        self.results = results
+        self.results = run_results + verify_results
+        self.run_results = run_results
+        self.verify_results = verify_results
 
     def success(self):
-        return all([ r.success() for r in self.results])
+        return self.success_run() and self.success_verify()
 
-    def skipped(self):
-        return all([ r.skipped() for r in self.results])
+    def success_run(self):
+        return all([ r.success() for r in self.run_results ])
 
-    def changed(self):
-        return self.results[0].success() and self.failed()
+    def success_verify(self):
+        return all([ r.success() for r in self.verify_results ])
 
     def failed(self):
-        return any([ not r for r in self.results])
+        return self.failed_run() or self.failed_verify()
+
+    def failed_run(self):
+        return any([ not r for r in self.run_results ])
+
+    def failed_verify(self):
+        return any([ not r for r in self.verify_results ])
+
+    def skipped(self):
+        return all([ r.skipped() for r in self.run_results ])
+
+    def changed(self):
+        return self.success_run() and self.failed_verify()
 
     def runtime(self):
         return sum([ r.runtime for r in self.results ])
 
     def __nonzero__(self):
-        return all([r for r in self.results])
+        return all([ r for r in self.results ])
 
 class ResultFormatter(object):
     __metaclass__ = ABCMeta
diff --git a/tests/testing/tests.py b/tests/testing/tests.py
index ae15d6d89..b2b821fd6 100644
--- a/tests/testing/tests.py
+++ b/tests/testing/tests.py
@@ -213,7 +213,9 @@ class Test(object):
             for u in self.verify_units()
         ]
 
-        return TestResult(self.test_name, run_results + verify_results)
+        return TestResult(self.test_name,
+                          run_results=run_results,
+                          verify_results=verify_results)
 
     def __str__(self):
         return self.test_name