tests: Split test results into running and verification

The test base class already assumes that test cases consists of a run
stage and a verification stage. Reflect this in the results class to
make it possible to detect cases where a run was successful, but
didn't verify.

Change-Id: I31ef393e496671221c5408aca41649cd8dda74ca
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-by: Curtis Dunham <curtis.dunham@arm.com>

3 files changed, 33 insertions, 23 deletions

diff --git a/tests/SConscript b/tests/SConscript
index e47d394f2..d1fee7ad1 100644
--- a/tests/SConscript
+++ b/tests/SConscript
@@ -120,19 +120,16 @@ def print_test(target, source, env):
     assert len(result) == 1
     result = result[0]
 
-    run = result.results[0]
-    assert run.name == "gem5"
-
     formatter = None
-    if not run:
-        status = color_message(termcap.Red, "FAILED!")
-        formatter = results.Text()
-    elif run.skipped():
+    if result.skipped():
         status = color_message(termcap.Cyan, "skipped.")
+    elif result.changed():
+        status = color_message(termcap.Yellow, "CHANGED!")
+        formatter = results.Text()
     elif result:
         status = color_message(termcap.Green, "passed.")
     else:
-        status = color_message(termcap.Yellow, "CHANGED!")
+        status = color_message(termcap.Red, "FAILED!")
         formatter = results.Text()
 
     if formatter:
@@ -164,10 +161,7 @@ def update_test(target, source, env):
     assert len(result) == 1
     result = result[0]
 
-    run = result.results[0]
-    assert run.name == "gem5"
-
-    if run.skipped():
+    if result.skipped():
         print "*** %s: %s: Test skipped, not updating." % (
             source[0].dir, color_message(termcap.Yellow, "WARNING"), )
         return 0
@@ -175,7 +169,7 @@ def update_test(target, source, env):
         print "*** %s: %s: Test successful, not updating." % (
             source[0].dir, color_message(termcap.Green, "skipped"), )
         return 0
-    elif not run.success():
+    elif result.failed_run():
         print "*** %s: %s: Test failed, not updating." % (
             source[0].dir, color_message(termcap.Red, "ERROR"), )
         return 1
diff --git a/tests/testing/results.py b/tests/testing/results.py
index 9432a0f10..1df14bc4f 100644
--- a/tests/testing/results.py
+++ b/tests/testing/results.py
@@ -105,27 +105,41 @@ class UnitResult(object):
 class TestResult(object):
     """Results for from a single test consisting of one or more units."""
 
-    def __init__(self, name, results=[]):
+    def __init__(self, name, run_results=[], verify_results=[]):
         self.name = name
-        self.results = results
+        self.results = run_results + verify_results
+        self.run_results = run_results
+        self.verify_results = verify_results
 
     def success(self):
-        return all([ r.success() for r in self.results])
+        return self.success_run() and self.success_verify()
 
-    def skipped(self):
-        return all([ r.skipped() for r in self.results])
+    def success_run(self):
+        return all([ r.success() for r in self.run_results ])
 
-    def changed(self):
-        return self.results[0].success() and self.failed()
+    def success_verify(self):
+        return all([ r.success() for r in self.verify_results ])
 
     def failed(self):
-        return any([ not r for r in self.results])
+        return self.failed_run() or self.failed_verify()
+
+    def failed_run(self):
+        return any([ not r for r in self.run_results ])
+
+    def failed_verify(self):
+        return any([ not r for r in self.verify_results ])
+
+    def skipped(self):
+        return all([ r.skipped() for r in self.run_results ])
+
+    def changed(self):
+        return self.success_run() and self.failed_verify()
 
     def runtime(self):
         return sum([ r.runtime for r in self.results ])
 
     def __nonzero__(self):
-        return all([r for r in self.results])
+        return all([ r for r in self.results ])
 
 class ResultFormatter(object):
     __metaclass__ = ABCMeta
diff --git a/tests/testing/tests.py b/tests/testing/tests.py
index ae15d6d89..b2b821fd6 100644
--- a/tests/testing/tests.py
+++ b/tests/testing/tests.py
@@ -213,7 +213,9 @@ class Test(object):
             for u in self.verify_units()
         ]
 
-        return TestResult(self.test_name, run_results + verify_results)
+        return TestResult(self.test_name,
+                          run_results=run_results,
+                          verify_results=verify_results)
 
     def __str__(self):
         return self.test_name