mem: Fix SenderState related cache deadlock

This patch fixes a potential deadlock in the caches. This deadlock
could occur when more than one cache is used in a system, and
pkt->senderState is modified in between the two caches. This happened
as the caches relied on the senderState remaining unchanged, and used
it for instantaneous upstream communication with other caches.

This issue has been addressed by iterating over the linked list of
senderStates until we are either able to cast to a MSHR* or
senderState is NULL. If the cast is successful, we know that the
packet has previously passed through another cache, and therefore
update the downstreamPending flag accordingly. Otherwise, we do
nothing.

2 files changed, 30 insertions, 2 deletions

diff --git a/src/mem/cache/mshr.cc b/src/mem/cache/mshr.cc
index 6fa22c9b4..86e2bebad 100644
--- a/src/mem/cache/mshr.cc
+++ b/src/mem/cache/mshr.cc
@@ -82,7 +82,10 @@ MSHR::TargetList::add(PacketPtr pkt, Tick readyTime,
     }
 
     if (markPending) {
-        MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
+        // Iterate over the SenderState stack and see if we find
+        // an MSHR entry. If we do, set the downstreamPending
+        // flag. Otherwise, do nothing.
+        MSHR *mshr = pkt->findNextSenderState<MSHR>();
         if (mshr != NULL) {
             assert(!mshr->downstreamPending);
             mshr->downstreamPending = true;
@@ -130,7 +133,13 @@ MSHR::TargetList::clearDownstreamPending()
     Iterator end_i = end();
     for (Iterator i = begin(); i != end_i; ++i) {
         if (i->markedPending) {
-            MSHR *mshr = dynamic_cast<MSHR*>(i->pkt->senderState);
+            // Iterate over the SenderState stack and see if we find
+            // an MSHR entry. If we find one, clear the
+            // downstreamPending flag by calling
+            // clearDownstreamPending(). This recursively clears the
+            // downstreamPending flag in all caches this packet has
+            // passed through.
+            MSHR *mshr = i->pkt->findNextSenderState<MSHR>();
             if (mshr != NULL) {
                 mshr->clearDownstreamPending();
             }
diff --git a/src/mem/packet.hh b/src/mem/packet.hh
index 6da1fe97d..6d9e88b8f 100644
--- a/src/mem/packet.hh
+++ b/src/mem/packet.hh
@@ -454,6 +454,25 @@ class Packet : public Printable
      */
     SenderState *popSenderState();
 
+    /**
+     * Go through the sender state stack and return the first instance
+     * that is of type T (as determined by a dynamic_cast). If there
+     * is no sender state of type T, NULL is returned.
+     *
+     * @return The topmost state of type T
+     */
+    template <typename T>
+    T * findNextSenderState() const
+    {
+        T *t = NULL;
+        SenderState* sender_state = senderState;
+        while (t == NULL && sender_state != NULL) {
+            t = dynamic_cast<T*>(sender_state);
+            sender_state = sender_state->predecessor;
+        }
+        return t;
+    }
+
     /// Return the string name of the cmd field (for debugging and
     /// tracing).
     const std::string &cmdString() const { return cmd.toString(); }