cache: fail SC when invalidated while waiting for bus

Corrects an oversight in cset f97b62be544f.  The fix there only
failed queued SCUpgradeReq packets that encountered an
invalidation, which meant that the upgrade had to reach the L2
cache.  To handle pending requests in the L1 we must similarly
fail StoreCondReq packets too.

4 files changed, 20 insertions, 7 deletions

diff --git a/src/mem/cache/cache_impl.hh b/src/mem/cache/cache_impl.hh
index 3f3dc6877..4f4d720b6 100644
--- a/src/mem/cache/cache_impl.hh
+++ b/src/mem/cache/cache_impl.hh
@@ -900,9 +900,10 @@ Cache<TagStore>::handleResponse(PacketPtr pkt)
                 assert(!target->pkt->req->isUncacheable());
                 missLatency[target->pkt->cmdToIndex()][0/*pkt->req->threadId()*/] +=
                     completion_time - target->recvTime;
-            } else if (target->pkt->cmd == MemCmd::StoreCondReq &&
-                       pkt->cmd == MemCmd::UpgradeFailResp) {
+            } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
                 // failed StoreCond upgrade
+                assert(target->pkt->cmd == MemCmd::StoreCondReq ||
+                       target->pkt->cmd == MemCmd::StoreCondFailReq);
                 completion_time = tags->getHitLatency() + pkt->finishTime;
                 target->pkt->req->setExtraData(0);
             } else {
@@ -1443,10 +1444,11 @@ Cache<TagStore>::getTimingPacket()
     PacketPtr tgt_pkt = mshr->getTarget()->pkt;
     PacketPtr pkt = NULL;
 
-    if (tgt_pkt->cmd == MemCmd::SCUpgradeFailReq) {
-        // SCUpgradeReq saw invalidation while queued in MSHR, so now
-        // that we are getting around to processing it, just treat it
-        // as if we got a failure response
+    if (tgt_pkt->cmd == MemCmd::SCUpgradeFailReq ||
+        tgt_pkt->cmd == MemCmd::StoreCondFailReq) {
+        // SCUpgradeReq or StoreCondReq saw invalidation while queued
+        // in MSHR, so now that we are getting around to processing
+        // it, just treat it as if we got a failure response
         pkt = new Packet(tgt_pkt);
         pkt->cmd = MemCmd::UpgradeFailResp;
         pkt->senderState = mshr;
diff --git a/src/mem/cache/mshr.cc b/src/mem/cache/mshr.cc
index bbddf2aee..54977346f 100644
--- a/src/mem/cache/mshr.cc
+++ b/src/mem/cache/mshr.cc
@@ -72,7 +72,10 @@ MSHR::TargetList::add(PacketPtr pkt, Tick readyTime,
             needsExclusive = true;
         }
 
-        if (pkt->isUpgrade()) {
+        // StoreCondReq is effectively an upgrade if it's in an MSHR
+        // since it would have been failed already if we didn't have a
+        // read-only copy
+        if (pkt->isUpgrade() || pkt->cmd == MemCmd::StoreCondReq) {
             hasUpgrade = true;
         }
     }
@@ -98,6 +101,9 @@ replaceUpgrade(PacketPtr pkt)
     } else if (pkt->cmd == MemCmd::SCUpgradeReq) {
         pkt->cmd = MemCmd::SCUpgradeFailReq;
         DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n");
+    } else if (pkt->cmd == MemCmd::StoreCondReq) {
+        pkt->cmd = MemCmd::StoreCondFailReq;
+        DPRINTF(Cache, "Replacing StoreCondReq with StoreCondFailReq\n");
     }
 }
 
diff --git a/src/mem/packet.cc b/src/mem/packet.cc
index 5eb2ecc4b..d0b1fed83 100644
--- a/src/mem/packet.cc
+++ b/src/mem/packet.cc
@@ -123,6 +123,10 @@ MemCmd::commandInfo[] =
     { SET6(IsWrite, NeedsExclusive, IsLlsc,
            IsRequest, NeedsResponse, HasData),
             StoreCondResp, "StoreCondReq" },
+    /* StoreCondFailReq: generates failing StoreCondResp ASAP */
+    { SET6(IsWrite, NeedsExclusive, IsLlsc,
+           IsRequest, NeedsResponse, HasData),
+            StoreCondResp, "StoreCondFailReq" },
     /* StoreCondResp */
     { SET4(IsWrite, NeedsExclusive, IsLlsc, IsResponse),
             InvalidCmd, "StoreCondResp" },
diff --git a/src/mem/packet.hh b/src/mem/packet.hh
index cc2a39d0e..cefb7c2ed 100644
--- a/src/mem/packet.hh
+++ b/src/mem/packet.hh
@@ -90,6 +90,7 @@ class MemCmd
         ReadExResp,
         LoadLockedReq,
         StoreCondReq,
+        StoreCondFailReq,       // Failed StoreCondReq in MSHR (never sent)
         StoreCondResp,
         SwapReq,
         SwapResp,