Add ReadRespWithInvalidate to handle multi-level coherence situation

where we defer a response to a read from a far-away cache A, then later
defer a ReadExcl from a cache B on the same bus as us.  We'll assert
MemInhibit in both cases, but in the latter case MemInhibit will keep
the invalidation from reaching cache A.  This special response tells
cache A that it gets the block to satisfy its read, but must immediately
invalidate it.

--HG--
extra : convert_revision : f85c8b47bb30232da37ac861b50a6539dc81161b

4 files changed, 49 insertions, 16 deletions

diff --git a/src/mem/cache/cache.hh b/src/mem/cache/cache.hh
index 4602fd835..170ba0cd1 100644
--- a/src/mem/cache/cache.hh
+++ b/src/mem/cache/cache.hh
@@ -186,7 +186,7 @@ class Cache : public BaseCache
     bool satisfyMSHR(MSHR *mshr, PacketPtr pkt, BlkType *blk);
 
     void doTimingSupplyResponse(PacketPtr req_pkt, uint8_t *blk_data,
-                                bool already_copied);
+                                bool already_copied, bool pending_inval);
 
     /**
      * Sets the blk to the new state.
@@ -194,7 +194,7 @@ class Cache : public BaseCache
      * @param new_state The new coherence state for the block.
      */
     void handleSnoop(PacketPtr ptk, BlkType *blk,
-                     bool is_timing, bool is_deferred);
+                     bool is_timing, bool is_deferred, bool pending_inval);
 
     /**
      * Create a writeback request for the given block.
diff --git a/src/mem/cache/cache_impl.hh b/src/mem/cache/cache_impl.hh
index 130a909cc..ed65fbabb 100644
--- a/src/mem/cache/cache_impl.hh
+++ b/src/mem/cache/cache_impl.hh
@@ -775,18 +775,31 @@ Cache<TagStore>::handleResponse(PacketPtr pkt)
             // if this packet is an error copy that to the new packet
             if (is_error)
                 target->pkt->copyError(pkt);
+            if (pkt->isInvalidate()) {
+                // If intermediate cache got ReadRespWithInvalidate,
+                // propagate that.  Response should not have
+                // isInvalidate() set otherwise.
+                assert(target->pkt->cmd == MemCmd::ReadResp);
+                assert(pkt->cmd == MemCmd::ReadRespWithInvalidate);
+                target->pkt->cmd = MemCmd::ReadRespWithInvalidate;
+            }
             cpuSidePort->respond(target->pkt, completion_time);
         } else {
             // I don't believe that a snoop can be in an error state
             assert(!is_error);
             // response to snoop request
             DPRINTF(Cache, "processing deferred snoop...\n");
-            handleSnoop(target->pkt, blk, true, true);
+            handleSnoop(target->pkt, blk, true, true,
+                        mshr->pendingInvalidate || pkt->isInvalidate());
         }
 
         mshr->popTarget();
     }
 
+    if (pkt->isInvalidate()) {
+        tags->invalidateBlk(blk);
+    }
+
     if (mshr->promoteDeferredTargets()) {
         MSHRQueue *mq = mshr->queue;
         mq->markPending(mshr);
@@ -854,7 +867,7 @@ Cache<TagStore>::handleFill(PacketPtr pkt, BlkType *blk,
 
     if (blk == NULL) {
         // better have read new data...
-        assert(pkt->isRead());
+        assert(pkt->hasData());
 
         // need to do a replacement
         blk = tags->findReplacement(addr, writebacks);
@@ -890,7 +903,7 @@ Cache<TagStore>::handleFill(PacketPtr pkt, BlkType *blk,
         // existing block... probably an upgrade
         assert(blk->tag == tags->extractTag(addr));
         // either we're getting new data or the block should already be valid
-        assert(pkt->isRead() || blk->isValid());
+        assert(pkt->hasData() || blk->isValid());
     }
 
     if (!pkt->sharedAsserted()) {
@@ -922,9 +935,9 @@ Cache<TagStore>::handleFill(PacketPtr pkt, BlkType *blk,
 
 template<class TagStore>
 void
-Cache<TagStore>::doTimingSupplyResponse(PacketPtr req_pkt,
-                                        uint8_t *blk_data,
-                                        bool already_copied)
+Cache<TagStore>::
+doTimingSupplyResponse(PacketPtr req_pkt, uint8_t *blk_data,
+                       bool already_copied, bool pending_inval)
 {
     // timing-mode snoop responses require a new packet, unless we
     // already made a copy...
@@ -941,14 +954,29 @@ Cache<TagStore>::doTimingSupplyResponse(PacketPtr req_pkt,
     if (pkt->isRead()) {
         pkt->setDataFromBlock(blk_data, blkSize);
     }
+    if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
+        // Assume we defer a response to a read from a far-away cache
+        // A, then later defer a ReadExcl from a cache B on the same
+        // bus as us.  We'll assert MemInhibit in both cases, but in
+        // the latter case MemInhibit will keep the invalidation from
+        // reaching cache A.  This special response tells cache A that
+        // it gets the block to satisfy its read, but must immediately
+        // invalidate it.
+        pkt->cmd = MemCmd::ReadRespWithInvalidate;
+    }
     memSidePort->respond(pkt, curTick + hitLatency);
 }
 
 template<class TagStore>
 void
 Cache<TagStore>::handleSnoop(PacketPtr pkt, BlkType *blk,
-                             bool is_timing, bool is_deferred)
+                             bool is_timing, bool is_deferred,
+                             bool pending_inval)
 {
+    // deferred snoops can only happen in timing mode
+    assert(!(is_deferred && !is_timing));
+    // pending_inval only makes sense on deferred snoops
+    assert(!(pending_inval && !is_deferred));
     assert(pkt->isRequest());
 
     // first propagate snoop upward to see if anyone above us wants to
@@ -1018,7 +1046,7 @@ Cache<TagStore>::handleSnoop(PacketPtr pkt, BlkType *blk,
             pkt->setSupplyExclusive();
         }
         if (is_timing) {
-            doTimingSupplyResponse(pkt, blk->data, is_deferred);
+            doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
         } else {
             pkt->makeAtomicResponse();
             pkt->setDataFromBlock(blk->data, blkSize);
@@ -1085,7 +1113,8 @@ Cache<TagStore>::snoopTiming(PacketPtr pkt)
                 // the packet's invalidate flag is set...
                 assert(pkt->isInvalidate());
             }
-            doTimingSupplyResponse(pkt, wb_pkt->getPtr<uint8_t>(), false);
+            doTimingSupplyResponse(pkt, wb_pkt->getPtr<uint8_t>(),
+                                   false, false);
 
             if (pkt->isInvalidate()) {
                 // Invalidation trumps our writeback... discard here
@@ -1101,7 +1130,7 @@ Cache<TagStore>::snoopTiming(PacketPtr pkt)
         }
     }
 
-    handleSnoop(pkt, blk, true, false);
+    handleSnoop(pkt, blk, true, false, false);
 }
 
 
@@ -1116,7 +1145,7 @@ Cache<TagStore>::snoopAtomic(PacketPtr pkt)
     }
 
     BlkType *blk = tags->findBlock(pkt->getAddr());
-    handleSnoop(pkt, blk, false, false);
+    handleSnoop(pkt, blk, false, false, false);
     return hitLatency;
 }
 
diff --git a/src/mem/packet.cc b/src/mem/packet.cc
index 164363860..f3bd06f36 100644
--- a/src/mem/packet.cc
+++ b/src/mem/packet.cc
@@ -59,6 +59,9 @@ MemCmd::commandInfo[] =
     { SET3(IsRead, IsRequest, NeedsResponse), ReadResp, "ReadReq" },
     /* ReadResp */
     { SET3(IsRead, IsResponse, HasData), InvalidCmd, "ReadResp" },
+    /* ReadRespWithInvalidate */
+    { SET4(IsRead, IsResponse, HasData, IsInvalidate),
+            InvalidCmd, "ReadRespWithInvalidate" },
     /* WriteReq */
     { SET5(IsWrite, NeedsExclusive, IsRequest, NeedsResponse, HasData),
             WriteResp, "WriteReq" },
@@ -84,19 +87,19 @@ MemCmd::commandInfo[] =
            IsRequest, HasData, NeedsResponse),
             WriteInvalidateResp, "WriteInvalidateReq" },
     /* WriteInvalidateResp */
-    { SET4(IsWrite, NeedsExclusive, IsInvalidate, IsResponse),
+    { SET3(IsWrite, NeedsExclusive, IsResponse),
             InvalidCmd, "WriteInvalidateResp" },
     /* UpgradeReq */
     { SET4(IsInvalidate, NeedsExclusive, IsRequest, NeedsResponse),
             UpgradeResp, "UpgradeReq" },
     /* UpgradeResp */
-    { SET3(IsInvalidate, NeedsExclusive, IsResponse),
+    { SET2(NeedsExclusive, IsResponse),
             InvalidCmd, "UpgradeResp" },
     /* ReadExReq */
     { SET5(IsRead, NeedsExclusive, IsInvalidate, IsRequest, NeedsResponse),
             ReadExResp, "ReadExReq" },
     /* ReadExResp */
-    { SET5(IsRead, NeedsExclusive, IsInvalidate, IsResponse, HasData),
+    { SET4(IsRead, NeedsExclusive, IsResponse, HasData),
             InvalidCmd, "ReadExResp" },
     /* LoadLockedReq */
     { SET4(IsRead, IsLocked, IsRequest, NeedsResponse),
diff --git a/src/mem/packet.hh b/src/mem/packet.hh
index 30ef71507..05442b369 100644
--- a/src/mem/packet.hh
+++ b/src/mem/packet.hh
@@ -66,6 +66,7 @@ class MemCmd
         InvalidCmd,
         ReadReq,
         ReadResp,
+        ReadRespWithInvalidate,
         WriteReq,
         WriteResp,
         Writeback,