3 files changed, 23 insertions, 1 deletions

diff --git a/src/mem/cache/cache_impl.hh b/src/mem/cache/cache_impl.hh
index 6db40b609..2ead54ba6 100644
--- a/src/mem/cache/cache_impl.hh
+++ b/src/mem/cache/cache_impl.hh
@@ -371,6 +371,17 @@ Cache<TagStore>::timingAccess(PacketPtr pkt)
         DPRINTF(Cache, "mem inhibited on 0x%x: not responding\n",
                 pkt->getAddr());
         assert(!pkt->req->isUncacheable());
+        // Special tweak for multilevel coherence: snoop downward here
+        // on invalidates since there may be other caches below here
+        // that have shared copies.  Not necessary if we know that
+        // supplier had exclusive copy to begin with.
+        if (pkt->needsExclusive() && !pkt->isSupplyExclusive()) {
+            Packet *snoopPkt = new Packet(pkt, true);  // clear flags
+            snoopPkt->setExpressSnoop();
+            snoopPkt->assertMemInhibit();
+            memSidePort->sendTiming(snoopPkt);
+            // main memory will delete snoopPkt
+        }
         return true;
     }
 
@@ -966,6 +977,7 @@ Cache<TagStore>::handleSnoop(PacketPtr pkt, BlkType *blk,
     // we respond in atomic mode), so just figure out what to do now
     // and then do it later
     bool supply = blk->isDirty() && pkt->isRead() && !upperSupply;
+    bool have_exclusive = blk->isWritable();
     bool invalidate = pkt->isInvalidate();
 
     if (pkt->isRead() && !pkt->isInvalidate()) {
@@ -985,6 +997,9 @@ Cache<TagStore>::handleSnoop(PacketPtr pkt, BlkType *blk,
     if (supply) {
         assert(!pkt->memInhibitAsserted());
         pkt->assertMemInhibit();
+        if (have_exclusive) {
+            pkt->setSupplyExclusive();
+        }
         if (is_timing) {
             doTimingSupplyResponse(pkt, blk->data, is_deferred);
         } else {
diff --git a/src/mem/cache/miss/mshr.cc b/src/mem/cache/miss/mshr.cc
index 7f216ad39..5ba3d1ec5 100644
--- a/src/mem/cache/miss/mshr.cc
+++ b/src/mem/cache/miss/mshr.cc
@@ -257,6 +257,7 @@ MSHR::handleSnoop(PacketPtr pkt, Counter _order)
             // We're awaiting an exclusive copy, so ownership is pending.
             // It's up to us to respond once the data arrives.
             pkt->assertMemInhibit();
+            pkt->setSupplyExclusive();
         } else {
             // Someone else may respond before we get around to
             // processing this snoop, which means the copied request
diff --git a/src/mem/packet.hh b/src/mem/packet.hh
index 036bd3fd7..c6534d6c9 100644
--- a/src/mem/packet.hh
+++ b/src/mem/packet.hh
@@ -256,7 +256,11 @@ class Packet : public FastAlloc
         MemInhibit,
         Shared,
         // Special control flags
+        /// Special timing-mode atomic snoop for multi-level coherence.
         ExpressSnoop,
+        /// Does supplier have exclusive copy?
+        /// Useful for multi-level coherence.
+        SupplyExclusive,
         NUM_PACKET_FLAGS
     };
 
@@ -315,13 +319,15 @@ class Packet : public FastAlloc
 
     // Snoop flags
     void assertMemInhibit()     { flags[MemInhibit] = true; }
-    void assertShared()         { flags[Shared] = true; }
     bool memInhibitAsserted()   { return flags[MemInhibit]; }
+    void assertShared()         { flags[Shared] = true; }
     bool sharedAsserted()       { return flags[Shared]; }
 
     // Special control flags
     void setExpressSnoop()      { flags[ExpressSnoop] = true; }
     bool isExpressSnoop()       { return flags[ExpressSnoop]; }
+    void setSupplyExclusive()   { flags[SupplyExclusive] = true; }
+    bool isSupplyExclusive()    { return flags[SupplyExclusive]; }
 
     // Network error conditions... encapsulate them as methods since
     // their encoding keeps changing (from result field to command