Get rid of requestCauses. Use timestamped queue to make

sure we don't re-request bus prematurely.  Use callback to
avoid calling sendRetry() recursively within recvTiming.

--HG--
extra : convert_revision : a907a2781b4b00aa8eb1ea7147afc81d6b424140

9 files changed, 137 insertions, 97 deletions

diff --git a/src/mem/cache/base_cache.cc b/src/mem/cache/base_cache.cc
index ac577f5a2..5062d6e87 100644
--- a/src/mem/cache/base_cache.cc
+++ b/src/mem/cache/base_cache.cc
@@ -42,7 +42,7 @@ using namespace std;
 
 BaseCache::CachePort::CachePort(const std::string &_name, BaseCache *_cache)
     : SimpleTimingPort(_name, _cache), cache(_cache), otherPort(NULL),
-      blocked(false), mustSendRetry(false), requestCauses(0)
+      blocked(false), mustSendRetry(false)
 {
 }
 
@@ -116,7 +116,9 @@ BaseCache::CachePort::clearBlocked()
     {
         DPRINTF(Cache, "Cache Sending Retry\n");
         mustSendRetry = false;
-        sendRetry();
+        SendRetryEvent *ev = new SendRetryEvent(this, true);
+        // @TODO: need to find a better time (next bus cycle?)
+        ev->schedule(curTick + 1);
     }
 }
 
diff --git a/src/mem/cache/base_cache.hh b/src/mem/cache/base_cache.hh
index b35fc0811..09484a14a 100644
--- a/src/mem/cache/base_cache.hh
+++ b/src/mem/cache/base_cache.hh
@@ -41,6 +41,7 @@
 #include <vector>
 #include <string>
 #include <list>
+#include <algorithm>
 #include <inttypes.h>
 
 #include "base/misc.hh"
@@ -105,6 +106,9 @@ class BaseCache : public MemObject
 
         bool recvRetryCommon();
 
+        typedef EventWrapper<Port, &Port::sendRetry>
+            SendRetryEvent;
+
       public:
         void setOtherPort(CachePort *_otherPort) { otherPort = _otherPort; }
 
@@ -120,27 +124,12 @@ class BaseCache : public MemObject
 
         bool mustSendRetry;
 
-        /**
-         * Bit vector for the outstanding requests for the master interface.
-         */
-        uint8_t requestCauses;
-
-        bool isBusRequested() { return requestCauses != 0; }
-
         void requestBus(RequestCause cause, Tick time)
         {
             DPRINTF(Cache, "Asserting bus request for cause %d\n", cause);
-            if (!isBusRequested() && !waitingOnRetry) {
-                assert(!sendEvent->scheduled());
-                sendEvent->schedule(time);
+            if (!waitingOnRetry) {
+                schedSendEvent(time);
             }
-            requestCauses |= (1 << cause);
-        }
-
-        void deassertBusRequest(RequestCause cause)
-        {
-            DPRINTF(Cache, "Deasserting bus request for cause %d\n", cause);
-            requestCauses &= ~(1 << cause);
         }
 
         void respond(PacketPtr pkt, Tick time) {
@@ -163,8 +152,7 @@ class BaseCache : public MemObject
     MSHR *allocateBufferInternal(MSHRQueue *mq, Addr addr, int size,
                                  PacketPtr pkt, Tick time, bool requestBus)
     {
-        MSHR *mshr = mq->allocate(addr, size, pkt);
-        mshr->order = order++;
+        MSHR *mshr = mq->allocate(addr, size, pkt, time, order++);
 
         if (mq->isFull()) {
             setBlocked((BlockedCause)mq->index);
@@ -182,9 +170,6 @@ class BaseCache : public MemObject
         MSHRQueue *mq = mshr->queue;
         bool wasFull = mq->isFull();
         mq->markInService(mshr);
-        if (!mq->havePending()) {
-            deassertMemSideBusRequest((RequestCause)mq->index);
-        }
         if (wasFull && !mq->isFull()) {
             clearBlocked((BlockedCause)mq->index);
         }
@@ -491,13 +476,10 @@ class BaseCache : public MemObject
         }
     }
 
-    /**
-     * True if the memory-side bus should be requested.
-     * @return True if there are outstanding requests for the master bus.
-     */
-    bool isMemSideBusRequested()
+    Tick nextMSHRReadyTick()
     {
-        return memSidePort->isBusRequested();
+        return std::min(mshrQueue.nextMSHRReadyTick(),
+                        writeBuffer.nextMSHRReadyTick());
     }
 
     /**
@@ -516,7 +498,9 @@ class BaseCache : public MemObject
      */
     void deassertMemSideBusRequest(RequestCause cause)
     {
-        memSidePort->deassertBusRequest(cause);
+        // obsolete!!
+        assert(false);
+        // memSidePort->deassertBusRequest(cause);
         // checkDrain();
     }
 
diff --git a/src/mem/cache/cache_impl.hh b/src/mem/cache/cache_impl.hh
index b4d334249..7610b5a41 100644
--- a/src/mem/cache/cache_impl.hh
+++ b/src/mem/cache/cache_impl.hh
@@ -185,9 +185,6 @@ Cache<TagStore,Coherence>::squash(int threadNum)
         cause = Blocked_NoMSHRs;
     }
     mshrQueue.squash(threadNum);
-    if (!mshrQueue.havePending()) {
-        deassertMemSideBusRequest(Request_MSHR);
-    }
     if (unblock && !mshrQueue.isFull()) {
         clearBlocked(cause);
     }
@@ -368,11 +365,14 @@ Cache<TagStore,Coherence>::timingAccess(PacketPtr pkt)
             if (mshr->threadNum != 0/*pkt->req->getThreadNum()*/) {
                 mshr->threadNum = -1;
             }
-            mshr->allocateTarget(pkt);
+            mshr->allocateTarget(pkt, time, order++);
             if (mshr->getNumTargets() == numTarget) {
                 noTargetMSHR = mshr;
                 setBlocked(Blocked_NoTargets);
-                mshrQueue.moveToFront(mshr);
+                // need to be careful with this... if this mshr isn't
+                // ready yet (i.e. time > curTick_, we don't want to
+                // move it ahead of mshrs that are ready
+                // mshrQueue.moveToFront(mshr);
             }
         } else {
             // no MSHR
@@ -630,7 +630,6 @@ Cache<TagStore,Coherence>::satisfyMSHR(MSHR *mshr, PacketPtr pkt,
     if (mshr->promoteDeferredTargets()) {
         MSHRQueue *mq = mshr->queue;
         mq->markPending(mshr);
-        mshr->order = order++;
         requestMemSideBus((RequestCause)mq->index, pkt->finishTime);
         return false;
     }
@@ -879,7 +878,7 @@ Cache<TagStore,Coherence>::snoopTiming(PacketPtr pkt)
     // we have outstanding...
     if (mshr && mshr->inService) {
         assert(mshr->getNumTargets() < numTarget); //handle later
-        mshr->allocateSnoopTarget(pkt);
+        mshr->allocateSnoopTarget(pkt, curTick, order++);
         assert(mshr->getNumTargets() < numTarget); //handle later
         return;
     }
@@ -1202,6 +1201,7 @@ Cache<TagStore,Coherence>::MemSidePort::sendPacket()
     } else {
         // check for non-response packets (requests & writebacks)
         PacketPtr pkt = myCache()->getTimingPacket();
+        assert(pkt != NULL);
         MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
 
         bool success = sendTiming(pkt);
@@ -1220,14 +1220,12 @@ Cache<TagStore,Coherence>::MemSidePort::sendPacket()
     // tried to send packet... if it was successful (no retry), see if
     // we need to rerequest bus or not
     if (!waitingOnRetry) {
-        if (isBusRequested()) {
-            // more requests/writebacks: rerequest ASAP
-            DPRINTF(CachePort, "still more MSHR requests to send\n");
-            sendEvent->schedule(curTick+1);
-        } else if (!transmitList.empty()) {
-            // deferred packets: rerequest bus, but possibly not until later
-            Tick time = transmitList.front().tick;
-            sendEvent->schedule(time <= curTick ? curTick+1 : time);
+        Tick nextReady = std::min(deferredPacketReadyTick(),
+                                  myCache()->nextMSHRReadyTick());
+        // @TODO: need to facotr in prefetch requests here somehow
+        if (nextReady != MaxTick) {
+            DPRINTF(CachePort, "more packets to send @ %d\n", nextReady);
+            sendEvent->schedule(std::max(nextReady, curTick + 1));
         } else {
             // no more to send right now: if we're draining, we may be done
             if (drainEvent) {
diff --git a/src/mem/cache/miss/mshr.cc b/src/mem/cache/miss/mshr.cc
index 24ff3b33c..8fa11ab2e 100644
--- a/src/mem/cache/miss/mshr.cc
+++ b/src/mem/cache/miss/mshr.cc
@@ -37,6 +37,7 @@
 #include <assert.h>
 #include <string>
 #include <vector>
+#include <algorithm>
 
 #include "mem/cache/miss/mshr.hh"
 #include "sim/core.hh" // for curTick
@@ -54,10 +55,13 @@ MSHR::MSHR()
 }
 
 void
-MSHR::allocate(Addr _addr, int _size, PacketPtr target)
+MSHR::allocate(Addr _addr, int _size, PacketPtr target,
+               Tick when, Counter _order)
 {
     addr = _addr;
     size = _size;
+    readyTick = when;
+    order = _order;
     assert(target);
     isCacheFill = false;
     needsExclusive = target->needsExclusive();
@@ -66,8 +70,8 @@ MSHR::allocate(Addr _addr, int _size, PacketPtr target)
     threadNum = 0;
     ntargets = 1;
     // Don't know of a case where we would allocate a new MSHR for a
-    // snoop (mem0-side request), so set cpuSide to true here.
-    targets.push_back(Target(target, true));
+    // snoop (mem-side request), so set cpuSide to true here.
+    targets.push_back(Target(target, when, _order, true));
     assert(deferredTargets.empty());
     deferredNeedsExclusive = false;
     pendingInvalidate = false;
@@ -88,33 +92,33 @@ MSHR::deallocate()
  * Adds a target to an MSHR
  */
 void
-MSHR::allocateTarget(PacketPtr target)
+MSHR::allocateTarget(PacketPtr target, Tick when, Counter _order)
 {
     if (inService) {
         if (!deferredTargets.empty() || pendingInvalidate ||
             (!needsExclusive && target->needsExclusive())) {
             // need to put on deferred list
-            deferredTargets.push_back(Target(target, true));
+            deferredTargets.push_back(Target(target, when, _order, true));
             if (target->needsExclusive()) {
                 deferredNeedsExclusive = true;
             }
         } else {
             // still OK to append to outstanding request
-            targets.push_back(Target(target, true));
+            targets.push_back(Target(target, when, _order, true));
         }
     } else {
         if (target->needsExclusive()) {
             needsExclusive = true;
         }
 
-        targets.push_back(Target(target, true));
+        targets.push_back(Target(target, when, _order, true));
     }
 
     ++ntargets;
 }
 
 void
-MSHR::allocateSnoopTarget(PacketPtr target)
+MSHR::allocateSnoopTarget(PacketPtr target, Tick when, Counter _order)
 {
     assert(inService); // don't bother to call otherwise
 
@@ -137,7 +141,7 @@ MSHR::allocateSnoopTarget(PacketPtr target)
         return;
     }
 
-    targets.push_back(Target(target, false));
+    targets.push_back(Target(target, when, _order, false));
     ++ntargets;
 }
 
@@ -157,6 +161,8 @@ MSHR::promoteDeferredTargets()
     needsExclusive = deferredNeedsExclusive;
     pendingInvalidate = false;
     deferredNeedsExclusive = false;
+    order = targets.front().order;
+    readyTick = std::max(curTick, targets.front().time);
 
     return true;
 }
diff --git a/src/mem/cache/miss/mshr.hh b/src/mem/cache/miss/mshr.hh
index f4e090a12..92288cf52 100644
--- a/src/mem/cache/miss/mshr.hh
+++ b/src/mem/cache/miss/mshr.hh
@@ -55,13 +55,14 @@ class MSHR : public Packet::SenderState
     class Target {
       public:
         Tick time;      //!< Time when request was received (for stats)
+        Counter order;  //!< Global order (for memory consistency mgmt)
         PacketPtr pkt;  //!< Pending request packet.
         bool cpuSide;   //!< Did request come from cpu side or mem side?
 
         bool isCpuSide() { return cpuSide; }
 
-        Target(PacketPtr _pkt, bool _cpuSide, Tick _time = curTick)
-            : time(_time), pkt(_pkt), cpuSide(_cpuSide)
+        Target(PacketPtr _pkt, Tick _time, Counter _order, bool _cpuSide)
+            : time(_time), order(_order), pkt(_pkt), cpuSide(_cpuSide)
         {}
     };
 
@@ -79,6 +80,12 @@ class MSHR : public Packet::SenderState
     /** Pointer to queue containing this MSHR. */
     MSHRQueue *queue;
 
+    /** Cycle when ready to issue */
+    Tick readyTick;
+
+    /** Order number assigned by the miss queue. */
+    Counter order;
+
     /** Address of the request. */
     Addr addr;
 
@@ -103,8 +110,6 @@ class MSHR : public Packet::SenderState
     short threadNum;
     /** The number of currently allocated targets. */
     short ntargets;
-    /** Order number of assigned by the miss queue. */
-    uint64_t order;
 
     /**
      * Pointer to this MSHR on the ready list.
@@ -136,13 +141,8 @@ public:
      * @param size The number of bytes to request.
      * @param pkt  The original miss.
      */
-    void allocate(Addr addr, int size, PacketPtr pkt);
-
-    /**
-     * Allocate this MSHR as a buffer for the given request.
-     * @param target The memory request to buffer.
-     */
-    void allocateAsBuffer(PacketPtr target);
+    void allocate(Addr addr, int size, PacketPtr pkt,
+                  Tick when, Counter _order);
 
     /**
      * Mark this MSHR as free.
@@ -153,8 +153,8 @@ public:
      * Add a request to the list of targets.
      * @param target The target.
      */
-    void allocateTarget(PacketPtr target);
-    void allocateSnoopTarget(PacketPtr target);
+    void allocateTarget(PacketPtr target, Tick when, Counter order);
+    void allocateSnoopTarget(PacketPtr target, Tick when, Counter order);
 
     /** A simple constructor. */
     MSHR();
diff --git a/src/mem/cache/miss/mshr_queue.cc b/src/mem/cache/miss/mshr_queue.cc
index 3407e2588..18184bd20 100644
--- a/src/mem/cache/miss/mshr_queue.cc
+++ b/src/mem/cache/miss/mshr_queue.cc
@@ -90,8 +90,8 @@ MSHRQueue::findMatches(Addr addr, vector<MSHR*>& matches) const
 MSHR *
 MSHRQueue::findPending(Addr addr, int size) const
 {
-    MSHR::ConstIterator i = pendingList.begin();
-    MSHR::ConstIterator end = pendingList.end();
+    MSHR::ConstIterator i = readyList.begin();
+    MSHR::ConstIterator end = readyList.end();
     for (; i != end; ++i) {
         MSHR *mshr = *i;
         if (mshr->addr < addr) {
@@ -107,17 +107,37 @@ MSHRQueue::findPending(Addr addr, int size) const
     return NULL;
 }
 
+
+MSHR::Iterator
+MSHRQueue::addToReadyList(MSHR *mshr)
+{
+    if (readyList.empty() || readyList.back()->readyTick <= mshr->readyTick) {
+        return readyList.insert(readyList.end(), mshr);
+    }
+
+    MSHR::Iterator i = readyList.begin();
+    MSHR::Iterator end = readyList.end();
+    for (; i != end; ++i) {
+        if ((*i)->readyTick > mshr->readyTick) {
+            return readyList.insert(i, mshr);
+        }
+    }
+    assert(false);
+}
+
+
 MSHR *
-MSHRQueue::allocate(Addr addr, int size, PacketPtr &pkt)
+MSHRQueue::allocate(Addr addr, int size, PacketPtr &pkt,
+                    Tick when, Counter order)
 {
     assert(!freeList.empty());
     MSHR *mshr = freeList.front();
     assert(mshr->getNumTargets() == 0);
     freeList.pop_front();
 
-    mshr->allocate(addr, size, pkt);
+    mshr->allocate(addr, size, pkt, when, order);
     mshr->allocIter = allocatedList.insert(allocatedList.end(), mshr);
-    mshr->readyIter = pendingList.insert(pendingList.end(), mshr);
+    mshr->readyIter = addToReadyList(mshr);
 
     allocated += 1;
     return mshr;
@@ -139,7 +159,7 @@ MSHRQueue::deallocateOne(MSHR *mshr)
     if (mshr->inService) {
         inServiceEntries--;
     } else {
-        pendingList.erase(mshr->readyIter);
+        readyList.erase(mshr->readyIter);
     }
     mshr->deallocate();
     return retval;
@@ -150,14 +170,15 @@ MSHRQueue::moveToFront(MSHR *mshr)
 {
     if (!mshr->inService) {
         assert(mshr == *(mshr->readyIter));
-        pendingList.erase(mshr->readyIter);
-        mshr->readyIter = pendingList.insert(pendingList.begin(), mshr);
+        readyList.erase(mshr->readyIter);
+        mshr->readyIter = readyList.insert(readyList.begin(), mshr);
     }
 }
 
 void
 MSHRQueue::markInService(MSHR *mshr)
 {
+    assert(!mshr->inService);
     if (mshr->isSimpleForward()) {
         // we just forwarded the request packet & don't expect a
         // response, so get rid of it
@@ -167,23 +188,23 @@ MSHRQueue::markInService(MSHR *mshr)
         return;
     }
     mshr->inService = true;
-    pendingList.erase(mshr->readyIter);
+    readyList.erase(mshr->readyIter);
     //mshr->readyIter = NULL;
     inServiceEntries += 1;
-    //pendingList.pop_front();
+    //readyList.pop_front();
 }
 
 void
 MSHRQueue::markPending(MSHR *mshr)
 {
-    //assert(mshr->readyIter == NULL);
+    assert(mshr->inService);
     mshr->inService = false;
     --inServiceEntries;
     /**
      * @ todo might want to add rerequests to front of pending list for
      * performance.
      */
-    mshr->readyIter = pendingList.insert(pendingList.end(), mshr);
+    mshr->readyIter = addToReadyList(mshr);
 }
 
 void
diff --git a/src/mem/cache/miss/mshr_queue.hh b/src/mem/cache/miss/mshr_queue.hh
index 806aa9c64..fd61dec8b 100644
--- a/src/mem/cache/miss/mshr_queue.hh
+++ b/src/mem/cache/miss/mshr_queue.hh
@@ -51,7 +51,7 @@ class MSHRQueue
     /** Holds pointers to all allocated entries. */
     MSHR::List allocatedList;
     /** Holds pointers to entries that haven't been sent to the bus. */
-    MSHR::List pendingList;
+    MSHR::List readyList;
     /** Holds non allocated entries. */
     MSHR::List freeList;
 
@@ -69,6 +69,9 @@ class MSHRQueue
      */
     const int numReserve;
 
+    MSHR::Iterator addToReadyList(MSHR *mshr);
+
+
   public:
     /** The number of allocated entries. */
     int allocated;
@@ -121,7 +124,8 @@ class MSHRQueue
      *
      * @pre There are free entries.
      */
-    MSHR *allocate(Addr addr, int size, PacketPtr &pkt);
+    MSHR *allocate(Addr addr, int size, PacketPtr &pkt,
+                   Tick when, Counter order);
 
     /**
      * Removes the given MSHR from the queue. This places the MSHR on the
@@ -147,7 +151,7 @@ class MSHRQueue
 
     /**
      * Mark the given MSHR as in service. This removes the MSHR from the
-     * pendingList. Deallocates the MSHR if it does not expect a response.
+     * readyList. Deallocates the MSHR if it does not expect a response.
      * @param mshr The MSHR to mark in service.
      */
     void markInService(MSHR *mshr);
@@ -171,7 +175,7 @@ class MSHRQueue
      */
     bool havePending() const
     {
-        return !pendingList.empty();
+        return !readyList.empty();
     }
 
     /**
@@ -184,15 +188,20 @@ class MSHRQueue
     }
 
     /**
-     * Returns the MSHR at the head of the pendingList.
+     * Returns the MSHR at the head of the readyList.
      * @return The next request to service.
      */
     MSHR *getNextMSHR() const
     {
-        if (pendingList.empty()) {
+        if (readyList.empty() || readyList.front()->readyTick > curTick) {
             return NULL;
         }
-        return pendingList.front();
+        return readyList.front();
+    }
+
+    Tick nextMSHRReadyTick() const
+    {
+        return readyList.empty() ? MaxTick : readyList.front()->readyTick;
     }
 };
 
diff --git a/src/mem/tport.cc b/src/mem/tport.cc
index 2644a504c..0a2127490 100644
--- a/src/mem/tport.cc
+++ b/src/mem/tport.cc
@@ -91,28 +91,30 @@ SimpleTimingPort::schedSendTiming(PacketPtr pkt, Tick when)
     assert(when > curTick);
 
     // Nothing is on the list: add it and schedule an event
-    if (transmitList.empty()) {
-        assert(!sendEvent->scheduled());
-        sendEvent->schedule(when);
-        transmitList.push_back(DeferredPacket(when, pkt));
+    if (transmitList.empty() || when < transmitList.front().tick) {
+        transmitList.push_front(DeferredPacket(when, pkt));
+        schedSendEvent(when);
         return;
     }
 
-    // something is on the list and this belongs at the end
+    // list is non-empty and this is not the head, so event should
+    // already be scheduled
+    assert(waitingOnRetry ||
+           (sendEvent->scheduled() && sendEvent->when() <= when));
+
+    // list is non-empty & this belongs at the end
     if (when >= transmitList.back().tick) {
         transmitList.push_back(DeferredPacket(when, pkt));
         return;
     }
-    // Something is on the list and this belongs somewhere else
+
+    // this belongs in the middle somewhere
     DeferredPacketIterator i = transmitList.begin();
+    i++; // already checked for insertion at front
     DeferredPacketIterator end = transmitList.end();
 
     for (; i != end; ++i) {
         if (when < i->tick) {
-            if (i == transmitList.begin()) {
-                //Inserting at begining, reschedule
-                sendEvent->reschedule(when);
-            }
             transmitList.insert(i, DeferredPacket(when, pkt));
             return;
         }
diff --git a/src/mem/tport.hh b/src/mem/tport.hh
index ea0f05ed1..bfed29f34 100644
--- a/src/mem/tport.hh
+++ b/src/mem/tport.hh
@@ -105,6 +105,24 @@ class SimpleTimingPort : public Port
     bool deferredPacketReady()
     { return !transmitList.empty() && transmitList.front().tick <= curTick; }
 
+    Tick deferredPacketReadyTick()
+    { return transmitList.empty() ? MaxTick : transmitList.front().tick; }
+
+    void schedSendEvent(Tick when)
+    {
+        if (waitingOnRetry) {
+            assert(!sendEvent->scheduled());
+            return;
+        }
+
+        if (!sendEvent->scheduled()) {
+            sendEvent->schedule(when);
+        } else if (sendEvent->when() > when) {
+            sendEvent->reschedule(when);
+        }
+    }
+
+
     /** Schedule a sendTiming() event to be called in the future.
      * @param pkt packet to send
      * @param absolute time (in ticks) to send packet