diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/mem/cache/base.hh | 4 | ||||
| -rw-r--r-- | src/mem/cache/cache.hh | 6 | ||||
| -rw-r--r-- | src/mem/cache/cache_impl.hh | 82 | ||||
| -rw-r--r-- | src/mem/cache/mshr.cc | 97 | ||||
| -rw-r--r-- | src/mem/cache/mshr.hh | 26 | ||||
| -rw-r--r-- | src/mem/cache/mshr_queue.cc | 4 | ||||
| -rw-r--r-- | src/mem/cache/mshr_queue.hh | 2 |
7 files changed, 146 insertions, 75 deletions
diff --git a/src/mem/cache/base.hh b/src/mem/cache/base.hh index 2f1088609..94cdc959c 100644 --- a/src/mem/cache/base.hh +++ b/src/mem/cache/base.hh @@ -170,11 +170,11 @@ class BaseCache : public MemObject return mshr; } - void markInServiceInternal(MSHR *mshr) + void markInServiceInternal(MSHR *mshr, PacketPtr pkt) { MSHRQueue *mq = mshr->queue; bool wasFull = mq->isFull(); - mq->markInService(mshr); + mq->markInService(mshr, pkt); if (wasFull && !mq->isFull()) { clearBlocked((BlockedCause)mq->index); } diff --git a/src/mem/cache/cache.hh b/src/mem/cache/cache.hh index 6cb6233f5..e15747c3f 100644 --- a/src/mem/cache/cache.hh +++ b/src/mem/cache/cache.hh @@ -178,7 +178,9 @@ class Cache : public BaseCache BlkType *handleFill(PacketPtr pkt, BlkType *blk, PacketList &writebacks); - void satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk); + void satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk, + bool deferred_response = false, + bool pending_downgrade = false); bool satisfyMSHR(MSHR *mshr, PacketPtr pkt, BlkType *blk); void doTimingSupplyResponse(PacketPtr req_pkt, uint8_t *blk_data, @@ -292,7 +294,7 @@ class Cache : public BaseCache * are successfully sent. * @param pkt The request that was sent on the bus. */ - void markInService(MSHR *mshr); + void markInService(MSHR *mshr, PacketPtr pkt = 0); /** * Perform the given writeback request. diff --git a/src/mem/cache/cache_impl.hh b/src/mem/cache/cache_impl.hh index 9e3374fea..8d2806b8d 100644 --- a/src/mem/cache/cache_impl.hh +++ b/src/mem/cache/cache_impl.hh @@ -165,16 +165,18 @@ Cache<TagStore>::cmpAndSwap(BlkType *blk, PacketPtr pkt) template<class TagStore> void -Cache<TagStore>::satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk) +Cache<TagStore>::satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk, + bool deferred_response, + bool pending_downgrade) { - assert(blk); + assert(blk && blk->isValid()); // Occasionally this is not true... if we are a lower-level cache // satisfying a string of Read and ReadEx requests from // upper-level caches, a Read will mark the block as shared but we // can satisfy a following ReadEx anyway since we can rely on the // Read requester(s) to have buffered the ReadEx snoop and to // invalidate their blocks after receiving them. - // assert(pkt->needsExclusive() ? blk->isWritable() : blk->isValid()); + // assert(!pkt->needsExclusive() || blk->isWritable()); assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize); // Check RMW operations first since both isRead() and @@ -195,13 +197,43 @@ Cache<TagStore>::satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk) // special handling for coherent block requests from // upper-level caches if (pkt->needsExclusive()) { - // on ReadExReq we give up our copy + // if we have a dirty copy, make sure the recipient + // keeps it marked dirty + if (blk->isDirty()) { + pkt->assertMemInhibit(); + } + // on ReadExReq we give up our copy unconditionally tags->invalidateBlk(blk); + } else if (blk->isWritable() && !pending_downgrade + && !pkt->sharedAsserted()) { + // we can give the requester an exclusive copy (by not + // asserting shared line) on a read request if: + // - we have an exclusive copy at this level (& below) + // - we don't have a pending snoop from below + // signaling another read request + // - no other cache above has a copy (otherwise it + // would have asseretd shared line on request) + + if (blk->isDirty()) { + // special considerations if we're owner: + if (!deferred_response) { + // if we are responding immediately and can + // signal that we're transferring ownership + // along with exclusivity, do so + pkt->assertMemInhibit(); + blk->status &= ~BlkDirty; + } else { + // if we're responding after our own miss, + // there's a window where the recipient didn't + // know it was getting ownership and may not + // have responded to snoops correctly, so we + // can't pass off ownership *or* exclusivity + pkt->assertShared(); + } + } } else { - // on ReadReq we create shareable copies here and in - // the requester + // otherwise only respond with a shared copy pkt->assertShared(); - blk->status &= ~BlkWritable; } } } else { @@ -223,9 +255,9 @@ Cache<TagStore>::satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk) template<class TagStore> void -Cache<TagStore>::markInService(MSHR *mshr) +Cache<TagStore>::markInService(MSHR *mshr, PacketPtr pkt) { - markInServiceInternal(mshr); + markInServiceInternal(mshr, pkt); #if 0 if (mshr->originalCmd == MemCmd::HardPFReq) { DPRINTF(HWPrefetch, "%s:Marking a HW_PF in service\n", @@ -829,7 +861,8 @@ Cache<TagStore>::handleResponse(PacketPtr pkt) case MSHR::Target::FromCPU: Tick completion_time; if (is_fill) { - satisfyCpuSideRequest(target->pkt, blk); + satisfyCpuSideRequest(target->pkt, blk, + true, mshr->hasPostDowngrade()); // How many bytes past the first request is this one int transfer_offset = target->pkt->getOffset(blkSize) - initial_offset; @@ -860,12 +893,11 @@ Cache<TagStore>::handleResponse(PacketPtr pkt) // if this packet is an error copy that to the new packet if (is_error) target->pkt->copyError(pkt); - if (pkt->isInvalidate()) { + if (target->pkt->cmd == MemCmd::ReadResp && + (pkt->isInvalidate() || mshr->hasPostInvalidate())) { // If intermediate cache got ReadRespWithInvalidate, // propagate that. Response should not have // isInvalidate() set otherwise. - assert(target->pkt->cmd == MemCmd::ReadResp); - assert(pkt->cmd == MemCmd::ReadRespWithInvalidate); target->pkt->cmd = MemCmd::ReadRespWithInvalidate; } cpuSidePort->respond(target->pkt, completion_time); @@ -884,8 +916,9 @@ Cache<TagStore>::handleResponse(PacketPtr pkt) assert(!is_error); // response to snoop request DPRINTF(Cache, "processing deferred snoop...\n"); + assert(!(pkt->isInvalidate() && !mshr->hasPostInvalidate())); handleSnoop(target->pkt, blk, true, true, - mshr->pendingInvalidate || pkt->isInvalidate()); + mshr->hasPostInvalidate()); break; default: @@ -895,14 +928,20 @@ Cache<TagStore>::handleResponse(PacketPtr pkt) mshr->popTarget(); } - if (pkt->isInvalidate()) { - tags->invalidateBlk(blk); + if (blk) { + if (pkt->isInvalidate() || mshr->hasPostInvalidate()) { + tags->invalidateBlk(blk); + } else if (mshr->hasPostDowngrade()) { + blk->status &= ~BlkWritable; + } } if (mshr->promoteDeferredTargets()) { // avoid later read getting stale data while write miss is // outstanding.. see comment in timingAccess() - blk->status &= ~BlkReadable; + if (blk) { + blk->status &= ~BlkReadable; + } MSHRQueue *mq = mshr->queue; mq->markPending(mshr); requestMemSideBus((RequestCause)mq->index, pkt->finishTime); @@ -1017,14 +1056,19 @@ Cache<TagStore>::handleFill(PacketPtr pkt, BlkType *blk, int id = pkt->req->hasContextId() ? pkt->req->contextId() : -1; tags->insertBlock(pkt->getAddr(), blk, id); } + + // starting from scratch with a new block + blk->status = 0; } else { // existing block... probably an upgrade assert(blk->tag == tags->extractTag(addr)); // either we're getting new data or the block should already be valid assert(pkt->hasData() || blk->isValid()); + // don't clear block status... if block is already dirty we + // don't want to lose that } - blk->status = BlkValid | BlkReadable; + blk->status |= BlkValid | BlkReadable; if (!pkt->sharedAsserted()) { blk->status |= BlkWritable; @@ -1587,7 +1631,7 @@ Cache<TagStore>::MemSidePort::sendPacket() delete pkt; } } else { - myCache()->markInService(mshr); + myCache()->markInService(mshr, pkt); } } } diff --git a/src/mem/cache/mshr.cc b/src/mem/cache/mshr.cc index 42cbcd372..bbddf2aee 100644 --- a/src/mem/cache/mshr.cc +++ b/src/mem/cache/mshr.cc @@ -89,6 +89,19 @@ MSHR::TargetList::add(PacketPtr pkt, Tick readyTime, } +static void +replaceUpgrade(PacketPtr pkt) +{ + if (pkt->cmd == MemCmd::UpgradeReq) { + pkt->cmd = MemCmd::ReadExReq; + DPRINTF(Cache, "Replacing UpgradeReq with ReadExReq\n"); + } else if (pkt->cmd == MemCmd::SCUpgradeReq) { + pkt->cmd = MemCmd::SCUpgradeFailReq; + DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n"); + } +} + + void MSHR::TargetList::replaceUpgrades() { @@ -97,13 +110,7 @@ MSHR::TargetList::replaceUpgrades() Iterator end_i = end(); for (Iterator i = begin(); i != end_i; ++i) { - if (i->pkt->cmd == MemCmd::UpgradeReq) { - i->pkt->cmd = MemCmd::ReadExReq; - DPRINTF(Cache, "Replacing UpgradeReq with ReadExReq\n"); - } else if (i->pkt->cmd == MemCmd::SCUpgradeReq) { - i->pkt->cmd = MemCmd::SCUpgradeFailReq; - DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n"); - } + replaceUpgrade(i->pkt); } hasUpgrade = false; @@ -180,8 +187,6 @@ MSHR::allocate(Addr _addr, int _size, PacketPtr target, Target::FromPrefetcher : Target::FromCPU; targets->add(target, whenReady, _order, source, true); assert(deferredTargets->isReset()); - pendingInvalidate = false; - pendingShared = false; data = NULL; } @@ -197,7 +202,7 @@ MSHR::clearDownstreamPending() } bool -MSHR::markInService() +MSHR::markInService(PacketPtr pkt) { assert(!inService); if (isForwardNoResponse()) { @@ -208,6 +213,10 @@ MSHR::markInService() return true; } inService = true; + pendingDirty = (targets->needsExclusive || + (!pkt->sharedAsserted() && pkt->memInhibitAsserted())); + postInvalidate = postDowngrade = false; + if (!downstreamPending) { // let upstream caches know that the request has made it to a // level where it's going to get a response @@ -225,8 +234,6 @@ MSHR::deallocate() assert(deferredTargets->isReset()); assert(ntargets == 0); inService = false; - //allocIter = NULL; - //readyIter = NULL; } /* @@ -241,17 +248,22 @@ MSHR::allocateTarget(PacketPtr pkt, Tick whenReady, Counter _order) // - there are other targets already deferred // - there's a pending invalidate to be applied after the response // comes back (but before this target is processed) - // - the outstanding request is for a non-exclusive block and this - // target requires an exclusive block + // - this target requires an exclusive block and either we're not + // getting an exclusive block back or we have already snooped + // another read request that will downgrade our exclusive block + // to shared // assume we'd never issue a prefetch when we've got an // outstanding miss assert(pkt->cmd != MemCmd::HardPFReq); if (inService && - (!deferredTargets->empty() || pendingInvalidate || - (!targets->needsExclusive && pkt->needsExclusive()))) { + (!deferredTargets->empty() || hasPostInvalidate() || + (pkt->needsExclusive() && + (!isPendingDirty() || hasPostDowngrade() || isForward)))) { // need to put on deferred list + if (hasPostInvalidate()) + replaceUpgrade(pkt); deferredTargets->add(pkt, whenReady, _order, Target::FromCPU, true); } else { // No request outstanding, or still OK to append to @@ -291,49 +303,49 @@ MSHR::handleSnoop(PacketPtr pkt, Counter _order) // From here on down, the request issued by this MSHR logically // precedes the request we're snooping. - if (pkt->needsExclusive()) { // snooped request still precedes the re-request we'll have to // issue for deferred targets, if any... deferredTargets->replaceUpgrades(); } - if (pendingInvalidate) { + if (hasPostInvalidate()) { // a prior snoop has already appended an invalidation, so // logically we don't have the block anymore; no need for // further snooping. return true; } - if (targets->needsExclusive || pkt->needsExclusive()) { - // actual target device (typ. PhysicalMemory) will delete the - // packet on reception, so we need to save a copy here + if (isPendingDirty() || pkt->isInvalidate()) { + // We need to save and replay the packet in two cases: + // 1. We're awaiting an exclusive copy, so ownership is pending, + // and we need to respond after we receive data. + // 2. It's an invalidation (e.g., UpgradeReq), and we need + // to forward the snoop up the hierarchy after the current + // transaction completes. + + // Actual target device (typ. PhysicalMemory) will delete the + // packet on reception, so we need to save a copy here. PacketPtr cp_pkt = new Packet(pkt, true); targets->add(cp_pkt, curTick, _order, Target::FromSnoop, downstreamPending && targets->needsExclusive); ++ntargets; - if (targets->needsExclusive) { - // We're awaiting an exclusive copy, so ownership is pending. - // It's up to us to respond once the data arrives. + if (isPendingDirty()) { pkt->assertMemInhibit(); pkt->setSupplyExclusive(); - } else { - // Someone else may respond before we get around to - // processing this snoop, which means the copied request - // pointer will no longer be valid - cp_pkt->req = NULL; } if (pkt->needsExclusive()) { // This transaction will take away our pending copy - pendingInvalidate = true; + postInvalidate = true; } - } else { - // Read to a read: no conflict, so no need to record as - // target, but make sure neither reader thinks he's getting an - // exclusive copy - pendingShared = true; + } + + if (!pkt->needsExclusive()) { + // This transaction will get a read-shared copy, downgrading + // our copy if we had an exclusive one + postDowngrade = true; pkt->assertShared(); } @@ -359,8 +371,6 @@ MSHR::promoteDeferredTargets() // clear deferredTargets flags deferredTargets->resetFlags(); - pendingInvalidate = false; - pendingShared = false; order = targets->front().order; readyTime = std::max(curTick, targets->front().readyTime); @@ -371,13 +381,8 @@ MSHR::promoteDeferredTargets() void MSHR::handleFill(Packet *pkt, CacheBlk *blk) { - if (pendingShared) { - // we snooped another read while this read was in - // service... assert shared line on its behalf - pkt->assertShared(); - } - - if (!pkt->sharedAsserted() && !pendingInvalidate + if (!pkt->sharedAsserted() + && !(hasPostInvalidate() || hasPostDowngrade()) && deferredTargets->needsExclusive) { // We got an exclusive response, but we have deferred targets // which are waiting to request an exclusive copy (not because @@ -427,8 +432,8 @@ MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const _isUncacheable ? "Unc" : "", inService ? "InSvc" : "", downstreamPending ? "DwnPend" : "", - pendingInvalidate ? "PendInv" : "", - pendingShared ? "PendShared" : ""); + hasPostInvalidate() ? "PostInv" : "", + hasPostDowngrade() ? "PostDowngr" : ""); ccprintf(os, "%s Targets:\n", prefix); targets->print(os, verbosity, prefix + " "); diff --git a/src/mem/cache/mshr.hh b/src/mem/cache/mshr.hh index 26eef2cac..9b55e70ef 100644 --- a/src/mem/cache/mshr.hh +++ b/src/mem/cache/mshr.hh @@ -134,8 +134,28 @@ class MSHR : public Packet::SenderState, public Printable bool downstreamPending; - bool pendingInvalidate; - bool pendingShared; + /** The pending* and post* flags are only valid if inService is + * true. Using the accessor functions lets us detect if these + * flags are accessed improperly. + */ + + /** Will we have a dirty copy after this request? */ + bool pendingDirty; + bool isPendingDirty() const { + assert(inService); return pendingDirty; + } + + /** Did we snoop an invalidate while waiting for data? */ + bool postInvalidate; + bool hasPostInvalidate() const { + assert(inService); return postInvalidate; + } + + /** Did we snoop a read while waiting for data? */ + bool postDowngrade; + bool hasPostDowngrade() const { + assert(inService); return postDowngrade; + } /** Thread number of the miss. */ ThreadID threadNum; @@ -180,7 +200,7 @@ public: void allocate(Addr addr, int size, PacketPtr pkt, Tick when, Counter _order); - bool markInService(); + bool markInService(PacketPtr pkt); void clearDownstreamPending(); diff --git a/src/mem/cache/mshr_queue.cc b/src/mem/cache/mshr_queue.cc index b5c6cc7b8..b412891d9 100644 --- a/src/mem/cache/mshr_queue.cc +++ b/src/mem/cache/mshr_queue.cc @@ -197,9 +197,9 @@ MSHRQueue::moveToFront(MSHR *mshr) } void -MSHRQueue::markInService(MSHR *mshr) +MSHRQueue::markInService(MSHR *mshr, PacketPtr pkt) { - if (mshr->markInService()) { + if (mshr->markInService(pkt)) { deallocate(mshr); } else { readyList.erase(mshr->readyIter); diff --git a/src/mem/cache/mshr_queue.hh b/src/mem/cache/mshr_queue.hh index f481ca471..d8c495679 100644 --- a/src/mem/cache/mshr_queue.hh +++ b/src/mem/cache/mshr_queue.hh @@ -160,7 +160,7 @@ class MSHRQueue * readyList. Deallocates the MSHR if it does not expect a response. * @param mshr The MSHR to mark in service. */ - void markInService(MSHR *mshr); + void markInService(MSHR *mshr, PacketPtr pkt); /** * Mark an in service entry as pending, used to resend a request. |