From f24ae2ec2a43a5197934668b6f9638ed118994d8 Mon Sep 17 00:00:00 2001 From: Steve Reinhardt Date: Wed, 16 Jun 2010 15:25:57 -0700 Subject: cache: fail store conditionals when upgrade loses race Requires new "SCUpgradeReq" message that marks upgrades for store conditionals, so downstream caches can fail these when they run into invalidations. See http://www.m5sim.org/flyspray/task/197 --- src/mem/cache/cache_impl.hh | 23 ++++++++++++++++++----- src/mem/cache/mshr.cc | 6 +++++- src/mem/packet.cc | 16 ++++++++++++++-- src/mem/packet.hh | 11 +++++++++-- 4 files changed, 46 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/mem/cache/cache_impl.hh b/src/mem/cache/cache_impl.hh index e4341af26..d5eb38e48 100644 --- a/src/mem/cache/cache_impl.hh +++ b/src/mem/cache/cache_impl.hh @@ -196,7 +196,7 @@ Cache::satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk) // Not a read or write... must be an upgrade. it's OK // to just ack those as long as we have an exclusive // copy at this level. - assert(pkt->cmd == MemCmd::UpgradeReq); + assert(pkt->isUpgrade()); tags->invalidateBlk(blk); } } @@ -536,8 +536,8 @@ Cache::getBusPacket(PacketPtr cpu_pkt, BlkType *blk, return NULL; } - if (!blkValid && (cpu_pkt->cmd == MemCmd::Writeback || - cpu_pkt->cmd == MemCmd::UpgradeReq)) { + if (!blkValid && + (cpu_pkt->cmd == MemCmd::Writeback || cpu_pkt->isUpgrade())) { // Writebacks that weren't allocated in access() and upgrades // from upper-level caches that missed completely just go // through. @@ -556,7 +556,7 @@ Cache::getBusPacket(PacketPtr cpu_pkt, BlkType *blk, // only reason to be here is that blk is shared // (read-only) and we need exclusive assert(needsExclusive && !blk->isWritable()); - cmd = MemCmd::UpgradeReq; + cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq; } else { // block is invalid cmd = needsExclusive ? MemCmd::ReadExReq : MemCmd::ReadReq; @@ -814,6 +814,11 @@ Cache::handleResponse(PacketPtr pkt) assert(!target->pkt->req->isUncacheable()); missLatency[target->pkt->cmdToIndex()][0/*pkt->req->threadId()*/] += completion_time - target->recvTime; + } else if (target->pkt->cmd == MemCmd::StoreCondReq && + pkt->cmd == MemCmd::UpgradeFailResp) { + // failed StoreCond upgrade + completion_time = tags->getHitLatency() + pkt->finishTime; + target->pkt->req->setExtraData(0); } else { // not a cache fill, just forwarding response completion_time = tags->getHitLatency() + pkt->finishTime; @@ -1336,7 +1341,15 @@ Cache::getTimingPacket() PacketPtr tgt_pkt = mshr->getTarget()->pkt; PacketPtr pkt = NULL; - if (mshr->isForwardNoResponse()) { + if (tgt_pkt->cmd == MemCmd::SCUpgradeFailReq) { + // SCUpgradeReq saw invalidation while queued in MSHR, so now + // that we are getting around to processing it, just treat it + // as if we got a failure response + pkt = new Packet(tgt_pkt); + pkt->cmd = MemCmd::UpgradeFailResp; + handleResponse(pkt); + return NULL; + } else if (mshr->isForwardNoResponse()) { // no response expected, just forward packet as it is assert(tags->findBlock(mshr->addr) == NULL); pkt = tgt_pkt; diff --git a/src/mem/cache/mshr.cc b/src/mem/cache/mshr.cc index 100e6126a..6333191a4 100644 --- a/src/mem/cache/mshr.cc +++ b/src/mem/cache/mshr.cc @@ -1,5 +1,6 @@ /* * Copyright (c) 2002-2005 The Regents of The University of Michigan + * Copyright (c) 2010 Advancec Micro Devices, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -71,7 +72,7 @@ MSHR::TargetList::add(PacketPtr pkt, Tick readyTime, needsExclusive = true; } - if (pkt->cmd == MemCmd::UpgradeReq) { + if (pkt->isUpgrade()) { hasUpgrade = true; } } @@ -99,6 +100,9 @@ MSHR::TargetList::replaceUpgrades() if (i->pkt->cmd == MemCmd::UpgradeReq) { i->pkt->cmd = MemCmd::ReadExReq; DPRINTF(Cache, "Replacing UpgradeReq with ReadExReq\n"); + } else if (i->pkt->cmd == MemCmd::SCUpgradeReq) { + i->pkt->cmd = MemCmd::SCUpgradeFailReq; + DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n"); } } diff --git a/src/mem/packet.cc b/src/mem/packet.cc index 2f84e4414..c54a52438 100644 --- a/src/mem/packet.cc +++ b/src/mem/packet.cc @@ -1,5 +1,6 @@ /* * Copyright (c) 2006 The Regents of The University of Michigan + * Copyright (c) 2010 Advancec Micro Devices, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -92,11 +93,22 @@ MemCmd::commandInfo[] = { SET3(IsWrite, NeedsExclusive, IsResponse), InvalidCmd, "WriteInvalidateResp" }, /* UpgradeReq */ - { SET4(IsInvalidate, NeedsExclusive, IsRequest, NeedsResponse), + { SET5(IsInvalidate, NeedsExclusive, IsUpgrade, IsRequest, NeedsResponse), UpgradeResp, "UpgradeReq" }, + /* SCUpgradeReq: response could be UpgradeResp or UpgradeFailResp */ + { SET6(IsInvalidate, NeedsExclusive, IsUpgrade, IsLlsc, + IsRequest, NeedsResponse), + UpgradeResp, "SCUpgradeReq" }, /* UpgradeResp */ - { SET2(NeedsExclusive, IsResponse), + { SET3(NeedsExclusive, IsUpgrade, IsResponse), InvalidCmd, "UpgradeResp" }, + /* SCUpgradeFailReq: generates UpgradeFailResp ASAP */ + { SET5(IsInvalidate, NeedsExclusive, IsLlsc, + IsRequest, NeedsResponse), + UpgradeFailResp, "SCUpgradeFailReq" }, + /* UpgradeFailResp */ + { SET2(NeedsExclusive, IsResponse), + InvalidCmd, "UpgradeFailResp" }, /* ReadExReq */ { SET5(IsRead, NeedsExclusive, IsInvalidate, IsRequest, NeedsResponse), ReadExResp, "ReadExReq" }, diff --git a/src/mem/packet.hh b/src/mem/packet.hh index 17af558b5..2c94da8bd 100644 --- a/src/mem/packet.hh +++ b/src/mem/packet.hh @@ -1,5 +1,6 @@ /* * Copyright (c) 2006 The Regents of The University of Michigan + * Copyright (c) 2010 Advancec Micro Devices, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -81,7 +82,10 @@ class MemCmd WriteInvalidateReq, WriteInvalidateResp, UpgradeReq, + SCUpgradeReq, // Special "weak" upgrade for StoreCond UpgradeResp, + SCUpgradeFailReq, // Failed SCUpgradeReq in MSHR (never sent) + UpgradeFailResp, // Valid for SCUpgradeReq only ReadExReq, ReadExResp, LoadLockedReq, @@ -111,6 +115,7 @@ class MemCmd { IsRead, //!< Data flows from responder to requester IsWrite, //!< Data flows from requester to responder + IsUpgrade, IsPrefetch, //!< Not a demand access IsInvalidate, NeedsExclusive, //!< Requires exclusive copy to complete in-cache @@ -157,7 +162,8 @@ class MemCmd public: bool isRead() const { return testCmdAttrib(IsRead); } - bool isWrite() const { return testCmdAttrib(IsWrite); } + bool isWrite() const { return testCmdAttrib(IsWrite); } + bool isUpgrade() const { return testCmdAttrib(IsUpgrade); } bool isRequest() const { return testCmdAttrib(IsRequest); } bool isResponse() const { return testCmdAttrib(IsResponse); } bool needsExclusive() const { return testCmdAttrib(NeedsExclusive); } @@ -392,7 +398,8 @@ class Packet : public FastAlloc, public Printable inline int cmdToIndex() const { return cmd.toInt(); } bool isRead() const { return cmd.isRead(); } - bool isWrite() const { return cmd.isWrite(); } + bool isWrite() const { return cmd.isWrite(); } + bool isUpgrade() const { return cmd.isUpgrade(); } bool isRequest() const { return cmd.isRequest(); } bool isResponse() const { return cmd.isResponse(); } bool needsExclusive() const { return cmd.needsExclusive(); } -- cgit v1.2.3