MEM: Remove the Broadcast destination from the packet

This patch simplifies the packet by removing the broadcast flag and
instead more firmly relying on (and enforcing) the semantics of
transactions in the classic memory system, i.e. request packets are
routed from a master to a slave based on the address, and when they
are created they have neither a valid source, nor destination. On
their way to the slave, the request packet is updated with a source
field for all modules that multiplex packets from multiple master
(e.g. a bus). When a request packet is turned into a response packet
(at the final slave), it moves the potentially populated source field
to the destination field, and the response packet is routed through
any multiplexing components back to the master based on the
destination field.

Modules that connect multiplexing components, such as caches and
bridges store any existing source and destination field in the sender
state as a stack (just as before).

The packet constructor is simplified in that there is no longer a need
to pass the Packet::Broadcast as the destination (this was always the
case for the classic memory system). In the case of Ruby, rather than
using the parameter to the constructor we now rely on setDest, as
there is already another three-argument constructor in the packet
class.

In many places where the packet information was printed as part of
DPRINTFs, request packets would be printed with a numeric "dest" that
would always be -1 (Broadcast) and that field is now removed from the
printing.

11 files changed, 67 insertions, 61 deletions

diff --git a/src/mem/bridge.cc b/src/mem/bridge.cc
index 4dcb2a537..ddbc154c0 100644
--- a/src/mem/bridge.cc
+++ b/src/mem/bridge.cc
@@ -144,9 +144,8 @@ Bridge::BridgeMasterPort::recvTiming(PacketPtr pkt)
 
     // all checks are done when the request is accepted on the slave
     // side, so we are guaranteed to have space for the response
-
-    DPRINTF(BusBridge, "recvTiming: src %d dest %d addr 0x%x\n",
-            pkt->getSrc(), pkt->getDest(), pkt->getAddr());
+    DPRINTF(BusBridge, "recvTiming: response %s addr 0x%x\n",
+            pkt->cmdString(), pkt->getAddr());
 
     DPRINTF(BusBridge, "Request queue size: %d\n", requestQueue.size());
 
@@ -161,8 +160,9 @@ Bridge::BridgeSlavePort::recvTiming(PacketPtr pkt)
     // should only see requests on the slave side
     assert(pkt->isRequest());
 
-    DPRINTF(BusBridge, "recvTiming: src %d dest %d addr 0x%x\n",
-            pkt->getSrc(), pkt->getDest(), pkt->getAddr());
+
+    DPRINTF(BusBridge, "recvTiming: request %s addr 0x%x\n",
+            pkt->cmdString(), pkt->getAddr());
 
     DPRINTF(BusBridge, "Response queue size: %d outresp: %d\n",
             responseQueue.size(), outstandingResponses);
@@ -277,6 +277,9 @@ Bridge::BridgeSlavePort::queueForSendTiming(PacketPtr pkt)
     // from original request
     buf->fixResponse(pkt);
 
+    // the bridge assumes that at least one bus has set the
+    // destination field of the packet
+    assert(pkt->isDestValid());
     DPRINTF(BusBridge, "response, new dest %d\n", pkt->getDest());
     delete buf;
 
@@ -304,8 +307,8 @@ Bridge::BridgeMasterPort::trySend()
 
     PacketPtr pkt = buf->pkt;
 
-    DPRINTF(BusBridge, "trySend: origSrc %d dest %d addr 0x%x\n",
-            buf->origSrc, pkt->getDest(), pkt->getAddr());
+    DPRINTF(BusBridge, "trySend: origSrc %d addr 0x%x\n",
+            buf->origSrc, pkt->getAddr());
 
     // If the send was successful, make sure sender state was set to NULL
     // otherwise we could get a NACK back of a packet that didn't expect a
diff --git a/src/mem/bridge.hh b/src/mem/bridge.hh
index 87b327ca3..45c7e3057 100644
--- a/src/mem/bridge.hh
+++ b/src/mem/bridge.hh
@@ -91,7 +91,7 @@ class Bridge : public MemObject
         PacketPtr pkt;
         bool nackedHere;
         Packet::SenderState *origSenderState;
-        short origSrc;
+        Packet::NodeID origSrc;
         bool expectResponse;
 
         PacketBuffer(PacketPtr _pkt, Tick t, bool nack = false)
diff --git a/src/mem/bus.cc b/src/mem/bus.cc
index eb26e268b..daf69c6df 100644
--- a/src/mem/bus.cc
+++ b/src/mem/bus.cc
@@ -213,24 +213,21 @@ Bus::recvTiming(PacketPtr pkt)
     // test if the bus should be considered occupied for the current
     // packet, and exclude express snoops from the check
     if (!pkt->isExpressSnoop() && isOccupied(pkt, src_port)) {
-        DPRINTF(Bus, "recvTiming: src %d dst %d %s 0x%x BUSY\n",
-                src_id, pkt->getDest(), pkt->cmdString(), pkt->getAddr());
+        DPRINTF(Bus, "recvTiming: src %s %s 0x%x BUSY\n",
+                src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
     }
 
-    DPRINTF(Bus, "recvTiming: src %d dst %d %s 0x%x\n",
-            src_id, pkt->getDest(), pkt->cmdString(), pkt->getAddr());
+    DPRINTF(Bus, "recvTiming: src %s %s 0x%x\n",
+            src_port->name(), pkt->cmdString(), pkt->getAddr());
 
     Tick headerFinishTime = pkt->isExpressSnoop() ? 0 : calcPacketTiming(pkt);
     Tick packetFinishTime = pkt->isExpressSnoop() ? 0 : pkt->finishTime;
 
     // decide what to do based on the direction
     if (pkt->isRequest()) {
-        // the packet is a memory-mapped request and should be broadcasted to
-        // our snoopers
-        assert(pkt->getDest() == Packet::Broadcast);
-
-        // forward to all snoopers but the source
+        // the packet is a memory-mapped request and should be
+        // broadcasted to our snoopers but the source
         forwardTiming(pkt, src_id);
 
         // remember if we add an outstanding req so we can undo it if
@@ -262,8 +259,8 @@ Bus::recvTiming(PacketPtr pkt)
             if (add_outstanding)
                 outstandingReq.erase(pkt->req);
 
-            DPRINTF(Bus, "recvTiming: src %d dst %d %s 0x%x RETRY\n",
-                    src_id, pkt->getDest(), pkt->cmdString(), pkt->getAddr());
+            DPRINTF(Bus, "recvTiming: src %s %s 0x%x RETRY\n",
+                    src_port->name(), pkt->cmdString(), pkt->getAddr());
 
             addToRetryList(src_port);
             occupyBus(headerFinishTime);
@@ -299,12 +296,11 @@ Bus::recvTimingSnoop(PacketPtr pkt)
     Packet::NodeID src_id = pkt->getSrc();
 
     if (pkt->isRequest()) {
-        DPRINTF(Bus, "recvTimingSnoop: src %d dst %d %s 0x%x\n",
-                src_id, pkt->getDest(), pkt->cmdString(), pkt->getAddr());
+        DPRINTF(Bus, "recvTimingSnoop: src %d %s 0x%x\n",
+                src_id, pkt->cmdString(), pkt->getAddr());
 
         // the packet is an express snoop request and should be
         // broadcasted to our snoopers
-        assert(pkt->getDest() == Packet::Broadcast);
         assert(pkt->isExpressSnoop());
 
         // forward to all snoopers
@@ -326,13 +322,13 @@ Bus::recvTimingSnoop(PacketPtr pkt)
         SlavePort* src_port = slavePorts[src_id];
 
         if (isOccupied(pkt, src_port)) {
-            DPRINTF(Bus, "recvTimingSnoop: src %d dst %d %s 0x%x BUSY\n",
-                    src_id, pkt->getDest(), pkt->cmdString(), pkt->getAddr());
+            DPRINTF(Bus, "recvTimingSnoop: src %s %s 0x%x BUSY\n",
+                    src_port->name(), pkt->cmdString(), pkt->getAddr());
             return false;
         }
 
-        DPRINTF(Bus, "recvTimingSnoop: src %d dst %d %s 0x%x\n",
-                src_id, pkt->getDest(), pkt->cmdString(), pkt->getAddr());
+        DPRINTF(Bus, "recvTimingSnoop: src %s %s 0x%x\n",
+                src_port->name(), pkt->cmdString(), pkt->getAddr());
 
         // get the destination from the packet
         Packet::NodeID dest = pkt->getDest();
@@ -532,11 +528,11 @@ Bus::findPort(Addr addr)
 Tick
 Bus::recvAtomic(PacketPtr pkt)
 {
-    DPRINTF(Bus, "recvAtomic: packet src %d dest %d addr 0x%x cmd %s\n",
-            pkt->getSrc(), pkt->getDest(), pkt->getAddr(), pkt->cmdString());
+    DPRINTF(Bus, "recvAtomic: packet src %s addr 0x%x cmd %s\n",
+            slavePorts[pkt->getSrc()]->name(), pkt->getAddr(),
+            pkt->cmdString());
 
     // we should always see a request routed based on the address
-    assert(pkt->getDest() == Packet::Broadcast);
     assert(pkt->isRequest());
 
     // forward to all snoopers but the source
@@ -566,11 +562,11 @@ Bus::recvAtomic(PacketPtr pkt)
 Tick
 Bus::recvAtomicSnoop(PacketPtr pkt)
 {
-    DPRINTF(Bus, "recvAtomicSnoop: packet src %d dest %d addr 0x%x cmd %s\n",
-            pkt->getSrc(), pkt->getDest(), pkt->getAddr(), pkt->cmdString());
+    DPRINTF(Bus, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n",
+            masterPorts[pkt->getSrc()]->name(), pkt->getAddr(),
+            pkt->cmdString());
 
     // we should always see a request routed based on the address
-    assert(pkt->getDest() == Packet::Broadcast);
     assert(pkt->isRequest());
 
     // forward to all snoopers
@@ -621,7 +617,7 @@ Bus::forwardAtomic(PacketPtr pkt, int exclude_slave_port_id)
                 // restore original packet state for remaining snoopers
                 pkt->cmd = orig_cmd;
                 pkt->setSrc(orig_src_id);
-                pkt->setDest(Packet::Broadcast);
+                pkt->clearDest();
             }
         }
     }
@@ -637,13 +633,12 @@ Bus::recvFunctional(PacketPtr pkt)
     if (!pkt->isPrint()) {
         // don't do DPRINTFs on PrintReq as it clutters up the output
         DPRINTF(Bus,
-                "recvFunctional: packet src %d dest %d addr 0x%x cmd %s\n",
-                pkt->getSrc(), pkt->getDest(), pkt->getAddr(),
+                "recvFunctional: packet src %s addr 0x%x cmd %s\n",
+                slavePorts[pkt->getSrc()]->name(), pkt->getAddr(),
                 pkt->cmdString());
     }
 
     // we should always see a request routed based on the address
-    assert(pkt->getDest() == Packet::Broadcast);
     assert(pkt->isRequest());
 
     // forward to all snoopers but the source
@@ -664,13 +659,12 @@ Bus::recvFunctionalSnoop(PacketPtr pkt)
     if (!pkt->isPrint()) {
         // don't do DPRINTFs on PrintReq as it clutters up the output
         DPRINTF(Bus,
-                "recvFunctionalSnoop: packet src %d dest %d addr 0x%x cmd %s\n",
-                pkt->getSrc(), pkt->getDest(), pkt->getAddr(),
+                "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n",
+                masterPorts[pkt->getSrc()]->name(), pkt->getAddr(),
                 pkt->cmdString());
     }
 
     // we should always see a request routed based on the address
-    assert(pkt->getDest() == Packet::Broadcast);
     assert(pkt->isRequest());
 
     // forward to all snoopers
diff --git a/src/mem/cache/cache_impl.hh b/src/mem/cache/cache_impl.hh
index fa6f6c860..fcdac1116 100644
--- a/src/mem/cache/cache_impl.hh
+++ b/src/mem/cache/cache_impl.hh
@@ -352,7 +352,7 @@ Cache<TagStore>::access(PacketPtr pkt, BlkType *&blk,
 class ForwardResponseRecord : public Packet::SenderState, public FastAlloc
 {
     Packet::SenderState *prevSenderState;
-    int prevSrc;
+    Packet::NodeID prevSrc;
 #ifndef NDEBUG
     BaseCache *cache;
 #endif
@@ -606,7 +606,7 @@ Cache<TagStore>::getBusPacket(PacketPtr cpu_pkt, BlkType *blk,
         // block is invalid
         cmd = needsExclusive ? MemCmd::ReadExReq : MemCmd::ReadReq;
     }
-    PacketPtr pkt = new Packet(cpu_pkt->req, cmd, Packet::Broadcast, blkSize);
+    PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
 
     pkt->allocate();
     return pkt;
@@ -1002,7 +1002,7 @@ Cache<TagStore>::writebackBlk(BlkType *blk)
     Request *writebackReq =
         new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0,
                 Request::wbMasterId);
-    PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback, -1);
+    PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback);
     if (blk->isWritable()) {
         writeback->setSupplyExclusive();
     }
diff --git a/src/mem/cache/prefetch/base.cc b/src/mem/cache/prefetch/base.cc
index 467550823..4d1f7c81c 100644
--- a/src/mem/cache/prefetch/base.cc
+++ b/src/mem/cache/prefetch/base.cc
@@ -234,7 +234,7 @@ BasePrefetcher::notify(PacketPtr &pkt, Tick time)
             // create a prefetch memreq
             Request *prefetchReq = new Request(*addrIter, blkSize, 0, masterId);
             PacketPtr prefetch =
-                new Packet(prefetchReq, MemCmd::HardPFReq, Packet::Broadcast);
+                new Packet(prefetchReq, MemCmd::HardPFReq);
             prefetch->allocate();
             prefetch->req->setThreadContext(pkt->req->contextId(),
                                             pkt->req->threadId());
diff --git a/src/mem/cache/tags/iic.cc b/src/mem/cache/tags/iic.cc
index 3501ec378..260b89194 100644
--- a/src/mem/cache/tags/iic.cc
+++ b/src/mem/cache/tags/iic.cc
@@ -370,8 +370,7 @@ IIC::freeReplacementBlock(PacketList & writebacks)
 */
             Request *writebackReq = new Request(regenerateBlkAddr(tag_ptr->tag, 0),
                                            blkSize, 0, Request::wbMasterId);
-            PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback,
-                                             -1);
+            PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback);
             writeback->allocate();
             memcpy(writeback->getPtr<uint8_t>(), tag_ptr->data, blkSize);
 
diff --git a/src/mem/packet.hh b/src/mem/packet.hh
index ce5748c24..e49fa67b8 100644
--- a/src/mem/packet.hh
+++ b/src/mem/packet.hh
@@ -1,4 +1,16 @@
 /*
+ * Copyright (c) 2012 ARM Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
  * Copyright (c) 2006 The Regents of The University of Michigan
  * Copyright (c) 2010 Advanced Micro Devices, Inc.
  * All rights reserved.
@@ -317,10 +329,6 @@ class Packet : public FastAlloc, public Printable
     /// The time at which the first chunk of the packet will be transmitted
     Tick firstWordTime;
 
-    /// The special destination address indicating that the packet
-    /// should be routed based on its address.
-    static const NodeID Broadcast = -1;
-
     /**
      * A virtual base opaque structure used to hold state associated
      * with the packet but specific to the sending device (e.g., an
@@ -478,6 +486,8 @@ class Packet : public FastAlloc, public Printable
     NodeID getDest() const     { assert(flags.isSet(VALID_DST)); return dest; }
     /// Accessor function to set the destination index of the packet.
     void setDest(NodeID _dest) { dest = _dest; flags.set(VALID_DST); }
+    /// Reset destination field, e.g. to turn a response into a request again.
+    void clearDest() { flags.clear(VALID_DST); }
 
     Addr getAddr() const { assert(flags.isSet(VALID_ADDR)); return addr; }
     unsigned getSize() const  { assert(flags.isSet(VALID_SIZE)); return size; }
@@ -513,9 +523,9 @@ class Packet : public FastAlloc, public Printable
      * not be valid. The command and destination addresses must be
      * supplied.
      */
-    Packet(Request *_req, MemCmd _cmd, NodeID _dest)
-        :  flags(VALID_DST), cmd(_cmd), req(_req), data(NULL),
-           dest(_dest), bytesValidStart(0), bytesValidEnd(0),
+    Packet(Request *_req, MemCmd _cmd)
+        :  cmd(_cmd), req(_req), data(NULL),
+           bytesValidStart(0), bytesValidEnd(0),
            time(curTick()), senderState(NULL)
     {
         if (req->hasPaddr()) {
@@ -533,9 +543,9 @@ class Packet : public FastAlloc, public Printable
      * a request that is for a whole block, not the address from the
      * req.  this allows for overriding the size/addr of the req.
      */
-    Packet(Request *_req, MemCmd _cmd, NodeID _dest, int _blkSize)
-        :  flags(VALID_DST), cmd(_cmd), req(_req), data(NULL),
-           dest(_dest), bytesValidStart(0), bytesValidEnd(0),
+    Packet(Request *_req, MemCmd _cmd, int _blkSize)
+        :  cmd(_cmd), req(_req), data(NULL),
+           bytesValidStart(0), bytesValidEnd(0),
            time(curTick()), senderState(NULL)
     {
         if (req->hasPaddr()) {
@@ -659,7 +669,7 @@ class Packet : public FastAlloc, public Printable
         assert(wasNacked());
         cmd = origCmd;
         assert(needsResponse());
-        setDest(Broadcast);
+        clearDest();
     }
 
     void
diff --git a/src/mem/port.cc b/src/mem/port.cc
index 92c6aaab8..4b7b040cb 100644
--- a/src/mem/port.cc
+++ b/src/mem/port.cc
@@ -120,7 +120,7 @@ void
 MasterPort::printAddr(Addr a)
 {
     Request req(a, 1, 0, Request::funcMasterId);
-    Packet pkt(&req, MemCmd::PrintReq, Packet::Broadcast);
+    Packet pkt(&req, MemCmd::PrintReq);
     Packet::PrintReqState prs(std::cerr);
     pkt.senderState = &prs;
 
diff --git a/src/mem/port_proxy.cc b/src/mem/port_proxy.cc
index c1fd41d3c..38162e1bf 100644
--- a/src/mem/port_proxy.cc
+++ b/src/mem/port_proxy.cc
@@ -48,7 +48,7 @@ PortProxy::blobHelper(Addr addr, uint8_t *p, int size, MemCmd cmd) const
     for (ChunkGenerator gen(addr, size, _port.peerBlockSize());
          !gen.done(); gen.next()) {
         req.setPhys(gen.addr(), gen.size(), 0, Request::funcMasterId);
-        Packet pkt(&req, cmd, Packet::Broadcast);
+        Packet pkt(&req, cmd);
         pkt.dataStatic(p);
         _port.sendFunctional(&pkt);
         p += gen.size();
diff --git a/src/mem/ruby/recorder/CacheRecorder.cc b/src/mem/ruby/recorder/CacheRecorder.cc
index a886f3238..c5593d945 100644
--- a/src/mem/ruby/recorder/CacheRecorder.cc
+++ b/src/mem/ruby/recorder/CacheRecorder.cc
@@ -77,7 +77,7 @@ CacheRecorder::enqueueNextFlushRequest()
                                    RubySystem::getBlockSizeBytes(),0,
                                    Request::funcMasterId);
         MemCmd::Command requestType = MemCmd::FlushReq;
-        Packet *pkt = new Packet(req, requestType, -1);
+        Packet *pkt = new Packet(req, requestType);
 
         Sequencer* m_sequencer_ptr = m_seq_map[rec->m_cntrl_id];
         assert(m_sequencer_ptr != NULL);
@@ -113,7 +113,7 @@ CacheRecorder::enqueueNextFetchRequest()
                     RubySystem::getBlockSizeBytes(),0, Request::funcMasterId);
         }
 
-        Packet *pkt = new Packet(req, requestType, -1);
+        Packet *pkt = new Packet(req, requestType);
         pkt->dataStatic(traceRecord->m_data);
 
         Sequencer* m_sequencer_ptr = m_seq_map[traceRecord->m_cntrl_id];
diff --git a/src/mem/ruby/system/RubyPort.cc b/src/mem/ruby/system/RubyPort.cc
index 74a60f863..ef9f59645 100644
--- a/src/mem/ruby/system/RubyPort.cc
+++ b/src/mem/ruby/system/RubyPort.cc
@@ -699,7 +699,7 @@ RubyPort::ruby_eviction_callback(const Address& address)
     Request req(address.getAddress(), 0, 0, Request::funcMasterId);
     for (CpuPortIter p = slave_ports.begin(); p != slave_ports.end(); ++p) {
         if ((*p)->getMasterPort().isSnooping()) {
-            Packet *pkt = new Packet(&req, MemCmd::InvalidationReq, -1);
+            Packet *pkt = new Packet(&req, MemCmd::InvalidationReq);
             // send as a snoop request
             (*p)->sendNextCycle(pkt, true);
         }