mem: Make the XBar responsible for tracking response routing

This patch removes the need for a source and destination field in the
packet by shifting the onus of the tracking to the crossbar, much like
a real implementation. This change in behaviour also means we no
longer need a SenderState to remember the source/dest when ever we
have multiple crossbars in the system. Thus, the stack that was
created by the SenderState is not needed, and each crossbar locally
tracks the response routing.

The fields in the packet are still left behind as the RubyPort (which
also acts as a crossbar) does routing based on them. In the succeeding
patches the uses of the src and dest field will be removed. Combined,
these patches improve the simulation performance by roughly 2%.

1 files changed, 18 insertions, 5 deletions

diff --git a/src/mem/noncoherent_xbar.cc b/src/mem/noncoherent_xbar.cc
index 476b8314f..e93446b47 100644
--- a/src/mem/noncoherent_xbar.cc
+++ b/src/mem/noncoherent_xbar.cc
@@ -127,12 +127,14 @@ NoncoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
     unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
     unsigned int pkt_cmd = pkt->cmdToIndex();
 
-    // set the source port for routing of the response
-    pkt->setSrc(slave_port_id);
-
     calcPacketTiming(pkt);
     Tick packetFinishTime = pkt->lastWordDelay + curTick();
 
+    // before forwarding the packet (and possibly altering it),
+    // remember if we are expecting a response
+    const bool expect_response = pkt->needsResponse() &&
+        !pkt->memInhibitAsserted();
+
     // since it is a normal request, attempt to send the packet
     bool success = masterPorts[master_port_id]->sendTimingReq(pkt);
 
@@ -153,6 +155,12 @@ NoncoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
         return false;
     }
 
+    // remember where to route the response to
+    if (expect_response) {
+        assert(routeTo.find(pkt->req) == routeTo.end());
+        routeTo[pkt->req] = slave_port_id;
+    }
+
     reqLayers[master_port_id]->succeededTiming(packetFinishTime);
 
     // stats updates
@@ -169,8 +177,10 @@ NoncoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id)
     // determine the source port based on the id
     MasterPort *src_port = masterPorts[master_port_id];
 
-    // determine the destination based on what is stored in the packet
-    PortID slave_port_id = pkt->getDest();
+    // determine the destination
+    const auto route_lookup = routeTo.find(pkt->req);
+    assert(route_lookup != routeTo.end());
+    const PortID slave_port_id = route_lookup->second;
     assert(slave_port_id != InvalidPortID);
     assert(slave_port_id < respLayers.size());
 
@@ -200,6 +210,9 @@ NoncoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id)
     // deadlock
     assert(success);
 
+    // remove the request from the routing table
+    routeTo.erase(route_lookup);
+
     respLayers[slave_port_id]->succeededTiming(packetFinishTime);
 
     // stats updates