mem: Make the buses multi layered

This patch makes the buses multi layered, and effectively creates a
crossbar structure with distributed contention ports at the
destination ports. Before this patch, a bus could have a single
request, response and snoop response in flight at any time, and with
these changes there can be as many requests as connected slaves (bus
master ports), and as many responses as connected masters (bus slave
ports).

Together with address interleaving, this patch enables us to create
high-throughput memory interconnects, e.g. 50+ GByte/s.

6 files changed, 151 insertions, 108 deletions

diff --git a/src/mem/bus.cc b/src/mem/bus.cc
index 3fa7c7231..855f82db4 100644
--- a/src/mem/bus.cc
+++ b/src/mem/bus.cc
@@ -156,18 +156,17 @@ BaseBus::calcPacketTiming(PacketPtr pkt)
         offset;
 }
 
-template <typename PortClass>
-BaseBus::Layer<PortClass>::Layer(BaseBus& _bus, const std::string& _name,
-                                 uint16_t num_dest_ports) :
-    Drainable(),
-    bus(_bus), _name(_name), state(IDLE), drainManager(NULL),
-    retryingPort(NULL), waitingForPeer(num_dest_ports, NULL),
+template <typename SrcType, typename DstType>
+BaseBus::Layer<SrcType,DstType>::Layer(DstType& _port, BaseBus& _bus,
+                                       const std::string& _name) :
+    port(_port), bus(_bus), _name(_name), state(IDLE), drainManager(NULL),
+    retryingPort(NULL), waitingForPeer(NULL),
     releaseEvent(this)
 {
 }
 
-template <typename PortClass>
-void BaseBus::Layer<PortClass>::occupyLayer(Tick until)
+template <typename SrcType, typename DstType>
+void BaseBus::Layer<SrcType,DstType>::occupyLayer(Tick until)
 {
     // ensure the state is busy at this point, as the bus should
     // transition from idle as soon as it has decided to forward the
@@ -186,21 +185,21 @@ void BaseBus::Layer<PortClass>::occupyLayer(Tick until)
             curTick(), until);
 }
 
-template <typename PortClass>
+template <typename SrcType, typename DstType>
 bool
-BaseBus::Layer<PortClass>::tryTiming(PortClass* port, PortID dest_port_id)
+BaseBus::Layer<SrcType,DstType>::tryTiming(SrcType* src_port)
 {
-    // first we see if the bus is busy, next we check if we are in a
+    // first we see if the layer is busy, next we check if we are in a
     // retry with a port other than the current one, lastly we check
     // if the destination port is already engaged in a transaction
     // waiting for a retry from the peer
-    if (state == BUSY || (state == RETRY && port != retryingPort) ||
-        waitingForPeer[dest_port_id] != NULL) {
+    if (state == BUSY || (state == RETRY && src_port != retryingPort) ||
+        waitingForPeer != NULL) {
         // put the port at the end of the retry list waiting for the
         // layer to be freed up (and in the case of a busy peer, for
         // that transaction to go through, and then the bus to free
         // up)
-        waitingForLayer.push_back(port);
+        waitingForLayer.push_back(src_port);
         return false;
     }
 
@@ -213,9 +212,9 @@ BaseBus::Layer<PortClass>::tryTiming(PortClass* port, PortID dest_port_id)
     return true;
 }
 
-template <typename PortClass>
+template <typename SrcType, typename DstType>
 void
-BaseBus::Layer<PortClass>::succeededTiming(Tick busy_time)
+BaseBus::Layer<SrcType,DstType>::succeededTiming(Tick busy_time)
 {
     // we should have gone from idle or retry to busy in the tryTiming
     // test
@@ -225,19 +224,19 @@ BaseBus::Layer<PortClass>::succeededTiming(Tick busy_time)
     occupyLayer(busy_time);
 }
 
-template <typename PortClass>
+template <typename SrcType, typename DstType>
 void
-BaseBus::Layer<PortClass>::failedTiming(PortClass* src_port,
-                                        PortID dest_port_id, Tick busy_time)
+BaseBus::Layer<SrcType,DstType>::failedTiming(SrcType* src_port,
+                                              Tick busy_time)
 {
     // ensure no one got in between and tried to send something to
     // this port
-    assert(waitingForPeer[dest_port_id] == NULL);
+    assert(waitingForPeer == NULL);
 
     // if the source port is the current retrying one or not, we have
     // failed in forwarding and should track that we are now waiting
     // for the peer to send a retry
-    waitingForPeer[dest_port_id] = src_port;
+    waitingForPeer = src_port;
 
     // we should have gone from idle or retry to busy in the tryTiming
     // test
@@ -247,9 +246,9 @@ BaseBus::Layer<PortClass>::failedTiming(PortClass* src_port,
     occupyLayer(busy_time);
 }
 
-template <typename PortClass>
+template <typename SrcType, typename DstType>
 void
-BaseBus::Layer<PortClass>::releaseLayer()
+BaseBus::Layer<SrcType,DstType>::releaseLayer()
 {
     // releasing the bus means we should now be idle
     assert(state == BUSY);
@@ -270,9 +269,9 @@ BaseBus::Layer<PortClass>::releaseLayer()
     }
 }
 
-template <typename PortClass>
+template <typename SrcType, typename DstType>
 void
-BaseBus::Layer<PortClass>::retryWaiting()
+BaseBus::Layer<SrcType,DstType>::retryWaiting()
 {
     // this should never be called with no one waiting
     assert(!waitingForLayer.empty());
@@ -306,23 +305,21 @@ BaseBus::Layer<PortClass>::retryWaiting()
     }
 }
 
-template <typename PortClass>
+template <typename SrcType, typename DstType>
 void
-BaseBus::Layer<PortClass>::recvRetry(PortID port_id)
+BaseBus::Layer<SrcType,DstType>::recvRetry()
 {
     // we should never get a retry without having failed to forward
     // something to this port
-    assert(waitingForPeer[port_id] != NULL);
+    assert(waitingForPeer != NULL);
 
-    // find the port where the failed packet originated and remove the
-    // item from the waiting list
-    PortClass* retry_port = waitingForPeer[port_id];
-    waitingForPeer[port_id] = NULL;
+    // add the port where the failed packet originated to the front of
+    // the waiting ports for the layer, this allows us to call retry
+    // on the port immediately if the bus layer is idle
+    waitingForLayer.push_front(waitingForPeer);
 
-    // add this port at the front of the waiting ports for the layer,
-    // this allows us to call retry on the port immediately if the bus
-    // layer is idle
-    waitingForLayer.push_front(retry_port);
+    // we are no longer waiting for the peer
+    waitingForPeer = NULL;
 
     // if the bus layer is idle, retry this port straight away, if we
     // are busy, then simply let the port wait for its turn
@@ -606,9 +603,9 @@ BaseBus::regStats()
     }
 }
 
-template <typename PortClass>
+template <typename SrcType, typename DstType>
 unsigned int
-BaseBus::Layer<PortClass>::drain(DrainManager *dm)
+BaseBus::Layer<SrcType,DstType>::drain(DrainManager *dm)
 {
     //We should check that we're not "doing" anything, and that noone is
     //waiting. We might be idle but have someone waiting if the device we
@@ -621,9 +618,9 @@ BaseBus::Layer<PortClass>::drain(DrainManager *dm)
     return 0;
 }
 
-template <typename PortClass>
+template <typename SrcType, typename DstType>
 void
-BaseBus::Layer<PortClass>::regStats()
+BaseBus::Layer<SrcType,DstType>::regStats()
 {
     using namespace Stats;
 
@@ -646,5 +643,5 @@ BaseBus::Layer<PortClass>::regStats()
  * file, but since there are only two given options (MasterPort and
  * SlavePort) it seems a bit excessive at this point.
  */
-template class BaseBus::Layer<SlavePort>;
-template class BaseBus::Layer<MasterPort>;
+template class BaseBus::Layer<SlavePort,MasterPort>;
+template class BaseBus::Layer<MasterPort,SlavePort>;
diff --git a/src/mem/bus.hh b/src/mem/bus.hh
index 079d6a08d..fc240a22b 100644
--- a/src/mem/bus.hh
+++ b/src/mem/bus.hh
@@ -93,7 +93,7 @@ class BaseBus : public MemObject
      * a request layer has a retry list containing slave ports,
      * whereas a response layer holds master ports.
      */
-    template <typename PortClass>
+    template <typename SrcType, typename DstType>
     class Layer : public Drainable
     {
 
@@ -103,11 +103,11 @@ class BaseBus : public MemObject
          * Create a bus layer and give it a name. The bus layer uses
          * the bus an event manager.
          *
+         * @param _port destination port the layer converges at
          * @param _bus the bus this layer belongs to
          * @param _name the layer's name
-         * @param num_dest_ports number of destination ports
          */
-        Layer(BaseBus& _bus, const std::string& _name, uint16_t num_dest_ports);
+        Layer(DstType& _port, BaseBus& _bus, const std::string& _name);
 
         /**
          * Drain according to the normal semantics, so that the bus
@@ -133,11 +133,10 @@ class BaseBus : public MemObject
          * updated accordingly.
          *
          * @param port Source port presenting the packet
-         * @param dest_port_id Destination port id
          *
          * @return True if the bus layer accepts the packet
          */
-        bool tryTiming(PortClass* port, PortID dest_port_id);
+        bool tryTiming(SrcType* src_port);
 
         /**
          * Deal with a destination port accepting a packet by potentially
@@ -155,11 +154,9 @@ class BaseBus : public MemObject
          * accordingly.
          *
          * @param src_port Source port
-         * @param dest_port_id Destination port id
          * @param busy_time Time to spend as a result of a failed send
          */
-        void failedTiming(PortClass* src_port, PortID dest_port_id,
-                          Tick busy_time);
+        void failedTiming(SrcType* src_port, Tick busy_time);
 
         /** Occupy the bus layer until until */
         void occupyLayer(Tick until);
@@ -174,10 +171,8 @@ class BaseBus : public MemObject
          * Handle a retry from a neighbouring module. This wraps
          * retryWaiting by verifying that there are ports waiting
          * before calling retryWaiting.
-         *
-         * @param port_id Id of the port that received the retry
          */
-        void recvRetry(PortID port_id);
+        void recvRetry();
 
         /**
          * Register stats for the layer
@@ -186,6 +181,9 @@ class BaseBus : public MemObject
 
       private:
 
+        /** The destination port this layer converges at. */
+        DstType& port;
+
         /** The bus this layer is a part of. */
         BaseBus& bus;
 
@@ -220,7 +218,7 @@ class BaseBus : public MemObject
          * A deque of ports that retry should be called on because
          * the original send was delayed due to a busy layer.
          */
-        std::deque<PortClass*> waitingForLayer;
+        std::deque<SrcType*> waitingForLayer;
 
         /**
          * Port that we are currently in the process of telling to
@@ -228,18 +226,13 @@ class BaseBus : public MemObject
          * transaction. This is a valid port when in the retry state,
          * and NULL when in busy or idle.
          */
-        PortClass* retryingPort;
+        SrcType* retryingPort;
 
         /**
-         * A vector that tracks who is waiting for the retry when
-         * receiving it from a peer. The vector indices are port ids
-         * of the outgoing ports for the specific layer. The values
-         * are the incoming ports that tried to forward something to
-         * the outgoing port, but was told to wait and is now waiting
-         * for a retry. If no port is waiting NULL is stored on the
-         * location in question.
+         * Track who is waiting for the retry when receiving it from a
+         * peer. If no port is waiting NULL is stored.
          */
-        std::vector<PortClass*> waitingForPeer;
+        SrcType* waitingForPeer;
 
         /**
          * Release the bus layer after being occupied and return to an
diff --git a/src/mem/coherent_bus.cc b/src/mem/coherent_bus.cc
index 923fa08d6..20597bc3a 100644
--- a/src/mem/coherent_bus.cc
+++ b/src/mem/coherent_bus.cc
@@ -55,14 +55,7 @@
 #include "sim/system.hh"
 
 CoherentBus::CoherentBus(const CoherentBusParams *p)
-    : BaseBus(p),
-      reqLayer(*this, ".reqLayer", p->port_master_connection_count +
-               p->port_default_connection_count),
-      respLayer(*this, ".respLayer", p->port_slave_connection_count),
-      snoopRespLayer(*this, ".snoopRespLayer",
-                     p->port_master_connection_count +
-                     p->port_default_connection_count),
-      system(p->system)
+    : BaseBus(p), system(p->system)
 {
     // create the ports based on the size of the master and slave
     // vector ports, and the presence of the default port, the ports
@@ -71,6 +64,10 @@ CoherentBus::CoherentBus(const CoherentBusParams *p)
         std::string portName = csprintf("%s.master[%d]", name(), i);
         MasterPort* bp = new CoherentBusMasterPort(portName, *this, i);
         masterPorts.push_back(bp);
+        reqLayers.push_back(new ReqLayer(*bp, *this,
+                                         csprintf(".reqLayer%d", i)));
+        snoopLayers.push_back(new SnoopLayer(*bp, *this,
+                                             csprintf(".snoopLayer%d", i)));
     }
 
     // see if we have a default slave device connected and if so add
@@ -81,6 +78,11 @@ CoherentBus::CoherentBus(const CoherentBusParams *p)
         MasterPort* bp = new CoherentBusMasterPort(portName, *this,
                                                    defaultPortID);
         masterPorts.push_back(bp);
+        reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d",
+                                             defaultPortID)));
+        snoopLayers.push_back(new SnoopLayer(*bp, *this,
+                                             csprintf(".snoopLayer%d",
+                                                      defaultPortID)));
     }
 
     // create the slave ports, once again starting at zero
@@ -88,11 +90,23 @@ CoherentBus::CoherentBus(const CoherentBusParams *p)
         std::string portName = csprintf("%s.slave[%d]", name(), i);
         SlavePort* bp = new CoherentBusSlavePort(portName, *this, i);
         slavePorts.push_back(bp);
+        respLayers.push_back(new RespLayer(*bp, *this,
+                                           csprintf(".respLayer%d", i)));
     }
 
     clearPortCache();
 }
 
+CoherentBus::~CoherentBus()
+{
+    for (auto l = reqLayers.begin(); l != reqLayers.end(); ++l)
+        delete *l;
+    for (auto l = respLayers.begin(); l != respLayers.end(); ++l)
+        delete *l;
+    for (auto l = snoopLayers.begin(); l != snoopLayers.end(); ++l)
+        delete *l;
+}
+
 void
 CoherentBus::init()
 {
@@ -129,7 +143,7 @@ CoherentBus::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
 
     // test if the bus should be considered occupied for the current
     // port, and exclude express snoops from the check
-    if (!is_express_snoop && !reqLayer.tryTiming(src_port, master_port_id)) {
+    if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) {
         DPRINTF(CoherentBus, "recvTimingReq: src %s %s 0x%x BUS BUSY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
@@ -198,11 +212,11 @@ CoherentBus::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
                     src_port->name(), pkt->cmdString(), pkt->getAddr());
 
             // update the bus state and schedule an idle event
-            reqLayer.failedTiming(src_port, master_port_id,
-                                  clockEdge(headerCycles));
+            reqLayers[master_port_id]->failedTiming(src_port,
+                                                    clockEdge(headerCycles));
         } else {
             // update the bus state and schedule an idle event
-            reqLayer.succeededTiming(packetFinishTime);
+            reqLayers[master_port_id]->succeededTiming(packetFinishTime);
             dataThroughBus += pkt_size;
         }
     }
@@ -228,7 +242,7 @@ CoherentBus::recvTimingResp(PacketPtr pkt, PortID master_port_id)
 
     // test if the bus should be considered occupied for the current
     // port
-    if (!respLayer.tryTiming(src_port, slave_port_id)) {
+    if (!respLayers[slave_port_id]->tryTiming(src_port)) {
         DPRINTF(CoherentBus, "recvTimingResp: src %s %s 0x%x BUSY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
@@ -259,7 +273,7 @@ CoherentBus::recvTimingResp(PacketPtr pkt, PortID master_port_id)
     // deadlock
     assert(success);
 
-    respLayer.succeededTiming(packetFinishTime);
+    respLayers[slave_port_id]->succeededTiming(packetFinishTime);
 
     // stats updates
     dataThroughBus += pkt_size;
@@ -318,10 +332,12 @@ CoherentBus::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
     // port, note that the check is bypassed if the response is being
     // passed on as a normal response since this is occupying the
     // response layer rather than the snoop response layer
-    if (forwardAsSnoop && !snoopRespLayer.tryTiming(src_port, dest_port_id)) {
-        DPRINTF(CoherentBus, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n",
-                src_port->name(), pkt->cmdString(), pkt->getAddr());
-        return false;
+    if (forwardAsSnoop) {
+        if (!snoopLayers[dest_port_id]->tryTiming(src_port)) {
+            DPRINTF(CoherentBus, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n",
+                    src_port->name(), pkt->cmdString(), pkt->getAddr());
+            return false;
+        }
     }
 
     DPRINTF(CoherentBus, "recvTimingSnoopResp: src %s %s 0x%x\n",
@@ -349,7 +365,7 @@ CoherentBus::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
         totPktSize[slave_port_id][dest_port_id] += pkt_size;
         assert(success);
 
-        snoopRespLayer.succeededTiming(packetFinishTime);
+        snoopLayers[dest_port_id]->succeededTiming(packetFinishTime);
     } else {
         // we got a snoop response on one of our slave ports,
         // i.e. from a coherent master connected to the bus, and
@@ -416,7 +432,7 @@ CoherentBus::recvRetry(PortID master_port_id)
     // responses and snoop responses never block on forwarding them,
     // so the retry will always be coming from a port to which we
     // tried to forward a request
-    reqLayer.recvRetry(master_port_id);
+    reqLayers[master_port_id]->recvRetry();
 }
 
 Tick
@@ -606,7 +622,14 @@ unsigned int
 CoherentBus::drain(DrainManager *dm)
 {
     // sum up the individual layers
-    return reqLayer.drain(dm) + respLayer.drain(dm) + snoopRespLayer.drain(dm);
+    unsigned int total = 0;
+    for (auto l = reqLayers.begin(); l != reqLayers.end(); ++l)
+        total += (*l)->drain(dm);
+    for (auto l = respLayers.begin(); l != respLayers.end(); ++l)
+        total += (*l)->drain(dm);
+    for (auto l = snoopLayers.begin(); l != snoopLayers.end(); ++l)
+        total += (*l)->drain(dm);
+    return total;
 }
 
 void
@@ -614,9 +637,12 @@ CoherentBus::regStats()
 {
     // register the stats of the base class and our three bus layers
     BaseBus::regStats();
-    reqLayer.regStats();
-    respLayer.regStats();
-    snoopRespLayer.regStats();
+    for (auto l = reqLayers.begin(); l != reqLayers.end(); ++l)
+        (*l)->regStats();
+    for (auto l = respLayers.begin(); l != respLayers.end(); ++l)
+        (*l)->regStats();
+    for (auto l = snoopLayers.begin(); l != snoopLayers.end(); ++l)
+        (*l)->regStats();
 
     dataThroughBus
         .name(name() + ".data_through_bus")
diff --git a/src/mem/coherent_bus.hh b/src/mem/coherent_bus.hh
index eb8b41e6a..d8ddd507c 100644
--- a/src/mem/coherent_bus.hh
+++ b/src/mem/coherent_bus.hh
@@ -72,12 +72,15 @@ class CoherentBus : public BaseBus
   protected:
 
     /**
-     * Declare the three layers of this bus, one for requests, one
+     * Declare the layers of this bus, one vector for requests, one
      * for responses, and one for snoop responses
      */
-    Layer<SlavePort> reqLayer;
-    Layer<MasterPort> respLayer;
-    Layer<SlavePort> snoopRespLayer;
+    typedef Layer<SlavePort,MasterPort> ReqLayer;
+    typedef Layer<MasterPort,SlavePort> RespLayer;
+    typedef Layer<SlavePort,MasterPort> SnoopLayer;
+    std::vector<ReqLayer*> reqLayers;
+    std::vector<RespLayer*> respLayers;
+    std::vector<SnoopLayer*> snoopLayers;
 
     /**
      * Declaration of the coherent bus slave port type, one will be
@@ -309,6 +312,8 @@ class CoherentBus : public BaseBus
 
     CoherentBus(const CoherentBusParams *p);
 
+    virtual ~CoherentBus();
+
     unsigned int drain(DrainManager *dm);
 
     virtual void regStats();
diff --git a/src/mem/noncoherent_bus.cc b/src/mem/noncoherent_bus.cc
index cc5d49cab..d34ae6d20 100644
--- a/src/mem/noncoherent_bus.cc
+++ b/src/mem/noncoherent_bus.cc
@@ -55,10 +55,7 @@
 #include "mem/noncoherent_bus.hh"
 
 NoncoherentBus::NoncoherentBus(const NoncoherentBusParams *p)
-    : BaseBus(p),
-      reqLayer(*this, ".reqLayer", p->port_master_connection_count +
-               p->port_default_connection_count),
-      respLayer(*this, ".respLayer", p->port_slave_connection_count)
+    : BaseBus(p)
 {
     // create the ports based on the size of the master and slave
     // vector ports, and the presence of the default port, the ports
@@ -67,6 +64,8 @@ NoncoherentBus::NoncoherentBus(const NoncoherentBusParams *p)
         std::string portName = csprintf("%s.master[%d]", name(), i);
         MasterPort* bp = new NoncoherentBusMasterPort(portName, *this, i);
         masterPorts.push_back(bp);
+        reqLayers.push_back(new ReqLayer(*bp, *this,
+                                         csprintf(".reqLayer%d", i)));
     }
 
     // see if we have a default slave device connected and if so add
@@ -77,6 +76,8 @@ NoncoherentBus::NoncoherentBus(const NoncoherentBusParams *p)
         MasterPort* bp = new NoncoherentBusMasterPort(portName, *this,
                                                       defaultPortID);
         masterPorts.push_back(bp);
+        reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d",
+                                                              defaultPortID)));
     }
 
     // create the slave ports, once again starting at zero
@@ -84,11 +85,21 @@ NoncoherentBus::NoncoherentBus(const NoncoherentBusParams *p)
         std::string portName = csprintf("%s.slave[%d]", name(), i);
         SlavePort* bp = new NoncoherentBusSlavePort(portName, *this, i);
         slavePorts.push_back(bp);
+        respLayers.push_back(new RespLayer(*bp, *this,
+                                           csprintf(".respLayer%d", i)));
     }
 
     clearPortCache();
 }
 
+NoncoherentBus::~NoncoherentBus()
+{
+    for (auto l = reqLayers.begin(); l != reqLayers.end(); ++l)
+        delete *l;
+    for (auto l = respLayers.begin(); l != respLayers.end(); ++l)
+        delete *l;
+}
+
 bool
 NoncoherentBus::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
 {
@@ -103,7 +114,7 @@ NoncoherentBus::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
 
     // test if the bus should be considered occupied for the current
     // port
-    if (!reqLayer.tryTiming(src_port, master_port_id)) {
+    if (!reqLayers[master_port_id]->tryTiming(src_port)) {
         DPRINTF(NoncoherentBus, "recvTimingReq: src %s %s 0x%x BUSY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
@@ -137,13 +148,13 @@ NoncoherentBus::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
         pkt->busFirstWordDelay = pkt->busLastWordDelay = 0;
 
         // occupy until the header is sent
-        reqLayer.failedTiming(src_port, master_port_id,
-                              clockEdge(headerCycles));
+        reqLayers[master_port_id]->failedTiming(src_port,
+                                                clockEdge(headerCycles));
 
         return false;
     }
 
-    reqLayer.succeededTiming(packetFinishTime);
+    reqLayers[master_port_id]->succeededTiming(packetFinishTime);
 
     // stats updates
     dataThroughBus += pkt_size;
@@ -165,7 +176,7 @@ NoncoherentBus::recvTimingResp(PacketPtr pkt, PortID master_port_id)
 
     // test if the bus should be considered occupied for the current
     // port
-    if (!respLayer.tryTiming(src_port, slave_port_id)) {
+    if (!respLayers[slave_port_id]->tryTiming(src_port)) {
         DPRINTF(NoncoherentBus, "recvTimingResp: src %s %s 0x%x BUSY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
@@ -189,7 +200,7 @@ NoncoherentBus::recvTimingResp(PacketPtr pkt, PortID master_port_id)
     // deadlock
     assert(success);
 
-    respLayer.succeededTiming(packetFinishTime);
+    respLayers[slave_port_id]->succeededTiming(packetFinishTime);
 
     // stats updates
     dataThroughBus += pkt_size;
@@ -206,7 +217,7 @@ NoncoherentBus::recvRetry(PortID master_port_id)
     // responses never block on forwarding them, so the retry will
     // always be coming from a port to which we tried to forward a
     // request
-    reqLayer.recvRetry(master_port_id);
+    reqLayers[master_port_id]->recvRetry();
 }
 
 Tick
@@ -256,7 +267,12 @@ unsigned int
 NoncoherentBus::drain(DrainManager *dm)
 {
     // sum up the individual layers
-    return reqLayer.drain(dm) + respLayer.drain(dm);
+    unsigned int total = 0;
+    for (auto l = reqLayers.begin(); l != reqLayers.end(); ++l)
+        total += (*l)->drain(dm);
+    for (auto l = respLayers.begin(); l != respLayers.end(); ++l)
+        total += (*l)->drain(dm);
+    return total;
 }
 
 NoncoherentBus*
@@ -270,8 +286,10 @@ NoncoherentBus::regStats()
 {
     // register the stats of the base class and our two bus layers
     BaseBus::regStats();
-    reqLayer.regStats();
-    respLayer.regStats();
+    for (auto l = reqLayers.begin(); l != reqLayers.end(); ++l)
+        (*l)->regStats();
+    for (auto l = respLayers.begin(); l != respLayers.end(); ++l)
+        (*l)->regStats();
 
     dataThroughBus
         .name(name() + ".data_through_bus")
diff --git a/src/mem/noncoherent_bus.hh b/src/mem/noncoherent_bus.hh
index 8fc2c40d5..e2148c60e 100644
--- a/src/mem/noncoherent_bus.hh
+++ b/src/mem/noncoherent_bus.hh
@@ -73,11 +73,13 @@ class NoncoherentBus : public BaseBus
   protected:
 
     /**
-     * Declare the two layers of this bus, one for requests and one
+     * Declare the layers of this bus, one vector for requests and one
      * for responses.
      */
-    Layer<SlavePort> reqLayer;
-    Layer<MasterPort> respLayer;
+    typedef Layer<SlavePort,MasterPort> ReqLayer;
+    typedef Layer<MasterPort,SlavePort> RespLayer;
+    std::vector<ReqLayer*> reqLayers;
+    std::vector<RespLayer*> respLayers;
 
     /**
      * Declaration of the non-coherent bus slave port type, one will
@@ -207,6 +209,8 @@ class NoncoherentBus : public BaseBus
 
     NoncoherentBus(const NoncoherentBusParams *p);
 
+    virtual ~NoncoherentBus();
+
     unsigned int drain(DrainManager *dm);
 
     /**