Bus: Add a notion of layers to the buses

This patch moves all flow control, arbitration and state information
into a bus layer. The layer is thus responsible for all the state
transitions, and for keeping hold of the retry list. Consequently the
layer is also responsible for the draining.

With this change, the non-coherent and coherent bus are given a single
layer to avoid changing any temporal behaviour, but the patch opens up
for adding more layers.

6 files changed, 232 insertions, 109 deletions

diff --git a/src/mem/bus.cc b/src/mem/bus.cc
index a2ddc0b69..16b581a7e 100644
--- a/src/mem/bus.cc
+++ b/src/mem/bus.cc
@@ -55,9 +55,8 @@
 #include "mem/bus.hh"
 
 BaseBus::BaseBus(const BaseBusParams *p)
-    : MemObject(p), state(IDLE), clock(p->clock),
+    : MemObject(p), clock(p->clock),
       headerCycles(p->header_cycles), width(p->width),
-      drainEvent(NULL), busIdleEvent(this),
       defaultPortID(InvalidPortID),
       useDefaultRange(p->use_default_range),
       defaultBlockSize(p->block_size),
@@ -138,7 +137,13 @@ BaseBus::calcPacketTiming(PacketPtr pkt)
     return headerTime;
 }
 
-void BaseBus::occupyBus(Tick until)
+BaseBus::Layer::Layer(BaseBus& _bus, const std::string& _name, Tick _clock) :
+    bus(_bus), _name(_name), state(IDLE), clock(_clock), drainEvent(NULL),
+    releaseEvent(this)
+{
+}
+
+void BaseBus::Layer::occupyLayer(Tick until)
 {
     // ensure the state is busy or in retry and never idle at this
     // point, as the bus should transition from idle as soon as it has
@@ -153,14 +158,14 @@ void BaseBus::occupyBus(Tick until)
 
     // until should never be 0 as express snoops never occupy the bus
     assert(until != 0);
-    schedule(busIdleEvent, until);
+    bus.schedule(releaseEvent, until);
 
     DPRINTF(BaseBus, "The bus is now busy from tick %d to %d\n",
             curTick(), until);
 }
 
 bool
-BaseBus::tryTiming(Port* port)
+BaseBus::Layer::tryTiming(Port* port)
 {
     // first we see if the bus is busy, next we check if we are in a
     // retry with a port other than the current one
@@ -180,7 +185,7 @@ BaseBus::tryTiming(Port* port)
 }
 
 void
-BaseBus::succeededTiming(Tick busy_time)
+BaseBus::Layer::succeededTiming(Tick busy_time)
 {
     // if a retrying port succeeded, also take it off the retry list
     if (state == RETRY) {
@@ -195,11 +200,11 @@ BaseBus::succeededTiming(Tick busy_time)
     assert(state == BUSY);
 
     // occupy the bus accordingly
-    occupyBus(busy_time);
+    occupyLayer(busy_time);
 }
 
 void
-BaseBus::failedTiming(SlavePort* port, Tick busy_time)
+BaseBus::Layer::failedTiming(SlavePort* port, Tick busy_time)
 {
     // if we are not in a retry, i.e. busy (but never idle), or we are
     // in a retry but not for the current port, then add the port at
@@ -213,15 +218,15 @@ BaseBus::failedTiming(SlavePort* port, Tick busy_time)
     state = BUSY;
 
     // occupy the bus accordingly
-    occupyBus(busy_time);
+    occupyLayer(busy_time);
 }
 
 void
-BaseBus::releaseBus()
+BaseBus::Layer::releaseLayer()
 {
     // releasing the bus means we should now be idle
     assert(state == BUSY);
-    assert(!busIdleEvent.scheduled());
+    assert(!releaseEvent.scheduled());
 
     // update the state
     state = IDLE;
@@ -242,7 +247,7 @@ BaseBus::releaseBus()
 }
 
 void
-BaseBus::retryWaiting()
+BaseBus::Layer::retryWaiting()
 {
     // this should never be called with an empty retry list
     assert(!retryList.empty());
@@ -277,23 +282,23 @@ BaseBus::retryWaiting()
         // clock edge
         Tick now = divCeil(curTick(), clock) * clock;
 
-        occupyBus(now + clock);
+        occupyLayer(now + clock);
     }
 }
 
 void
-BaseBus::recvRetry()
+BaseBus::Layer::recvRetry()
 {
     // we got a retry from a peer that we tried to send something to
     // and failed, but we sent it on the account of someone else, and
     // that source port should be on our retry list, however if the
-    // bus is released before this happens and the retry (from the bus
-    // point of view) is successful then this no longer holds and we
-    // could in fact have an empty retry list
+    // bus layer is released before this happens and the retry (from
+    // the bus point of view) is successful then this no longer holds
+    // and we could in fact have an empty retry list
     if (retryList.empty())
         return;
 
-    // if the bus is idle
+    // if the bus layer is idle
     if (state == IDLE) {
         // note that we do not care who told us to retry at the moment, we
         // merely let the first one on the retry list go
@@ -481,7 +486,7 @@ BaseBus::findBlockSize()
 
 
 unsigned int
-BaseBus::drain(Event * de)
+BaseBus::Layer::drain(Event * de)
 {
     //We should check that we're not "doing" anything, and that noone is
     //waiting. We might be idle but have someone waiting if the device we
diff --git a/src/mem/bus.hh b/src/mem/bus.hh
index db0686683..c54532c65 100644
--- a/src/mem/bus.hh
+++ b/src/mem/bus.hh
@@ -75,34 +75,158 @@ class BaseBus : public MemObject
   protected:
 
     /**
-     * We declare an enum to track the state of the bus. The starting
-     * point is an idle state where the bus is waiting for a packet to
-     * arrive. Upon arrival, the bus transitions to the busy state,
-     * where it remains either until the packet transfer is done, or
-     * the header time is spent. Once the bus leaves the busy state,
-     * it can either go back to idle, if no packets have arrived while
-     * it was busy, or the bus goes on to retry the first port on the
-     * retryList. A similar transition takes place from idle to retry
-     * if the bus receives a retry from one of its connected
-     * ports. The retry state lasts until the port in questions calls
-     * sendTiming and returns control to the bus, or goes to a busy
-     * state if the port does not immediately react to the retry by
-     * calling sendTiming.
+     * A bus layer is an internal bus structure with its own flow
+     * control and arbitration. Hence, a single-layer bus mimics a
+     * traditional off-chip tri-state bus (like PCI), where only one
+     * set of wires are shared. For on-chip buses, a good starting
+     * point is to have three layers, for requests, responses, and
+     * snoop responses respectively (snoop requests are instantaneous
+     * and do not need any flow control or arbitration). This case is
+     * similar to AHB and some OCP configurations. As a further
+     * extensions beyond the three-layer bus, a future multi-layer bus
+     * has with one layer per connected slave port provides a full or
+     * partial crossbar, like AXI, OCP, PCIe etc.
      */
-    enum State { IDLE, BUSY, RETRY };
+    class Layer
+    {
+
+      public:
+
+        /**
+         * Create a bus layer and give it a name. The bus layer uses
+         * the bus an event manager.
+         *
+         * @param _bus the bus this layer belongs to
+         * @param _name the layer's name
+         * @param _clock clock period in ticks
+         */
+        Layer(BaseBus& _bus, const std::string& _name, Tick _clock);
+
+        /**
+         * Drain according to the normal semantics, so that the bus
+         * can tell the layer to drain, and pass an event to signal
+         * back when drained.
+         *
+         * @param de drain event to call once drained
+         *
+         * @return 1 if busy or waiting to retry, or 0 if idle
+         */
+        unsigned int drain(Event *de);
+
+        /**
+         * Get the bus layer's name
+         */
+        const std::string name() const { return bus.name() + _name; }
+
+
+        /**
+         * Determine if the bus layer accepts a packet from a specific
+         * port. If not, the port in question is also added to the
+         * retry list. In either case the state of the layer is updated
+         * accordingly.
+         *
+         * @param port Source port resenting the packet
+         *
+         * @return True if the bus layer accepts the packet
+         */
+        bool tryTiming(Port* port);
+
+        /**
+         * Deal with a destination port accepting a packet by potentially
+         * removing the source port from the retry list (if retrying) and
+         * occupying the bus layer accordingly.
+         *
+         * @param busy_time Time to spend as a result of a successful send
+         */
+        void succeededTiming(Tick busy_time);
+
+        /**
+         * Deal with a destination port not accepting a packet by
+         * potentially adding the source port to the retry list (if
+         * not already at the front) and occupying the bus layer
+         * accordingly.
+         *
+         * @param busy_time Time to spend as a result of a failed send
+         */
+        void failedTiming(SlavePort* port, Tick busy_time);
+
+        /** Occupy the bus layer until until */
+        void occupyLayer(Tick until);
+
+        /**
+         * Send a retry to the port at the head of the retryList. The
+         * caller must ensure that the list is not empty.
+         */
+        void retryWaiting();
+
+        /**
+         * Handler a retry from a neighbouring module. Eventually this
+         * should be all encapsulated in the bus. This wraps
+         * retryWaiting by verifying that there are ports waiting
+         * before calling retryWaiting.
+         */
+        void recvRetry();
+
+      private:
+
+        /** The bus this layer is a part of. */
+        BaseBus& bus;
+
+        /** A name for this layer. */
+        std::string _name;
+
+        /**
+         * We declare an enum to track the state of the bus layer. The
+         * starting point is an idle state where the bus layer is
+         * waiting for a packet to arrive. Upon arrival, the bus layer
+         * transitions to the busy state, where it remains either
+         * until the packet transfer is done, or the header time is
+         * spent. Once the bus layer leaves the busy state, it can
+         * either go back to idle, if no packets have arrived while it
+         * was busy, or the bus layer goes on to retry the first port
+         * on the retryList. A similar transition takes place from
+         * idle to retry if the bus layer receives a retry from one of
+         * its connected ports. The retry state lasts until the port
+         * in questions calls sendTiming and returns control to the
+         * bus layer, or goes to a busy state if the port does not
+         * immediately react to the retry by calling sendTiming.
+         */
+        enum State { IDLE, BUSY, RETRY };
+
+        /** track the state of the bus layer */
+        State state;
+
+        /** the clock speed for the bus layer */
+        Tick clock;
+
+        /** event for signalling when drained */
+        Event * drainEvent;
+
+        /**
+         * An array of pointers to ports that retry should be called
+         * on because the original send failed for whatever reason.
+         */
+        std::list<Port*> retryList;
+
+        /**
+         * Release the bus layer after being occupied and return to an
+         * idle state where we proceed to send a retry to any
+         * potential waiting port, or drain if asked to do so.
+         */
+        void releaseLayer();
+
+        /** event used to schedule a release of the layer */
+        EventWrapper<Layer, &Layer::releaseLayer> releaseEvent;
 
-    /** track the state of the bus */
-    State state;
+    };
 
     /** the clock speed for the bus */
-    int clock;
+    Tick clock;
     /** cycles of overhead per transaction */
     int headerCycles;
     /** the width of the bus in bytes */
     int width;
 
-    Event * drainEvent;
-
     typedef range_map<Addr, PortID>::iterator PortMapIter;
     typedef range_map<Addr, PortID>::const_iterator PortMapConstIter;
     range_map<Addr, PortID> portMap;
@@ -110,39 +234,6 @@ class BaseBus : public MemObject
     AddrRangeList defaultRange;
 
     /**
-     * Determine if the bus accepts a packet from a specific port. If
-     * not, the port in question is also added to the retry list. In
-     * either case the state of the bus is updated accordingly.
-     *
-     * @param port Source port on the bus presenting the packet
-     *
-     * @return True if the bus accepts the packet
-     */
-    bool tryTiming(Port* port);
-
-    /**
-     * Deal with a destination port accepting a packet by potentially
-     * removing the source port from the retry list (if retrying) and
-     * occupying the bus accordingly.
-     *
-     * @param busy_time Time to spend as a result of a successful send
-     */
-    void succeededTiming(Tick busy_time);
-
-    /**
-     * Deal with a destination port not accepting a packet by
-     * potentially adding the source port to the retry list (if
-     * not already at the front) and occupying the bus accordingly.
-     *
-     * @param busy_time Time to spend as a result of a failed send
-     */
-    void failedTiming(SlavePort* port, Tick busy_time);
-
-    /** Timing function called by port when it is once again able to process
-     * requests. */
-    void recvRetry();
-
-    /**
      * Function called by the port when the bus is recieving a range change.
      *
      * @param master_port_id id of the port that received the change
@@ -224,22 +315,6 @@ class BaseBus : public MemObject
      */
     Tick calcPacketTiming(PacketPtr pkt);
 
-    /** Occupy the bus until until */
-    void occupyBus(Tick until);
-
-    /**
-     * Release the bus after being occupied and return to an idle
-     * state where we proceed to send a retry to any potential waiting
-     * port, or drain if asked to do so.
-     */
-    void releaseBus();
-
-    /**
-     * Send a retry to the port at the head of the retryList. The
-     * caller must ensure that the list is not empty.
-     */
-    void retryWaiting();
-
     /**
      * Ask everyone on the bus what their size is
      *
@@ -247,9 +322,6 @@ class BaseBus : public MemObject
      */
     unsigned findBlockSize();
 
-    // event used to schedule a release of the bus
-    EventWrapper<BaseBus, &BaseBus::releaseBus> busIdleEvent;
-
     std::set<PortID> inRecvRangeChange;
 
     /** The master and slave ports of the bus */
@@ -262,10 +334,6 @@ class BaseBus : public MemObject
     typedef std::vector<SlavePort*>::const_iterator SlavePortConstIter;
     typedef std::vector<MasterPort*>::const_iterator MasterPortConstIter;
 
-    /** An array of pointers to ports that retry should be called on because the
-     * original send failed for whatever reason.*/
-    std::list<Port*> retryList;
-
     /** Port that handles requests that don't match any of the interfaces.*/
     PortID defaultPortID;
 
@@ -289,7 +357,7 @@ class BaseBus : public MemObject
     virtual MasterPort& getMasterPort(const std::string& if_name, int idx = -1);
     virtual SlavePort& getSlavePort(const std::string& if_name, int idx = -1);
 
-    unsigned int drain(Event *de);
+    virtual unsigned int drain(Event *de) = 0;
 
 };
 
diff --git a/src/mem/coherent_bus.cc b/src/mem/coherent_bus.cc
index f7d4b9d11..956654981 100644
--- a/src/mem/coherent_bus.cc
+++ b/src/mem/coherent_bus.cc
@@ -54,7 +54,7 @@
 #include "mem/coherent_bus.hh"
 
 CoherentBus::CoherentBus(const CoherentBusParams *p)
-    : BaseBus(p)
+    : BaseBus(p), layer(*this, ".layer", p->clock)
 {
     // create the ports based on the size of the master and slave
     // vector ports, and the presence of the default port, the ports
@@ -115,7 +115,7 @@ CoherentBus::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
 
     // test if the bus should be considered occupied for the current
     // port, and exclude express snoops from the check
-    if (!is_express_snoop && !tryTiming(src_port)) {
+    if (!is_express_snoop && !layer.tryTiming(src_port)) {
         DPRINTF(CoherentBus, "recvTimingReq: src %s %s 0x%x BUSY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
@@ -176,10 +176,10 @@ CoherentBus::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
                     src_port->name(), pkt->cmdString(), pkt->getAddr());
 
             // update the bus state and schedule an idle event
-            failedTiming(src_port, headerFinishTime);
+            layer.failedTiming(src_port, headerFinishTime);
         } else {
             // update the bus state and schedule an idle event
-            succeededTiming(packetFinishTime);
+            layer.succeededTiming(packetFinishTime);
         }
     }
 
@@ -194,7 +194,7 @@ CoherentBus::recvTimingResp(PacketPtr pkt, PortID master_port_id)
 
     // test if the bus should be considered occupied for the current
     // port
-    if (!tryTiming(src_port)) {
+    if (!layer.tryTiming(src_port)) {
         DPRINTF(CoherentBus, "recvTimingResp: src %s %s 0x%x BUSY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
@@ -221,7 +221,7 @@ CoherentBus::recvTimingResp(PacketPtr pkt, PortID master_port_id)
     // deadlock
     assert(success);
 
-    succeededTiming(packetFinishTime);
+    layer.succeededTiming(packetFinishTime);
 
     return true;
 }
@@ -258,7 +258,7 @@ CoherentBus::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
 
     // test if the bus should be considered occupied for the current
     // port
-    if (!tryTiming(src_port)) {
+    if (!layer.tryTiming(src_port)) {
         DPRINTF(CoherentBus, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
@@ -309,7 +309,7 @@ CoherentBus::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
         assert(success);
     }
 
-    succeededTiming(packetFinishTime);
+    layer.succeededTiming(packetFinishTime);
 
     return true;
 }
@@ -332,6 +332,13 @@ CoherentBus::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id)
     }
 }
 
+void
+CoherentBus::recvRetry()
+{
+    // only one layer that can deal with it
+    layer.recvRetry();
+}
+
 Tick
 CoherentBus::recvAtomic(PacketPtr pkt, PortID slave_port_id)
 {
@@ -493,6 +500,13 @@ CoherentBus::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id)
     }
 }
 
+unsigned int
+CoherentBus::drain(Event *de)
+{
+    // only one layer to worry about at the moment
+    return layer.drain(de);
+}
+
 CoherentBus *
 CoherentBusParams::create()
 {
diff --git a/src/mem/coherent_bus.hh b/src/mem/coherent_bus.hh
index 460afd828..b5f0cdee5 100644
--- a/src/mem/coherent_bus.hh
+++ b/src/mem/coherent_bus.hh
@@ -70,6 +70,11 @@ class CoherentBus : public BaseBus
   protected:
 
     /**
+     * Declare the single layer of this bus.
+     */
+    Layer layer;
+
+    /**
      * Declaration of the coherent bus slave port type, one will be
      * instantiated for each of the master ports connecting to the
      * bus.
@@ -231,6 +236,10 @@ class CoherentBus : public BaseBus
         snoop response.*/
     virtual bool recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id);
 
+    /** Timing function called by port when it is once again able to process
+     * requests. */
+    void recvRetry();
+
     /**
      * Forward a timing packet to our snoopers, potentially excluding
      * one of the connected coherent masters to avoid sending a packet
@@ -285,6 +294,8 @@ class CoherentBus : public BaseBus
     virtual void init();
 
     CoherentBus(const CoherentBusParams *p);
+
+    unsigned int drain(Event *de);
 };
 
 #endif //__MEM_COHERENT_BUS_HH__
diff --git a/src/mem/noncoherent_bus.cc b/src/mem/noncoherent_bus.cc
index 718bbebdd..e502a78a8 100644
--- a/src/mem/noncoherent_bus.cc
+++ b/src/mem/noncoherent_bus.cc
@@ -55,7 +55,7 @@
 #include "mem/noncoherent_bus.hh"
 
 NoncoherentBus::NoncoherentBus(const NoncoherentBusParams *p)
-    : BaseBus(p)
+    : BaseBus(p), layer(*this, ".layer", p->clock)
 {
     // create the ports based on the size of the master and slave
     // vector ports, and the presence of the default port, the ports
@@ -97,7 +97,7 @@ NoncoherentBus::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
 
     // test if the bus should be considered occupied for the current
     // port
-    if (!tryTiming(src_port)) {
+    if (!layer.tryTiming(src_port)) {
         DPRINTF(NoncoherentBus, "recvTimingReq: src %s %s 0x%x BUSY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
@@ -123,12 +123,12 @@ NoncoherentBus::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
         DPRINTF(NoncoherentBus, "recvTimingReq: src %s %s 0x%x RETRY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
 
-        failedTiming(src_port, headerFinishTime);
+        layer.failedTiming(src_port, headerFinishTime);
 
         return false;
     }
 
-    succeededTiming(packetFinishTime);
+    layer.succeededTiming(packetFinishTime);
 
     return true;
 }
@@ -141,7 +141,7 @@ NoncoherentBus::recvTimingResp(PacketPtr pkt, PortID master_port_id)
 
     // test if the bus should be considered occupied for the current
     // port
-    if (!tryTiming(src_port)) {
+    if (!layer.tryTiming(src_port)) {
         DPRINTF(NoncoherentBus, "recvTimingResp: src %s %s 0x%x BUSY\n",
                 src_port->name(), pkt->cmdString(), pkt->getAddr());
         return false;
@@ -161,11 +161,18 @@ NoncoherentBus::recvTimingResp(PacketPtr pkt, PortID master_port_id)
     // deadlock
     assert(success);
 
-    succeededTiming(packetFinishTime);
+    layer.succeededTiming(packetFinishTime);
 
     return true;
 }
 
+void
+NoncoherentBus::recvRetry()
+{
+    // only one layer that can deal with it
+    layer.recvRetry();
+}
+
 Tick
 NoncoherentBus::recvAtomic(PacketPtr pkt, PortID slave_port_id)
 {
@@ -201,6 +208,13 @@ NoncoherentBus::recvFunctional(PacketPtr pkt, PortID slave_port_id)
     masterPorts[dest_id]->sendFunctional(pkt);
 }
 
+unsigned int
+NoncoherentBus::drain(Event *de)
+{
+    // only one layer to worry about at the moment
+    return layer.drain(de);
+}
+
 NoncoherentBus*
 NoncoherentBusParams::create()
 {
diff --git a/src/mem/noncoherent_bus.hh b/src/mem/noncoherent_bus.hh
index 46fc65fad..dd43d8c19 100644
--- a/src/mem/noncoherent_bus.hh
+++ b/src/mem/noncoherent_bus.hh
@@ -73,6 +73,11 @@ class NoncoherentBus : public BaseBus
   protected:
 
     /**
+     * Declare the single layer of this bus.
+     */
+    Layer layer;
+
+    /**
      * Declaration of the non-coherent bus slave port type, one will
      * be instantiated for each of the master ports connecting to the
      * bus.
@@ -184,6 +189,10 @@ class NoncoherentBus : public BaseBus
       response packet.*/
     virtual bool recvTimingResp(PacketPtr pkt, PortID master_port_id);
 
+    /** Timing function called by port when it is once again able to process
+     * requests. */
+    void recvRetry();
+
     /** Function called by the port when the bus is recieving a Atomic
       transaction.*/
     Tick recvAtomic(PacketPtr pkt, PortID slave_port_id);
@@ -196,6 +205,8 @@ class NoncoherentBus : public BaseBus
 
     NoncoherentBus(const NoncoherentBusParams *p);
 
+    unsigned int drain(Event *de);
+
 };
 
 #endif //__MEM_NONCOHERENT_BUS_HH__