ruby: Expose MessageBuffers as SimObjects

Expose MessageBuffers from SLICC controllers as SimObjects that can be
manipulated in Python. This patch has numerous benefits:
1) First and foremost, it exposes MessageBuffers as SimObjects that can be
manipulated in Python code. This allows parameters to be set and checked in
Python code to avoid obfuscating parameters within protocol files. Further, now
as SimObjects, MessageBuffer parameters are printed to config output files as a
way to track parameters across simulations (e.g. buffer sizes)

2) Cleans up special-case code for responseFromMemory buffers, and aligns their
instantiation and use with mandatoryQueue buffers. These two special buffers
are the only MessageBuffers that are exposed to components outside of SLICC
controllers, and they're both slave ends of these buffers. They should be
exposed outside of SLICC in the same way, and this patch does it.

3) Distinguishes buffer-specific parameters from buffer-to-network parameters.
Specifically, buffer size, randomization, ordering, recycle latency, and ports
are all specific to a MessageBuffer, while the virtual network ID and type are
intrinsics of how the buffer is connected to network ports. The former are
specified in the Python object, while the latter are specified in the
controller *.sm files. Unlike buffer-specific parameters, which may need to
change depending on the simulated system structure, buffer-to-network
parameters can be specified statically for most or all different simulated
systems.

11 files changed, 141 insertions, 76 deletions

diff --git a/src/mem/ruby/network/MessageBuffer.cc b/src/mem/ruby/network/MessageBuffer.cc
index 484d2876b..e9bbd0059 100644
--- a/src/mem/ruby/network/MessageBuffer.cc
+++ b/src/mem/ruby/network/MessageBuffer.cc
@@ -39,25 +39,23 @@
 using namespace std;
 using m5::stl_helpers::operator<<;
 
-MessageBuffer::MessageBuffer(const string &name)
-    : m_time_last_time_size_checked(0), m_time_last_time_enqueue(0),
-      m_time_last_time_pop(0), m_last_arrival_time(0)
+MessageBuffer::MessageBuffer(const Params *p)
+    : SimObject(p), m_recycle_latency(p->recycle_latency),
+    m_max_size(p->buffer_size), m_time_last_time_size_checked(0),
+    m_time_last_time_enqueue(0), m_time_last_time_pop(0),
+    m_last_arrival_time(0), m_strict_fifo(p->ordered),
+    m_randomization(p->randomization)
 {
     m_msg_counter = 0;
     m_consumer = NULL;
     m_sender = NULL;
     m_receiver = NULL;
 
-    m_ordering_set = false;
-    m_strict_fifo = true;
-    m_max_size = 0;
-    m_randomization = true;
     m_size_last_time_size_checked = 0;
     m_size_at_cycle_start = 0;
     m_msgs_this_cycle = 0;
     m_not_avail_count = 0;
     m_priority_rank = 0;
-    m_name = name;
 
     m_stall_msg_map.clear();
     m_input_link_id = 0;
@@ -144,8 +142,6 @@ random_time()
 void
 MessageBuffer::enqueue(MsgPtr message, Cycles delta)
 {
-    assert(m_ordering_set);
-
     // record current time incase we have a pop that also adjusts my size
     if (m_time_last_time_enqueue < m_sender->curCycle()) {
         m_msgs_this_cycle = 0;  // first msg this cycle
@@ -184,7 +180,7 @@ MessageBuffer::enqueue(MsgPtr message, Cycles delta)
         if (arrival_time < m_last_arrival_time) {
             panic("FIFO ordering violated: %s name: %s current time: %d "
                   "delta: %d arrival_time: %d last arrival_time: %d\n",
-                  *this, m_name, current_time,
+                  *this, name(), current_time,
                   delta * m_sender->clockPeriod(),
                   arrival_time, m_last_arrival_time);
         }
@@ -356,7 +352,7 @@ MessageBuffer::print(ostream& out) const
 
     vector<MsgPtr> copy(m_prio_heap);
     sort_heap(copy.begin(), copy.end(), greater<MsgPtr>());
-    ccprintf(out, "%s] %s", copy, m_name);
+    ccprintf(out, "%s] %s", copy, name());
 }
 
 bool
@@ -424,3 +420,9 @@ MessageBuffer::functionalWrite(Packet *pkt)
 
     return num_functional_writes;
 }
+
+MessageBuffer *
+MessageBufferParams::create()
+{
+    return new MessageBuffer(this);
+}
diff --git a/src/mem/ruby/network/MessageBuffer.hh b/src/mem/ruby/network/MessageBuffer.hh
index 3ced44250..d47284eb1 100644
--- a/src/mem/ruby/network/MessageBuffer.hh
+++ b/src/mem/ruby/network/MessageBuffer.hh
@@ -41,20 +41,19 @@
 #include <string>
 #include <vector>
 
+#include "debug/RubyQueue.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/common/Consumer.hh"
 #include "mem/ruby/slicc_interface/Message.hh"
 #include "mem/packet.hh"
+#include "params/MessageBuffer.hh"
+#include "sim/sim_object.hh"
 
-class MessageBuffer
+class MessageBuffer : public SimObject
 {
   public:
-    MessageBuffer(const std::string &name = "");
-
-    std::string name() const { return m_name; }
-
-    void setRecycleLatency(Cycles recycle_latency)
-    { m_recycle_latency = recycle_latency; }
+    typedef MessageBufferParams Params;
+    MessageBuffer(const Params *p);
 
     void reanalyzeMessages(const Address& addr);
     void reanalyzeAllMessages();
@@ -78,6 +77,7 @@ class MessageBuffer
     void setPriority(int rank) { m_priority_rank = rank; }
     void setConsumer(Consumer* consumer)
     {
+        DPRINTF(RubyQueue, "Setting consumer: %s\n", *consumer);
         if (m_consumer != NULL) {
             fatal("Trying to connect %s to MessageBuffer %s. \
                   \n%s already connected. Check the cntrl_id's.\n",
@@ -88,21 +88,22 @@ class MessageBuffer
 
     void setSender(ClockedObject* obj)
     {
+        DPRINTF(RubyQueue, "Setting sender: %s\n", obj->name());
         assert(m_sender == NULL || m_sender == obj);
         m_sender = obj;
     }
 
     void setReceiver(ClockedObject* obj)
     {
+        DPRINTF(RubyQueue, "Setting receiver: %s\n", obj->name());
         assert(m_receiver == NULL || m_receiver == obj);
         m_receiver = obj;
     }
 
-    void setDescription(const std::string& name) { m_name = name; }
-    std::string getDescription() { return m_name;}
-
     Consumer* getConsumer() { return m_consumer; }
 
+    bool getOrdered() { return m_strict_fifo; }
+
     //! Function for extracting the message at the head of the
     //! message queue.  The function assumes that the queue is nonempty.
     const Message* peek() const;
@@ -126,16 +127,7 @@ class MessageBuffer
     bool isStallMapEmpty() { return m_stall_msg_map.size() == 0; }
     unsigned int getStallMapSize() { return m_stall_msg_map.size(); }
 
-    void
-    setOrdering(bool order)
-    {
-        m_strict_fifo = order;
-        m_ordering_set = true;
-    }
-
-    void resize(unsigned int size) { m_max_size = size; }
     unsigned int getSize();
-    void setRandomization(bool random_flag) { m_randomization = random_flag; }
 
     void clear();
     void print(std::ostream& out) const;
@@ -156,12 +148,12 @@ class MessageBuffer
     uint32_t functionalWrite(Packet *pkt);
 
   private:
+    //added by SS
+    const Cycles m_recycle_latency;
+
     void reanalyzeList(std::list<MsgPtr> &, Tick);
 
   private:
-    //added by SS
-    Cycles m_recycle_latency;
-
     // Data Members (m_ prefix)
     //! The two ends of the buffer.
     ClockedObject* m_sender;
@@ -176,9 +168,8 @@ class MessageBuffer
     typedef std::map< Address, std::list<MsgPtr> > StallMsgMapType;
 
     StallMsgMapType m_stall_msg_map;
-    std::string m_name;
 
-    unsigned int m_max_size;
+    const unsigned int m_max_size;
     Cycles m_time_last_time_size_checked;
     unsigned int m_size_last_time_size_checked;
 
@@ -195,9 +186,8 @@ class MessageBuffer
                             // slots available
     uint64 m_msg_counter;
     int m_priority_rank;
-    bool m_strict_fifo;
-    bool m_ordering_set;
-    bool m_randomization;
+    const bool m_strict_fifo;
+    const bool m_randomization;
 
     int m_input_link_id;
     int m_vnet_id;
diff --git a/src/mem/ruby/network/MessageBuffer.py b/src/mem/ruby/network/MessageBuffer.py
new file mode 100644
index 000000000..88c528e30
--- /dev/null
+++ b/src/mem/ruby/network/MessageBuffer.py
@@ -0,0 +1,44 @@
+# Copyright (c) 2015 Mark D. Hill and David A. Wood.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Joel Hestness
+
+from m5.params import *
+from m5.proxy import *
+from m5.SimObject import SimObject
+
+class MessageBuffer(SimObject):
+    type = 'MessageBuffer'
+    cxx_class = 'MessageBuffer'
+    cxx_header = "mem/ruby/network/MessageBuffer.hh"
+    ordered = Param.Bool(False, "Whether the buffer is ordered")
+    buffer_size = Param.Unsigned(0, "Maximum number of entries to buffer \
+                                     (0 allows infinite entries)")
+    recycle_latency = Param.Cycles(Parent.recycle_latency, "")
+    randomization = Param.Bool(False, "")
+
+    master = MasterPort("Master port to MessageBuffer receiver")
+    slave = SlavePort("Slave port from MessageBuffer sender")
diff --git a/src/mem/ruby/network/SConscript b/src/mem/ruby/network/SConscript
index 282d47647..36b02432f 100644
--- a/src/mem/ruby/network/SConscript
+++ b/src/mem/ruby/network/SConscript
@@ -35,6 +35,7 @@ if env['PROTOCOL'] == 'None':
 
 SimObject('BasicLink.py')
 SimObject('BasicRouter.py')
+SimObject('MessageBuffer.py')
 SimObject('Network.py')
 
 Source('BasicLink.cc')
diff --git a/src/mem/ruby/network/simple/PerfectSwitch.cc b/src/mem/ruby/network/simple/PerfectSwitch.cc
index 06072724e..de038d211 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.cc
+++ b/src/mem/ruby/network/simple/PerfectSwitch.cc
@@ -76,13 +76,6 @@ PerfectSwitch::addInPort(const vector<MessageBuffer*>& in)
     for (int i = 0; i < in.size(); ++i) {
         if (in[i] != nullptr) {
             in[i]->setConsumer(this);
-
-            string desc =
-                csprintf("[Queue from port %s %s %s to PerfectSwitch]",
-                         to_string(m_switch_id), to_string(port),
-                         to_string(i));
-
-            in[i]->setDescription(desc);
             in[i]->setIncomingLink(port);
             in[i]->setVnet(i);
         }
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc
index 7426e9691..5b7d7ebad 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.cc
+++ b/src/mem/ruby/network/simple/SimpleNetwork.cc
@@ -62,6 +62,9 @@ SimpleNetwork::SimpleNetwork(const Params *p)
         m_switches.push_back(s);
         s->init_net_ptr(this);
     }
+
+    m_int_link_buffers = p->int_link_buffers;
+    m_num_connected_buffers = 0;
 }
 
 void
@@ -78,7 +81,7 @@ SimpleNetwork::init()
 SimpleNetwork::~SimpleNetwork()
 {
     deletePointers(m_switches);
-    deletePointers(m_buffers_to_free);
+    deletePointers(m_int_link_buffers);
 }
 
 // From a switch to an endpoint node
@@ -121,16 +124,10 @@ SimpleNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
 
     for (int i = 0; i < m_virtual_networks; i++) {
         // allocate a buffer
-        MessageBuffer* buffer_ptr = new MessageBuffer;
-        buffer_ptr->setOrdering(true);
-
-        if (m_buffer_size > 0) {
-            buffer_ptr->resize(m_buffer_size);
-        }
-
+        assert(m_num_connected_buffers < m_int_link_buffers.size());
+        MessageBuffer* buffer_ptr = m_int_link_buffers[m_num_connected_buffers];
+        m_num_connected_buffers++;
         queues[i] = buffer_ptr;
-        // remember to deallocate it
-        m_buffers_to_free.push_back(buffer_ptr);
     }
 
     // Connect it to the two switches
@@ -236,8 +233,8 @@ SimpleNetwork::functionalRead(Packet *pkt)
         }
     }
 
-    for (unsigned int i = 0; i < m_buffers_to_free.size(); ++i) {
-        if (m_buffers_to_free[i]->functionalRead(pkt)) {
+    for (unsigned int i = 0; i < m_int_link_buffers.size(); ++i) {
+        if (m_int_link_buffers[i]->functionalRead(pkt)) {
             return true;
         }
     }
@@ -254,8 +251,8 @@ SimpleNetwork::functionalWrite(Packet *pkt)
         num_functional_writes += m_switches[i]->functionalWrite(pkt);
     }
 
-    for (unsigned int i = 0; i < m_buffers_to_free.size(); ++i) {
-        num_functional_writes += m_buffers_to_free[i]->functionalWrite(pkt);
+    for (unsigned int i = 0; i < m_int_link_buffers.size(); ++i) {
+        num_functional_writes += m_int_link_buffers[i]->functionalWrite(pkt);
     }
     return num_functional_writes;
 }
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.hh b/src/mem/ruby/network/simple/SimpleNetwork.hh
index a2723c715..fe0c1838b 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.hh
+++ b/src/mem/ruby/network/simple/SimpleNetwork.hh
@@ -93,7 +93,8 @@ class SimpleNetwork : public Network
     SimpleNetwork& operator=(const SimpleNetwork& obj);
 
     std::vector<Switch*> m_switches;
-    std::vector<MessageBuffer*> m_buffers_to_free;
+    std::vector<MessageBuffer*> m_int_link_buffers;
+    int m_num_connected_buffers;
     std::vector<Switch*> m_endpoint_switches;
 
     int m_buffer_size;
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.py b/src/mem/ruby/network/simple/SimpleNetwork.py
index 8d0442b7d..f4ec440a3 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.py
+++ b/src/mem/ruby/network/simple/SimpleNetwork.py
@@ -31,6 +31,7 @@ from m5.params import *
 from m5.proxy import *
 from Network import RubyNetwork
 from BasicRouter import BasicRouter
+from MessageBuffer import MessageBuffer
 
 class SimpleNetwork(RubyNetwork):
     type = 'SimpleNetwork'
@@ -39,9 +40,48 @@ class SimpleNetwork(RubyNetwork):
         "default buffer size; 0 indicates infinite buffering");
     endpoint_bandwidth = Param.Int(1000, "bandwidth adjustment factor");
     adaptive_routing = Param.Bool(False, "enable adaptive routing");
+    int_link_buffers = VectorParam.MessageBuffer("Buffers for int_links")
+    # int_links do not recycle buffers, so this parameter is not used.
+    # TODO: Move recycle_latency out of MessageBuffers and into controllers
+    recycle_latency = Param.Cycles(0, "")
+
+    def setup_buffers(self):
+        # Note that all SimpleNetwork MessageBuffers are currently ordered
+        network_buffers = []
+        for link in self.int_links:
+            # The network needs number_of_virtual_networks buffers per
+            # int_link port
+            for i in xrange(self.number_of_virtual_networks):
+                network_buffers.append(MessageBuffer(ordered = True))
+                network_buffers.append(MessageBuffer(ordered = True))
+        self.int_link_buffers = network_buffers
+
+        # Also add buffers for all router-link connections
+        for router in self.routers:
+            router_buffers = []
+            # Add message buffers to routers for each internal link connection
+            for link in self.int_links:
+                if link.node_a == router:
+                    for i in xrange(self.number_of_virtual_networks):
+                        router_buffers.append(MessageBuffer(ordered = True))
+                if link.node_b == router:
+                    for i in xrange(self.number_of_virtual_networks):
+                        router_buffers.append(MessageBuffer(ordered = True))
+
+            # Add message buffers to routers for each external link connection
+            for link in self.ext_links:
+                # Routers can only be int_nodes on ext_links
+                if link.int_node in self.routers:
+                    for i in xrange(self.number_of_virtual_networks):
+                        router_buffers.append(MessageBuffer(ordered = True))
+            router.port_buffers = router_buffers
 
 class Switch(BasicRouter):
     type = 'Switch'
     cxx_header = 'mem/ruby/network/simple/Switch.hh'
     virt_nets = Param.Int(Parent.number_of_virtual_networks,
                           "number of virtual networks")
+    port_buffers = VectorParam.MessageBuffer("Port buffers")
+    # Ports do not recycle buffers, so this parameter is not used.
+    # TODO: Move recycle_latency out of MessageBuffers and into controllers
+    recycle_latency = Param.Cycles(0, "")
diff --git a/src/mem/ruby/network/simple/Switch.cc b/src/mem/ruby/network/simple/Switch.cc
index 431a7b28f..b9d0b8010 100644
--- a/src/mem/ruby/network/simple/Switch.cc
+++ b/src/mem/ruby/network/simple/Switch.cc
@@ -43,6 +43,8 @@ using m5::stl_helpers::operator<<;
 Switch::Switch(const Params *p) : BasicRouter(p)
 {
     m_perfect_switch = new PerfectSwitch(m_id, this, p->virt_nets);
+    m_port_buffers = p->port_buffers;
+    m_num_connected_buffers = 0;
 }
 
 Switch::~Switch()
@@ -53,7 +55,7 @@ Switch::~Switch()
     deletePointers(m_throttles);
 
     // Delete MessageBuffers
-    deletePointers(m_buffers_to_free);
+    deletePointers(m_port_buffers);
 }
 
 void
@@ -97,15 +99,10 @@ Switch::addOutPort(const vector<MessageBuffer*>& out,
             out[i]->setSender(this);
         }
 
-        MessageBuffer* buffer_ptr = new MessageBuffer;
-        // Make these queues ordered
-        buffer_ptr->setOrdering(true);
-        if (m_network_ptr->getBufferSize() > 0) {
-            buffer_ptr->resize(m_network_ptr->getBufferSize());
-        }
-
+        assert(m_num_connected_buffers < m_port_buffers.size());
+        MessageBuffer* buffer_ptr = m_port_buffers[m_num_connected_buffers];
+        m_num_connected_buffers++;
         intermediateBuffers.push_back(buffer_ptr);
-        m_buffers_to_free.push_back(buffer_ptr);
 
         buffer_ptr->setSender(this);
         buffer_ptr->setReceiver(this);
@@ -188,8 +185,8 @@ bool
 Switch::functionalRead(Packet *pkt)
 {
     // Access the buffers in the switch for performing a functional read
-    for (unsigned int i = 0; i < m_buffers_to_free.size(); ++i) {
-        if (m_buffers_to_free[i]->functionalRead(pkt)) {
+    for (unsigned int i = 0; i < m_port_buffers.size(); ++i) {
+        if (m_port_buffers[i]->functionalRead(pkt)) {
             return true;
         }
     }
@@ -201,8 +198,8 @@ Switch::functionalWrite(Packet *pkt)
 {
     // Access the buffers in the switch for performing a functional write
     uint32_t num_functional_writes = 0;
-    for (unsigned int i = 0; i < m_buffers_to_free.size(); ++i) {
-        num_functional_writes += m_buffers_to_free[i]->functionalWrite(pkt);
+    for (unsigned int i = 0; i < m_port_buffers.size(); ++i) {
+        num_functional_writes += m_port_buffers[i]->functionalWrite(pkt);
     }
     return num_functional_writes;
 }
diff --git a/src/mem/ruby/network/simple/Switch.hh b/src/mem/ruby/network/simple/Switch.hh
index 3c11a5fa0..dbb1bbd05 100644
--- a/src/mem/ruby/network/simple/Switch.hh
+++ b/src/mem/ruby/network/simple/Switch.hh
@@ -89,7 +89,8 @@ class Switch : public BasicRouter
     PerfectSwitch* m_perfect_switch;
     SimpleNetwork* m_network_ptr;
     std::vector<Throttle*> m_throttles;
-    std::vector<MessageBuffer*> m_buffers_to_free;
+    std::vector<MessageBuffer*> m_port_buffers;
+    unsigned m_num_connected_buffers;
 
     // Statistical variables
     Stats::Formula m_avg_utilization;
diff --git a/src/mem/ruby/network/simple/Throttle.cc b/src/mem/ruby/network/simple/Throttle.cc
index 2164d76de..785e09aa2 100644
--- a/src/mem/ruby/network/simple/Throttle.cc
+++ b/src/mem/ruby/network/simple/Throttle.cc
@@ -100,7 +100,6 @@ Throttle::addLinks(const vector<MessageBuffer*>& in_vec,
         in_ptr->setConsumer(this);
         string desc = "[Queue to Throttle " + to_string(m_sID) + " " +
             to_string(m_node) + "]";
-        in_ptr->setDescription(desc);
     }
 }