ruby: message buffers: significant changes

This patch is the final patch in a series of patches.  The aim of the series
is to make ruby more configurable than it was.  More specifically, the
connections between controllers are not at all possible (unless one is ready
to make significant changes to the coherence protocol).  Moreover the buffers
themselves are magically connected to the network inside the slicc code.
These connections are not part of the configuration file.

This patch makes changes so that these connections will now be made in the
python configuration files associated with the protocols.  This requires
each state machine to expose the message buffers it uses for input and output.
So, the patch makes these buffers configurable members of the machines.

The patch drops the slicc code that usd to connect these buffers to the
network.  Now these buffers are exposed to the python configuration system
as Master and Slave ports.  In the configuration files, any master port
can be connected any slave port.  The file pyobject.cc has been modified to
take care of allocating the actual message buffer.  This is inline with how
other port connections work.

8 files changed, 160 insertions, 136 deletions

diff --git a/src/mem/ruby/network/simple/PerfectSwitch.cc b/src/mem/ruby/network/simple/PerfectSwitch.cc
index 0c6111c48..4565711a2 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.cc
+++ b/src/mem/ruby/network/simple/PerfectSwitch.cc
@@ -61,36 +61,33 @@ PerfectSwitch::init(SimpleNetwork *network_ptr)
 {
     m_network_ptr = network_ptr;
 
-    for(int i = 0;i < m_virtual_networks;++i)
-    {
+    for(int i = 0;i < m_virtual_networks;++i) {
         m_pending_message_count.push_back(0);
     }
 }
 
 void
-PerfectSwitch::addInPort(const vector<MessageBuffer*>& in)
+PerfectSwitch::addInPort(const map<int, MessageBuffer*>& in)
 {
-    assert(in.size() == m_virtual_networks);
     NodeID port = m_in.size();
     m_in.push_back(in);
 
-    for (int j = 0; j < m_virtual_networks; j++) {
-        m_in[port][j]->setConsumer(this);
+    for (auto& it : in) {
+        it.second->setConsumer(this);
 
         string desc = csprintf("[Queue from port %s %s %s to PerfectSwitch]",
-            to_string(m_switch_id), to_string(port), to_string(j));
-        m_in[port][j]->setDescription(desc);
-        m_in[port][j]->setIncomingLink(port);
-        m_in[port][j]->setVnet(j);
+            to_string(m_switch_id), to_string(port), to_string(it.first));
+
+        it.second->setDescription(desc);
+        it.second->setIncomingLink(port);
+        it.second->setVnet(it.first);
     }
 }
 
 void
-PerfectSwitch::addOutPort(const vector<MessageBuffer*>& out,
+PerfectSwitch::addOutPort(const map<int, MessageBuffer*>& out,
     const NetDest& routing_table_entry)
 {
-    assert(out.size() == m_virtual_networks);
-
     // Setup link order
     LinkOrder l;
     l.m_value = 0;
@@ -152,11 +149,16 @@ PerfectSwitch::wakeup()
                 vector<NetDest> output_link_destinations;
 
                 // Is there a message waiting?
-                while (m_in[incoming][vnet]->isReady()) {
+                auto it = m_in[incoming].find(vnet);
+                if (it == m_in[incoming].end())
+                    continue;
+                MessageBuffer *buffer = (*it).second;
+
+                while (buffer->isReady()) {
                     DPRINTF(RubyNetwork, "incoming: %d\n", incoming);
 
                     // Peek at message
-                    msg_ptr = m_in[incoming][vnet]->peekMsgPtr();
+                    msg_ptr = buffer->peekMsgPtr();
                     net_msg_ptr = safe_cast<NetworkMessage*>(msg_ptr.get());
                     DPRINTF(RubyNetwork, "Message: %s\n", (*net_msg_ptr));
 
@@ -261,7 +263,7 @@ PerfectSwitch::wakeup()
                     }
 
                     // Dequeue msg
-                    m_in[incoming][vnet]->dequeue();
+                    buffer->dequeue();
                     m_pending_message_count[vnet]--;
 
                     // Enqueue it - for all outgoing queues
diff --git a/src/mem/ruby/network/simple/PerfectSwitch.hh b/src/mem/ruby/network/simple/PerfectSwitch.hh
index c01c50a3b..25e3e2754 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.hh
+++ b/src/mem/ruby/network/simple/PerfectSwitch.hh
@@ -65,9 +65,10 @@ class PerfectSwitch : public Consumer
     { return csprintf("PerfectSwitch-%i", m_switch_id); }
 
     void init(SimpleNetwork *);
-    void addInPort(const std::vector<MessageBuffer*>& in);
-    void addOutPort(const std::vector<MessageBuffer*>& out,
+    void addInPort(const std::map<int, MessageBuffer*>& in);
+    void addOutPort(const std::map<int, MessageBuffer*>& out,
                     const NetDest& routing_table_entry);
+
     int getInLinks() const { return m_in.size(); }
     int getOutLinks() const { return m_out.size(); }
 
@@ -86,8 +87,9 @@ class PerfectSwitch : public Consumer
     SwitchID m_switch_id;
 
     // vector of queues from the components
-    std::vector<std::vector<MessageBuffer*> > m_in;
-    std::vector<std::vector<MessageBuffer*> > m_out;
+    std::vector<std::map<int, MessageBuffer*> > m_in;
+    std::vector<std::map<int, MessageBuffer*> > m_out;
+
     std::vector<NetDest> m_routing_table;
     std::vector<LinkOrder> m_link_order;
 
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc
index 2d08f9fa4..f51a0c891 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.cc
+++ b/src/mem/ruby/network/simple/SimpleNetwork.cc
@@ -93,10 +93,9 @@ SimpleNetwork::makeOutLink(SwitchID src, NodeID dest, BasicLink* link,
 
     SimpleExtLink *simple_link = safe_cast<SimpleExtLink*>(link);
 
-    m_switches[src]->addOutPort(m_fromNetQueues[dest],
-                                         routing_table_entry,
-                                         simple_link->m_latency,
-                                         simple_link->m_bw_multiplier);
+    m_switches[src]->addOutPort(m_fromNetQueues[dest], routing_table_entry,
+                                simple_link->m_latency,
+                                simple_link->m_bw_multiplier);
 
     m_endpoint_switches[dest] = m_switches[src];
 }
@@ -118,25 +117,28 @@ SimpleNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
                                 const NetDest& routing_table_entry)
 {
     // Create a set of new MessageBuffers
-    std::vector<MessageBuffer*> queues;
+    std::map<int, MessageBuffer*> queues;
     for (int i = 0; i < m_virtual_networks; i++) {
         // allocate a buffer
         MessageBuffer* buffer_ptr = new MessageBuffer;
         buffer_ptr->setOrdering(true);
+
         if (m_buffer_size > 0) {
             buffer_ptr->resize(m_buffer_size);
         }
-        queues.push_back(buffer_ptr);
+
+        queues[i] = buffer_ptr;
         // remember to deallocate it
         m_buffers_to_free.push_back(buffer_ptr);
     }
+
     // Connect it to the two switches
     SimpleIntLink *simple_link = safe_cast<SimpleIntLink*>(link);
 
     m_switches[dest]->addInPort(queues);
     m_switches[src]->addOutPort(queues, routing_table_entry,
-                                         simple_link->m_latency, 
-                                         simple_link->m_bw_multiplier);
+                                simple_link->m_latency,
+                                simple_link->m_bw_multiplier);
 }
 
 void
@@ -151,20 +153,20 @@ SimpleNetwork::checkNetworkAllocation(NodeID id, bool ordered, int network_num)
     m_in_use[network_num] = true;
 }
 
-MessageBuffer*
-SimpleNetwork::getToNetQueue(NodeID id, bool ordered, int network_num,
-                             std::string vnet_type)
+void
+SimpleNetwork::setToNetQueue(NodeID id, bool ordered, int network_num,
+                             std::string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num);
-    return m_toNetQueues[id][network_num];
+    m_toNetQueues[id][network_num] = b;
 }
 
-MessageBuffer*
-SimpleNetwork::getFromNetQueue(NodeID id, bool ordered, int network_num,
-                               std::string vnet_type)
+void
+SimpleNetwork::setFromNetQueue(NodeID id, bool ordered, int network_num,
+                               std::string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num);
-    return m_fromNetQueues[id][network_num];
+    m_fromNetQueues[id][network_num] = b;
 }
 
 void
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.hh b/src/mem/ruby/network/simple/SimpleNetwork.hh
index 90560c267..a2723c715 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.hh
+++ b/src/mem/ruby/network/simple/SimpleNetwork.hh
@@ -56,9 +56,11 @@ class SimpleNetwork : public Network
     void collateStats();
     void regStats();
 
-    // returns the queue requested for the given component
-    MessageBuffer* getToNetQueue(NodeID id, bool ordered, int network_num, std::string vnet_type);
-    MessageBuffer* getFromNetQueue(NodeID id, bool ordered, int network_num, std::string vnet_type);
+    // sets the queue requested
+    void setToNetQueue(NodeID id, bool ordered, int network_num,
+                       std::string vnet_type, MessageBuffer *b);
+    void setFromNetQueue(NodeID id, bool ordered, int network_num,
+                         std::string vnet_type, MessageBuffer *b);
 
     bool isVNetOrdered(int vnet) { return m_ordered[vnet]; }
     bool validVirtualNetwork(int vnet) { return m_in_use[vnet]; }
@@ -89,6 +91,7 @@ class SimpleNetwork : public Network
     // Private copy constructor and assignment operator
     SimpleNetwork(const SimpleNetwork& obj);
     SimpleNetwork& operator=(const SimpleNetwork& obj);
+
     std::vector<Switch*> m_switches;
     std::vector<MessageBuffer*> m_buffers_to_free;
     std::vector<Switch*> m_endpoint_switches;
diff --git a/src/mem/ruby/network/simple/Switch.cc b/src/mem/ruby/network/simple/Switch.cc
index 6e116d82c..e028de02a 100644
--- a/src/mem/ruby/network/simple/Switch.cc
+++ b/src/mem/ruby/network/simple/Switch.cc
@@ -64,29 +64,33 @@ Switch::init()
 }
 
 void
-Switch::addInPort(const vector<MessageBuffer*>& in)
+Switch::addInPort(const map<int, MessageBuffer*>& in)
 {
     m_perfect_switch->addInPort(in);
 
-    for (int i = 0; i < in.size(); i++) {
-        in[i]->setReceiver(this);
+    for (auto& it : in) {
+        it.second->setReceiver(this);
     }
 }
 
 void
-Switch::addOutPort(const vector<MessageBuffer*>& out,
-    const NetDest& routing_table_entry, Cycles link_latency, int bw_multiplier)
+Switch::addOutPort(const map<int, MessageBuffer*>& out,
+                   const NetDest& routing_table_entry,
+                   Cycles link_latency, int bw_multiplier)
 {
     // Create a throttle
     Throttle* throttle_ptr = new Throttle(m_id, m_throttles.size(),
-            link_latency, bw_multiplier, m_network_ptr->getEndpointBandwidth(),
-            this);
+                                          link_latency, bw_multiplier,
+                                          m_network_ptr->getEndpointBandwidth(),
+                                          this);
+
     m_throttles.push_back(throttle_ptr);
 
     // Create one buffer per vnet (these are intermediaryQueues)
-    vector<MessageBuffer*> intermediateBuffers;
-    for (int i = 0; i < out.size(); i++) {
-        out[i]->setSender(this);
+    map<int, MessageBuffer*> intermediateBuffers;
+
+    for (auto& it : out) {
+        it.second->setSender(this);
 
         MessageBuffer* buffer_ptr = new MessageBuffer;
         // Make these queues ordered
@@ -95,7 +99,7 @@ Switch::addOutPort(const vector<MessageBuffer*>& out,
             buffer_ptr->resize(m_network_ptr->getBufferSize());
         }
 
-        intermediateBuffers.push_back(buffer_ptr);
+        intermediateBuffers[it.first] = buffer_ptr;
         m_buffers_to_free.push_back(buffer_ptr);
 
         buffer_ptr->setSender(this);
diff --git a/src/mem/ruby/network/simple/Switch.hh b/src/mem/ruby/network/simple/Switch.hh
index 58193d42d..d4e5c5eba 100644
--- a/src/mem/ruby/network/simple/Switch.hh
+++ b/src/mem/ruby/network/simple/Switch.hh
@@ -60,12 +60,13 @@ class Switch : public BasicRouter
     typedef SwitchParams Params;
     Switch(const Params *p);
     ~Switch();
-
     void init();
-    void addInPort(const std::vector<MessageBuffer*>& in);
-    void addOutPort(const std::vector<MessageBuffer*>& out,
-        const NetDest& routing_table_entry, Cycles link_latency,
-        int bw_multiplier);
+
+    void addInPort(const std::map<int, MessageBuffer*>& in);
+    void addOutPort(const std::map<int, MessageBuffer*>& out,
+                    const NetDest& routing_table_entry,
+                    Cycles link_latency, int bw_multiplier);
+
     const Throttle* getThrottle(LinkID link_number) const;
 
     void resetStats();
diff --git a/src/mem/ruby/network/simple/Throttle.cc b/src/mem/ruby/network/simple/Throttle.cc
index 40958a6da..91bad217b 100644
--- a/src/mem/ruby/network/simple/Throttle.cc
+++ b/src/mem/ruby/network/simple/Throttle.cc
@@ -69,42 +69,92 @@ Throttle::init(NodeID node, Cycles link_latency,
                int link_bandwidth_multiplier, int endpoint_bandwidth)
 {
     m_node = node;
-    m_vnets = 0;
-
     assert(link_bandwidth_multiplier > 0);
     m_link_bandwidth_multiplier = link_bandwidth_multiplier;
+
     m_link_latency = link_latency;
     m_endpoint_bandwidth = endpoint_bandwidth;
 
     m_wakeups_wo_switch = 0;
-
     m_link_utilization_proxy = 0;
 }
 
 void
-Throttle::addLinks(const std::vector<MessageBuffer*>& in_vec,
-    const std::vector<MessageBuffer*>& out_vec)
+Throttle::addLinks(const map<int, MessageBuffer*>& in_vec,
+                   const map<int, MessageBuffer*>& out_vec)
 {
     assert(in_vec.size() == out_vec.size());
-    for (int i=0; i<in_vec.size(); i++) {
-        addVirtualNetwork(in_vec[i], out_vec[i]);
+
+    for (auto& it : in_vec) {
+        int vnet = it.first;
+
+        auto jt = out_vec.find(vnet);
+        assert(jt != out_vec.end());
+
+        MessageBuffer *in_ptr = it.second;
+        MessageBuffer *out_ptr = (*jt).second;
+
+        m_in[vnet] = in_ptr;
+        m_out[vnet] = out_ptr;
+        m_units_remaining[vnet] = 0;
+
+        // Set consumer and description
+        in_ptr->setConsumer(this);
+        string desc = "[Queue to Throttle " + to_string(m_sID) + " " +
+            to_string(m_node) + "]";
+        in_ptr->setDescription(desc);
     }
 }
 
 void
-Throttle::addVirtualNetwork(MessageBuffer* in_ptr, MessageBuffer* out_ptr)
+Throttle::operateVnet(int vnet, int &bw_remaining, bool &schedule_wakeup,
+                      MessageBuffer *in, MessageBuffer *out)
 {
-    m_units_remaining.push_back(0);
-    m_in.push_back(in_ptr);
-    m_out.push_back(out_ptr);
+    assert(out != NULL);
+    assert(in != NULL);
+    assert(m_units_remaining[vnet] >= 0);
+
+    while (bw_remaining > 0 && (in->isReady() || m_units_remaining[vnet] > 0) &&
+                                out->areNSlotsAvailable(1)) {
+
+        // See if we are done transferring the previous message on
+        // this virtual network
+        if (m_units_remaining[vnet] == 0 && in->isReady()) {
+            // Find the size of the message we are moving
+            MsgPtr msg_ptr = in->peekMsgPtr();
+            NetworkMessage* net_msg_ptr =
+                safe_cast<NetworkMessage*>(msg_ptr.get());
+            m_units_remaining[vnet] +=
+                network_message_to_size(net_msg_ptr);
+
+            DPRINTF(RubyNetwork, "throttle: %d my bw %d bw spent "
+                    "enqueueing net msg %d time: %lld.\n",
+                    m_node, getLinkBandwidth(), m_units_remaining[vnet],
+                    g_system_ptr->curCycle());
+
+            // Move the message
+            in->dequeue();
+            out->enqueue(msg_ptr, m_link_latency);
+
+            // Count the message
+            m_msg_counts[net_msg_ptr->getMessageSize()][vnet]++;
+            DPRINTF(RubyNetwork, "%s\n", *out);
+        }
+
+        // Calculate the amount of bandwidth we spent on this message
+        int diff = m_units_remaining[vnet] - bw_remaining;
+        m_units_remaining[vnet] = max(0, diff);
+        bw_remaining = max(0, -diff);
+    }
 
-    // Set consumer and description
-    m_in[m_vnets]->setConsumer(this);
+    if (bw_remaining > 0 && (in->isReady() || m_units_remaining[vnet] > 0) &&
+                             !out->areNSlotsAvailable(1)) {
+        DPRINTF(RubyNetwork, "vnet: %d", vnet);
 
-    string desc = "[Queue to Throttle " + to_string(m_sID) + " " +
-        to_string(m_node) + "]";
-    m_in[m_vnets]->setDescription(desc);
-    m_vnets++;
+        // schedule me to wakeup again because I'm waiting for my
+        // output queue to become available
+        schedule_wakeup = true;
+    }
 }
 
 void
@@ -114,71 +164,30 @@ Throttle::wakeup()
     assert(getLinkBandwidth() > 0);
     int bw_remaining = getLinkBandwidth();
 
-    // Give the highest numbered link priority most of the time
     m_wakeups_wo_switch++;
-    int highest_prio_vnet = m_vnets-1;
-    int lowest_prio_vnet = 0;
-    int counter = 1;
     bool schedule_wakeup = false;
 
+    // variable for deciding the direction in which to iterate
+    bool iteration_direction = false;
+
+
     // invert priorities to avoid starvation seen in the component network
     if (m_wakeups_wo_switch > PRIORITY_SWITCH_LIMIT) {
         m_wakeups_wo_switch = 0;
-        highest_prio_vnet = 0;
-        lowest_prio_vnet = m_vnets-1;
-        counter = -1;
+        iteration_direction = true;
     }
 
-    for (int vnet = highest_prio_vnet;
-         (vnet * counter) >= (counter * lowest_prio_vnet);
-         vnet -= counter) {
-
-        assert(m_out[vnet] != NULL);
-        assert(m_in[vnet] != NULL);
-        assert(m_units_remaining[vnet] >= 0);
-
-        while (bw_remaining > 0 &&
-            (m_in[vnet]->isReady() || m_units_remaining[vnet] > 0) &&
-            m_out[vnet]->areNSlotsAvailable(1)) {
-
-            // See if we are done transferring the previous message on
-            // this virtual network
-            if (m_units_remaining[vnet] == 0 && m_in[vnet]->isReady()) {
-                // Find the size of the message we are moving
-                MsgPtr msg_ptr = m_in[vnet]->peekMsgPtr();
-                NetworkMessage* net_msg_ptr =
-                    safe_cast<NetworkMessage*>(msg_ptr.get());
-                m_units_remaining[vnet] +=
-                    network_message_to_size(net_msg_ptr);
-
-                DPRINTF(RubyNetwork, "throttle: %d my bw %d bw spent "
-                        "enqueueing net msg %d time: %lld.\n",
-                        m_node, getLinkBandwidth(), m_units_remaining[vnet],
-                        g_system_ptr->curCycle());
-
-                // Move the message
-                m_in[vnet]->dequeue();
-                m_out[vnet]->enqueue(msg_ptr, m_link_latency);
-
-                // Count the message
-                m_msg_counts[net_msg_ptr->getMessageSize()][vnet]++;
-
-                DPRINTF(RubyNetwork, "%s\n", *m_out[vnet]);
-            }
-
-            // Calculate the amount of bandwidth we spent on this message
-            int diff = m_units_remaining[vnet] - bw_remaining;
-            m_units_remaining[vnet] = max(0, diff);
-            bw_remaining = max(0, -diff);
+    if (iteration_direction) {
+        for (auto& it : m_in) {
+            int vnet = it.first;
+            operateVnet(vnet, bw_remaining, schedule_wakeup,
+                        it.second, m_out[vnet]);
         }
-
-        if (bw_remaining > 0 &&
-            (m_in[vnet]->isReady() || m_units_remaining[vnet] > 0) &&
-            !m_out[vnet]->areNSlotsAvailable(1)) {
-            DPRINTF(RubyNetwork, "vnet: %d", vnet);
-            // schedule me to wakeup again because I'm waiting for my
-            // output queue to become available
-            schedule_wakeup = true;
+    } else {
+        for (auto it = m_in.rbegin(); it != m_in.rend(); ++it) {
+            int vnet = (*it).first;
+            operateVnet(vnet, bw_remaining, schedule_wakeup,
+                        (*it).second, m_out[vnet]);
         }
     }
 
@@ -215,7 +224,7 @@ Throttle::regStats(string parent)
     for (MessageSizeType type = MessageSizeType_FIRST;
          type < MessageSizeType_NUM; ++type) {
         m_msg_counts[(unsigned int)type]
-            .init(m_vnets)
+            .init(Network::getNumberOfVirtualNetworks())
             .name(parent + csprintf(".throttle%i", m_node) + ".msg_count." +
                     MessageSizeType_to_string(type))
             .flags(Stats::nozero)
diff --git a/src/mem/ruby/network/simple/Throttle.hh b/src/mem/ruby/network/simple/Throttle.hh
index cdc627bb7..d978f14fd 100644
--- a/src/mem/ruby/network/simple/Throttle.hh
+++ b/src/mem/ruby/network/simple/Throttle.hh
@@ -62,8 +62,8 @@ class Throttle : public Consumer
     std::string name()
     { return csprintf("Throttle-%i", m_sID); }
 
-    void addLinks(const std::vector<MessageBuffer*>& in_vec,
-                  const std::vector<MessageBuffer*>& out_vec);
+    void addLinks(const std::map<int, MessageBuffer*>& in_vec,
+                  const std::map<int, MessageBuffer*>& out_vec);
     void wakeup();
 
     // The average utilization (a fraction) since last clearStats()
@@ -85,16 +85,17 @@ class Throttle : public Consumer
   private:
     void init(NodeID node, Cycles link_latency, int link_bandwidth_multiplier,
               int endpoint_bandwidth);
-    void addVirtualNetwork(MessageBuffer* in_ptr, MessageBuffer* out_ptr);
+    void operateVnet(int vnet, int &bw_remainin, bool &schedule_wakeup,
+                     MessageBuffer *in, MessageBuffer *out);
 
     // Private copy constructor and assignment operator
     Throttle(const Throttle& obj);
     Throttle& operator=(const Throttle& obj);
 
-    std::vector<MessageBuffer*> m_in;
-    std::vector<MessageBuffer*> m_out;
-    unsigned int m_vnets;
-    std::vector<int> m_units_remaining;
+    std::map<int, MessageBuffer*> m_in;
+    std::map<int, MessageBuffer*> m_out;
+    std::map<int, int> m_units_remaining;
+
     int m_sID;
     NodeID m_node;
     int m_link_bandwidth_multiplier;