ruby: message buffers: significant changes

This patch is the final patch in a series of patches.  The aim of the series
is to make ruby more configurable than it was.  More specifically, the
connections between controllers are not at all possible (unless one is ready
to make significant changes to the coherence protocol).  Moreover the buffers
themselves are magically connected to the network inside the slicc code.
These connections are not part of the configuration file.

This patch makes changes so that these connections will now be made in the
python configuration files associated with the protocols.  This requires
each state machine to expose the message buffers it uses for input and output.
So, the patch makes these buffers configurable members of the machines.

The patch drops the slicc code that usd to connect these buffers to the
network.  Now these buffers are exposed to the python configuration system
as Master and Slave ports.  In the configuration files, any master port
can be connected any slave port.  The file pyobject.cc has been modified to
take care of allocating the actual message buffer.  This is inline with how
other port connections work.

22 files changed, 247 insertions, 253 deletions

diff --git a/src/mem/ruby/network/Network.cc b/src/mem/ruby/network/Network.cc
index 60531a423..2d9376b08 100644
--- a/src/mem/ruby/network/Network.cc
+++ b/src/mem/ruby/network/Network.cc
@@ -57,19 +57,6 @@ Network::Network(const Params *p)
     // Queues that are feeding the protocol
     m_fromNetQueues.resize(m_nodes);
 
-    for (int node = 0; node < m_nodes; node++) {
-        // Setting number of virtual message buffers per Network Queue
-        m_toNetQueues[node].resize(m_virtual_networks);
-        m_fromNetQueues[node].resize(m_virtual_networks);
-
-        // Instantiating the Message Buffers that
-        // interact with the coherence protocol
-        for (int j = 0; j < m_virtual_networks; j++) {
-            m_toNetQueues[node][j] = new MessageBuffer();
-            m_fromNetQueues[node][j] = new MessageBuffer();
-        }
-    }
-
     m_in_use.resize(m_virtual_networks);
     m_ordered.resize(m_virtual_networks);
 
@@ -95,10 +82,14 @@ Network::Network(const Params *p)
 Network::~Network()
 {
     for (int node = 0; node < m_nodes; node++) {
+
         // Delete the Message Buffers
-        for (int j = 0; j < m_virtual_networks; j++) {
-            delete m_toNetQueues[node][j];
-            delete m_fromNetQueues[node][j];
+        for (auto& it : m_toNetQueues[node]) {
+            delete it.second;
+        }
+
+        for (auto& it : m_fromNetQueues[node]) {
+            delete it.second;
         }
     }
 
diff --git a/src/mem/ruby/network/Network.hh b/src/mem/ruby/network/Network.hh
index dcdd791e7..d595ca285 100644
--- a/src/mem/ruby/network/Network.hh
+++ b/src/mem/ruby/network/Network.hh
@@ -72,11 +72,10 @@ class Network : public ClockedObject
     static uint32_t MessageSizeType_to_int(MessageSizeType size_type);
 
     // returns the queue requested for the given component
-    virtual MessageBuffer* getToNetQueue(NodeID id, bool ordered,
-        int netNumber, std::string vnet_type) = 0;
-    virtual MessageBuffer* getFromNetQueue(NodeID id, bool ordered,
-        int netNumber, std::string vnet_type) = 0;
-
+    virtual void setToNetQueue(NodeID id, bool ordered, int netNumber,
+                               std::string vnet_type, MessageBuffer *b) = 0;
+    virtual void setFromNetQueue(NodeID id, bool ordered, int netNumber,
+                                 std::string vnet_type, MessageBuffer *b) = 0;
 
     virtual void makeOutLink(SwitchID src, NodeID dest, BasicLink* link,
                              LinkDirection direction,
@@ -113,8 +112,8 @@ class Network : public ClockedObject
     static uint32_t m_data_msg_size;
 
     // vector of queues from the components
-    std::vector<std::vector<MessageBuffer*> > m_toNetQueues;
-    std::vector<std::vector<MessageBuffer*> > m_fromNetQueues;
+    std::vector<std::map<int, MessageBuffer*> > m_toNetQueues;
+    std::vector<std::map<int, MessageBuffer*> > m_fromNetQueues;
 
     std::vector<bool> m_in_use;
     std::vector<bool> m_ordered;
diff --git a/src/mem/ruby/network/Network.py b/src/mem/ruby/network/Network.py
index 4f33dd196..8cc38f26f 100644
--- a/src/mem/ruby/network/Network.py
+++ b/src/mem/ruby/network/Network.py
@@ -28,7 +28,6 @@
 #          Brad Beckmann
 
 from m5.params import *
-from m5.SimObject import SimObject
 from ClockedObject import ClockedObject
 from BasicLink import BasicLink
 
@@ -48,3 +47,6 @@ class RubyNetwork(ClockedObject):
     netifs = VectorParam.ClockedObject("Network Interfaces")
     ext_links = VectorParam.BasicExtLink("Links to external nodes")
     int_links = VectorParam.BasicIntLink("Links between internal nodes")
+
+    slave = VectorSlavePort("CPU slave port")
+    master = VectorMasterPort("CPU master port")
diff --git a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
index 92e45c36d..2aeddad37 100644
--- a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
+++ b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
@@ -66,20 +66,20 @@ BaseGarnetNetwork::init()
     Network::init();
 }
 
-MessageBuffer*
-BaseGarnetNetwork::getToNetQueue(NodeID id, bool ordered, int network_num,
-                                 string vnet_type)
+void
+BaseGarnetNetwork::setToNetQueue(NodeID id, bool ordered, int network_num,
+                                 string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    return m_toNetQueues[id][network_num];
+    m_toNetQueues[id][network_num] = b;
 }
 
-MessageBuffer*
-BaseGarnetNetwork::getFromNetQueue(NodeID id, bool ordered, int network_num,  
-                                   string vnet_type)
+void
+BaseGarnetNetwork::setFromNetQueue(NodeID id, bool ordered, int network_num,
+                                   string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    return m_fromNetQueues[id][network_num];
+    m_fromNetQueues[id][network_num] = b;
 }
 
 void
diff --git a/src/mem/ruby/network/garnet/BaseGarnetNetwork.hh b/src/mem/ruby/network/garnet/BaseGarnetNetwork.hh
index c4bb9f5b1..cc1d4d929 100644
--- a/src/mem/ruby/network/garnet/BaseGarnetNetwork.hh
+++ b/src/mem/ruby/network/garnet/BaseGarnetNetwork.hh
@@ -68,12 +68,11 @@ class BaseGarnetNetwork : public Network
         m_queueing_latency[vnet] += latency;
     }
 
-    // returns the queue requested for the given component
-    MessageBuffer* getToNetQueue(NodeID id, bool ordered, int network_num,
-                                 std::string vnet_type);
-    MessageBuffer* getFromNetQueue(NodeID id, bool ordered, int network_num,
-                                   std::string vnet_type);
-
+    // set the queue
+    void setToNetQueue(NodeID id, bool ordered, int network_num,
+                       std::string vnet_type, MessageBuffer *b);
+    void setFromNetQueue(NodeID id, bool ordered, int network_num,
+                         std::string vnet_type, MessageBuffer *b);
 
     bool isVNetOrdered(int vnet) { return m_ordered[vnet]; }
     bool validVirtualNetwork(int vnet) { return m_in_use[vnet]; }
diff --git a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc
index 2f1b5ee46..7384cc6a7 100644
--- a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc
+++ b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc
@@ -53,8 +53,6 @@ NetworkInterface_d::NetworkInterface_d(const Params *p)
     m_vc_round_robin = 0;
     m_ni_buffers.resize(m_num_vcs);
     m_ni_enqueue_time.resize(m_num_vcs);
-    inNode_ptr.resize(m_virtual_networks);
-    outNode_ptr.resize(m_virtual_networks);
     creditQueue = new flitBuffer_d();
 
     // instantiating the NI flit buffers
@@ -108,18 +106,20 @@ NetworkInterface_d::addOutPort(NetworkLink_d *out_link,
 }
 
 void
-NetworkInterface_d::addNode(vector<MessageBuffer *>& in,
-                            vector<MessageBuffer *>& out)
+NetworkInterface_d::addNode(map<int, MessageBuffer *>& in,
+                            map<int, MessageBuffer *>& out)
 {
-    assert(in.size() == m_virtual_networks);
     inNode_ptr = in;
     outNode_ptr = out;
 
-    for (int j = 0; j < m_virtual_networks; j++) {
+    for (auto& it : in) {
         // the protocol injects messages into the NI
-        inNode_ptr[j]->setConsumer(this);
-        inNode_ptr[j]->setReceiver(this);
-        outNode_ptr[j]->setSender(this);
+        it.second->setConsumer(this);
+        it.second->setReceiver(this);
+    }
+
+    for (auto& it : out) {
+        it.second->setSender(this);
     }
 }
 
@@ -223,11 +223,14 @@ NetworkInterface_d::wakeup()
 
     // Checking for messages coming from the protocol
     // can pick up a message/cycle for each virtual net
-    for (int vnet = 0; vnet < m_virtual_networks; vnet++) {
-        while (inNode_ptr[vnet]->isReady()) { // Is there a message waiting
-            msg_ptr = inNode_ptr[vnet]->peekMsgPtr();
+    for (auto it = inNode_ptr.begin(); it != inNode_ptr.end(); ++it) {
+        int vnet = (*it).first;
+        MessageBuffer *b = (*it).second;
+
+        while (b->isReady()) { // Is there a message waiting
+            msg_ptr = b->peekMsgPtr();
             if (flitisizeMessage(msg_ptr, vnet)) {
-                inNode_ptr[vnet]->dequeue();
+                b->dequeue();
             } else {
                 break;
             }
@@ -351,12 +354,15 @@ NetworkInterface_d::get_vnet(int vc)
 void
 NetworkInterface_d::checkReschedule()
 {
-    for (int vnet = 0; vnet < m_virtual_networks; vnet++) {
-        if (inNode_ptr[vnet]->isReady()) { // Is there a message waiting
+    for (const auto& it : inNode_ptr) {
+        MessageBuffer *b = it.second;
+
+        while (b->isReady()) { // Is there a message waiting
             scheduleEvent(Cycles(1));
             return;
         }
     }
+
     for (int vc = 0; vc < m_num_vcs; vc++) {
         if (m_ni_buffers[vc]->isReady(curCycle() + Cycles(1))) {
             scheduleEvent(Cycles(1));
diff --git a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh
index 05142cd28..2494d05d1 100644
--- a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh
+++ b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh
@@ -60,8 +60,9 @@ class NetworkInterface_d : public ClockedObject, public Consumer
     void addOutPort(NetworkLink_d *out_link, CreditLink_d *credit_link);
 
     void wakeup();
-    void addNode(std::vector<MessageBuffer *> &inNode,
-                 std::vector<MessageBuffer *> &outNode);
+    void addNode(std::map<int, MessageBuffer *> &inNode,
+                 std::map<int, MessageBuffer *> &outNode);
+
     void print(std::ostream& out) const;
     int get_vnet(int vc);
     void init_net_ptr(GarnetNetwork_d *net_ptr) { m_net_ptr = net_ptr; }
@@ -89,9 +90,9 @@ class NetworkInterface_d : public ClockedObject, public Consumer
     std::vector<Cycles> m_ni_enqueue_time;
 
     // The Message buffers that takes messages from the protocol
-    std::vector<MessageBuffer *> inNode_ptr;
+    std::map<int, MessageBuffer *> inNode_ptr;
     // The Message buffers that provides messages to the protocol
-    std::vector<MessageBuffer *> outNode_ptr;
+    std::map<int, MessageBuffer *> outNode_ptr;
 
     bool flitisizeMessage(MsgPtr msg_ptr, int vnet);
     int calculateVC(int vnet);
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc
index 13bbe2b08..26d2423e8 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc
@@ -49,13 +49,10 @@ NetworkInterface::NetworkInterface(const Params *p)
     m_virtual_networks  = p->virt_nets;
     m_vc_per_vnet = p->vcs_per_vnet;
     m_num_vcs = m_vc_per_vnet*m_virtual_networks;
-
     m_vc_round_robin = 0;
-    m_ni_buffers.resize(m_num_vcs);
-    inNode_ptr.resize(m_virtual_networks);
-    outNode_ptr.resize(m_virtual_networks);
 
     // instantiating the NI flit buffers
+    m_ni_buffers.resize(m_num_vcs);
     for (int i =0; i < m_num_vcs; i++)
         m_ni_buffers[i] = new flitBuffer();
 
@@ -93,18 +90,20 @@ NetworkInterface::addOutPort(NetworkLink *out_link)
 }
 
 void
-NetworkInterface::addNode(vector<MessageBuffer*>& in,
-    vector<MessageBuffer*>& out)
+NetworkInterface::addNode(map<int, MessageBuffer*>& in,
+    map<int, MessageBuffer*>& out)
 {
-    assert(in.size() == m_virtual_networks);
     inNode_ptr = in;
     outNode_ptr = out;
 
-    // protocol injects messages into the NI
-    for (int j = 0; j < m_virtual_networks; j++) {
-        inNode_ptr[j]->setConsumer(this);
-        inNode_ptr[j]->setReceiver(this);
-        outNode_ptr[j]->setSender(this);
+    for (auto& it: in) {
+        // the protocol injects messages into the NI
+        it.second->setConsumer(this);
+        it.second->setReceiver(this);
+    }
+
+    for (auto& it : out) {
+        it.second->setSender(this);
     }
 }
 
@@ -243,12 +242,14 @@ NetworkInterface::wakeup()
 
     //Checking for messages coming from the protocol
     // can pick up a message/cycle for each virtual net
-    for (int vnet = 0; vnet < m_virtual_networks; vnet++) {
-        while (inNode_ptr[vnet]->isReady()) // Is there a message waiting
-        {
-            msg_ptr = inNode_ptr[vnet]->peekMsgPtr();
+    for (auto it = inNode_ptr.begin(); it != inNode_ptr.end(); ++it) {
+        int vnet = (*it).first;
+        MessageBuffer *b = (*it).second;
+
+        while (b->isReady()) { // Is there a message waiting
+            msg_ptr = b->peekMsgPtr();
             if (flitisizeMessage(msg_ptr, vnet)) {
-                inNode_ptr[vnet]->dequeue();
+                b->dequeue();
             } else {
                 break;
             }
@@ -324,14 +325,17 @@ NetworkInterface::scheduleOutputLink()
 void
 NetworkInterface::checkReschedule()
 {
-    for (int vnet = 0; vnet < m_virtual_networks; vnet++) {
-        if (inNode_ptr[vnet]->isReady()) { // Is there a message waiting
+    for (const auto& it : inNode_ptr) {
+        MessageBuffer *b = it.second;
+
+        while (b->isReady()) { // Is there a message waiting
             scheduleEvent(Cycles(1));
             return;
         }
     }
+
     for (int vc = 0; vc < m_num_vcs; vc++) {
-        if (m_ni_buffers[vc]->isReadyForNext(curCycle())) {
+        if (m_ni_buffers[vc]->isReady(curCycle() + Cycles(1))) {
             scheduleEvent(Cycles(1));
             return;
         }
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh
index 0af538bf2..aa30bd758 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh
@@ -56,10 +56,10 @@ class NetworkInterface : public ClockedObject, public FlexibleConsumer
 
     void addInPort(NetworkLink *in_link);
     void addOutPort(NetworkLink *out_link);
+    void addNode(std::map<int, MessageBuffer *> &inNode,
+                 std::map<int, MessageBuffer *> &outNode);
 
     void wakeup();
-    void addNode(std::vector<MessageBuffer *> &inNode,
-                 std::vector<MessageBuffer *> &outNode);
     void grant_vc(int out_port, int vc, Cycles grant_time);
     void release_vc(int out_port, int vc, Cycles release_time);
 
@@ -93,10 +93,10 @@ class NetworkInterface : public ClockedObject, public FlexibleConsumer
     std::vector<flitBuffer *>   m_ni_buffers;
 
     // The Message buffers that takes messages from the protocol
-    std::vector<MessageBuffer *> inNode_ptr;
+    std::map<int, MessageBuffer *> inNode_ptr;
 
     // The Message buffers that provides messages to the protocol
-    std::vector<MessageBuffer *> outNode_ptr;
+    std::map<int, MessageBuffer *> outNode_ptr;
 
     bool flitisizeMessage(MsgPtr msg_ptr, int vnet);
     int calculateVC(int vnet);
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/Router.cc b/src/mem/ruby/network/garnet/flexible-pipeline/Router.cc
index 851ababc4..0fc2c6be3 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/Router.cc
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/Router.cc
@@ -387,7 +387,7 @@ Router::checkReschedule()
 {
     for (int port = 0; port < m_out_link.size(); port++) {
         for (int vc = 0; vc < m_num_vcs; vc++) {
-            if (m_router_buffers[port][vc]->isReadyForNext(curCycle())) {
+            if (m_router_buffers[port][vc]->isReady(curCycle() + Cycles(1))) {
                 scheduleEvent(Cycles(1));
                 return;
             }
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/flitBuffer.cc b/src/mem/ruby/network/garnet/flexible-pipeline/flitBuffer.cc
index 7a8ea8c23..ee31ac3d2 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/flitBuffer.cc
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/flitBuffer.cc
@@ -62,17 +62,6 @@ flitBuffer::isReady(Cycles curTime)
 }
 
 bool
-flitBuffer::isReadyForNext(Cycles curTime)
-{
-    if (m_buffer.size() != 0 ) {
-        flit *t_flit = m_buffer.front();
-        if (t_flit->get_time() <= (curTime + 1))
-            return true;
-    }
-    return false;
-}
-
-bool
 flitBuffer::isFull()
 {
     return (m_buffer.size() >= max_size);
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/flitBuffer.hh b/src/mem/ruby/network/garnet/flexible-pipeline/flitBuffer.hh
index 609c5a9b6..99fa2678b 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/flitBuffer.hh
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/flitBuffer.hh
@@ -44,7 +44,6 @@ class flitBuffer
     flitBuffer(int maximum_size);
 
     bool isReady(Cycles curTime);
-    bool isReadyForNext(Cycles curTime);
     bool isFull();
     bool isEmpty();
     void setMaxSize(int maximum);
diff --git a/src/mem/ruby/network/simple/PerfectSwitch.cc b/src/mem/ruby/network/simple/PerfectSwitch.cc
index 0c6111c48..4565711a2 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.cc
+++ b/src/mem/ruby/network/simple/PerfectSwitch.cc
@@ -61,36 +61,33 @@ PerfectSwitch::init(SimpleNetwork *network_ptr)
 {
     m_network_ptr = network_ptr;
 
-    for(int i = 0;i < m_virtual_networks;++i)
-    {
+    for(int i = 0;i < m_virtual_networks;++i) {
         m_pending_message_count.push_back(0);
     }
 }
 
 void
-PerfectSwitch::addInPort(const vector<MessageBuffer*>& in)
+PerfectSwitch::addInPort(const map<int, MessageBuffer*>& in)
 {
-    assert(in.size() == m_virtual_networks);
     NodeID port = m_in.size();
     m_in.push_back(in);
 
-    for (int j = 0; j < m_virtual_networks; j++) {
-        m_in[port][j]->setConsumer(this);
+    for (auto& it : in) {
+        it.second->setConsumer(this);
 
         string desc = csprintf("[Queue from port %s %s %s to PerfectSwitch]",
-            to_string(m_switch_id), to_string(port), to_string(j));
-        m_in[port][j]->setDescription(desc);
-        m_in[port][j]->setIncomingLink(port);
-        m_in[port][j]->setVnet(j);
+            to_string(m_switch_id), to_string(port), to_string(it.first));
+
+        it.second->setDescription(desc);
+        it.second->setIncomingLink(port);
+        it.second->setVnet(it.first);
     }
 }
 
 void
-PerfectSwitch::addOutPort(const vector<MessageBuffer*>& out,
+PerfectSwitch::addOutPort(const map<int, MessageBuffer*>& out,
     const NetDest& routing_table_entry)
 {
-    assert(out.size() == m_virtual_networks);
-
     // Setup link order
     LinkOrder l;
     l.m_value = 0;
@@ -152,11 +149,16 @@ PerfectSwitch::wakeup()
                 vector<NetDest> output_link_destinations;
 
                 // Is there a message waiting?
-                while (m_in[incoming][vnet]->isReady()) {
+                auto it = m_in[incoming].find(vnet);
+                if (it == m_in[incoming].end())
+                    continue;
+                MessageBuffer *buffer = (*it).second;
+
+                while (buffer->isReady()) {
                     DPRINTF(RubyNetwork, "incoming: %d\n", incoming);
 
                     // Peek at message
-                    msg_ptr = m_in[incoming][vnet]->peekMsgPtr();
+                    msg_ptr = buffer->peekMsgPtr();
                     net_msg_ptr = safe_cast<NetworkMessage*>(msg_ptr.get());
                     DPRINTF(RubyNetwork, "Message: %s\n", (*net_msg_ptr));
 
@@ -261,7 +263,7 @@ PerfectSwitch::wakeup()
                     }
 
                     // Dequeue msg
-                    m_in[incoming][vnet]->dequeue();
+                    buffer->dequeue();
                     m_pending_message_count[vnet]--;
 
                     // Enqueue it - for all outgoing queues
diff --git a/src/mem/ruby/network/simple/PerfectSwitch.hh b/src/mem/ruby/network/simple/PerfectSwitch.hh
index c01c50a3b..25e3e2754 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.hh
+++ b/src/mem/ruby/network/simple/PerfectSwitch.hh
@@ -65,9 +65,10 @@ class PerfectSwitch : public Consumer
     { return csprintf("PerfectSwitch-%i", m_switch_id); }
 
     void init(SimpleNetwork *);
-    void addInPort(const std::vector<MessageBuffer*>& in);
-    void addOutPort(const std::vector<MessageBuffer*>& out,
+    void addInPort(const std::map<int, MessageBuffer*>& in);
+    void addOutPort(const std::map<int, MessageBuffer*>& out,
                     const NetDest& routing_table_entry);
+
     int getInLinks() const { return m_in.size(); }
     int getOutLinks() const { return m_out.size(); }
 
@@ -86,8 +87,9 @@ class PerfectSwitch : public Consumer
     SwitchID m_switch_id;
 
     // vector of queues from the components
-    std::vector<std::vector<MessageBuffer*> > m_in;
-    std::vector<std::vector<MessageBuffer*> > m_out;
+    std::vector<std::map<int, MessageBuffer*> > m_in;
+    std::vector<std::map<int, MessageBuffer*> > m_out;
+
     std::vector<NetDest> m_routing_table;
     std::vector<LinkOrder> m_link_order;
 
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc
index 2d08f9fa4..f51a0c891 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.cc
+++ b/src/mem/ruby/network/simple/SimpleNetwork.cc
@@ -93,10 +93,9 @@ SimpleNetwork::makeOutLink(SwitchID src, NodeID dest, BasicLink* link,
 
     SimpleExtLink *simple_link = safe_cast<SimpleExtLink*>(link);
 
-    m_switches[src]->addOutPort(m_fromNetQueues[dest],
-                                         routing_table_entry,
-                                         simple_link->m_latency,
-                                         simple_link->m_bw_multiplier);
+    m_switches[src]->addOutPort(m_fromNetQueues[dest], routing_table_entry,
+                                simple_link->m_latency,
+                                simple_link->m_bw_multiplier);
 
     m_endpoint_switches[dest] = m_switches[src];
 }
@@ -118,25 +117,28 @@ SimpleNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
                                 const NetDest& routing_table_entry)
 {
     // Create a set of new MessageBuffers
-    std::vector<MessageBuffer*> queues;
+    std::map<int, MessageBuffer*> queues;
     for (int i = 0; i < m_virtual_networks; i++) {
         // allocate a buffer
         MessageBuffer* buffer_ptr = new MessageBuffer;
         buffer_ptr->setOrdering(true);
+
         if (m_buffer_size > 0) {
             buffer_ptr->resize(m_buffer_size);
         }
-        queues.push_back(buffer_ptr);
+
+        queues[i] = buffer_ptr;
         // remember to deallocate it
         m_buffers_to_free.push_back(buffer_ptr);
     }
+
     // Connect it to the two switches
     SimpleIntLink *simple_link = safe_cast<SimpleIntLink*>(link);
 
     m_switches[dest]->addInPort(queues);
     m_switches[src]->addOutPort(queues, routing_table_entry,
-                                         simple_link->m_latency, 
-                                         simple_link->m_bw_multiplier);
+                                simple_link->m_latency,
+                                simple_link->m_bw_multiplier);
 }
 
 void
@@ -151,20 +153,20 @@ SimpleNetwork::checkNetworkAllocation(NodeID id, bool ordered, int network_num)
     m_in_use[network_num] = true;
 }
 
-MessageBuffer*
-SimpleNetwork::getToNetQueue(NodeID id, bool ordered, int network_num,
-                             std::string vnet_type)
+void
+SimpleNetwork::setToNetQueue(NodeID id, bool ordered, int network_num,
+                             std::string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num);
-    return m_toNetQueues[id][network_num];
+    m_toNetQueues[id][network_num] = b;
 }
 
-MessageBuffer*
-SimpleNetwork::getFromNetQueue(NodeID id, bool ordered, int network_num,
-                               std::string vnet_type)
+void
+SimpleNetwork::setFromNetQueue(NodeID id, bool ordered, int network_num,
+                               std::string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num);
-    return m_fromNetQueues[id][network_num];
+    m_fromNetQueues[id][network_num] = b;
 }
 
 void
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.hh b/src/mem/ruby/network/simple/SimpleNetwork.hh
index 90560c267..a2723c715 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.hh
+++ b/src/mem/ruby/network/simple/SimpleNetwork.hh
@@ -56,9 +56,11 @@ class SimpleNetwork : public Network
     void collateStats();
     void regStats();
 
-    // returns the queue requested for the given component
-    MessageBuffer* getToNetQueue(NodeID id, bool ordered, int network_num, std::string vnet_type);
-    MessageBuffer* getFromNetQueue(NodeID id, bool ordered, int network_num, std::string vnet_type);
+    // sets the queue requested
+    void setToNetQueue(NodeID id, bool ordered, int network_num,
+                       std::string vnet_type, MessageBuffer *b);
+    void setFromNetQueue(NodeID id, bool ordered, int network_num,
+                         std::string vnet_type, MessageBuffer *b);
 
     bool isVNetOrdered(int vnet) { return m_ordered[vnet]; }
     bool validVirtualNetwork(int vnet) { return m_in_use[vnet]; }
@@ -89,6 +91,7 @@ class SimpleNetwork : public Network
     // Private copy constructor and assignment operator
     SimpleNetwork(const SimpleNetwork& obj);
     SimpleNetwork& operator=(const SimpleNetwork& obj);
+
     std::vector<Switch*> m_switches;
     std::vector<MessageBuffer*> m_buffers_to_free;
     std::vector<Switch*> m_endpoint_switches;
diff --git a/src/mem/ruby/network/simple/Switch.cc b/src/mem/ruby/network/simple/Switch.cc
index 6e116d82c..e028de02a 100644
--- a/src/mem/ruby/network/simple/Switch.cc
+++ b/src/mem/ruby/network/simple/Switch.cc
@@ -64,29 +64,33 @@ Switch::init()
 }
 
 void
-Switch::addInPort(const vector<MessageBuffer*>& in)
+Switch::addInPort(const map<int, MessageBuffer*>& in)
 {
     m_perfect_switch->addInPort(in);
 
-    for (int i = 0; i < in.size(); i++) {
-        in[i]->setReceiver(this);
+    for (auto& it : in) {
+        it.second->setReceiver(this);
     }
 }
 
 void
-Switch::addOutPort(const vector<MessageBuffer*>& out,
-    const NetDest& routing_table_entry, Cycles link_latency, int bw_multiplier)
+Switch::addOutPort(const map<int, MessageBuffer*>& out,
+                   const NetDest& routing_table_entry,
+                   Cycles link_latency, int bw_multiplier)
 {
     // Create a throttle
     Throttle* throttle_ptr = new Throttle(m_id, m_throttles.size(),
-            link_latency, bw_multiplier, m_network_ptr->getEndpointBandwidth(),
-            this);
+                                          link_latency, bw_multiplier,
+                                          m_network_ptr->getEndpointBandwidth(),
+                                          this);
+
     m_throttles.push_back(throttle_ptr);
 
     // Create one buffer per vnet (these are intermediaryQueues)
-    vector<MessageBuffer*> intermediateBuffers;
-    for (int i = 0; i < out.size(); i++) {
-        out[i]->setSender(this);
+    map<int, MessageBuffer*> intermediateBuffers;
+
+    for (auto& it : out) {
+        it.second->setSender(this);
 
         MessageBuffer* buffer_ptr = new MessageBuffer;
         // Make these queues ordered
@@ -95,7 +99,7 @@ Switch::addOutPort(const vector<MessageBuffer*>& out,
             buffer_ptr->resize(m_network_ptr->getBufferSize());
         }
 
-        intermediateBuffers.push_back(buffer_ptr);
+        intermediateBuffers[it.first] = buffer_ptr;
         m_buffers_to_free.push_back(buffer_ptr);
 
         buffer_ptr->setSender(this);
diff --git a/src/mem/ruby/network/simple/Switch.hh b/src/mem/ruby/network/simple/Switch.hh
index 58193d42d..d4e5c5eba 100644
--- a/src/mem/ruby/network/simple/Switch.hh
+++ b/src/mem/ruby/network/simple/Switch.hh
@@ -60,12 +60,13 @@ class Switch : public BasicRouter
     typedef SwitchParams Params;
     Switch(const Params *p);
     ~Switch();
-
     void init();
-    void addInPort(const std::vector<MessageBuffer*>& in);
-    void addOutPort(const std::vector<MessageBuffer*>& out,
-        const NetDest& routing_table_entry, Cycles link_latency,
-        int bw_multiplier);
+
+    void addInPort(const std::map<int, MessageBuffer*>& in);
+    void addOutPort(const std::map<int, MessageBuffer*>& out,
+                    const NetDest& routing_table_entry,
+                    Cycles link_latency, int bw_multiplier);
+
     const Throttle* getThrottle(LinkID link_number) const;
 
     void resetStats();
diff --git a/src/mem/ruby/network/simple/Throttle.cc b/src/mem/ruby/network/simple/Throttle.cc
index 40958a6da..91bad217b 100644
--- a/src/mem/ruby/network/simple/Throttle.cc
+++ b/src/mem/ruby/network/simple/Throttle.cc
@@ -69,42 +69,92 @@ Throttle::init(NodeID node, Cycles link_latency,
                int link_bandwidth_multiplier, int endpoint_bandwidth)
 {
     m_node = node;
-    m_vnets = 0;
-
     assert(link_bandwidth_multiplier > 0);
     m_link_bandwidth_multiplier = link_bandwidth_multiplier;
+
     m_link_latency = link_latency;
     m_endpoint_bandwidth = endpoint_bandwidth;
 
     m_wakeups_wo_switch = 0;
-
     m_link_utilization_proxy = 0;
 }
 
 void
-Throttle::addLinks(const std::vector<MessageBuffer*>& in_vec,
-    const std::vector<MessageBuffer*>& out_vec)
+Throttle::addLinks(const map<int, MessageBuffer*>& in_vec,
+                   const map<int, MessageBuffer*>& out_vec)
 {
     assert(in_vec.size() == out_vec.size());
-    for (int i=0; i<in_vec.size(); i++) {
-        addVirtualNetwork(in_vec[i], out_vec[i]);
+
+    for (auto& it : in_vec) {
+        int vnet = it.first;
+
+        auto jt = out_vec.find(vnet);
+        assert(jt != out_vec.end());
+
+        MessageBuffer *in_ptr = it.second;
+        MessageBuffer *out_ptr = (*jt).second;
+
+        m_in[vnet] = in_ptr;
+        m_out[vnet] = out_ptr;
+        m_units_remaining[vnet] = 0;
+
+        // Set consumer and description
+        in_ptr->setConsumer(this);
+        string desc = "[Queue to Throttle " + to_string(m_sID) + " " +
+            to_string(m_node) + "]";
+        in_ptr->setDescription(desc);
     }
 }
 
 void
-Throttle::addVirtualNetwork(MessageBuffer* in_ptr, MessageBuffer* out_ptr)
+Throttle::operateVnet(int vnet, int &bw_remaining, bool &schedule_wakeup,
+                      MessageBuffer *in, MessageBuffer *out)
 {
-    m_units_remaining.push_back(0);
-    m_in.push_back(in_ptr);
-    m_out.push_back(out_ptr);
+    assert(out != NULL);
+    assert(in != NULL);
+    assert(m_units_remaining[vnet] >= 0);
+
+    while (bw_remaining > 0 && (in->isReady() || m_units_remaining[vnet] > 0) &&
+                                out->areNSlotsAvailable(1)) {
+
+        // See if we are done transferring the previous message on
+        // this virtual network
+        if (m_units_remaining[vnet] == 0 && in->isReady()) {
+            // Find the size of the message we are moving
+            MsgPtr msg_ptr = in->peekMsgPtr();
+            NetworkMessage* net_msg_ptr =
+                safe_cast<NetworkMessage*>(msg_ptr.get());
+            m_units_remaining[vnet] +=
+                network_message_to_size(net_msg_ptr);
+
+            DPRINTF(RubyNetwork, "throttle: %d my bw %d bw spent "
+                    "enqueueing net msg %d time: %lld.\n",
+                    m_node, getLinkBandwidth(), m_units_remaining[vnet],
+                    g_system_ptr->curCycle());
+
+            // Move the message
+            in->dequeue();
+            out->enqueue(msg_ptr, m_link_latency);
+
+            // Count the message
+            m_msg_counts[net_msg_ptr->getMessageSize()][vnet]++;
+            DPRINTF(RubyNetwork, "%s\n", *out);
+        }
+
+        // Calculate the amount of bandwidth we spent on this message
+        int diff = m_units_remaining[vnet] - bw_remaining;
+        m_units_remaining[vnet] = max(0, diff);
+        bw_remaining = max(0, -diff);
+    }
 
-    // Set consumer and description
-    m_in[m_vnets]->setConsumer(this);
+    if (bw_remaining > 0 && (in->isReady() || m_units_remaining[vnet] > 0) &&
+                             !out->areNSlotsAvailable(1)) {
+        DPRINTF(RubyNetwork, "vnet: %d", vnet);
 
-    string desc = "[Queue to Throttle " + to_string(m_sID) + " " +
-        to_string(m_node) + "]";
-    m_in[m_vnets]->setDescription(desc);
-    m_vnets++;
+        // schedule me to wakeup again because I'm waiting for my
+        // output queue to become available
+        schedule_wakeup = true;
+    }
 }
 
 void
@@ -114,71 +164,30 @@ Throttle::wakeup()
     assert(getLinkBandwidth() > 0);
     int bw_remaining = getLinkBandwidth();
 
-    // Give the highest numbered link priority most of the time
     m_wakeups_wo_switch++;
-    int highest_prio_vnet = m_vnets-1;
-    int lowest_prio_vnet = 0;
-    int counter = 1;
     bool schedule_wakeup = false;
 
+    // variable for deciding the direction in which to iterate
+    bool iteration_direction = false;
+
+
     // invert priorities to avoid starvation seen in the component network
     if (m_wakeups_wo_switch > PRIORITY_SWITCH_LIMIT) {
         m_wakeups_wo_switch = 0;
-        highest_prio_vnet = 0;
-        lowest_prio_vnet = m_vnets-1;
-        counter = -1;
+        iteration_direction = true;
     }
 
-    for (int vnet = highest_prio_vnet;
-         (vnet * counter) >= (counter * lowest_prio_vnet);
-         vnet -= counter) {
-
-        assert(m_out[vnet] != NULL);
-        assert(m_in[vnet] != NULL);
-        assert(m_units_remaining[vnet] >= 0);
-
-        while (bw_remaining > 0 &&
-            (m_in[vnet]->isReady() || m_units_remaining[vnet] > 0) &&
-            m_out[vnet]->areNSlotsAvailable(1)) {
-
-            // See if we are done transferring the previous message on
-            // this virtual network
-            if (m_units_remaining[vnet] == 0 && m_in[vnet]->isReady()) {
-                // Find the size of the message we are moving
-                MsgPtr msg_ptr = m_in[vnet]->peekMsgPtr();
-                NetworkMessage* net_msg_ptr =
-                    safe_cast<NetworkMessage*>(msg_ptr.get());
-                m_units_remaining[vnet] +=
-                    network_message_to_size(net_msg_ptr);
-
-                DPRINTF(RubyNetwork, "throttle: %d my bw %d bw spent "
-                        "enqueueing net msg %d time: %lld.\n",
-                        m_node, getLinkBandwidth(), m_units_remaining[vnet],
-                        g_system_ptr->curCycle());
-
-                // Move the message
-                m_in[vnet]->dequeue();
-                m_out[vnet]->enqueue(msg_ptr, m_link_latency);
-
-                // Count the message
-                m_msg_counts[net_msg_ptr->getMessageSize()][vnet]++;
-
-                DPRINTF(RubyNetwork, "%s\n", *m_out[vnet]);
-            }
-
-            // Calculate the amount of bandwidth we spent on this message
-            int diff = m_units_remaining[vnet] - bw_remaining;
-            m_units_remaining[vnet] = max(0, diff);
-            bw_remaining = max(0, -diff);
+    if (iteration_direction) {
+        for (auto& it : m_in) {
+            int vnet = it.first;
+            operateVnet(vnet, bw_remaining, schedule_wakeup,
+                        it.second, m_out[vnet]);
         }
-
-        if (bw_remaining > 0 &&
-            (m_in[vnet]->isReady() || m_units_remaining[vnet] > 0) &&
-            !m_out[vnet]->areNSlotsAvailable(1)) {
-            DPRINTF(RubyNetwork, "vnet: %d", vnet);
-            // schedule me to wakeup again because I'm waiting for my
-            // output queue to become available
-            schedule_wakeup = true;
+    } else {
+        for (auto it = m_in.rbegin(); it != m_in.rend(); ++it) {
+            int vnet = (*it).first;
+            operateVnet(vnet, bw_remaining, schedule_wakeup,
+                        (*it).second, m_out[vnet]);
         }
     }
 
@@ -215,7 +224,7 @@ Throttle::regStats(string parent)
     for (MessageSizeType type = MessageSizeType_FIRST;
          type < MessageSizeType_NUM; ++type) {
         m_msg_counts[(unsigned int)type]
-            .init(m_vnets)
+            .init(Network::getNumberOfVirtualNetworks())
             .name(parent + csprintf(".throttle%i", m_node) + ".msg_count." +
                     MessageSizeType_to_string(type))
             .flags(Stats::nozero)
diff --git a/src/mem/ruby/network/simple/Throttle.hh b/src/mem/ruby/network/simple/Throttle.hh
index cdc627bb7..d978f14fd 100644
--- a/src/mem/ruby/network/simple/Throttle.hh
+++ b/src/mem/ruby/network/simple/Throttle.hh
@@ -62,8 +62,8 @@ class Throttle : public Consumer
     std::string name()
     { return csprintf("Throttle-%i", m_sID); }
 
-    void addLinks(const std::vector<MessageBuffer*>& in_vec,
-                  const std::vector<MessageBuffer*>& out_vec);
+    void addLinks(const std::map<int, MessageBuffer*>& in_vec,
+                  const std::map<int, MessageBuffer*>& out_vec);
     void wakeup();
 
     // The average utilization (a fraction) since last clearStats()
@@ -85,16 +85,17 @@ class Throttle : public Consumer
   private:
     void init(NodeID node, Cycles link_latency, int link_bandwidth_multiplier,
               int endpoint_bandwidth);
-    void addVirtualNetwork(MessageBuffer* in_ptr, MessageBuffer* out_ptr);
+    void operateVnet(int vnet, int &bw_remainin, bool &schedule_wakeup,
+                     MessageBuffer *in, MessageBuffer *out);
 
     // Private copy constructor and assignment operator
     Throttle(const Throttle& obj);
     Throttle& operator=(const Throttle& obj);
 
-    std::vector<MessageBuffer*> m_in;
-    std::vector<MessageBuffer*> m_out;
-    unsigned int m_vnets;
-    std::vector<int> m_units_remaining;
+    std::map<int, MessageBuffer*> m_in;
+    std::map<int, MessageBuffer*> m_out;
+    std::map<int, int> m_units_remaining;
+
     int m_sID;
     NodeID m_node;
     int m_link_bandwidth_multiplier;
diff --git a/src/mem/ruby/slicc_interface/AbstractController.cc b/src/mem/ruby/slicc_interface/AbstractController.cc
index 0f5a70a6e..366ea04ce 100644
--- a/src/mem/ruby/slicc_interface/AbstractController.cc
+++ b/src/mem/ruby/slicc_interface/AbstractController.cc
@@ -89,13 +89,6 @@ AbstractController::profileMsgDelay(uint32_t virtualNetwork, Cycles delay)
 }
 
 void
-AbstractController::connectWithPeer(AbstractController *c)
-{
-    getQueuesFromPeer(c);
-    c->getQueuesFromPeer(this);
-}
-
-void
 AbstractController::stallBuffer(MessageBuffer* buf, Address addr)
 {
     if (m_waiting_buffers.count(addr) == 0) {
diff --git a/src/mem/ruby/slicc_interface/AbstractController.hh b/src/mem/ruby/slicc_interface/AbstractController.hh
index 36b4665c3..42d158653 100644
--- a/src/mem/ruby/slicc_interface/AbstractController.hh
+++ b/src/mem/ruby/slicc_interface/AbstractController.hh
@@ -96,6 +96,9 @@ class AbstractController : public ClockedObject, public Consumer
     virtual void collateStats()
     {fatal("collateStats() should be overridden!");}
 
+    //! Set the message buffer with given name.
+    virtual void setNetQueue(const std::string& name, MessageBuffer *b) = 0;
+
   public:
     MachineID getMachineID() const { return m_machineID; }
 
@@ -103,25 +106,12 @@ class AbstractController : public ClockedObject, public Consumer
     Stats::Histogram& getDelayVCHist(uint32_t index)
     { return *(m_delayVCHistogram[index]); }
 
-    MessageBuffer *getPeerQueue(uint32_t pid)
-    {
-        std::map<uint32_t, MessageBuffer *>::iterator it =
-                                        peerQueueMap.find(pid);
-        assert(it != peerQueueMap.end());
-        return (*it).second;
-    }
-
   protected:
     //! Profiles original cache requests including PUTs
     void profileRequest(const std::string &request);
     //! Profiles the delay associated with messages.
     void profileMsgDelay(uint32_t virtualNetwork, Cycles delay);
 
-    //! Function for connecting peer controllers
-    void connectWithPeer(AbstractController *);
-    virtual void getQueuesFromPeer(AbstractController *)
-    { fatal("getQueuesFromPeer() should be called only if implemented!"); }
-
     void stallBuffer(MessageBuffer* buf, Address addr);
     void wakeUpBuffers(Address addr);
     void wakeUpAllBuffers(Address addr);
@@ -147,9 +137,6 @@ class AbstractController : public ClockedObject, public Consumer
     unsigned int m_buffer_size;
     Cycles m_recycle_latency;
 
-    //! Map from physical network number to the Message Buffer.
-    std::map<uint32_t, MessageBuffer*> peerQueueMap;
-
     //! Counter for the number of cycles when the transitions carried out
     //! were equal to the maximum allowed
     Stats::Scalar m_fully_busy_cycles;