ruby: fix message buffer init order

The recent changes to make MessageBuffers SimObjects required them to be
initialized in a particular order, which could break some protocols. Fix this
by calling initNetQueues on the external nodes of each external link in the
constructor of Network.

This patch also refactors the duplicated code for checking network allocation
and setting net queues (which are called by initNetQueues) from the simple and
garnet networks to be in Network.

10 files changed, 62 insertions, 115 deletions

diff --git a/src/mem/ruby/network/Network.cc b/src/mem/ruby/network/Network.cc
index d35b909d9..721dbbabe 100644
--- a/src/mem/ruby/network/Network.cc
+++ b/src/mem/ruby/network/Network.cc
@@ -58,6 +58,7 @@ Network::Network(const Params *p)
     m_fromNetQueues.resize(m_nodes);
 
     m_ordered.resize(m_virtual_networks);
+    m_vnet_type_names.resize(m_virtual_networks);
 
     for (int i = 0; i < m_virtual_networks; i++) {
         m_ordered[i] = false;
@@ -75,6 +76,10 @@ Network::Network(const Params *p)
 
     // Register a callback function for combining the statistics
     Stats::registerDumpCallback(new StatsCallback(this));
+
+    for (auto &it : dynamic_cast<Network *>(this)->params()->ext_links) {
+        it->params()->ext_node->initNetQueues();
+    }
 }
 
 Network::~Network()
@@ -128,3 +133,41 @@ Network::MessageSizeType_to_int(MessageSizeType size_type)
         break;
     }
 }
+
+void
+Network::checkNetworkAllocation(NodeID id, bool ordered,
+                                        int network_num,
+                                        std::string vnet_type)
+{
+    fatal_if(id >= m_nodes, "Node ID is out of range");
+    fatal_if(network_num >= m_virtual_networks, "Network id is out of range");
+
+    if (ordered) {
+        m_ordered[network_num] = true;
+    }
+
+    m_vnet_type_names[network_num] = vnet_type;
+}
+
+
+void
+Network::setToNetQueue(NodeID id, bool ordered, int network_num,
+                                 std::string vnet_type, MessageBuffer *b)
+{
+    checkNetworkAllocation(id, ordered, network_num, vnet_type);
+    while (m_toNetQueues[id].size() <= network_num) {
+        m_toNetQueues[id].push_back(nullptr);
+    }
+    m_toNetQueues[id][network_num] = b;
+}
+
+void
+Network::setFromNetQueue(NodeID id, bool ordered, int network_num,
+                                   std::string vnet_type, MessageBuffer *b)
+{
+    checkNetworkAllocation(id, ordered, network_num, vnet_type);
+    while (m_fromNetQueues[id].size() <= network_num) {
+        m_fromNetQueues[id].push_back(nullptr);
+    }
+    m_fromNetQueues[id][network_num] = b;
+}
diff --git a/src/mem/ruby/network/Network.hh b/src/mem/ruby/network/Network.hh
index 26221521f..c06ee5a1a 100644
--- a/src/mem/ruby/network/Network.hh
+++ b/src/mem/ruby/network/Network.hh
@@ -72,10 +72,13 @@ class Network : public ClockedObject
     static uint32_t MessageSizeType_to_int(MessageSizeType size_type);
 
     // returns the queue requested for the given component
-    virtual void setToNetQueue(NodeID id, bool ordered, int netNumber,
-                               std::string vnet_type, MessageBuffer *b) = 0;
+    void setToNetQueue(NodeID id, bool ordered, int netNumber,
+                               std::string vnet_type, MessageBuffer *b);
     virtual void setFromNetQueue(NodeID id, bool ordered, int netNumber,
-                                 std::string vnet_type, MessageBuffer *b) = 0;
+                                 std::string vnet_type, MessageBuffer *b);
+
+    virtual void checkNetworkAllocation(NodeID id, bool ordered,
+        int network_num, std::string vnet_type);
 
     virtual void makeOutLink(SwitchID src, NodeID dest, BasicLink* link,
                              LinkDirection direction,
@@ -107,6 +110,7 @@ class Network : public ClockedObject
 
     uint32_t m_nodes;
     static uint32_t m_virtual_networks;
+    std::vector<std::string> m_vnet_type_names;
     Topology* m_topology_ptr;
     static uint32_t m_control_msg_size;
     static uint32_t m_data_msg_size;
diff --git a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
index ed4679c63..1213073e9 100644
--- a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
+++ b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
@@ -67,28 +67,6 @@ BaseGarnetNetwork::init()
 }
 
 void
-BaseGarnetNetwork::setToNetQueue(NodeID id, bool ordered, int network_num,
-                                 string vnet_type, MessageBuffer *b)
-{
-    checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    while (m_toNetQueues[id].size() <= network_num) {
-        m_toNetQueues[id].push_back(nullptr);
-    }
-    m_toNetQueues[id][network_num] = b;
-}
-
-void
-BaseGarnetNetwork::setFromNetQueue(NodeID id, bool ordered, int network_num,
-                                   string vnet_type, MessageBuffer *b)
-{
-    checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    while (m_fromNetQueues[id].size() <= network_num) {
-        m_fromNetQueues[id].push_back(nullptr);
-    }
-    m_fromNetQueues[id][network_num] = b;
-}
-
-void
 BaseGarnetNetwork::regStats()
 {
     m_flits_received
diff --git a/src/mem/ruby/network/garnet/BaseGarnetNetwork.hh b/src/mem/ruby/network/garnet/BaseGarnetNetwork.hh
index e2c843e10..fe2cbacc3 100644
--- a/src/mem/ruby/network/garnet/BaseGarnetNetwork.hh
+++ b/src/mem/ruby/network/garnet/BaseGarnetNetwork.hh
@@ -68,15 +68,7 @@ class BaseGarnetNetwork : public Network
         m_queueing_latency[vnet] += latency;
     }
 
-    // set the queue
-    void setToNetQueue(NodeID id, bool ordered, int network_num,
-                       std::string vnet_type, MessageBuffer *b);
-    void setFromNetQueue(NodeID id, bool ordered, int network_num,
-                         std::string vnet_type, MessageBuffer *b);
-
     bool isVNetOrdered(int vnet) const { return m_ordered[vnet]; }
-    virtual void checkNetworkAllocation(NodeID id, bool ordered,
-        int network_num, std::string vnet_type) = 0;
 
     virtual void regStats();
     virtual void collateStats() {}
diff --git a/src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.cc b/src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.cc
index 4d5846f55..f6fe6f586 100644
--- a/src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.cc
+++ b/src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.cc
@@ -51,8 +51,13 @@ GarnetNetwork_d::GarnetNetwork_d(const Params *p)
     m_buffers_per_ctrl_vc = p->buffers_per_ctrl_vc;
 
     m_vnet_type.resize(m_virtual_networks);
-    for (int i = 0; i < m_vnet_type.size(); i++) {
-        m_vnet_type[i] = NULL_VNET_; // default
+
+    for(int i = 0 ; i < m_virtual_networks ; i++)
+    {
+        if (m_vnet_type_names[i] == "response")
+            m_vnet_type[i] = DATA_VNET_; // carries data (and ctrl) packets
+        else
+            m_vnet_type[i] = CTRL_VNET_; // carries only ctrl packets
     }
 
     // record the routers
@@ -188,23 +193,6 @@ GarnetNetwork_d::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
 }
 
 void
-GarnetNetwork_d::checkNetworkAllocation(NodeID id, bool ordered,
-                                        int network_num, string vnet_type)
-{
-    assert(id < m_nodes);
-    assert(network_num < m_virtual_networks);
-
-    if (ordered) {
-        m_ordered[network_num] = true;
-    }
-
-    if (vnet_type == "response")
-        m_vnet_type[network_num] = DATA_VNET_; // carries data (and ctrl) packets
-    else
-        m_vnet_type[network_num] = CTRL_VNET_; // carries only ctrl packets
-}
-
-void
 GarnetNetwork_d::regStats()
 {
     BaseGarnetNetwork::regStats();
diff --git a/src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.hh b/src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.hh
index efd70c3a0..99ecc02da 100644
--- a/src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.hh
+++ b/src/mem/ruby/network/garnet/fixed-pipeline/GarnetNetwork_d.hh
@@ -84,9 +84,6 @@ class GarnetNetwork_d : public BaseGarnetNetwork
     uint32_t functionalWrite(Packet *pkt);
 
   private:
-    void checkNetworkAllocation(NodeID id, bool ordered, int network_num,
-                                std::string vnet_type);
-
     GarnetNetwork_d(const GarnetNetwork_d& obj);
     GarnetNetwork_d& operator=(const GarnetNetwork_d& obj);
 
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.cc b/src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.cc
index 83cbe8d51..bd96fc2ed 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.cc
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.cc
@@ -143,18 +143,6 @@ GarnetNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
                                          link->m_weight);
 }
 
-void
-GarnetNetwork::checkNetworkAllocation(NodeID id, bool ordered,
-    int network_num, std::string vnet_type)
-{
-    assert(id < m_nodes);
-    assert(network_num < m_virtual_networks);
-
-    if (ordered) {
-        m_ordered[network_num] = true;
-    }
-}
-
 /*
  * Go through all the routers, network interfaces and the interconnecting
  * links for reading/writing all the messages.
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.hh b/src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.hh
index 906e7eb3a..46f62c04f 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.hh
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/GarnetNetwork.hh
@@ -79,9 +79,6 @@ class GarnetNetwork : public BaseGarnetNetwork
     uint32_t functionalWrite(Packet *pkt);
 
   private:
-    void checkNetworkAllocation(NodeID id, bool ordered, int network_num,
-                                std::string vnet_type);
-
     GarnetNetwork(const GarnetNetwork& obj);
     GarnetNetwork& operator=(const GarnetNetwork& obj);
 
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc
index c10d1cce8..457c3248e 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.cc
+++ b/src/mem/ruby/network/simple/SimpleNetwork.cc
@@ -140,39 +140,6 @@ SimpleNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
 }
 
 void
-SimpleNetwork::checkNetworkAllocation(NodeID id, bool ordered, int network_num)
-{
-    assert(id < m_nodes);
-    assert(network_num < m_virtual_networks);
-
-    if (ordered) {
-        m_ordered[network_num] = true;
-    }
-}
-
-void
-SimpleNetwork::setToNetQueue(NodeID id, bool ordered, int network_num,
-                             std::string vnet_type, MessageBuffer *b)
-{
-    checkNetworkAllocation(id, ordered, network_num);
-    while (m_toNetQueues[id].size() <= network_num) {
-        m_toNetQueues[id].push_back(nullptr);
-    }
-    m_toNetQueues[id][network_num] = b;
-}
-
-void
-SimpleNetwork::setFromNetQueue(NodeID id, bool ordered, int network_num,
-                               std::string vnet_type, MessageBuffer *b)
-{
-    checkNetworkAllocation(id, ordered, network_num);
-    while (m_fromNetQueues[id].size() <= network_num) {
-        m_fromNetQueues[id].push_back(nullptr);
-    }
-    m_fromNetQueues[id][network_num] = b;
-}
-
-void
 SimpleNetwork::regStats()
 {
     for (MessageSizeType type = MessageSizeType_FIRST;
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.hh b/src/mem/ruby/network/simple/SimpleNetwork.hh
index 2d9b48dea..c5e56b1ec 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.hh
+++ b/src/mem/ruby/network/simple/SimpleNetwork.hh
@@ -56,23 +56,17 @@ class SimpleNetwork : public Network
     void collateStats();
     void regStats();
 
-    // sets the queue requested
-    void setToNetQueue(NodeID id, bool ordered, int network_num,
-                       std::string vnet_type, MessageBuffer *b);
-    void setFromNetQueue(NodeID id, bool ordered, int network_num,
-                         std::string vnet_type, MessageBuffer *b);
-
     bool isVNetOrdered(int vnet) const { return m_ordered[vnet]; }
 
     // Methods used by Topology to setup the network
-    void makeOutLink(SwitchID src, NodeID dest, BasicLink* link, 
-                     LinkDirection direction, 
+    void makeOutLink(SwitchID src, NodeID dest, BasicLink* link,
+                     LinkDirection direction,
                      const NetDest& routing_table_entry);
     void makeInLink(NodeID src, SwitchID dest, BasicLink* link,
-                    LinkDirection direction, 
+                    LinkDirection direction,
                     const NetDest& routing_table_entry);
     void makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
-                          LinkDirection direction, 
+                          LinkDirection direction,
                           const NetDest& routing_table_entry);
 
     void print(std::ostream& out) const;
@@ -81,7 +75,6 @@ class SimpleNetwork : public Network
     uint32_t functionalWrite(Packet *pkt);
 
   private:
-    void checkNetworkAllocation(NodeID id, bool ordered, int network_num);
     void addLink(SwitchID src, SwitchID dest, int link_latency);
     void makeLink(SwitchID src, SwitchID dest,
         const NetDest& routing_table_entry, int link_latency);
@@ -98,7 +91,7 @@ class SimpleNetwork : public Network
 
     int m_buffer_size;
     int m_endpoint_bandwidth;
-    bool m_adaptive_routing;    
+    bool m_adaptive_routing;
 
     //Statistical variables
     Stats::Formula m_msg_counts[MessageSizeType_NUM];