ruby: network: revert some of the changes from ad9c042dce54

The changeset ad9c042dce54 made changes to the structures under the network
directory to use a map of buffers instead of vector of buffers.
The reasoning was that not all vnets that are created are used and we
needlessly allocate more buffers than required and then iterate over them
while processing network messages.  But the move to map resulted in a slow
down which was pointed out by Andreas Hansson.  This patch moves things
back to using vector of message buffers.

5 files changed, 50 insertions, 30 deletions

diff --git a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
index 2aeddad37..ed4679c63 100644
--- a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
+++ b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
@@ -71,6 +71,9 @@ BaseGarnetNetwork::setToNetQueue(NodeID id, bool ordered, int network_num,
                                  string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num, vnet_type);
+    while (m_toNetQueues[id].size() <= network_num) {
+        m_toNetQueues[id].push_back(nullptr);
+    }
     m_toNetQueues[id][network_num] = b;
 }
 
@@ -79,6 +82,9 @@ BaseGarnetNetwork::setFromNetQueue(NodeID id, bool ordered, int network_num,
                                    string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num, vnet_type);
+    while (m_fromNetQueues[id].size() <= network_num) {
+        m_fromNetQueues[id].push_back(nullptr);
+    }
     m_fromNetQueues[id][network_num] = b;
 }
 
diff --git a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc
index 7384cc6a7..47ded231c 100644
--- a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc
+++ b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc
@@ -106,20 +106,23 @@ NetworkInterface_d::addOutPort(NetworkLink_d *out_link,
 }
 
 void
-NetworkInterface_d::addNode(map<int, MessageBuffer *>& in,
-                            map<int, MessageBuffer *>& out)
+NetworkInterface_d::addNode(vector<MessageBuffer *>& in,
+                            vector<MessageBuffer *>& out)
 {
     inNode_ptr = in;
     outNode_ptr = out;
 
     for (auto& it : in) {
-        // the protocol injects messages into the NI
-        it.second->setConsumer(this);
-        it.second->setReceiver(this);
+        if (it != nullptr) {
+            it->setConsumer(this);
+            it->setReceiver(this);
+        }
     }
 
     for (auto& it : out) {
-        it.second->setSender(this);
+        if (it != nullptr) {
+            it->setSender(this);
+        }
     }
 }
 
@@ -223,9 +226,11 @@ NetworkInterface_d::wakeup()
 
     // Checking for messages coming from the protocol
     // can pick up a message/cycle for each virtual net
-    for (auto it = inNode_ptr.begin(); it != inNode_ptr.end(); ++it) {
-        int vnet = (*it).first;
-        MessageBuffer *b = (*it).second;
+    for (int vnet = 0; vnet < inNode_ptr.size(); ++vnet) {
+        MessageBuffer *b = inNode_ptr[vnet];
+        if (b == nullptr) {
+            continue;
+        }
 
         while (b->isReady()) { // Is there a message waiting
             msg_ptr = b->peekMsgPtr();
@@ -355,9 +360,11 @@ void
 NetworkInterface_d::checkReschedule()
 {
     for (const auto& it : inNode_ptr) {
-        MessageBuffer *b = it.second;
+        if (it == nullptr) {
+            continue;
+        }
 
-        while (b->isReady()) { // Is there a message waiting
+        while (it->isReady()) { // Is there a message waiting
             scheduleEvent(Cycles(1));
             return;
         }
diff --git a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh
index 2494d05d1..0cb928d82 100644
--- a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh
+++ b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh
@@ -60,8 +60,8 @@ class NetworkInterface_d : public ClockedObject, public Consumer
     void addOutPort(NetworkLink_d *out_link, CreditLink_d *credit_link);
 
     void wakeup();
-    void addNode(std::map<int, MessageBuffer *> &inNode,
-                 std::map<int, MessageBuffer *> &outNode);
+    void addNode(std::vector<MessageBuffer *> &inNode,
+                 std::vector<MessageBuffer *> &outNode);
 
     void print(std::ostream& out) const;
     int get_vnet(int vc);
@@ -90,9 +90,9 @@ class NetworkInterface_d : public ClockedObject, public Consumer
     std::vector<Cycles> m_ni_enqueue_time;
 
     // The Message buffers that takes messages from the protocol
-    std::map<int, MessageBuffer *> inNode_ptr;
+    std::vector<MessageBuffer *> inNode_ptr;
     // The Message buffers that provides messages to the protocol
-    std::map<int, MessageBuffer *> outNode_ptr;
+    std::vector<MessageBuffer *> outNode_ptr;
 
     bool flitisizeMessage(MsgPtr msg_ptr, int vnet);
     int calculateVC(int vnet);
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc
index 26d2423e8..32066f0e1 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc
@@ -90,20 +90,23 @@ NetworkInterface::addOutPort(NetworkLink *out_link)
 }
 
 void
-NetworkInterface::addNode(map<int, MessageBuffer*>& in,
-    map<int, MessageBuffer*>& out)
+NetworkInterface::addNode(vector<MessageBuffer*>& in,
+                          vector<MessageBuffer*>& out)
 {
     inNode_ptr = in;
     outNode_ptr = out;
 
     for (auto& it: in) {
-        // the protocol injects messages into the NI
-        it.second->setConsumer(this);
-        it.second->setReceiver(this);
+        if (it != nullptr) {
+            it->setConsumer(this);
+            it->setReceiver(this);
+        }
     }
 
     for (auto& it : out) {
-        it.second->setSender(this);
+        if (it != nullptr) {
+            it->setSender(this);
+        }
     }
 }
 
@@ -242,9 +245,11 @@ NetworkInterface::wakeup()
 
     //Checking for messages coming from the protocol
     // can pick up a message/cycle for each virtual net
-    for (auto it = inNode_ptr.begin(); it != inNode_ptr.end(); ++it) {
-        int vnet = (*it).first;
-        MessageBuffer *b = (*it).second;
+    for (int vnet = 0; vnet < inNode_ptr.size(); ++vnet) {
+        MessageBuffer *b = inNode_ptr[vnet];
+        if (b == nullptr) {
+            continue;
+        }
 
         while (b->isReady()) { // Is there a message waiting
             msg_ptr = b->peekMsgPtr();
@@ -326,9 +331,11 @@ void
 NetworkInterface::checkReschedule()
 {
     for (const auto& it : inNode_ptr) {
-        MessageBuffer *b = it.second;
+        if (it == nullptr) {
+            continue;
+        }
 
-        while (b->isReady()) { // Is there a message waiting
+        while (it->isReady()) { // Is there a message waiting
             scheduleEvent(Cycles(1));
             return;
         }
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh
index aa30bd758..03cdf3dc6 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh
@@ -56,8 +56,8 @@ class NetworkInterface : public ClockedObject, public FlexibleConsumer
 
     void addInPort(NetworkLink *in_link);
     void addOutPort(NetworkLink *out_link);
-    void addNode(std::map<int, MessageBuffer *> &inNode,
-                 std::map<int, MessageBuffer *> &outNode);
+    void addNode(std::vector<MessageBuffer *> &inNode,
+                 std::vector<MessageBuffer *> &outNode);
 
     void wakeup();
     void grant_vc(int out_port, int vc, Cycles grant_time);
@@ -93,10 +93,10 @@ class NetworkInterface : public ClockedObject, public FlexibleConsumer
     std::vector<flitBuffer *>   m_ni_buffers;
 
     // The Message buffers that takes messages from the protocol
-    std::map<int, MessageBuffer *> inNode_ptr;
+    std::vector<MessageBuffer *> inNode_ptr;
 
     // The Message buffers that provides messages to the protocol
-    std::map<int, MessageBuffer *> outNode_ptr;
+    std::vector<MessageBuffer *> outNode_ptr;
 
     bool flitisizeMessage(MsgPtr msg_ptr, int vnet);
     int calculateVC(int vnet);