ruby: network: revert some of the changes from ad9c042dce54

The changeset ad9c042dce54 made changes to the structures under the network
directory to use a map of buffers instead of vector of buffers.
The reasoning was that not all vnets that are created are used and we
needlessly allocate more buffers than required and then iterate over them
while processing network messages.  But the move to map resulted in a slow
down which was pointed out by Andreas Hansson.  This patch moves things
back to using vector of message buffers.

7 files changed, 70 insertions, 53 deletions

diff --git a/src/mem/ruby/network/simple/PerfectSwitch.cc b/src/mem/ruby/network/simple/PerfectSwitch.cc
index caf07b3cf..86cafbe15 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.cc
+++ b/src/mem/ruby/network/simple/PerfectSwitch.cc
@@ -68,26 +68,30 @@ PerfectSwitch::init(SimpleNetwork *network_ptr)
 }
 
 void
-PerfectSwitch::addInPort(const map<int, MessageBuffer*>& in)
+PerfectSwitch::addInPort(const vector<MessageBuffer*>& in)
 {
     NodeID port = m_in.size();
     m_in.push_back(in);
 
-    for (auto& it : in) {
-        it.second->setConsumer(this);
+    for (int i = 0; i < in.size(); ++i) {
+        if (in[i] != nullptr) {
+            in[i]->setConsumer(this);
 
-        string desc = csprintf("[Queue from port %s %s %s to PerfectSwitch]",
-            to_string(m_switch_id), to_string(port), to_string(it.first));
+            string desc =
+                csprintf("[Queue from port %s %s %s to PerfectSwitch]",
+                         to_string(m_switch_id), to_string(port),
+                         to_string(i));
 
-        it.second->setDescription(desc);
-        it.second->setIncomingLink(port);
-        it.second->setVnet(it.first);
+            in[i]->setDescription(desc);
+            in[i]->setIncomingLink(port);
+            in[i]->setVnet(i);
+        }
     }
 }
 
 void
-PerfectSwitch::addOutPort(const map<int, MessageBuffer*>& out,
-    const NetDest& routing_table_entry)
+PerfectSwitch::addOutPort(const vector<MessageBuffer*>& out,
+                          const NetDest& routing_table_entry)
 {
     // Setup link order
     LinkOrder l;
@@ -131,10 +135,14 @@ PerfectSwitch::operateVnet(int vnet)
             vector<NetDest> output_link_destinations;
 
             // Is there a message waiting?
-            auto it = m_in[incoming].find(vnet);
-            if (it == m_in[incoming].end())
+            if (m_in[incoming].size() <= vnet) {
                 continue;
-            MessageBuffer *buffer = (*it).second;
+            }
+
+            MessageBuffer *buffer = m_in[incoming][vnet];
+            if (buffer == nullptr) {
+                continue;
+            }
 
             while (buffer->isReady()) {
                 DPRINTF(RubyNetwork, "incoming: %d\n", incoming);
diff --git a/src/mem/ruby/network/simple/PerfectSwitch.hh b/src/mem/ruby/network/simple/PerfectSwitch.hh
index 161430bd1..f55281d54 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.hh
+++ b/src/mem/ruby/network/simple/PerfectSwitch.hh
@@ -65,8 +65,8 @@ class PerfectSwitch : public Consumer
     { return csprintf("PerfectSwitch-%i", m_switch_id); }
 
     void init(SimpleNetwork *);
-    void addInPort(const std::map<int, MessageBuffer*>& in);
-    void addOutPort(const std::map<int, MessageBuffer*>& out,
+    void addInPort(const std::vector<MessageBuffer*>& in);
+    void addOutPort(const std::vector<MessageBuffer*>& out,
                     const NetDest& routing_table_entry);
 
     int getInLinks() const { return m_in.size(); }
@@ -89,8 +89,8 @@ class PerfectSwitch : public Consumer
     SwitchID m_switch_id;
 
     // vector of queues from the components
-    std::vector<std::map<int, MessageBuffer*> > m_in;
-    std::vector<std::map<int, MessageBuffer*> > m_out;
+    std::vector<std::vector<MessageBuffer*> > m_in;
+    std::vector<std::vector<MessageBuffer*> > m_out;
 
     std::vector<NetDest> m_routing_table;
     std::vector<LinkOrder> m_link_order;
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc
index f51a0c891..074052cb9 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.cc
+++ b/src/mem/ruby/network/simple/SimpleNetwork.cc
@@ -117,7 +117,8 @@ SimpleNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
                                 const NetDest& routing_table_entry)
 {
     // Create a set of new MessageBuffers
-    std::map<int, MessageBuffer*> queues;
+    std::vector<MessageBuffer*> queues(m_virtual_networks);
+
     for (int i = 0; i < m_virtual_networks; i++) {
         // allocate a buffer
         MessageBuffer* buffer_ptr = new MessageBuffer;
@@ -158,6 +159,9 @@ SimpleNetwork::setToNetQueue(NodeID id, bool ordered, int network_num,
                              std::string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num);
+    while (m_toNetQueues[id].size() <= network_num) {
+        m_toNetQueues[id].push_back(nullptr);
+    }
     m_toNetQueues[id][network_num] = b;
 }
 
@@ -166,6 +170,9 @@ SimpleNetwork::setFromNetQueue(NodeID id, bool ordered, int network_num,
                                std::string vnet_type, MessageBuffer *b)
 {
     checkNetworkAllocation(id, ordered, network_num);
+    while (m_fromNetQueues[id].size() <= network_num) {
+        m_fromNetQueues[id].push_back(nullptr);
+    }
     m_fromNetQueues[id][network_num] = b;
 }
 
diff --git a/src/mem/ruby/network/simple/Switch.cc b/src/mem/ruby/network/simple/Switch.cc
index e028de02a..416a222bb 100644
--- a/src/mem/ruby/network/simple/Switch.cc
+++ b/src/mem/ruby/network/simple/Switch.cc
@@ -64,17 +64,19 @@ Switch::init()
 }
 
 void
-Switch::addInPort(const map<int, MessageBuffer*>& in)
+Switch::addInPort(const vector<MessageBuffer*>& in)
 {
     m_perfect_switch->addInPort(in);
 
     for (auto& it : in) {
-        it.second->setReceiver(this);
+        if (it != nullptr) {
+            it->setReceiver(this);
+        }
     }
 }
 
 void
-Switch::addOutPort(const map<int, MessageBuffer*>& out,
+Switch::addOutPort(const vector<MessageBuffer*>& out,
                    const NetDest& routing_table_entry,
                    Cycles link_latency, int bw_multiplier)
 {
@@ -87,10 +89,12 @@ Switch::addOutPort(const map<int, MessageBuffer*>& out,
     m_throttles.push_back(throttle_ptr);
 
     // Create one buffer per vnet (these are intermediaryQueues)
-    map<int, MessageBuffer*> intermediateBuffers;
+    vector<MessageBuffer*> intermediateBuffers;
 
-    for (auto& it : out) {
-        it.second->setSender(this);
+    for (int i = 0; i < out.size(); ++i) {
+        if (out[i] != nullptr) {
+            out[i]->setSender(this);
+        }
 
         MessageBuffer* buffer_ptr = new MessageBuffer;
         // Make these queues ordered
@@ -99,7 +103,7 @@ Switch::addOutPort(const map<int, MessageBuffer*>& out,
             buffer_ptr->resize(m_network_ptr->getBufferSize());
         }
 
-        intermediateBuffers[it.first] = buffer_ptr;
+        intermediateBuffers.push_back(buffer_ptr);
         m_buffers_to_free.push_back(buffer_ptr);
 
         buffer_ptr->setSender(this);
diff --git a/src/mem/ruby/network/simple/Switch.hh b/src/mem/ruby/network/simple/Switch.hh
index d4e5c5eba..3c11a5fa0 100644
--- a/src/mem/ruby/network/simple/Switch.hh
+++ b/src/mem/ruby/network/simple/Switch.hh
@@ -62,8 +62,8 @@ class Switch : public BasicRouter
     ~Switch();
     void init();
 
-    void addInPort(const std::map<int, MessageBuffer*>& in);
-    void addOutPort(const std::map<int, MessageBuffer*>& out,
+    void addInPort(const std::vector<MessageBuffer*>& in);
+    void addOutPort(const std::vector<MessageBuffer*>& out,
                     const NetDest& routing_table_entry,
                     Cycles link_latency, int bw_multiplier);
 
diff --git a/src/mem/ruby/network/simple/Throttle.cc b/src/mem/ruby/network/simple/Throttle.cc
index 91bad217b..2da810edb 100644
--- a/src/mem/ruby/network/simple/Throttle.cc
+++ b/src/mem/ruby/network/simple/Throttle.cc
@@ -69,6 +69,8 @@ Throttle::init(NodeID node, Cycles link_latency,
                int link_bandwidth_multiplier, int endpoint_bandwidth)
 {
     m_node = node;
+    m_vnets = 0;
+
     assert(link_bandwidth_multiplier > 0);
     m_link_bandwidth_multiplier = link_bandwidth_multiplier;
 
@@ -80,23 +82,19 @@ Throttle::init(NodeID node, Cycles link_latency,
 }
 
 void
-Throttle::addLinks(const map<int, MessageBuffer*>& in_vec,
-                   const map<int, MessageBuffer*>& out_vec)
+Throttle::addLinks(const vector<MessageBuffer*>& in_vec,
+                   const vector<MessageBuffer*>& out_vec)
 {
     assert(in_vec.size() == out_vec.size());
 
-    for (auto& it : in_vec) {
-        int vnet = it.first;
-
-        auto jt = out_vec.find(vnet);
-        assert(jt != out_vec.end());
+    for (int vnet = 0; vnet < in_vec.size(); ++vnet) {
+        MessageBuffer *in_ptr = in_vec[vnet];
+        MessageBuffer *out_ptr = out_vec[vnet];
 
-        MessageBuffer *in_ptr = it.second;
-        MessageBuffer *out_ptr = (*jt).second;
-
-        m_in[vnet] = in_ptr;
-        m_out[vnet] = out_ptr;
-        m_units_remaining[vnet] = 0;
+        m_vnets++;
+        m_units_remaining.push_back(0);
+        m_in.push_back(in_ptr);
+        m_out.push_back(out_ptr);
 
         // Set consumer and description
         in_ptr->setConsumer(this);
@@ -110,8 +108,9 @@ void
 Throttle::operateVnet(int vnet, int &bw_remaining, bool &schedule_wakeup,
                       MessageBuffer *in, MessageBuffer *out)
 {
-    assert(out != NULL);
-    assert(in != NULL);
+    if (out == nullptr || in == nullptr) {
+        return;
+    }
     assert(m_units_remaining[vnet] >= 0);
 
     while (bw_remaining > 0 && (in->isReady() || m_units_remaining[vnet] > 0) &&
@@ -178,16 +177,14 @@ Throttle::wakeup()
     }
 
     if (iteration_direction) {
-        for (auto& it : m_in) {
-            int vnet = it.first;
+        for (int vnet = 0; vnet < m_vnets; ++vnet) {
             operateVnet(vnet, bw_remaining, schedule_wakeup,
-                        it.second, m_out[vnet]);
+                        m_in[vnet], m_out[vnet]);
         }
     } else {
-        for (auto it = m_in.rbegin(); it != m_in.rend(); ++it) {
-            int vnet = (*it).first;
+        for (int vnet = m_vnets-1; vnet >= 0; --vnet) {
             operateVnet(vnet, bw_remaining, schedule_wakeup,
-                        (*it).second, m_out[vnet]);
+                        m_in[vnet], m_out[vnet]);
         }
     }
 
diff --git a/src/mem/ruby/network/simple/Throttle.hh b/src/mem/ruby/network/simple/Throttle.hh
index d978f14fd..797511702 100644
--- a/src/mem/ruby/network/simple/Throttle.hh
+++ b/src/mem/ruby/network/simple/Throttle.hh
@@ -62,8 +62,8 @@ class Throttle : public Consumer
     std::string name()
     { return csprintf("Throttle-%i", m_sID); }
 
-    void addLinks(const std::map<int, MessageBuffer*>& in_vec,
-                  const std::map<int, MessageBuffer*>& out_vec);
+    void addLinks(const std::vector<MessageBuffer*>& in_vec,
+                  const std::vector<MessageBuffer*>& out_vec);
     void wakeup();
 
     // The average utilization (a fraction) since last clearStats()
@@ -92,9 +92,10 @@ class Throttle : public Consumer
     Throttle(const Throttle& obj);
     Throttle& operator=(const Throttle& obj);
 
-    std::map<int, MessageBuffer*> m_in;
-    std::map<int, MessageBuffer*> m_out;
-    std::map<int, int> m_units_remaining;
+    std::vector<MessageBuffer*> m_in;
+    std::vector<MessageBuffer*> m_out;
+    unsigned int m_vnets;
+    std::vector<int> m_units_remaining;
 
     int m_sID;
     NodeID m_node;