ruby: Make MessageBuffers actually finite sized

When Ruby controllers stall messages in MessageBuffers, the buffer moves those
messages off the priority heap and into a per-address stall map. When buffers
are finite-sized, the test areNSlotsAvailable() only checks the size of the
priority heap, but ignores the stall map, so the map is allowed to grow
unbounded if the controller stalls numerous messages. This patch fixes the
problem by tracking the stall map size and testing the total number of messages
in the buffer appropriately.

2 files changed, 35 insertions, 2 deletions

diff --git a/src/mem/ruby/network/MessageBuffer.cc b/src/mem/ruby/network/MessageBuffer.cc
index 557e0e80e..b96b00504 100644
--- a/src/mem/ruby/network/MessageBuffer.cc
+++ b/src/mem/ruby/network/MessageBuffer.cc
@@ -40,7 +40,7 @@ using namespace std;
 using m5::stl_helpers::operator<<;
 
 MessageBuffer::MessageBuffer(const Params *p)
-    : SimObject(p),
+    : SimObject(p), m_stall_map_size(0),
     m_max_size(p->buffer_size), m_time_last_time_size_checked(0),
     m_time_last_time_enqueue(0), m_time_last_time_pop(0),
     m_last_arrival_time(0), m_strict_fifo(p->ordered),
@@ -99,7 +99,7 @@ MessageBuffer::areNSlotsAvailable(unsigned int n, Tick current_time)
     }
 
     // now compare the new size with our max size
-    if (current_size + n <= m_max_size) {
+    if (current_size + m_stall_map_size + n <= m_max_size) {
         return true;
     } else {
         DPRINTF(RubyQueue, "n: %d, current_size: %d, heap size: %d, "
@@ -289,6 +289,8 @@ MessageBuffer::reanalyzeMessages(Addr addr, Tick current_time)
     // scheduled for the current cycle so that the previously stalled messages
     // will be observed before any younger messages that may arrive this cycle
     //
+    m_stall_map_size -= m_stall_msg_map[addr].size();
+    assert(m_stall_map_size >= 0);
     reanalyzeList(m_stall_msg_map[addr], current_time);
     m_stall_msg_map.erase(addr);
 }
@@ -306,6 +308,8 @@ MessageBuffer::reanalyzeAllMessages(Tick current_time)
     //
     for (StallMsgMapType::iterator map_iter = m_stall_msg_map.begin();
          map_iter != m_stall_msg_map.end(); ++map_iter) {
+        m_stall_map_size -= map_iter->second.size();
+        assert(m_stall_map_size >= 0);
         reanalyzeList(map_iter->second, current_time);
     }
     m_stall_msg_map.clear();
@@ -327,6 +331,7 @@ MessageBuffer::stallMessage(Addr addr, Tick current_time)
     // these addresses change state.
     //
     (m_stall_msg_map[addr]).push_back(message);
+    m_stall_map_size++;
 }
 
 void
diff --git a/src/mem/ruby/network/MessageBuffer.hh b/src/mem/ruby/network/MessageBuffer.hh
index 2a82887cd..b1d04ab9a 100644
--- a/src/mem/ruby/network/MessageBuffer.hh
+++ b/src/mem/ruby/network/MessageBuffer.hh
@@ -137,9 +137,37 @@ class MessageBuffer : public SimObject
     // sorted and ensures a well-defined iteration order
     typedef std::map<Addr, std::list<MsgPtr> > StallMsgMapType;
 
+    /**
+     * A map from line addresses to lists of stalled messages for that line.
+     * If this buffer allows the receiver to stall messages, on a stall
+     * request, the stalled message is removed from the m_prio_heap and placed
+     * in the m_stall_msg_map. Messages are held there until the receiver
+     * requests they be reanalyzed, at which point they are moved back to
+     * m_prio_heap.
+     *
+     * NOTE: The stall map holds messages in the order in which they were
+     * initially received, and when a line is unblocked, the messages are
+     * moved back to the m_prio_heap in the same order. This prevents starving
+     * older requests with younger ones.
+     */
     StallMsgMapType m_stall_msg_map;
 
+    /**
+     * Current size of the stall map.
+     * Track the number of messages held in stall map lists. This is used to
+     * ensure that if the buffer is finite-sized, it blocks further requests
+     * when the m_prio_heap and m_stall_msg_map contain m_max_size messages.
+     */
+    int m_stall_map_size;
+
+    /**
+     * The maximum capacity. For finite-sized buffers, m_max_size stores a
+     * number greater than 0 to indicate the maximum allowed number of messages
+     * in the buffer at any time. To get infinitely-sized buffers, set buffer
+     * size: m_max_size = 0
+     */
     const unsigned int m_max_size;
+
     Tick m_time_last_time_size_checked;
     unsigned int m_size_last_time_size_checked;