ruby: Fix the way stall map size is checked for availability

To ensure that enqueuer observes the practical availability. We
check the message buffer queue size at the start of the cycle.
We also add the size of the stall queue to consider the total
queue size. However, messages can be moved from regular queue
to stall map. This leads to messages being considered twice leading
to false flow control. This patch fixes it by storing the stall map
size at the beginning of the cycle and considering it for checking
availability.

Change-Id: I6ea94f34fe5279b91f74e106d43263e55ec4bf06
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/20389
Maintainer: Jason Lowe-Power <jason@lowepower.com>
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>

2 files changed, 13 insertions, 3 deletions

diff --git a/src/mem/ruby/network/MessageBuffer.cc b/src/mem/ruby/network/MessageBuffer.cc
index 03d1bb003..f8cab3ceb 100644
--- a/src/mem/ruby/network/MessageBuffer.cc
+++ b/src/mem/ruby/network/MessageBuffer.cc
@@ -51,6 +51,7 @@ MessageBuffer::MessageBuffer(const Params *p)
     m_consumer = NULL;
     m_size_last_time_size_checked = 0;
     m_size_at_cycle_start = 0;
+    m_stalled_at_cycle_start = 0;
     m_msgs_this_cycle = 0;
     m_priority_rank = 0;
 
@@ -89,10 +90,12 @@ MessageBuffer::areNSlotsAvailable(unsigned int n, Tick current_time)
     // until schd cycle, but enqueue operations effect the visible
     // size immediately
     unsigned int current_size = 0;
+    unsigned int current_stall_size = 0;
 
     if (m_time_last_time_pop < current_time) {
-        // no pops this cycle - heap size is correct
+        // no pops this cycle - heap and stall queue size is correct
         current_size = m_prio_heap.size();
+        current_stall_size = m_stall_map_size;
     } else {
         if (m_time_last_time_enqueue < current_time) {
             // no enqueues this cycle - m_size_at_cycle_start is correct
@@ -102,15 +105,19 @@ MessageBuffer::areNSlotsAvailable(unsigned int n, Tick current_time)
             // enqueued msgs to m_size_at_cycle_start
             current_size = m_size_at_cycle_start + m_msgs_this_cycle;
         }
+
+        // Stall queue size at start is considered
+        current_stall_size = m_stalled_at_cycle_start;
     }
 
     // now compare the new size with our max size
-    if (current_size + m_stall_map_size + n <= m_max_size) {
+    if (current_size + current_stall_size + n <= m_max_size) {
         return true;
     } else {
         DPRINTF(RubyQueue, "n: %d, current_size: %d, heap size: %d, "
                 "m_max_size: %d\n",
-                n, current_size, m_prio_heap.size(), m_max_size);
+                n, current_size + current_stall_size,
+                m_prio_heap.size(), m_max_size);
         m_not_avail_count++;
         return false;
     }
@@ -234,6 +241,7 @@ MessageBuffer::dequeue(Tick current_time, bool decrement_messages)
     // adjusted until schd cycle
     if (m_time_last_time_pop < current_time) {
         m_size_at_cycle_start = m_prio_heap.size();
+        m_stalled_at_cycle_start = m_stall_map_size;
         m_time_last_time_pop = current_time;
     }
 
@@ -274,6 +282,7 @@ MessageBuffer::clear()
     m_time_last_time_enqueue = 0;
     m_time_last_time_pop = 0;
     m_size_at_cycle_start = 0;
+    m_stalled_at_cycle_start = 0;
     m_msgs_this_cycle = 0;
 }
 
diff --git a/src/mem/ruby/network/MessageBuffer.hh b/src/mem/ruby/network/MessageBuffer.hh
index 05821d5ac..f92d56501 100644
--- a/src/mem/ruby/network/MessageBuffer.hh
+++ b/src/mem/ruby/network/MessageBuffer.hh
@@ -192,6 +192,7 @@ class MessageBuffer : public SimObject
     Tick m_last_arrival_time;
 
     unsigned int m_size_at_cycle_start;
+    unsigned int m_stalled_at_cycle_start;
     unsigned int m_msgs_this_cycle;
 
     Stats::Scalar m_not_avail_count;  // count the # of times I didn't have N