ruby: enable multiple clock domains

This patch allows ruby to have multiple clock domains. As I understand
with this patch, controllers can have different frequencies. The entire
network needs to run at a single frequency.

The idea is that with in an object, time is treated in terms of cycles.
But the messages that are passed from one entity to another should contain
the time in Ticks. As of now, this is only true for the message buffers,
but not for the links in the network. As I understand the code, all the
entities in different networks (simple, garnet-fixed, garnet-flexible) should
be clocked at the same frequency.

Another problem is that the directory controller has to operate at the same
frequency as the ruby system. This is because the memory controller does
not make use of the Message Buffer, and instead implements a buffer of its
own. So, it has no idea of the frequency at which the directory controller
is operating and uses ruby system's frequency for scheduling events.

2 files changed, 49 insertions, 36 deletions

diff --git a/src/mem/ruby/buffers/MessageBuffer.cc b/src/mem/ruby/buffers/MessageBuffer.cc
index c9b255c02..92c989851 100644
--- a/src/mem/ruby/buffers/MessageBuffer.cc
+++ b/src/mem/ruby/buffers/MessageBuffer.cc
@@ -44,7 +44,8 @@ MessageBuffer::MessageBuffer(const string &name)
 {
     m_msg_counter = 0;
     m_consumer_ptr = NULL;
-    m_clockobj_ptr = NULL;
+    m_sender_ptr = NULL;
+    m_receiver_ptr = NULL;
 
     m_ordering_set = false;
     m_strict_fifo = true;
@@ -66,10 +67,10 @@ MessageBuffer::MessageBuffer(const string &name)
 int
 MessageBuffer::getSize()
 {
-    if (m_time_last_time_size_checked == m_clockobj_ptr->curCycle()) {
+    if (m_time_last_time_size_checked == m_receiver_ptr->curCycle()) {
         return m_size_last_time_size_checked;
     } else {
-        m_time_last_time_size_checked = m_clockobj_ptr->curCycle();
+        m_time_last_time_size_checked = m_receiver_ptr->curCycle();
         m_size_last_time_size_checked = m_size;
         return m_size;
     }
@@ -89,11 +90,11 @@ MessageBuffer::areNSlotsAvailable(int n)
     // until next cycle, but enqueue operations effect the visible
     // size immediately
     int current_size = max(m_size_at_cycle_start, m_size);
-    if (m_time_last_time_pop < m_clockobj_ptr->curCycle()) {
+    if (m_time_last_time_pop < m_receiver_ptr->curCycle()) {
         // no pops this cycle - m_size is correct
         current_size = m_size;
     } else {
-        if (m_time_last_time_enqueue < m_clockobj_ptr->curCycle()) {
+        if (m_time_last_time_enqueue < m_receiver_ptr->curCycle()) {
             // no enqueues this cycle - m_size_at_cycle_start is correct
             current_size = m_size_at_cycle_start;
         } else {
@@ -149,15 +150,15 @@ random_time()
 }
 
 void
-MessageBuffer::enqueue(MsgPtr message, Cycles delta)
+MessageBuffer::enqueue(MsgPtr message, Cycles delay)
 {
     m_msg_counter++;
     m_size++;
 
     // record current time incase we have a pop that also adjusts my size
-    if (m_time_last_time_enqueue < m_clockobj_ptr->curCycle()) {
+    if (m_time_last_time_enqueue < m_receiver_ptr->curCycle()) {
         m_msgs_this_cycle = 0;  // first msg this cycle
-        m_time_last_time_enqueue = m_clockobj_ptr->curCycle();
+        m_time_last_time_enqueue = m_receiver_ptr->curCycle();
     }
     m_msgs_this_cycle++;
 
@@ -167,8 +168,11 @@ MessageBuffer::enqueue(MsgPtr message, Cycles delta)
 
     // Calculate the arrival time of the message, that is, the first
     // cycle the message can be dequeued.
-    assert(delta>0);
-    Cycles current_time(m_clockobj_ptr->curCycle());
+    assert(delay > 0);
+    Cycles delta = m_receiver_ptr->ticksToCycles(delay *
+                                                 m_sender_ptr->clockPeriod());
+
+    Cycles current_time(m_receiver_ptr->curCycle());
     Cycles arrival_time(0);
 
     if (!RubySystem::getRandomization() || (m_randomization == false)) {
@@ -192,10 +196,10 @@ MessageBuffer::enqueue(MsgPtr message, Cycles delta)
         if (arrival_time < m_last_arrival_time) {
             panic("FIFO ordering violated: %s name: %s current time: %d "
                   "delta: %d arrival_time: %d last arrival_time: %d\n",
-                  *this, m_name, current_time * m_clockobj_ptr->clockPeriod(),
-                  delta * m_clockobj_ptr->clockPeriod(),
-                  arrival_time * m_clockobj_ptr->clockPeriod(),
-                  m_last_arrival_time * m_clockobj_ptr->clockPeriod());
+                  *this, m_name, current_time * m_receiver_ptr->clockPeriod(),
+                  delta * m_receiver_ptr->clockPeriod(),
+                  arrival_time * m_receiver_ptr->clockPeriod(),
+                  m_last_arrival_time * m_receiver_ptr->clockPeriod());
         }
     }
 
@@ -208,13 +212,13 @@ MessageBuffer::enqueue(MsgPtr message, Cycles delta)
     Message* msg_ptr = message.get();
     assert(msg_ptr != NULL);
 
-    assert(m_clockobj_ptr->curCycle() >= msg_ptr->getLastEnqueueTime() &&
+    assert(m_receiver_ptr->clockEdge() >= msg_ptr->getLastEnqueueTime() &&
            "ensure we aren't dequeued early");
 
-    msg_ptr->setDelayedCycles(m_clockobj_ptr->curCycle() -
+    msg_ptr->setDelayedTicks(m_receiver_ptr->clockEdge() -
                               msg_ptr->getLastEnqueueTime() +
-                              msg_ptr->getDelayedCycles());
-    msg_ptr->setLastEnqueueTime(arrival_time);
+                              msg_ptr->getDelayedTicks());
+    msg_ptr->setLastEnqueueTime(arrival_time * m_receiver_ptr->clockPeriod());
 
     // Insert the message into the priority heap
     MessageBufferNode thisNode(arrival_time, m_msg_counter, message);
@@ -223,7 +227,7 @@ MessageBuffer::enqueue(MsgPtr message, Cycles delta)
         greater<MessageBufferNode>());
 
     DPRINTF(RubyQueue, "Enqueue arrival_time: %lld, Message: %s\n",
-            arrival_time * m_clockobj_ptr->clockPeriod(), *(message.get()));
+            arrival_time * m_receiver_ptr->clockPeriod(), *(message.get()));
 
     // Schedule the wakeup
     if (m_consumer_ptr != NULL) {
@@ -275,9 +279,9 @@ MessageBuffer::pop()
 
     // record previous size and time so the current buffer size isn't
     // adjusted until next cycle
-    if (m_time_last_time_pop < m_clockobj_ptr->curCycle()) {
+    if (m_time_last_time_pop < m_receiver_ptr->curCycle()) {
         m_size_at_cycle_start = m_size;
-        m_time_last_time_pop = m_clockobj_ptr->curCycle();
+        m_time_last_time_pop = m_receiver_ptr->curCycle();
     }
     m_size--;
 }
@@ -304,11 +308,11 @@ MessageBuffer::recycle()
     pop_heap(m_prio_heap.begin(), m_prio_heap.end(),
         greater<MessageBufferNode>());
 
-    node.m_time = m_clockobj_ptr->curCycle() + m_recycle_latency;
+    node.m_time = m_receiver_ptr->curCycle() + m_recycle_latency;
     m_prio_heap.back() = node;
     push_heap(m_prio_heap.begin(), m_prio_heap.end(),
         greater<MessageBufferNode>());
-    m_consumer_ptr->scheduleEventAbsolute(m_clockobj_ptr->curCycle() +
+    m_consumer_ptr->scheduleEventAbsolute(m_receiver_ptr->curCycle() +
                                           m_recycle_latency);
 }
 
@@ -317,7 +321,7 @@ MessageBuffer::reanalyzeMessages(const Address& addr)
 {
     DPRINTF(RubyQueue, "ReanalyzeMessages\n");
     assert(m_stall_msg_map.count(addr) > 0);
-    Cycles nextCycle = m_clockobj_ptr->curCycle() + Cycles(1);
+    Cycles nextCycle = m_receiver_ptr->curCycle() + Cycles(1);
 
     //
     // Put all stalled messages associated with this address back on the
@@ -342,7 +346,7 @@ void
 MessageBuffer::reanalyzeAllMessages()
 {
     DPRINTF(RubyQueue, "ReanalyzeAllMessages %s\n");
-    Cycles nextCycle = m_clockobj_ptr->curCycle() + Cycles(1);
+    Cycles nextCycle = m_receiver_ptr->curCycle() + Cycles(1);
 
     //
     // Put all stalled messages associated with this address back on the
@@ -393,12 +397,13 @@ MessageBuffer::setAndReturnDelayCycles(MsgPtr msg_ptr)
 
     // this function should only be called on dequeue
     // ensure the msg hasn't been enqueued
-    assert(msg_ptr->getLastEnqueueTime() <= m_clockobj_ptr->curCycle());
-    msg_ptr->setDelayedCycles(m_clockobj_ptr->curCycle() -
-                             msg_ptr->getLastEnqueueTime() +
-                             msg_ptr->getDelayedCycles());
+    assert(msg_ptr->getLastEnqueueTime() <= m_receiver_ptr->clockEdge());
+
+    msg_ptr->setDelayedTicks(m_receiver_ptr->clockEdge() -
+                              msg_ptr->getLastEnqueueTime() +
+                              msg_ptr->getDelayedTicks());
 
-    return msg_ptr->getDelayedCycles();
+    return m_receiver_ptr->ticksToCycles(msg_ptr->getDelayedTicks());
 }
 
 void
@@ -425,7 +430,7 @@ bool
 MessageBuffer::isReady() const
 {
     return ((m_prio_heap.size() > 0) &&
-            (m_prio_heap.front().m_time <= m_clockobj_ptr->curCycle()));
+            (m_prio_heap.front().m_time <= m_receiver_ptr->curCycle()));
 }
 
 bool
diff --git a/src/mem/ruby/buffers/MessageBuffer.hh b/src/mem/ruby/buffers/MessageBuffer.hh
index e68a05786..74023a8ac 100644
--- a/src/mem/ruby/buffers/MessageBuffer.hh
+++ b/src/mem/ruby/buffers/MessageBuffer.hh
@@ -83,10 +83,16 @@ class MessageBuffer
         m_consumer_ptr = consumer_ptr;
     }
 
-    void setClockObj(ClockedObject* obj)
+    void setSender(ClockedObject* obj)
     {
-        assert(m_clockobj_ptr == NULL);
-        m_clockobj_ptr = obj;
+        assert(m_sender_ptr == NULL || m_sender_ptr == obj);
+        m_sender_ptr = obj;
+    }
+
+    void setReceiver(ClockedObject* obj)
+    {
+        assert(m_receiver_ptr == NULL || m_receiver_ptr == obj);
+        m_receiver_ptr = obj;
     }
 
     void setDescription(const std::string& name) { m_name = name; }
@@ -167,8 +173,10 @@ class MessageBuffer
     MessageBuffer& operator=(const MessageBuffer& obj);
 
     // Data Members (m_ prefix)
-    //! Object used for querying time.
-    ClockedObject* m_clockobj_ptr;
+    //! The two ends of the buffer.
+    ClockedObject* m_sender_ptr;
+    ClockedObject* m_receiver_ptr;
+
     //! Consumer to signal a wakeup(), can be NULL
     Consumer* m_consumer_ptr;
     std::vector<MessageBufferNode> m_prio_heap;