ruby: enable multiple clock domains

This patch allows ruby to have multiple clock domains. As I understand
with this patch, controllers can have different frequencies. The entire
network needs to run at a single frequency.

The idea is that with in an object, time is treated in terms of cycles.
But the messages that are passed from one entity to another should contain
the time in Ticks. As of now, this is only true for the message buffers,
but not for the links in the network. As I understand the code, all the
entities in different networks (simple, garnet-fixed, garnet-flexible) should
be clocked at the same frequency.

Another problem is that the directory controller has to operate at the same
frequency as the ruby system. This is because the memory controller does
not make use of the Message Buffer, and instead implements a buffer of its
own. So, it has no idea of the frequency at which the directory controller
is operating and uses ruby system's frequency for scheduling events.

7 files changed, 32 insertions, 20 deletions

diff --git a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc
index 3d915598e..ca4d6aabe 100644
--- a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc
+++ b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc
@@ -116,7 +116,9 @@ NetworkInterface_d::addNode(vector<MessageBuffer *>& in,
 
         // the protocol injects messages into the NI
         inNode_ptr[j]->setConsumer(this);
-        inNode_ptr[j]->setClockObj(m_net_ptr);
+        inNode_ptr[j]->setReceiver(m_net_ptr);
+
+        outNode_ptr[j]->setSender(m_net_ptr);
     }
 }
 
@@ -173,7 +175,8 @@ NetworkInterface_d::flitisizeMessage(MsgPtr msg_ptr, int vnet)
             flit_d *fl = new flit_d(i, vc, vnet, num_flits, new_msg_ptr,
                 m_net_ptr->curCycle());
 
-            fl->set_delay(m_net_ptr->curCycle() - msg_ptr->getTime());
+            fl->set_delay(m_net_ptr->curCycle() -
+                          m_net_ptr->ticksToCycles(msg_ptr->getTime()));
             m_ni_buffers[vc]->insert(fl);
         }
 
diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc
index ddb5da716..79e295601 100644
--- a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc
+++ b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc
@@ -106,7 +106,9 @@ NetworkInterface::addNode(vector<MessageBuffer*>& in,
     // protocol injects messages into the NI
     for (int j = 0; j < m_virtual_networks; j++) {
         inNode_ptr[j]->setConsumer(this);
-        inNode_ptr[j]->setClockObj(m_net_ptr);
+        inNode_ptr[j]->setReceiver(m_net_ptr);
+
+        outNode_ptr[j]->setSender(m_net_ptr);
     }
 }
 
@@ -169,7 +171,8 @@ NetworkInterface::flitisizeMessage(MsgPtr msg_ptr, int vnet)
             m_net_ptr->increment_injected_flits(vnet);
             flit *fl = new flit(i, vc, vnet, num_flits, new_msg_ptr,
                                 m_net_ptr->curCycle());
-            fl->set_delay(m_net_ptr->curCycle() - msg_ptr->getTime());
+            fl->set_delay(m_net_ptr->curCycle() -
+                          m_net_ptr->ticksToCycles(msg_ptr->getTime()));
             m_ni_buffers[vc]->insert(fl);
         }
 
diff --git a/src/mem/ruby/network/simple/PerfectSwitch.cc b/src/mem/ruby/network/simple/PerfectSwitch.cc
index 687bdbd86..ecfe12f15 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.cc
+++ b/src/mem/ruby/network/simple/PerfectSwitch.cc
@@ -68,7 +68,7 @@ PerfectSwitch::init(SimpleNetwork *network_ptr)
 }
 
 void
-PerfectSwitch::addInPort(const vector<MessageBuffer*>& in, Switch *sw)
+PerfectSwitch::addInPort(const vector<MessageBuffer*>& in)
 {
     assert(in.size() == m_virtual_networks);
     NodeID port = m_in.size();
@@ -76,7 +76,6 @@ PerfectSwitch::addInPort(const vector<MessageBuffer*>& in, Switch *sw)
 
     for (int j = 0; j < m_virtual_networks; j++) {
         m_in[port][j]->setConsumer(this);
-        m_in[port][j]->setClockObj(sw);
 
         string desc = csprintf("[Queue from port %s %s %s to PerfectSwitch]",
             to_string(m_switch_id), to_string(port), to_string(j));
diff --git a/src/mem/ruby/network/simple/PerfectSwitch.hh b/src/mem/ruby/network/simple/PerfectSwitch.hh
index ffd1f84ba..695c848bc 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.hh
+++ b/src/mem/ruby/network/simple/PerfectSwitch.hh
@@ -63,7 +63,7 @@ class PerfectSwitch : public Consumer
     { return csprintf("PerfectSwitch-%i", m_switch_id); }
 
     void init(SimpleNetwork *);
-    void addInPort(const std::vector<MessageBuffer*>& in, Switch *);
+    void addInPort(const std::vector<MessageBuffer*>& in);
     void addOutPort(const std::vector<MessageBuffer*>& out,
         const NetDest& routing_table_entry);
     void clearRoutingTables();
diff --git a/src/mem/ruby/network/simple/Switch.cc b/src/mem/ruby/network/simple/Switch.cc
index 76d37c321..d951ea38d 100644
--- a/src/mem/ruby/network/simple/Switch.cc
+++ b/src/mem/ruby/network/simple/Switch.cc
@@ -67,7 +67,11 @@ Switch::init()
 void
 Switch::addInPort(const vector<MessageBuffer*>& in)
 {
-    m_perfect_switch_ptr->addInPort(in, this);
+    m_perfect_switch_ptr->addInPort(in);
+
+    for (int i = 0; i < in.size(); i++) {
+        in[i]->setReceiver(this);
+    }
 }
 
 void
@@ -83,21 +87,27 @@ Switch::addOutPort(const vector<MessageBuffer*>& out,
     // Create one buffer per vnet (these are intermediaryQueues)
     vector<MessageBuffer*> intermediateBuffers;
     for (int i = 0; i < out.size(); i++) {
+        out[i]->setSender(this);
+
         MessageBuffer* buffer_ptr = new MessageBuffer;
         // Make these queues ordered
         buffer_ptr->setOrdering(true);
         if (m_network_ptr->getBufferSize() > 0) {
             buffer_ptr->resize(m_network_ptr->getBufferSize());
         }
+
         intermediateBuffers.push_back(buffer_ptr);
         m_buffers_to_free.push_back(buffer_ptr);
+
+        buffer_ptr->setSender(this);
+        buffer_ptr->setReceiver(this);
     }
 
     // Hook the queues to the PerfectSwitch
     m_perfect_switch_ptr->addOutPort(intermediateBuffers, routing_table_entry);
 
     // Hook the queues to the Throttle
-    throttle_ptr->addLinks(intermediateBuffers, out, this);
+    throttle_ptr->addLinks(intermediateBuffers, out);
 }
 
 void
diff --git a/src/mem/ruby/network/simple/Throttle.cc b/src/mem/ruby/network/simple/Throttle.cc
index bb5d9cf53..88abe4167 100644
--- a/src/mem/ruby/network/simple/Throttle.cc
+++ b/src/mem/ruby/network/simple/Throttle.cc
@@ -93,11 +93,11 @@ Throttle::clear()
 
 void
 Throttle::addLinks(const std::vector<MessageBuffer*>& in_vec,
-    const std::vector<MessageBuffer*>& out_vec, ClockedObject *em)
+    const std::vector<MessageBuffer*>& out_vec)
 {
     assert(in_vec.size() == out_vec.size());
     for (int i=0; i<in_vec.size(); i++) {
-        addVirtualNetwork(in_vec[i], out_vec[i], em);
+        addVirtualNetwork(in_vec[i], out_vec[i]);
     }
 
     m_message_counters.resize(MessageSizeType_NUM);
@@ -110,8 +110,7 @@ Throttle::addLinks(const std::vector<MessageBuffer*>& in_vec,
 }
 
 void
-Throttle::addVirtualNetwork(MessageBuffer* in_ptr, MessageBuffer* out_ptr,
-                            ClockedObject *em)
+Throttle::addVirtualNetwork(MessageBuffer* in_ptr, MessageBuffer* out_ptr)
 {
     m_units_remaining.push_back(0);
     m_in.push_back(in_ptr);
@@ -119,7 +118,6 @@ Throttle::addVirtualNetwork(MessageBuffer* in_ptr, MessageBuffer* out_ptr,
 
     // Set consumer and description
     m_in[m_vnets]->setConsumer(this);
-    m_in[m_vnets]->setClockObj(em);
 
     string desc = "[Queue to Throttle " + to_string(m_sID) + " " +
         to_string(m_node) + "]";
@@ -174,7 +172,7 @@ Throttle::wakeup()
                 DPRINTF(RubyNetwork, "throttle: %d my bw %d bw spent "
                         "enqueueing net msg %d time: %lld.\n",
                         m_node, getLinkBandwidth(), m_units_remaining[vnet],
-                        g_system_ptr->getTime());
+                        g_system_ptr->curCycle());
 
                 // Move the message
                 m_out[vnet]->enqueue(m_in[vnet]->peekMsgPtr(), m_link_latency);
@@ -235,7 +233,7 @@ Throttle::printStats(ostream& out) const
 void
 Throttle::clearStats()
 {
-    m_ruby_start = g_system_ptr->getTime();
+    m_ruby_start = g_system_ptr->curCycle();
     m_links_utilized = 0.0;
 
     for (int i = 0; i < m_message_counters.size(); i++) {
@@ -249,7 +247,7 @@ double
 Throttle::getUtilization() const
 {
     return 100.0 * double(m_links_utilized) /
-        double(g_system_ptr->getTime()-m_ruby_start);
+        double(g_system_ptr->curCycle()-m_ruby_start);
 }
 
 void
diff --git a/src/mem/ruby/network/simple/Throttle.hh b/src/mem/ruby/network/simple/Throttle.hh
index 4a84227fb..2ca474f41 100644
--- a/src/mem/ruby/network/simple/Throttle.hh
+++ b/src/mem/ruby/network/simple/Throttle.hh
@@ -63,7 +63,7 @@ class Throttle : public Consumer
     { return csprintf("Throttle-%i", m_sID); }
 
     void addLinks(const std::vector<MessageBuffer*>& in_vec,
-        const std::vector<MessageBuffer*>& out_vec, ClockedObject *em);
+        const std::vector<MessageBuffer*>& out_vec);
     void wakeup();
 
     void printStats(std::ostream& out) const;
@@ -87,8 +87,7 @@ class Throttle : public Consumer
   private:
     void init(NodeID node, Cycles link_latency, int link_bandwidth_multiplier,
               int endpoint_bandwidth);
-    void addVirtualNetwork(MessageBuffer* in_ptr, MessageBuffer* out_ptr,
-        ClockedObject *em);
+    void addVirtualNetwork(MessageBuffer* in_ptr, MessageBuffer* out_ptr);
     void linkUtilized(double ratio) { m_links_utilized += ratio; }
 
     // Private copy constructor and assignment operator