ruby: improved support for functional accesses

This patch adds support to different entities in the ruby memory system
for more reliable functional read/write accesses. Only the simple network
has been augmented as of now. Later on Garnet will also support functional
accesses.
The patch adds functional access code to all the different types of messages
that protocols can send around. These messages are functionally accessed
by going through the buffers maintained by the network entities.
The patch also rectifies some of the bugs found in coherence protocols while
testing the patch.

With this patch applied, functional writes always succeed. But functional
reads can still fail.

20 files changed, 422 insertions, 89 deletions

diff --git a/src/mem/ruby/buffers/MessageBuffer.cc b/src/mem/ruby/buffers/MessageBuffer.cc
index f0d372a52..f960cc033 100644
--- a/src/mem/ruby/buffers/MessageBuffer.cc
+++ b/src/mem/ruby/buffers/MessageBuffer.cc
@@ -435,3 +435,69 @@ MessageBuffer::printStats(ostream& out)
     out << "MessageBuffer: " << m_name << " stats - msgs:" << m_msg_counter
         << " full:" << m_not_avail_count << endl;
 }
+
+bool
+MessageBuffer::isReady() const
+{
+    return ((m_prio_heap.size() > 0) &&
+            (m_prio_heap.front().m_time <= g_system_ptr->getTime()));
+}
+
+bool
+MessageBuffer::functionalRead(Packet *pkt)
+{
+    // Check the priority heap and read any messages that may
+    // correspond to the address in the packet.
+    for (unsigned int i = 0; i < m_prio_heap.size(); ++i) {
+        Message *msg = m_prio_heap[i].m_msgptr.get();
+        if (msg->functionalRead(pkt)) return true;
+    }
+
+    // Read the messages in the stall queue that correspond
+    // to the address in the packet.
+    for (StallMsgMapType::iterator map_iter = m_stall_msg_map.begin();
+         map_iter != m_stall_msg_map.end();
+         ++map_iter) {
+
+        for (std::list<MsgPtr>::iterator it = (map_iter->second).begin();
+            it != (map_iter->second).end(); ++it) {
+
+            Message *msg = (*it).get();
+            if (msg->functionalRead(pkt)) return true;
+        }
+    }
+    return false;
+}
+
+uint32_t
+MessageBuffer::functionalWrite(Packet *pkt)
+{
+    uint32_t num_functional_writes = 0;
+
+    // Check the priority heap and write any messages that may
+    // correspond to the address in the packet.
+    for (unsigned int i = 0; i < m_prio_heap.size(); ++i) {
+        Message *msg = m_prio_heap[i].m_msgptr.get();
+        if (msg->functionalWrite(pkt)) {
+            num_functional_writes++;
+        }
+    }
+
+    // Check the stall queue and write any messages that may
+    // correspond to the address in the packet.
+    for (StallMsgMapType::iterator map_iter = m_stall_msg_map.begin();
+         map_iter != m_stall_msg_map.end();
+         ++map_iter) {
+
+        for (std::list<MsgPtr>::iterator it = (map_iter->second).begin();
+            it != (map_iter->second).end(); ++it) {
+
+            Message *msg = (*it).get();
+            if (msg->functionalWrite(pkt)) {
+                num_functional_writes++;
+            }
+        }
+    }
+
+    return num_functional_writes;
+}
diff --git a/src/mem/ruby/buffers/MessageBuffer.hh b/src/mem/ruby/buffers/MessageBuffer.hh
index cf7e77c2d..c4fd7165d 100644
--- a/src/mem/ruby/buffers/MessageBuffer.hh
+++ b/src/mem/ruby/buffers/MessageBuffer.hh
@@ -41,10 +41,10 @@
 #include <string>
 #include <vector>
 
+#include "mem/packet.hh"
 #include "mem/ruby/buffers/MessageBufferNode.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/common/Consumer.hh"
-#include "mem/ruby/common/Global.hh"
 #include "mem/ruby/slicc_interface/Message.hh"
 
 class MessageBuffer
@@ -65,12 +65,7 @@ class MessageBuffer
     void stallMessage(const Address& addr);
 
     // TRUE if head of queue timestamp <= SystemTime
-    bool
-    isReady() const
-    {
-        return ((m_prio_heap.size() > 0) &&
-                (m_prio_heap.front().m_time <= g_system_ptr->getTime()));
-    }
+    bool isReady() const;
 
     void
     delayHead()
@@ -145,6 +140,17 @@ class MessageBuffer
     void setIncomingLink(int link_id) { m_input_link_id = link_id; }
     void setVnet(int net) { m_vnet_id = net; }
 
+    // Function for figuring out if any of the messages in the buffer can
+    // satisfy the read request for the address in the packet.
+    // Return value, if true, indicates that the request was fulfilled.
+    bool functionalRead(Packet *pkt);
+
+    // Function for figuring out if any of the messages in the buffer need
+    // to be updated with the data from the packet.
+    // Return value indicates the number of messages that were updated.
+    // This required for debugging the code.
+    uint32_t functionalWrite(Packet *pkt);
+
   private:
     //added by SS
     int m_recycle_latency;
diff --git a/src/mem/ruby/buffers/MessageBufferNode.hh b/src/mem/ruby/buffers/MessageBufferNode.hh
index 70afa9831..5e7a52e12 100644
--- a/src/mem/ruby/buffers/MessageBufferNode.hh
+++ b/src/mem/ruby/buffers/MessageBufferNode.hh
@@ -31,7 +31,6 @@
 
 #include <iostream>
 
-#include "mem/ruby/common/Global.hh"
 #include "mem/ruby/slicc_interface/Message.hh"
 
 class MessageBufferNode
diff --git a/src/mem/ruby/network/Network.hh b/src/mem/ruby/network/Network.hh
index e9c5a98b7..9236a2207 100644
--- a/src/mem/ruby/network/Network.hh
+++ b/src/mem/ruby/network/Network.hh
@@ -44,6 +44,7 @@
 #include <string>
 #include <vector>
 
+#include "mem/packet.hh"
 #include "mem/protocol/LinkDirection.hh"
 #include "mem/protocol/MessageSizeType.hh"
 #include "mem/ruby/common/TypeDefines.hh"
@@ -81,7 +82,7 @@ class Network : public SimObject
                              bool isReconfiguration) = 0;
     virtual void makeInLink(NodeID src, SwitchID dest, BasicLink* link,
                             LinkDirection direction,
-                            const NetDest& routing_table_entry, 
+                            const NetDest& routing_table_entry,
                             bool isReconfiguration) = 0;
     virtual void makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
                                   LinkDirection direction,
@@ -94,6 +95,16 @@ class Network : public SimObject
     virtual void clearStats() = 0;
     virtual void print(std::ostream& out) const = 0;
 
+    /*
+     * Virtual functions for functionally reading and writing packets in
+     * the network. Each network needs to implement these for functional
+     * accesses to work correctly.
+     */
+    virtual bool functionalRead(Packet *pkt)
+    { fatal("Functional read not implemented.\n"); }
+    virtual uint32_t functionalWrite(Packet *pkt)
+    { fatal("Functional write not implemented.\n"); }
+
   protected:
     // Private copy constructor and assignment operator
     Network(const Network& obj);
diff --git a/src/mem/ruby/network/simple/PerfectSwitch.cc b/src/mem/ruby/network/simple/PerfectSwitch.cc
index 2c845683d..b38b6d539 100644
--- a/src/mem/ruby/network/simple/PerfectSwitch.cc
+++ b/src/mem/ruby/network/simple/PerfectSwitch.cc
@@ -343,4 +343,3 @@ PerfectSwitch::print(std::ostream& out) const
 {
     out << "[PerfectSwitch " << m_switch_id << "]";
 }
-
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc
index 7aa8e62f9..9df9ed3a5 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.cc
+++ b/src/mem/ruby/network/simple/SimpleNetwork.cc
@@ -327,3 +327,41 @@ SimpleNetworkParams::create()
 {
     return new SimpleNetwork(this);
 }
+
+/*
+ * The simple network has an array of switches. These switches have buffers
+ * that need to be accessed for functional reads and writes. Also the links
+ * between different switches have buffers that need to be accessed.
+ */
+bool
+SimpleNetwork::functionalRead(Packet *pkt)
+{
+    for (unsigned int i = 0; i < m_switch_ptr_vector.size(); i++) {
+        if (m_switch_ptr_vector[i]->functionalRead(pkt)) {
+            return true;
+        }
+    }
+
+    for (unsigned int i = 0; i < m_buffers_to_free.size(); ++i) {
+        if (m_buffers_to_free[i]->functionalRead(pkt)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+uint32_t
+SimpleNetwork::functionalWrite(Packet *pkt)
+{
+    uint32_t num_functional_writes = 0;
+
+    for (unsigned int i = 0; i < m_switch_ptr_vector.size(); i++) {
+        num_functional_writes += m_switch_ptr_vector[i]->functionalWrite(pkt);
+    }
+
+    for (unsigned int i = 0; i < m_buffers_to_free.size(); ++i) {
+        num_functional_writes += m_buffers_to_free[i]->functionalWrite(pkt);
+    }
+    return num_functional_writes;
+}
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.hh b/src/mem/ruby/network/simple/SimpleNetwork.hh
index 6dfaa2724..879446822 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.hh
+++ b/src/mem/ruby/network/simple/SimpleNetwork.hh
@@ -86,6 +86,9 @@ class SimpleNetwork : public Network
 
     void print(std::ostream& out) const;
 
+    bool functionalRead(Packet *pkt);
+    uint32_t functionalWrite(Packet *pkt);
+
   private:
     void checkNetworkAllocation(NodeID id, bool ordered, int network_num);
     void addLink(SwitchID src, SwitchID dest, int link_latency);
diff --git a/src/mem/ruby/network/simple/Switch.cc b/src/mem/ruby/network/simple/Switch.cc
index 5c2f5a717..a0a27f758 100644
--- a/src/mem/ruby/network/simple/Switch.cc
+++ b/src/mem/ruby/network/simple/Switch.cc
@@ -217,6 +217,28 @@ Switch::print(std::ostream& out) const
     // FIXME printing
     out << "[Switch]";
 }
+bool
+Switch::functionalRead(Packet *pkt)
+{
+    // Access the buffers in the switch for performing a functional read
+    for (unsigned int i = 0; i < m_buffers_to_free.size(); ++i) {
+        if (m_buffers_to_free[i]->functionalRead(pkt)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+uint32_t
+Switch::functionalWrite(Packet *pkt)
+{
+    // Access the buffers in the switch for performing a functional write
+    uint32_t num_functional_writes = 0;
+    for (unsigned int i = 0; i < m_buffers_to_free.size(); ++i) {
+        num_functional_writes += m_buffers_to_free[i]->functionalWrite(pkt);
+    }
+    return num_functional_writes;
+}
 
 Switch *
 SwitchParams::create()
diff --git a/src/mem/ruby/network/simple/Switch.hh b/src/mem/ruby/network/simple/Switch.hh
index 9946e5cc1..05e90f278 100644
--- a/src/mem/ruby/network/simple/Switch.hh
+++ b/src/mem/ruby/network/simple/Switch.hh
@@ -74,6 +74,9 @@ class Switch : public BasicRouter
     void print(std::ostream& out) const;
     void init_net_ptr(SimpleNetwork* net_ptr) { m_network_ptr = net_ptr; }
 
+    bool functionalRead(Packet *);
+    uint32_t functionalWrite(Packet *);
+
   private:
     // Private copy constructor and assignment operator
     Switch(const Switch& obj);
diff --git a/src/mem/ruby/slicc_interface/AbstractController.hh b/src/mem/ruby/slicc_interface/AbstractController.hh
index 70b9968c7..c5cb46f1e 100644
--- a/src/mem/ruby/slicc_interface/AbstractController.hh
+++ b/src/mem/ruby/slicc_interface/AbstractController.hh
@@ -32,6 +32,7 @@
 #include <iostream>
 #include <string>
 
+#include "mem/packet.hh"
 #include "mem/protocol/AccessPermission.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/common/Consumer.hh"
@@ -68,6 +69,15 @@ class AbstractController : public SimObject, public Consumer
     virtual void clearStats() = 0;
     virtual void recordCacheTrace(int cntrl, CacheRecorder* tr) = 0;
     virtual Sequencer* getSequencer() const = 0;
+
+    //! These functions are used by ruby system to read/write the message
+    //! queues that exist with in the controller.
+    //! The boolean return value indicates if the read was performed
+    //! successfully.
+    virtual bool functionalReadBuffers(PacketPtr&) = 0;
+    //! The return value indicates the number of messages written with the
+    //! data from the packet.
+    virtual uint32_t functionalWriteBuffers(PacketPtr&) = 0;
 };
 
 #endif // __MEM_RUBY_SLICC_INTERFACE_ABSTRACTCONTROLLER_HH__
diff --git a/src/mem/ruby/slicc_interface/Message.hh b/src/mem/ruby/slicc_interface/Message.hh
index 48156a99a..7b94a01e2 100644
--- a/src/mem/ruby/slicc_interface/Message.hh
+++ b/src/mem/ruby/slicc_interface/Message.hh
@@ -61,6 +61,18 @@ class Message : public RefCounted
     virtual void setIncomingLink(int) {}
     virtual void setVnet(int) {}
 
+    /**
+     * The two functions below are used for reading / writing the message
+     * functionally. The methods return true if the address in the packet
+     * matches the address / address range in the message. Each message
+     * class that can be potentially searched for the address needs to
+     * implement these methods.
+     */
+    virtual bool functionalRead(Packet *pkt) = 0;
+    //{ fatal("Read functional access not implemented!"); }
+    virtual bool functionalWrite(Packet *pkt) = 0;
+    //{ fatal("Write functional access not implemented!"); }
+
     void setDelayedCycles(const int& cycles) { m_DelayedCycles = cycles; }
     const int& getDelayedCycles() const {return m_DelayedCycles;}
     int& getDelayedCycles() {return m_DelayedCycles;}
diff --git a/src/mem/ruby/slicc_interface/NetworkMessage.hh b/src/mem/ruby/slicc_interface/NetworkMessage.hh
index a8f9c625b..d2bcb1241 100644
--- a/src/mem/ruby/slicc_interface/NetworkMessage.hh
+++ b/src/mem/ruby/slicc_interface/NetworkMessage.hh
@@ -33,7 +33,6 @@
 
 #include "base/refcnt.hh"
 #include "mem/protocol/MessageSizeType.hh"
-#include "mem/ruby/common/Global.hh"
 #include "mem/ruby/common/NetDest.hh"
 #include "mem/ruby/slicc_interface/Message.hh"
 
diff --git a/src/mem/ruby/slicc_interface/RubyRequest.cc b/src/mem/ruby/slicc_interface/RubyRequest.cc
index 2aae61d7b..7ff2b75d8 100644
--- a/src/mem/ruby/slicc_interface/RubyRequest.cc
+++ b/src/mem/ruby/slicc_interface/RubyRequest.cc
@@ -18,3 +18,40 @@ RubyRequest::print(ostream& out) const
 //  out << "Time = " << getTime() << " ";
   out << "]";
 }
+
+bool
+RubyRequest::functionalRead(Packet *pkt)
+{
+    // This needs a little explanation. Initially I thought that this
+    // message should be read. But the way the memtester works for now,
+    // we should not be reading this message as memtester updates the
+    // functional memory only after a write has actually taken place.
+    return false;
+}
+
+bool
+RubyRequest::functionalWrite(Packet *pkt)
+{
+    // This needs a little explanation. I am not sure if this message
+    // should be written. Essentially the question is how are writes
+    // ordered. I am assuming that if a functional write is issued after
+    // a timing write to the same address, then the functional write
+    // has to overwrite the data for the timing request, even if the
+    // timing request has still not been ordered globally.
+
+    Address pktLineAddr(pkt->getAddr());
+    pktLineAddr.makeLineAddress();
+
+    if (pktLineAddr == m_LineAddress) {
+        uint8_t *pktData = pkt->getPtr<uint8_t>(true);
+        unsigned int size_in_bytes = pkt->getSize();
+        unsigned startByte = pkt->getAddr() - m_LineAddress.getAddress();
+
+        for (unsigned i = 0; i < size_in_bytes; ++i) {
+            data[i + startByte] = pktData[i];
+        }
+
+        return true;
+    }
+    return false;
+}
diff --git a/src/mem/ruby/slicc_interface/RubyRequest.hh b/src/mem/ruby/slicc_interface/RubyRequest.hh
index 870ad1d0e..a4dadc7a7 100644
--- a/src/mem/ruby/slicc_interface/RubyRequest.hh
+++ b/src/mem/ruby/slicc_interface/RubyRequest.hh
@@ -126,6 +126,9 @@ class RubyRequest : public Message
     }
 
     void print(std::ostream& out) const;
+
+    bool functionalRead(Packet *pkt);
+    bool functionalWrite(Packet *pkt);
 };
 
 inline std::ostream&
diff --git a/src/mem/ruby/slicc_interface/RubySlicc_Util.hh b/src/mem/ruby/slicc_interface/RubySlicc_Util.hh
index 06372c9fc..06c540db5 100644
--- a/src/mem/ruby/slicc_interface/RubySlicc_Util.hh
+++ b/src/mem/ruby/slicc_interface/RubySlicc_Util.hh
@@ -35,6 +35,7 @@
 
 #include <cassert>
 
+#include "debug/RubySlicc.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/common/Global.hh"
 #include "mem/ruby/slicc_interface/RubySlicc_ComponentMapping.hh"
@@ -129,4 +130,60 @@ mod(int val, int mod)
     return val % mod;
 }
 
+/**
+ * This function accepts an address, a data block and a packet. If the address
+ * range for the data block contains the address which the packet needs to
+ * read, then the data from the data block is written to the packet. True is
+ * returned if the data block was read, otherwise false is returned.
+ */
+inline bool
+testAndRead(Address addr, DataBlock& blk, Packet *pkt)
+{
+    Address pktLineAddr(pkt->getAddr());
+    pktLineAddr.makeLineAddress();
+
+    Address lineAddr = addr;
+    lineAddr.makeLineAddress();
+
+    if (pktLineAddr == lineAddr) {
+        uint8_t *data = pkt->getPtr<uint8_t>(true);
+        unsigned int size_in_bytes = pkt->getSize();
+        unsigned startByte = pkt->getAddr() - lineAddr.getAddress();
+
+        for (unsigned i = 0; i < size_in_bytes; ++i) {
+            data[i] = blk.getByte(i + startByte);
+        }
+        return true;
+    }
+    return false;
+}
+
+/**
+ * This function accepts an address, a data block and a packet. If the address
+ * range for the data block contains the address which the packet needs to
+ * write, then the data from the packet is written to the data block. True is
+ * returned if the data block was written, otherwise false is returned.
+ */
+inline bool
+testAndWrite(Address addr, DataBlock& blk, Packet *pkt)
+{
+    Address pktLineAddr(pkt->getAddr());
+    pktLineAddr.makeLineAddress();
+
+    Address lineAddr = addr;
+    lineAddr.makeLineAddress();
+
+    if (pktLineAddr == lineAddr) {
+        uint8_t *data = pkt->getPtr<uint8_t>(true);
+        unsigned int size_in_bytes = pkt->getSize();
+        unsigned startByte = pkt->getAddr() - lineAddr.getAddress();
+
+        for (unsigned i = 0; i < size_in_bytes; ++i) {
+            blk.setByte(i + startByte, data[i]);
+        }
+        return true;
+    }
+    return false;
+}
+
 #endif // __MEM_RUBY_SLICC_INTERFACE_RUBYSLICCUTIL_HH__
diff --git a/src/mem/ruby/system/MemoryControl.cc b/src/mem/ruby/system/MemoryControl.cc
index c5ddb0c44..e58b36f63 100644
--- a/src/mem/ruby/system/MemoryControl.cc
+++ b/src/mem/ruby/system/MemoryControl.cc
@@ -27,18 +27,10 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "base/cast.hh"
-#include "base/cprintf.hh"
 #include "debug/RubyStats.hh"
-#include "mem/ruby/common/Address.hh"
-#include "mem/ruby/common/Consumer.hh"
 #include "mem/ruby/common/Global.hh"
-#include "mem/ruby/network/Network.hh"
-#include "mem/ruby/profiler/Profiler.hh"
-#include "mem/ruby/slicc_interface/NetworkMessage.hh"
 #include "mem/ruby/slicc_interface/RubySlicc_ComponentMapping.hh"
 #include "mem/ruby/system/MemoryControl.hh"
-#include "mem/ruby/system/RubyMemoryControl.hh"
 #include "mem/ruby/system/System.hh"
 
 using namespace std;
@@ -55,9 +47,3 @@ MemoryControl::recordRequestType(MemoryControlRequestType request) {
     DPRINTF(RubyStats, "Recorded request: %s\n",
             MemoryControlRequestType_to_string(request));
 }
-
-RubyMemoryControl *
-RubyMemoryControlParams::create()
-{
-    return new RubyMemoryControl(this);
-}
diff --git a/src/mem/ruby/system/MemoryControl.hh b/src/mem/ruby/system/MemoryControl.hh
index 7bdc14cfc..8d15b8dec 100644
--- a/src/mem/ruby/system/MemoryControl.hh
+++ b/src/mem/ruby/system/MemoryControl.hh
@@ -35,19 +35,14 @@
 #include <string>
 
 #include "mem/protocol/MemoryControlRequestType.hh"
-#include "mem/protocol/MemoryMsg.hh"
 #include "mem/ruby/common/Consumer.hh"
-#include "mem/ruby/profiler/MemCntrlProfiler.hh"
 #include "mem/ruby/slicc_interface/Message.hh"
 #include "mem/ruby/system/MemoryNode.hh"
-#include "mem/ruby/system/System.hh"
 #include "params/MemoryControl.hh"
 #include "sim/clocked_object.hh"
 
 //////////////////////////////////////////////////////////////////////////////
 
-class Consumer;
-
 class MemoryControl : public ClockedObject, public Consumer
 {
   public:
@@ -97,6 +92,11 @@ class MemoryControl : public ClockedObject, public Consumer
 
     virtual void recordRequestType(MemoryControlRequestType requestType);
 
+    virtual bool functionalReadBuffers(Packet *pkt)
+    { fatal("Functional read access not implemented!");}
+    virtual uint32_t functionalWriteBuffers(Packet *pkt)
+    { fatal("Functional read access not implemented!");}
+
 protected:
     class MemCntrlEvent : public Event
     {
diff --git a/src/mem/ruby/system/RubyMemoryControl.cc b/src/mem/ruby/system/RubyMemoryControl.cc
index cfdaaaef7..c0e91c28b 100644
--- a/src/mem/ruby/system/RubyMemoryControl.cc
+++ b/src/mem/ruby/system/RubyMemoryControl.cc
@@ -708,3 +708,91 @@ RubyMemoryControl::wakeup()
     }
 }
 
+/**
+ * This function reads the different buffers that exist in the Ruby Memory
+ * Controller, and figures out if any of the buffers hold a message that
+ * contains the data for the address provided in the packet. True is returned
+ * if any of the messages was read, otherwise false is returned.
+ *
+ * I think we should move these buffers to being message buffers, instead of
+ * being lists.
+ */
+bool
+RubyMemoryControl::functionalReadBuffers(Packet *pkt)
+{
+    for (std::list<MemoryNode>::iterator it = m_input_queue.begin();
+         it != m_input_queue.end(); ++it) {
+        Message* msg_ptr = (*it).m_msgptr.get();
+        if (msg_ptr->functionalRead(pkt)) {
+            return true;
+        }
+    }
+
+    for (std::list<MemoryNode>::iterator it = m_response_queue.begin();
+         it != m_response_queue.end(); ++it) {
+        Message* msg_ptr = (*it).m_msgptr.get();
+        if (msg_ptr->functionalRead(pkt)) {
+            return true;
+        }
+    }
+
+    for (uint32_t bank = 0; bank < m_total_banks; ++bank) {
+        for (std::list<MemoryNode>::iterator it = m_bankQueues[bank].begin();
+             it != m_bankQueues[bank].end(); ++it) {
+            Message* msg_ptr = (*it).m_msgptr.get();
+            if (msg_ptr->functionalRead(pkt)) {
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
+
+/**
+ * This function reads the different buffers that exist in the Ruby Memory
+ * Controller, and figures out if any of the buffers hold a message that
+ * needs to functionally written with the data in the packet.
+ *
+ * The number of messages written is returned at the end. This is required
+ * for debugging purposes.
+ */
+uint32_t
+RubyMemoryControl::functionalWriteBuffers(Packet *pkt)
+{
+    uint32_t num_functional_writes = 0;
+
+    for (std::list<MemoryNode>::iterator it = m_input_queue.begin();
+         it != m_input_queue.end(); ++it) {
+        Message* msg_ptr = (*it).m_msgptr.get();
+        if (msg_ptr->functionalWrite(pkt)) {
+            num_functional_writes++;
+        }
+    }
+
+    for (std::list<MemoryNode>::iterator it = m_response_queue.begin();
+         it != m_response_queue.end(); ++it) {
+        Message* msg_ptr = (*it).m_msgptr.get();
+        if (msg_ptr->functionalWrite(pkt)) {
+            num_functional_writes++;
+        }
+    }
+
+    for (uint32_t bank = 0; bank < m_total_banks; ++bank) {
+        for (std::list<MemoryNode>::iterator it = m_bankQueues[bank].begin();
+             it != m_bankQueues[bank].end(); ++it) {
+            Message* msg_ptr = (*it).m_msgptr.get();
+            if (msg_ptr->functionalWrite(pkt)) {
+                num_functional_writes++;
+            }
+        }
+    }
+
+    return num_functional_writes;
+}
+
+RubyMemoryControl *
+RubyMemoryControlParams::create()
+{
+    return new RubyMemoryControl(this);
+}
diff --git a/src/mem/ruby/system/RubyMemoryControl.hh b/src/mem/ruby/system/RubyMemoryControl.hh
index 98636e408..1f3a8acf5 100644
--- a/src/mem/ruby/system/RubyMemoryControl.hh
+++ b/src/mem/ruby/system/RubyMemoryControl.hh
@@ -96,6 +96,8 @@ class RubyMemoryControl : public MemoryControl
     int getRanksPerDimm() { return m_ranks_per_dimm; };
     int getDimmsPerChannel() { return m_dimms_per_channel; }
 
+    bool functionalReadBuffers(Packet *pkt);
+    uint32_t functionalWriteBuffers(Packet *pkt);
 
   private:
     void enqueueToDirectory(MemoryNode req, int latency);
diff --git a/src/mem/ruby/system/System.cc b/src/mem/ruby/system/System.cc
index b41f2d727..5ee22e9f5 100644
--- a/src/mem/ruby/system/System.cc
+++ b/src/mem/ruby/system/System.cc
@@ -417,7 +417,7 @@ RubySystem::functionalRead(PacketPtr pkt)
 
     // In this loop we count the number of controllers that have the given
     // address in read only, read write and busy states.
-    for (int i = 0; i < num_controllers; ++i) {
+    for (unsigned int i = 0; i < num_controllers; ++i) {
         access_perm = m_abs_cntrl_vec[i]-> getAccessPermission(line_address);
         if (access_perm == AccessPermission_Read_Only)
             num_ro++;
@@ -452,7 +452,7 @@ RubySystem::functionalRead(PacketPtr pkt)
     if (num_invalid == (num_controllers - 1) &&
             num_backing_store == 1) {
         DPRINTF(RubySystem, "only copy in Backing_Store memory, read from it\n");
-        for (int i = 0; i < num_controllers; ++i) {
+        for (unsigned int i = 0; i < num_controllers; ++i) {
             access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_address);
             if (access_perm == AccessPermission_Backing_Store) {
                 DataBlock& block = m_abs_cntrl_vec[i]->
@@ -466,7 +466,7 @@ RubySystem::functionalRead(PacketPtr pkt)
                 return true;
             }
         }
-    } else {
+    } else if (num_ro > 0 || num_rw == 1) {
         // In Broadcast/Snoop protocols, this covers if you know the block
         // exists somewhere in the caching hierarchy, then you want to read any
         // valid RO or RW block.  In directory protocols, same thing, you want
@@ -476,7 +476,7 @@ RubySystem::functionalRead(PacketPtr pkt)
         // In this loop, we try to figure which controller has a read only or
         // a read write copy of the given address. Any valid copy would suffice
         // for a functional read.
-        for (int i = 0;i < num_controllers;++i) {
+        for (unsigned int i = 0;i < num_controllers;++i) {
             access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_address);
             if (access_perm == AccessPermission_Read_Only ||
                 access_perm == AccessPermission_Read_Write) {
@@ -492,9 +492,34 @@ RubySystem::functionalRead(PacketPtr pkt)
             }
         }
     }
+
+    // Since we are here, this means that none of the controllers hold this
+    // address in a stable/base state. The function searches through all the
+    // buffers that exist in different cache, directory and memory
+    // controllers, and in the network components and reads the data portion
+    // of the first message that holds address specified in the packet.
+    for (unsigned int i = 0; i < num_controllers;++i) {
+        if (m_abs_cntrl_vec[i]->functionalReadBuffers(pkt)) {
+            return true;
+        }
+    }
+
+    for (unsigned int i = 0; i < m_memory_controller_vec.size(); ++i) {
+        if (m_memory_controller_vec[i]->functionalReadBuffers(pkt)) {
+            return true;
+        }
+    }
+
+    if (m_network_ptr->functionalRead(pkt)) {
+        return true;
+    }
     return false;
 }
 
+// The function searches through all the buffers that exist in different
+// cache, directory and memory controllers, and in the network components
+// and writes the data portion of those that hold the address specified
+// in the packet.
 bool
 RubySystem::functionalWrite(PacketPtr pkt)
 {
@@ -505,69 +530,36 @@ RubySystem::functionalWrite(PacketPtr pkt)
 
     DPRINTF(RubySystem, "Functional Write request for %s\n",addr);
 
-    unsigned int num_ro = 0;
-    unsigned int num_rw = 0;
-    unsigned int num_busy = 0;
-    unsigned int num_backing_store = 0;
-    unsigned int num_invalid = 0;
-
-    // In this loop we count the number of controllers that have the given
-    // address in read only, read write and busy states.
-    for (int i = 0;i < num_controllers;++i) {
-        access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_addr);
-        if (access_perm == AccessPermission_Read_Only)
-            num_ro++;
-        else if (access_perm == AccessPermission_Read_Write)
-            num_rw++;
-        else if (access_perm == AccessPermission_Busy)
-            num_busy++;
-        else if (access_perm == AccessPermission_Backing_Store)
-            // See RubySlicc_Exports.sm for details, but Backing_Store is meant
-            // to represent blocks in memory *for Broadcast/Snooping protocols*,
-            // where memory has no idea whether it has an exclusive copy of data
-            // or not.
-            num_backing_store++;
-        else if (access_perm == AccessPermission_Invalid ||
-                 access_perm == AccessPermission_NotPresent)
-            num_invalid++;
-    }
-
-    // If the number of read write copies is more than 1, then there is bug in
-    // coherence protocol. Otherwise, if all copies are in stable states, i.e.
-    // num_busy == 0, we update all the copies. If there is at least one copy
-    // in busy state, then we check if there is read write copy. If yes, then
-    // also we let the access go through. Or, if there is no copy in the cache
-    // hierarchy at all, we still want to do the write to the memory
-    // (Backing_Store) instead of failing.
-
-    DPRINTF(RubySystem, "num_busy = %d, num_ro = %d, num_rw = %d\n",
-            num_busy, num_ro, num_rw);
-    assert(num_rw <= 1);
-
     uint8_t *data = pkt->getPtr<uint8_t>(true);
     unsigned int size_in_bytes = pkt->getSize();
     unsigned startByte = addr.getAddress() - line_addr.getAddress();
 
-    if ((num_busy == 0 && num_ro > 0) || num_rw == 1 ||
-        (num_invalid == (num_controllers - 1) && num_backing_store == 1)) {
-        for (int i = 0; i < num_controllers;++i) {
-            access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_addr);
-            if (access_perm == AccessPermission_Read_Only ||
-                access_perm == AccessPermission_Read_Write||
-                access_perm == AccessPermission_Maybe_Stale ||
-                access_perm == AccessPermission_Backing_Store) {
+    for (unsigned int i = 0; i < num_controllers;++i) {
+        m_abs_cntrl_vec[i]->functionalWriteBuffers(pkt);
 
-                DataBlock& block = m_abs_cntrl_vec[i]->getDataBlock(line_addr);
-                DPRINTF(RubySystem, "%s\n",block);
-                for (unsigned i = 0; i < size_in_bytes; ++i) {
-                  block.setByte(i + startByte, data[i]);
-                }
-                DPRINTF(RubySystem, "%s\n",block);
+        access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_addr);
+        if (access_perm != AccessPermission_Invalid &&
+            access_perm != AccessPermission_NotPresent) {
+
+            DataBlock& block = m_abs_cntrl_vec[i]->getDataBlock(line_addr);
+            DPRINTF(RubySystem, "%s\n",block);
+            for (unsigned i = 0; i < size_in_bytes; ++i) {
+              block.setByte(i + startByte, data[i]);
             }
+            DPRINTF(RubySystem, "%s\n",block);
         }
-        return true;
     }
-    return false;
+
+    uint32_t M5_VAR_USED num_functional_writes = 0;
+    for (unsigned int i = 0; i < m_memory_controller_vec.size() ;++i) {
+        num_functional_writes +=
+            m_memory_controller_vec[i]->functionalWriteBuffers(pkt);
+    }
+
+    num_functional_writes += m_network_ptr->functionalWrite(pkt);
+    DPRINTF(RubySystem, "Messages written = %u\n", num_functional_writes);
+
+    return true;
 }
 
 #ifdef CHECK_COHERENCE