ruby: coherence protocols: remove data block from dirctory entry

This patch removes the data block present in the directory entry structure
of each protocol in gem5's mainline.  Firstly, this is required for moving
towards common set of memory controllers for classic and ruby memory systems.
Secondly, the data block was being misused in several places.  It was being
used for having free access to the physical memory instead of calling on the
memory controller.

From now on, the directory controller will not have a direct visibility into
the physical memory.  The Memory Vector object now resides in the
Memory Controller class.  This also means that some significant changes are
being made to the functional accesses in ruby.

32 files changed, 414 insertions, 398 deletions

diff --git a/src/mem/protocol/MESI_Three_Level-L0cache.sm b/src/mem/protocol/MESI_Three_Level-L0cache.sm
index 49b6aa7a9..c5802d776 100644
--- a/src/mem/protocol/MESI_Three_Level-L0cache.sm
+++ b/src/mem/protocol/MESI_Three_Level-L0cache.sm
@@ -205,13 +205,28 @@ machine(L0Cache, "MESI Directory L0 Cache")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   void setAccessPermission(Entry cache_entry, Address addr, State state) {
diff --git a/src/mem/protocol/MESI_Three_Level-L1cache.sm b/src/mem/protocol/MESI_Three_Level-L1cache.sm
index 59249d822..024f8f6da 100644
--- a/src/mem/protocol/MESI_Three_Level-L1cache.sm
+++ b/src/mem/protocol/MESI_Three_Level-L1cache.sm
@@ -205,13 +205,28 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   void setAccessPermission(Entry cache_entry, Address addr, State state) {
diff --git a/src/mem/protocol/MESI_Two_Level-L1cache.sm b/src/mem/protocol/MESI_Two_Level-L1cache.sm
index 080b0c0bb..b449c4f2b 100644
--- a/src/mem/protocol/MESI_Two_Level-L1cache.sm
+++ b/src/mem/protocol/MESI_Two_Level-L1cache.sm
@@ -224,13 +224,28 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   void setAccessPermission(Entry cache_entry, Address addr, State state) {
diff --git a/src/mem/protocol/MESI_Two_Level-L2cache.sm b/src/mem/protocol/MESI_Two_Level-L2cache.sm
index f4809959d..ede420626 100644
--- a/src/mem/protocol/MESI_Two_Level-L2cache.sm
+++ b/src/mem/protocol/MESI_Two_Level-L2cache.sm
@@ -212,13 +212,28 @@ machine(L2Cache, "MESI Directory L2 Cache CMP")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   void setAccessPermission(Entry cache_entry, Address addr, State state) {
diff --git a/src/mem/protocol/MESI_Two_Level-dir.sm b/src/mem/protocol/MESI_Two_Level-dir.sm
index dd0ecf49e..939ae2a36 100644
--- a/src/mem/protocol/MESI_Two_Level-dir.sm
+++ b/src/mem/protocol/MESI_Two_Level-dir.sm
@@ -73,7 +73,6 @@ machine(Directory, "MESI Two Level directory protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
     MachineID Owner;
   }
 
@@ -90,6 +89,8 @@ machine(Directory, "MESI Two Level directory protocol")
     void allocate(Address);
     void deallocate(Address);
     bool isPresent(Address);
+    bool functionalRead(Packet *pkt);
+    int functionalWrite(Packet *pkt);
   }
 
 
@@ -148,13 +149,22 @@ machine(Directory, "MESI Two Level directory protocol")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      memBuffer.functionalRead(pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndWrite(addr, tbe.DataBlk, pkt);
     }
 
-    return getDirectoryEntry(addr).DataBlk;
+    return memBuffer.functionalWrite(pkt);
   }
 
   void setAccessPermission(Address addr, State state) {
@@ -297,7 +307,6 @@ machine(Directory, "MESI Two Level directory protocol")
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
         out_msg.Prefetch := in_msg.Prefetch;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
 
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
@@ -320,13 +329,6 @@ machine(Directory, "MESI Two Level directory protocol")
     }
   }
 
-  action(m_writeDataToMemory, "m", desc="Write dirty writeback to memory") {
-    peek(responseNetwork_in, ResponseMsg) {
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
 //added by SS for dma
   action(qf_queueMemoryFetchRequestDMA, "qfd", desc="Queue off-chip fetch request") {
     peek(requestNetwork_in, RequestMsg) {
@@ -336,7 +338,6 @@ machine(Directory, "MESI Two Level directory protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := machineID;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -358,25 +359,14 @@ machine(Directory, "MESI Two Level directory protocol")
     }
   }
 
-  action(dw_writeDMAData, "dw", desc="DMA Write data to memory") {
-    peek(requestNetwork_in, RequestMsg) {
-      getDirectoryEntry(address).DataBlk.copyPartial(in_msg.DataBlk, addressOffset(in_msg.Addr), in_msg.Len);
-    }
-  }
-
   action(qw_queueMemoryWBRequest_partial, "qwp", desc="Queue off-chip writeback request") {
      peek(requestNetwork_in, RequestMsg) {
       enqueue(memQueue_out, MemoryMsg, to_mem_ctrl_latency) {
         out_msg.Addr := address;
         out_msg.Type := MemoryRequestType:MEMORY_WB;
         out_msg.OriginalRequestorMachId := machineID;
-        //out_msg.DataBlk := in_msg.DataBlk;
         out_msg.DataBlk.copyPartial(in_msg.DataBlk, addressOffset(address), in_msg.Len);
-
-
         out_msg.MessageSize := in_msg.MessageSize;
-        //out_msg.Prefetch := in_msg.Prefetch;
-
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -434,15 +424,6 @@ machine(Directory, "MESI Two Level directory protocol")
     }
   }
 
-  action(dwt_writeDMADataFromTBE, "dwt", desc="DMA Write data to memory from TBE") {
-    assert(is_valid(tbe));
-    //getDirectoryEntry(address).DataBlk.copyPartial(tbe.DataBlk, tbe.Offset, tbe.Len);
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
-
-
-  }
-
-
   action(qw_queueMemoryWBRequest_partialTBE, "qwt", desc="Queue off-chip writeback request") {
     peek(responseNetwork_in, ResponseMsg) {
       enqueue(memQueue_out, MemoryMsg, to_mem_ctrl_latency) {
@@ -493,7 +474,6 @@ machine(Directory, "MESI Two Level directory protocol")
   }
 
   transition(M, Data, MI) {
-    m_writeDataToMemory;
     qw_queueMemoryWBRequest;
     k_popIncomingResponseQueue;
   }
@@ -518,7 +498,6 @@ machine(Directory, "MESI Two Level directory protocol")
   }
 
   transition(I, DMA_WRITE, ID_W) {
-    dw_writeDMAData;
     qw_queueMemoryWBRequest_partial;
     j_popIncomingRequestQueue;
   }
@@ -545,7 +524,6 @@ machine(Directory, "MESI Two Level directory protocol")
 
   transition(M_DRD, Data, M_DRDI) {
     drp_sendDMAData;
-    m_writeDataToMemory;
     qw_queueMemoryWBRequest;
     k_popIncomingResponseQueue;
   }
@@ -563,13 +541,11 @@ machine(Directory, "MESI Two Level directory protocol")
   }
 
   transition(M_DWR, Data, M_DWRI) {
-    m_writeDataToMemory;
     qw_queueMemoryWBRequest_partialTBE;
     k_popIncomingResponseQueue;
   }
 
   transition(M_DWRI, Memory_Ack, I) {
-    dwt_writeDMADataFromTBE;
     aa_sendAck;
     da_sendDMAAck;
     w_deallocateTBE;
diff --git a/src/mem/protocol/MESI_Two_Level-dma.sm b/src/mem/protocol/MESI_Two_Level-dma.sm
index 845d4df2f..3d9f2336f 100644
--- a/src/mem/protocol/MESI_Two_Level-dma.sm
+++ b/src/mem/protocol/MESI_Two_Level-dma.sm
@@ -55,8 +55,9 @@ machine(DMA, "DMA Controller")
   State getState(Address addr) {
     return cur_state;
   }
+
   void setState(Address addr, State state) {
-  cur_state := state;
+    cur_state := state;
   }
 
   AccessPermission getAccessPermission(Address addr) {
@@ -66,8 +67,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("DMA does not support get data block.");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(requestToDir_out, RequestMsg, requestToDir, desc="...");
diff --git a/src/mem/protocol/MI_example-cache.sm b/src/mem/protocol/MI_example-cache.sm
index ee774f4c2..b0217ffea 100644
--- a/src/mem/protocol/MI_example-cache.sm
+++ b/src/mem/protocol/MI_example-cache.sm
@@ -171,13 +171,28 @@ machine(L1Cache, "MI Example L1 Cache")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-      return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   // NETWORK PORTS
diff --git a/src/mem/protocol/MI_example-dir.sm b/src/mem/protocol/MI_example-dir.sm
index cd12e3eb7..60662080a 100644
--- a/src/mem/protocol/MI_example-dir.sm
+++ b/src/mem/protocol/MI_example-dir.sm
@@ -84,7 +84,6 @@ machine(Directory, "Directory protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
     NetDest Sharers,                   desc="Sharers for this block";
     NetDest Owner,                     desc="Owner of this block";
   }
@@ -151,7 +150,6 @@ machine(Directory, "Directory protocol")
       if (state == State:I)  {
         assert(getDirectoryEntry(addr).Owner.count() == 0);
         assert(getDirectoryEntry(addr).Sharers.count() == 0);
-        directory.invalidateBlock(addr);
       }
     }
   }
@@ -175,13 +173,22 @@ machine(Directory, "Directory protocol")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-      return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      memBuffer.functionalRead(pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndWrite(addr, tbe.DataBlk, pkt);
     }
 
-    return getDirectoryEntry(addr).DataBlk;
+    return memBuffer.functionalWrite(pkt);
   }
 
   // ** OUT_PORTS **
@@ -326,7 +333,10 @@ machine(Directory, "Directory protocol")
         out_msg.PhysicalAddress := address;
         out_msg.LineAddress := address;
         out_msg.Type := DMAResponseType:DATA;
-        out_msg.DataBlk := in_msg.DataBlk;   // we send the entire data block and rely on the dma controller to split it up if need be
+
+        // we send the entire data block and rely on the dma controller
+        // to split it up if need be
+        out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Destination.add(tbe.DmaRequestor);
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
@@ -386,21 +396,7 @@ machine(Directory, "Directory protocol")
   action(p_popIncomingDMARequestQueue, "p", desc="Pop incoming DMA queue") {
     dmaRequestQueue_in.dequeue();
   }
-
-  action(l_writeDataToMemory, "pl", desc="Write PUTX data to memory") {
-    peek(requestQueue_in, RequestMsg) {
-      // assert(in_msg.Dirty);
-      // assert(in_msg.MessageSize == MessageSizeType:Writeback_Data);
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      //getDirectoryEntry(in_msg.Addr).DataBlk.copyPartial(in_msg.DataBlk, addressOffset(in_msg.Addr), in_msg.Len);
-    }
-  }
   
-  action(dwt_writeDMADataFromTBE, "dwt", desc="DMA Write data to memory from TBE") {
-    assert(is_valid(tbe));
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
-  }
-
   action(v_allocateTBE, "v", desc="Allocate TBE") {
     peek(dmaRequestQueue_in, DMARequestMsg) {
       TBEs.allocate(address);
@@ -450,7 +446,6 @@ machine(Directory, "Directory protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         DPRINTF(RubySlicc,"%s\n", out_msg);
       }
     }
@@ -464,7 +459,6 @@ machine(Directory, "Directory protocol")
         out_msg.Sender := machineID;
         //out_msg.OriginalRequestorMachId := machineID;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc,"%s\n", out_msg);
       }
     }
@@ -475,12 +469,9 @@ machine(Directory, "Directory protocol")
       enqueue(memQueue_out, MemoryMsg, 1) {
         out_msg.Addr := address;
         out_msg.Type := MemoryRequestType:MEMORY_WB;
-        //out_msg.OriginalRequestorMachId := machineID;
-        //out_msg.DataBlk := in_msg.DataBlk;
-        out_msg.DataBlk.copyPartial(in_msg.DataBlk, addressOffset(in_msg.PhysicalAddress), in_msg.Len);
+        out_msg.DataBlk.copyPartial(
+            in_msg.DataBlk, addressOffset(in_msg.PhysicalAddress), in_msg.Len);
         out_msg.MessageSize := in_msg.MessageSize;
-        //out_msg.Prefetch := in_msg.Prefetch;
-
         DPRINTF(RubySlicc,"%s\n", out_msg);
       }
     }
@@ -493,19 +484,17 @@ machine(Directory, "Directory protocol")
         out_msg.Addr := address;
         out_msg.Type := MemoryRequestType:MEMORY_WB;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
+
         // get incoming data
-        // out_msg.DataBlk := in_msg.DataBlk;
-        out_msg.DataBlk.copyPartial(tbe.DataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
+        out_msg.DataBlk.copyPartial(
+            tbe.DataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
         out_msg.MessageSize := in_msg.MessageSize;
-        //out_msg.Prefetch := in_msg.Prefetch;
-
         DPRINTF(RubySlicc,"%s\n", out_msg);
       }
     }
   }
 
 
-
   action(l_queueMemoryWBRequest, "lq", desc="Write PUTX data to memory") {
     peek(requestQueue_in, RequestMsg) {
       enqueue(memQueue_out, MemoryMsg, 1) {
@@ -525,13 +514,7 @@ machine(Directory, "Directory protocol")
     memQueue_in.dequeue();
   }
 
-  action(w_writeDataToMemoryFromTBE, "\w", desc="Write date to directory memory from TBE") {
-    assert(is_valid(tbe));
-    getDirectoryEntry(address).DataBlk := TBEs[address].DataBlk;
-  }
-
   // TRANSITIONS
-
   transition({M_DRD, M_DWR, M_DWRI, M_DRDI}, GETX) {
     z_recycleRequestQueue;
   }
@@ -582,7 +565,6 @@ machine(Directory, "Directory protocol")
   }
 
   transition(ID_W, Memory_Ack, I) {
-    dwt_writeDMADataFromTBE;
     da_sendDMAAck;
     w_deallocateTBE;
     l_popMemQueue;
@@ -595,7 +577,6 @@ machine(Directory, "Directory protocol")
   }
 
   transition(M_DRD, PUTX, M_DRDI) {     
-    l_writeDataToMemory;
     drp_sendDMAData;
     c_clearOwner;
     l_queueMemoryWBRequest;
@@ -616,14 +597,12 @@ machine(Directory, "Directory protocol")
   }
 
   transition(M_DWR, PUTX, M_DWRI) {
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest_partialTBE;
     c_clearOwner;
     i_popIncomingRequestQueue;
   }
 
   transition(M_DWRI, Memory_Ack, I) {
-    w_writeDataToMemoryFromTBE;
     l_sendWriteBackAck;
     da_sendDMAAck;
     w_deallocateTBE;
@@ -644,7 +623,6 @@ machine(Directory, "Directory protocol")
   }
 
   transition(MI, Memory_Ack, I) {
-    w_writeDataToMemoryFromTBE;
     l_sendWriteBackAck;
     w_deallocateTBE;
     l_popMemQueue;
@@ -659,5 +637,4 @@ machine(Directory, "Directory protocol")
     b_sendWriteBackNack;
     i_popIncomingRequestQueue;
   }
-
 }
diff --git a/src/mem/protocol/MI_example-dma.sm b/src/mem/protocol/MI_example-dma.sm
index e328d9e20..c3cc29ba2 100644
--- a/src/mem/protocol/MI_example-dma.sm
+++ b/src/mem/protocol/MI_example-dma.sm
@@ -66,8 +66,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("DMA Controller does not support getDataBlock function.\n");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(requestToDir_out, DMARequestMsg, requestToDir, desc="...");
diff --git a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
index 3cd87616f..e9b05a0c8 100644
--- a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
@@ -212,18 +212,34 @@ machine(L1Cache, "Directory protocol")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     Entry cache_entry := getCacheEntry(addr);
     if(is_valid(cache_entry)) {
-        return cache_entry.DataBlk;
+      testAndRead(addr, cache_entry.DataBlk, pkt);
+    } else {
+      TBE tbe := TBEs[addr];
+      if(is_valid(tbe)) {
+        testAndRead(addr, tbe.DataBlk, pkt);
+      } else {
+        error("Data block missing!");
+      }
     }
+  }
 
-    TBE tbe := TBEs[addr];
-    if(is_valid(tbe)) {
-      return tbe.DataBlk;
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    Entry cache_entry := getCacheEntry(addr);
+    if(is_valid(cache_entry)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, cache_entry.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    error("Data block missing!");
+    TBE tbe := TBEs[addr];
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+    return num_functional_writes;
   }
 
   Event mandatory_request_type_to_event(RubyRequestType type) {
diff --git a/src/mem/protocol/MOESI_CMP_directory-L2cache.sm b/src/mem/protocol/MOESI_CMP_directory-L2cache.sm
index 46fd12a3a..c01b9765d 100644
--- a/src/mem/protocol/MOESI_CMP_directory-L2cache.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-L2cache.sm
@@ -520,13 +520,28 @@ machine(L2Cache, "Token protocol")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   MessageBuffer triggerQueue, ordered="true";
diff --git a/src/mem/protocol/MOESI_CMP_directory-dir.sm b/src/mem/protocol/MOESI_CMP_directory-dir.sm
index 272a8c9ab..a6b93fa54 100644
--- a/src/mem/protocol/MOESI_CMP_directory-dir.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-dir.sm
@@ -96,7 +96,6 @@ machine(Directory, "Directory protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface='AbstractEntry') {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
     NetDest Sharers,                   desc="Sharers for this block";
     NetDest Owner,                     desc="Owner of this block";
     int WaitingUnblocks,           desc="Number of acks we're waiting for";
@@ -191,8 +190,12 @@ machine(Directory, "Directory protocol")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    return getDirectoryEntry(addr).DataBlk;
+  void functionalRead(Address addr, Packet *pkt) {
+    memBuffer.functionalRead(pkt);
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    return memBuffer.functionalWrite(pkt);
   }
 
   // if no sharers, then directory can be considered
@@ -346,7 +349,6 @@ machine(Directory, "Directory protocol")
         out_msg.Sender := machineID;
         out_msg.SenderMachine := MachineType:Directory;
         out_msg.Destination.add(in_msg.OriginalRequestorMachId);
-        //out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Dirty := false; // By definition, the block is now clean
         out_msg.Acks := in_msg.Acks;
@@ -367,7 +369,6 @@ machine(Directory, "Directory protocol")
         out_msg.Sender := machineID;
         out_msg.SenderMachine := MachineType:Directory;
         out_msg.Destination.add(in_msg.Requestor);
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         out_msg.Dirty := false; // By definition, the block is now clean
         out_msg.Type := CoherenceResponseType:DATA_EXCLUSIVE;
         out_msg.MessageSize := MessageSizeType:Response_Data;
@@ -375,8 +376,6 @@ machine(Directory, "Directory protocol")
     }
   }
 
-
-
   action(e_ownerIsUnblocker, "e", desc="The owner is now the unblocker") {
     peek(unblockNetwork_in, ResponseMsg) {
       getDirectoryEntry(address).Owner.clear();
@@ -445,40 +444,6 @@ machine(Directory, "Directory protocol")
     unblockNetwork_in.dequeue();
   }
 
-  action(l_writeDataToMemory, "l", desc="Write PUTX/PUTO data to memory") {
-    peek(unblockNetwork_in, ResponseMsg) {
-      assert(in_msg.Dirty);
-      assert(in_msg.MessageSize == MessageSizeType:Writeback_Data);
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
-
-  action(p_writeFwdDataToMemory, "p", desc="Write Response data to memory") {
-     peek(unblockNetwork_in, ResponseMsg) {
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
- }
-
-  action(ll_checkDataInMemory, "\ld", desc="Check PUTX/PUTO data is same as in the memory") {
-    peek(unblockNetwork_in, ResponseMsg) {
-      assert(in_msg.Dirty == false);
-      assert(in_msg.MessageSize == MessageSizeType:Writeback_Control);
-
-      // NOTE: The following check would not be valid in a real
-      // implementation.  We include the data in the "dataless"
-      // message so we can assert the clean data matches the datablock
-      // in memory
-      DPRINTF(RubySlicc, "Address: %s, MsgDataBlock: %s MemoryDataBlock: %s\n",
-              in_msg.Addr, in_msg.DataBlk,
-              getDirectoryEntry(in_msg.Addr).DataBlk);
-      assert(getDirectoryEntry(in_msg.Addr).DataBlk == in_msg.DataBlk);
-    }
-  }
-
   action(m_addUnlockerToSharers, "m", desc="Add the unlocker to the sharer list") {
     peek(unblockNetwork_in, ResponseMsg) {
       getDirectoryEntry(address).Sharers.add(in_msg.Sender);
@@ -505,7 +470,6 @@ machine(Directory, "Directory protocol")
         out_msg.Type := MemoryRequestType:MEMORY_READ;
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         out_msg.MessageSize := in_msg.MessageSize;
         //out_msg.Prefetch := false;
         // These are not used by memory but are passed back here with the read data:
@@ -540,6 +504,29 @@ machine(Directory, "Directory protocol")
     }
   }
 
+  action(qw_queueMemoryWBRequestFromMessageAndTBE, "qwmt",
+    desc="Queue off-chip writeback request") {
+    peek(unblockNetwork_in, ResponseMsg) {
+      enqueue(memQueue_out, MemoryMsg, 1) {
+        out_msg.Addr := address;
+        out_msg.Type := MemoryRequestType:MEMORY_WB;
+        out_msg.Sender := machineID;
+        if (is_valid(tbe)) {
+          out_msg.OriginalRequestorMachId := tbe.Requestor;
+        }
+        out_msg.DataBlk := in_msg.DataBlk;
+        out_msg.DataBlk.copyPartial(tbe.DataBlk,
+            addressOffset(tbe.PhysicalAddress), tbe.Len);
+
+        out_msg.MessageSize := in_msg.MessageSize;
+        // Not used:
+        out_msg.ReadX := false;
+        out_msg.Acks := getDirectoryEntry(address).Sharers.count();  // for dma requests
+        DPRINTF(RubySlicc, "%s\n", out_msg);
+      }
+    }
+  }
+
   action(qw_queueMemoryWBRequest2, "/qw", desc="Queue off-chip writeback request") {
     peek(requestQueue_in, RequestMsg) {
       enqueue(memQueue_out, MemoryMsg, 1) {
@@ -594,18 +581,6 @@ machine(Directory, "Directory protocol")
     }
   }
 
-  action(l_writeDMADataToMemory, "\l", desc="Write data from a DMA_WRITE to memory") {
-    peek(requestQueue_in, RequestMsg) {
-      getDirectoryEntry(address).DataBlk.copyPartial(in_msg.DataBlk, addressOffset(in_msg.Addr), in_msg.Len);
-    }
-  }
-
-  action(l_writeDMADataToMemoryFromTBE, "\ll", desc="Write data from a DMA_WRITE to memory") {
-    assert(is_valid(tbe));
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DataBlk,
-                      addressOffset(tbe.PhysicalAddress), tbe.Len);
-  }
-
   action(v_allocateTBE, "v", desc="Allocate TBE entry") {
     peek (requestQueue_in, RequestMsg) {
       TBEs.allocate(address);
@@ -623,9 +598,7 @@ machine(Directory, "Directory protocol")
   }
 
 
-
   // TRANSITIONS
-
   transition(I, GETX, MM) {
     qf_queueMemoryFetchRequest;
     i_popIncomingRequestQueue;
@@ -639,7 +612,6 @@ machine(Directory, "Directory protocol")
   transition(I, DMA_WRITE, XI_U) {
     qw_queueMemoryWBRequest2;
     a_sendDMAAck;  // ack count may be zero
-    l_writeDMADataToMemory;
     i_popIncomingRequestQueue;
   }
 
@@ -670,7 +642,6 @@ machine(Directory, "Directory protocol")
   transition(S, DMA_WRITE, XI_U) {
     qw_queueMemoryWBRequest2;
     a_sendDMAAck;  // ack count may be zero
-    l_writeDMADataToMemory;
     g_sendInvalidations;  // the DMA will collect invalidations
     i_popIncomingRequestQueue;
   }
@@ -720,10 +691,8 @@ machine(Directory, "Directory protocol")
   }
 
   transition(OI_D, Data, XI_U) {
-    qw_queueMemoryWBRequest;
+    qw_queueMemoryWBRequestFromMessageAndTBE;
     a_sendDMAAck2;  // ack count may be zero
-    p_writeFwdDataToMemory;
-    l_writeDMADataToMemoryFromTBE;
     w_deallocateTBE;
     j_popIncomingUnblockQueue;
   }
@@ -842,14 +811,12 @@ machine(Directory, "Directory protocol")
   transition(MI, Dirty_Writeback, I) {
     c_clearOwner;
     cc_clearSharers;
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest;
     j_popIncomingUnblockQueue;
   }
 
   transition(MIS, Dirty_Writeback, S) {
     c_moveOwnerToSharer;
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest;
     j_popIncomingUnblockQueue;
   }
@@ -861,14 +828,12 @@ machine(Directory, "Directory protocol")
 
   transition(OS, Dirty_Writeback, S) {
     c_clearOwner;
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest;
     j_popIncomingUnblockQueue;
   }
 
   transition(OSS, Dirty_Writeback, S) {
     c_moveOwnerToSharer;
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest;
     j_popIncomingUnblockQueue;
   }
@@ -881,13 +846,11 @@ machine(Directory, "Directory protocol")
   transition(MI, Clean_Writeback, I) {
     c_clearOwner;
     cc_clearSharers;
-    ll_checkDataInMemory;
     j_popIncomingUnblockQueue;
   }
 
   transition(OS, Clean_Writeback, S) {
     c_clearOwner;
-    ll_checkDataInMemory;
     j_popIncomingUnblockQueue;
   }
 
diff --git a/src/mem/protocol/MOESI_CMP_directory-dma.sm b/src/mem/protocol/MOESI_CMP_directory-dma.sm
index 9bdd791e1..d7e3a02d9 100644
--- a/src/mem/protocol/MOESI_CMP_directory-dma.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-dma.sm
@@ -91,8 +91,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-     error("DMA Controller does not support getDataBlock().\n");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(reqToDirectory_out, RequestMsg, reqToDir, desc="...");
diff --git a/src/mem/protocol/MOESI_CMP_token-L1cache.sm b/src/mem/protocol/MOESI_CMP_token-L1cache.sm
index 860744384..ebfa970ff 100644
--- a/src/mem/protocol/MOESI_CMP_token-L1cache.sm
+++ b/src/mem/protocol/MOESI_CMP_token-L1cache.sm
@@ -240,8 +240,15 @@ machine(L1Cache, "Token protocol")
     return L1Icache_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    return getCacheEntry(addr).DataBlk;
+  void functionalRead(Address addr, Packet *pkt) {
+    testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   Entry getL1DCacheEntry(Address addr), return_by_pointer="yes" {
diff --git a/src/mem/protocol/MOESI_CMP_token-L2cache.sm b/src/mem/protocol/MOESI_CMP_token-L2cache.sm
index a2488066a..6542ede49 100644
--- a/src/mem/protocol/MOESI_CMP_token-L2cache.sm
+++ b/src/mem/protocol/MOESI_CMP_token-L2cache.sm
@@ -157,8 +157,15 @@ machine(L2Cache, "Token protocol")
     return cache_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    return getCacheEntry(addr).DataBlk;
+  void functionalRead(Address addr, Packet *pkt) {
+    testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   int getTokens(Entry cache_entry) {
diff --git a/src/mem/protocol/MOESI_CMP_token-dir.sm b/src/mem/protocol/MOESI_CMP_token-dir.sm
index be5df02e0..8d6abd93c 100644
--- a/src/mem/protocol/MOESI_CMP_token-dir.sm
+++ b/src/mem/protocol/MOESI_CMP_token-dir.sm
@@ -121,7 +121,6 @@ machine(Directory, "Token protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
     int Tokens, default="max_tokens()", desc="Number of tokens for the line we're holding";
 
     // The following state is provided to allow for bandwidth
@@ -188,10 +187,6 @@ machine(Directory, "Token protocol")
     return dir_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    return getDirectoryEntry(addr).DataBlk;
-  }
-
   State getState(TBE tbe, Address addr) {
     if (is_valid(tbe)) {
       return tbe.TBEState;
@@ -250,6 +245,24 @@ machine(Directory, "Token protocol")
     persistentTable.markEntries(addr);
   }
 
+  void functionalRead(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      memBuffer.functionalRead(pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndWrite(addr, tbe.DataBlk, pkt);
+    }
+
+    return memBuffer.functionalWrite(pkt);
+  }
+
   // ** OUT_PORTS **
   out_port(responseNetwork_out, ResponseMsg, responseFromDir);
   out_port(persistentNetwork_out, PersistentMsg, persistentFromDir);
@@ -598,7 +611,7 @@ machine(Directory, "Token protocol")
         out_msg.Destination.add(in_msg.OriginalRequestorMachId);
         assert(getDirectoryEntry(address).Tokens > 0);
         out_msg.Tokens := getDirectoryEntry(in_msg.Addr).Tokens;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
+        out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Dirty := false;
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
@@ -615,7 +628,7 @@ machine(Directory, "Token protocol")
         out_msg.Destination.add(persistentTable.findSmallest(address));
         assert(getDirectoryEntry(address).Tokens > 0);
         out_msg.Tokens := getDirectoryEntry(address).Tokens;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
+        out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Dirty := false;
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
@@ -646,7 +659,6 @@ machine(Directory, "Token protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -659,7 +671,6 @@ machine(Directory, "Token protocol")
       out_msg.Sender := machineID;
       out_msg.OriginalRequestorMachId := persistentTable.findSmallest(address);
       out_msg.MessageSize := MessageSizeType:Request_Control;
-      out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
       DPRINTF(RubySlicc, "%s\n", out_msg);
     }
   }
@@ -672,18 +683,20 @@ machine(Directory, "Token protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
   }
 
   action(lq_queueMemoryWbRequest, "lq", desc="Write data to memory") {
-    enqueue(memQueue_out, MemoryMsg, 1) {
-      out_msg.Addr := address;
-      out_msg.Type := MemoryRequestType:MEMORY_WB;
-      out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
-      DPRINTF(RubySlicc, "%s\n", out_msg);
+    peek(responseNetwork_in, ResponseMsg) {
+      enqueue(memQueue_out, MemoryMsg, 1) {
+        out_msg.Addr := address;
+        out_msg.Type := MemoryRequestType:MEMORY_WB;
+        out_msg.MessageSize := in_msg.MessageSize;
+        out_msg.DataBlk := in_msg.DataBlk;
+        DPRINTF(RubySlicc, "%s\n", out_msg);
+      }
     }
   }
 
@@ -694,7 +707,8 @@ machine(Directory, "Token protocol")
       // first, initialize the data blk to the current version of system memory
       out_msg.DataBlk := tbe.DataBlk;
       // then add the dma write data
-      out_msg.DataBlk.copyPartial(tbe.DmaDataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
+      out_msg.DataBlk.copyPartial(
+        tbe.DmaDataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
       DPRINTF(RubySlicc, "%s\n", out_msg);
     }
   }
@@ -759,15 +773,6 @@ machine(Directory, "Token protocol")
     }
   }
 
-  action(cd_writeCleanDataToTbe, "cd", desc="Write clean memory data to TBE") {
-    tbe.DataBlk := getDirectoryEntry(address).DataBlk;
-  }
-
-  action(dwt_writeDmaDataFromTBE, "dwt", desc="DMA Write data to memory from TBE") {
-    getDirectoryEntry(address).DataBlk := tbe.DataBlk;
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DmaDataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
-  }
-
   action(f_incrementTokens, "f", desc="Increment the number of tokens we're tracking") {
     peek(responseNetwork_in, ResponseMsg) {
       assert(in_msg.Tokens >= 1);
@@ -811,20 +816,6 @@ machine(Directory, "Token protocol")
     memQueue_in.dequeue();
   }
 
-  action(m_writeDataToMemory, "m", desc="Write dirty writeback to memory") {
-    peek(responseNetwork_in, ResponseMsg) {
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
-
-  action(n_checkData, "n", desc="Check incoming clean data message") {
-    peek(responseNetwork_in, ResponseMsg) {
-      assert(getDirectoryEntry(in_msg.Addr).DataBlk == in_msg.DataBlk);
-    }
-  }
-
   action(r_bounceResponse, "r", desc="Bounce response to starving processor") {
     peek(responseNetwork_in, ResponseMsg) {
       enqueue(responseNetwork_out, ResponseMsg, 1) {
@@ -869,12 +860,6 @@ machine(Directory, "Token protocol")
       assert(in_msg.Dirty == false);
       assert(in_msg.MessageSize == MessageSizeType:Writeback_Control);
 
-      // NOTE: The following check would not be valid in a real
-      // implementation.  We include the data in the "dataless"
-      // message so we can assert the clean data matches the datablock
-      // in memory
-      assert(getDirectoryEntry(in_msg.Addr).DataBlk == in_msg.DataBlk);
-
       // Bounce the message, but "re-associate" the data and the owner
       // token.  In essence we're converting an ACK_OWNER message to a
       // DATA_OWNER message, keeping the number of tokens the same.
@@ -884,7 +869,6 @@ machine(Directory, "Token protocol")
         out_msg.Sender := machineID;
         out_msg.Destination.add(persistentTable.findSmallest(address));
         out_msg.Tokens := in_msg.Tokens;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         out_msg.Dirty := in_msg.Dirty;
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
@@ -948,7 +932,6 @@ machine(Directory, "Token protocol")
 
   transition(O, DMA_WRITE, O_DW) {
     vd_allocateDmaRequestInTBE;
-    cd_writeCleanDataToTbe;
     bw_broadcastWrite;
     st_scheduleTimeout;
     p_popDmaRequestQueue;
@@ -956,8 +939,6 @@ machine(Directory, "Token protocol")
 
   transition(O, DMA_WRITE_All_Tokens, O_DW_W) {
     vd_allocateDmaRequestInTBE;
-    cd_writeCleanDataToTbe;
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWriteFromTbe;
     p_popDmaRequestQueue;
   }
@@ -985,7 +966,6 @@ machine(Directory, "Token protocol")
   }
 
   transition(O, {Data_Owner, Data_All_Tokens}) {
-    n_checkData;
     f_incrementTokens;
     k_popIncomingResponseQueue;
   }
@@ -1026,7 +1006,6 @@ machine(Directory, "Token protocol")
 
   transition(O_DW, Ack_Owner) {
     f_incrementTokens;
-    cd_writeCleanDataToTbe;
     k_popIncomingResponseQueue;
   }
 
@@ -1038,7 +1017,6 @@ machine(Directory, "Token protocol")
   transition({NO_DW, O_DW}, Data_All_Tokens, O_DW_W) {
     f_incrementTokens;
     rd_recordDataInTbe;
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWriteFromTbe;
     ut_unsetReissueTimer;
     k_popIncomingResponseQueue;
@@ -1046,7 +1024,6 @@ machine(Directory, "Token protocol")
 
   transition(O_DW, Ack_All_Tokens, O_DW_W) {
     f_incrementTokens;
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWriteFromTbe;
     ut_unsetReissueTimer;
     k_popIncomingResponseQueue;
@@ -1054,8 +1031,6 @@ machine(Directory, "Token protocol")
 
   transition(O_DW, Ack_Owner_All_Tokens, O_DW_W) {
     f_incrementTokens;
-    cd_writeCleanDataToTbe;
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWriteFromTbe;
     ut_unsetReissueTimer;
     k_popIncomingResponseQueue;
@@ -1100,14 +1075,12 @@ machine(Directory, "Token protocol")
   }
 
   transition(NO, {Data_Owner, Data_All_Tokens}, O_W) {
-    m_writeDataToMemory;
     f_incrementTokens;
     lq_queueMemoryWbRequest;
     k_popIncomingResponseQueue;
   }
 
   transition(NO, {Ack_Owner, Ack_Owner_All_Tokens}, O) {
-    n_checkData;
     f_incrementTokens;
     k_popIncomingResponseQueue;
   }
@@ -1160,7 +1133,6 @@ machine(Directory, "Token protocol")
   }
 
   transition(NO_DR, {Data_Owner, Data_All_Tokens}, O_W) {
-    m_writeDataToMemory;
     f_incrementTokens;
     dd_sendDmaData;
     lr_queueMemoryDmaReadWriteback;
@@ -1195,11 +1167,17 @@ machine(Directory, "Token protocol")
     k_popIncomingResponseQueue;
   }
 
-  transition({DW_L, DR_L, L}, {Ack_Owner_All_Tokens, Ack_Owner}) {
+  transition({DW_L, DR_L}, {Ack_Owner_All_Tokens, Ack_Owner}) {
     bd_bounceDatalessOwnerToken;
     k_popIncomingResponseQueue;
   }
 
+  transition(L, {Ack_Owner_All_Tokens, Ack_Owner}, L_O_W) {
+    f_incrementTokens;
+    qp_queueMemoryForPersistent;
+    k_popIncomingResponseQueue;
+  }
+
   transition(L, {Unlockdown, Own_Lock_or_Unlock}, NO) {
     l_popIncomingPersistentQueue;
   }
diff --git a/src/mem/protocol/MOESI_CMP_token-dma.sm b/src/mem/protocol/MOESI_CMP_token-dma.sm
index 1c28971a1..f11e471b4 100644
--- a/src/mem/protocol/MOESI_CMP_token-dma.sm
+++ b/src/mem/protocol/MOESI_CMP_token-dma.sm
@@ -68,8 +68,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("DMA Controller does not support getDataBlock function.\n");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(reqToDirectory_out, DMARequestMsg, reqToDirectory, desc="...");
diff --git a/src/mem/protocol/MOESI_hammer-cache.sm b/src/mem/protocol/MOESI_hammer-cache.sm
index de502e118..badbe1d8b 100644
--- a/src/mem/protocol/MOESI_hammer-cache.sm
+++ b/src/mem/protocol/MOESI_hammer-cache.sm
@@ -205,18 +205,34 @@ machine({L1Cache, L2Cache}, "AMD Hammer-like protocol")
     return L1Icache_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     Entry cache_entry := getCacheEntry(addr);
     if(is_valid(cache_entry)) {
-        return cache_entry.DataBlk;
+      testAndRead(addr, cache_entry.DataBlk, pkt);
+    } else {
+      TBE tbe := TBEs[addr];
+      if(is_valid(tbe)) {
+        testAndRead(addr, tbe.DataBlk, pkt);
+      } else {
+        error("Missing data block");
+      }
     }
+  }
 
-    TBE tbe := TBEs[addr];
-    if(is_valid(tbe)) {
-      return tbe.DataBlk;
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    Entry cache_entry := getCacheEntry(addr);
+    if(is_valid(cache_entry)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, cache_entry.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    error("Missing data block");
+    TBE tbe := TBEs[addr];
+    num_functional_writes := num_functional_writes +
+      testAndWrite(addr, tbe.DataBlk, pkt);
+    return num_functional_writes;
   }
 
   Entry getL2CacheEntry(Address address), return_by_pointer="yes" {
diff --git a/src/mem/protocol/MOESI_hammer-dir.sm b/src/mem/protocol/MOESI_hammer-dir.sm
index db11b290f..43d48c6d2 100644
--- a/src/mem/protocol/MOESI_hammer-dir.sm
+++ b/src/mem/protocol/MOESI_hammer-dir.sm
@@ -147,14 +147,12 @@ machine(Directory, "AMD Hammer-like protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
   }
 
   // ProbeFilterEntry
   structure(PfEntry, desc="...", interface="AbstractCacheEntry") {
     State PfState,                 desc="Directory state";
     MachineID Owner,               desc="Owner node";
-    DataBlock DataBlk,             desc="data for the block";
     Set Sharers,                   desc="sharing vector for full bit directory";
   }
 
@@ -208,20 +206,6 @@ machine(Directory, "AMD Hammer-like protocol")
     return dir_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    Entry dir_entry := getDirectoryEntry(addr);
-    if(is_valid(dir_entry)) {
-        return dir_entry.DataBlk;
-    }
-
-    TBE tbe := TBEs[addr];
-    if(is_valid(tbe)) {
-      return tbe.DataBlk;
-    }
-
-    error("Data block missing!");
-  }
-
   PfEntry getProbeFilterEntry(Address addr), return_by_pointer="yes" {
     if (probe_filter_enabled || full_bit_dir_enabled) {
       PfEntry pfEntry := static_cast(PfEntry, "pointer", probeFilter.lookup(addr));
@@ -282,6 +266,24 @@ machine(Directory, "AMD Hammer-like protocol")
     getDirectoryEntry(addr).changePermission(Directory_State_to_permission(state));
   }
 
+  void functionalRead(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      memBuffer.functionalRead(pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndWrite(addr, tbe.DataBlk, pkt);
+    }
+
+    return memBuffer.functionalWrite(pkt);
+  }
+
   Event cache_request_to_event(CoherenceRequestType type) {
     if (type == CoherenceRequestType:GETS) {
       return Event:GETS;
@@ -851,7 +853,6 @@ machine(Directory, "AMD Hammer-like protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -865,7 +866,6 @@ machine(Directory, "AMD Hammer-like protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -1179,38 +1179,6 @@ machine(Directory, "AMD Hammer-like protocol")
     }
   }
 
-  action(wr_writeResponseDataToMemory, "wr", desc="Write response data to memory") {
-    peek(responseToDir_in, ResponseMsg) {
-      getDirectoryEntry(address).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
-
-  action(l_writeDataToMemory, "l", desc="Write PUTX/PUTO data to memory") {
-    peek(memQueue_in, MemoryMsg) {
-      getDirectoryEntry(address).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
-
-  action(dwt_writeDmaDataFromTBE, "dwt", desc="DMA Write data to memory from TBE") {
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-    assert(is_valid(tbe));
-    getDirectoryEntry(address).DataBlk := tbe.DataBlk;
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DmaDataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-  }
-
-  action(wdt_writeDataFromTBE, "wdt", desc="DMA Write data to memory from TBE") {
-    assert(is_valid(tbe));
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-    getDirectoryEntry(address).DataBlk := tbe.DataBlk;
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-  }
-
   action(a_assertCacheData, "ac", desc="Assert that a cache provided the data") {
     assert(is_valid(tbe));
     assert(tbe.CacheDirty);
@@ -1277,18 +1245,21 @@ machine(Directory, "AMD Hammer-like protocol")
     }
   }
 
+  action(ly_queueMemoryWriteFromTBE, "ly", desc="Write data to memory from TBE") {
+    enqueue(memQueue_out, MemoryMsg, 1) {
+      assert(is_valid(tbe));
+      out_msg.Addr := address;
+      out_msg.Type := MemoryRequestType:MEMORY_WB;
+      out_msg.DataBlk := tbe.DataBlk;
+      DPRINTF(RubySlicc, "%s\n", out_msg);
+    }
+  }
+
   action(ll_checkIncomingWriteback, "\l", desc="Check PUTX/PUTO response message") {
     peek(unblockNetwork_in, ResponseMsg) {
       assert(in_msg.Dirty == false);
       assert(in_msg.MessageSize == MessageSizeType:Writeback_Control);
       DPRINTF(RubySlicc, "%s\n", in_msg.DataBlk);
-      DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-
-      // NOTE: The following check would not be valid in a real
-      // implementation.  We include the data in the "dataless"
-      // message so we can assert the clean data matches the datablock
-      // in memory
-      assert(getDirectoryEntry(address).DataBlk == in_msg.DataBlk);
     }
   }
 
@@ -1651,20 +1622,26 @@ machine(Directory, "AMD Hammer-like protocol")
   }
 
   transition(S_R, Data) {
-    wr_writeResponseDataToMemory;
     m_decrementNumberOfMessages;
     o_checkForCompletion;
     n_popResponseQueue;
   }
 
   transition(NO_R, {Data, Exclusive_Data}) {
-    wr_writeResponseDataToMemory;
+    r_recordCacheData;
     m_decrementNumberOfMessages;
     o_checkForCompletion;
     n_popResponseQueue;
   }
 
-  transition({O_R, S_R, NO_R}, All_acks_and_data_no_sharers, E) {
+  transition({O_R, S_R}, All_acks_and_data_no_sharers, E) {
+    w_deallocateTBE;
+    k_wakeUpDependents;
+    g_popTriggerQueue;
+  }
+
+  transition(NO_R, All_acks_and_data_no_sharers, WB_E_W) {
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
@@ -1730,14 +1707,14 @@ machine(Directory, "AMD Hammer-like protocol")
     n_popResponseQueue;
   }
 
-  transition(NO_DR_B, All_acks_and_owner_data, O) {
+  transition(NO_DR_B, All_acks_and_owner_data, WB_O_W) {
     //
     // Note that the DMA consistency model allows us to send the DMA device
     // a response as soon as we receive valid data and prior to receiving
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
@@ -1750,20 +1727,19 @@ machine(Directory, "AMD Hammer-like protocol")
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(NO_DR_B_D, All_acks_and_owner_data, O) {
+  transition(NO_DR_B_D, All_acks_and_owner_data, WB_O_W) {
     //
     // Note that the DMA consistency model allows us to send the DMA device
     // a response as soon as we receive valid data and prior to receiving
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
@@ -1776,42 +1752,41 @@ machine(Directory, "AMD Hammer-like protocol")
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(O_DR_B, All_acks_and_owner_data, O) {
-    wdt_writeDataFromTBE;
+  transition(O_DR_B, All_acks_and_owner_data, WB_O_W) {
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(O_DR_B, All_acks_and_data_no_sharers, E) {
-    wdt_writeDataFromTBE;
+  transition(O_DR_B, All_acks_and_data_no_sharers, WB_E_W) {
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     pfd_probeFilterDeallocate;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(NO_DR_B, All_acks_and_data_no_sharers, E) {
+  transition(NO_DR_B, All_acks_and_data_no_sharers, WB_E_W) {
     //
     // Note that the DMA consistency model allows us to send the DMA device
     // a response as soon as we receive valid data and prior to receiving
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     ppfd_possibleProbeFilterDeallocate;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(NO_DR_B_D, All_acks_and_data_no_sharers, E) {
+  transition(NO_DR_B_D, All_acks_and_data_no_sharers, WB_E_W) {
     a_assertCacheData;
     //
     // Note that the DMA consistency model allows us to send the DMA device
@@ -1819,7 +1794,7 @@ machine(Directory, "AMD Hammer-like protocol")
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     ppfd_possibleProbeFilterDeallocate;
     k_wakeUpDependents;
@@ -1827,7 +1802,6 @@ machine(Directory, "AMD Hammer-like protocol")
   }
 
   transition(NO_DW_B_W, All_acks_and_data_no_sharers, NO_DW_W) {
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWrite;
     g_popTriggerQueue;
   }
@@ -1883,18 +1857,16 @@ machine(Directory, "AMD Hammer-like protocol")
   transition(WB, Writeback_Exclusive_Dirty, WB_E_W) {
     rs_removeSharer;
     l_queueMemoryWBRequest;
+    pfd_probeFilterDeallocate;
     j_popIncomingUnblockQueue;
   }
 
   transition(WB_E_W, Memory_Ack, E) {
-    l_writeDataToMemory;
-    pfd_probeFilterDeallocate;
     k_wakeUpDependents;
     l_popMemQueue;
   }
 
   transition(WB_O_W, Memory_Ack, O) {
-    l_writeDataToMemory;
     k_wakeUpDependents;
     l_popMemQueue;
   }
diff --git a/src/mem/protocol/MOESI_hammer-dma.sm b/src/mem/protocol/MOESI_hammer-dma.sm
index ab41adb4d..067ded0ca 100644
--- a/src/mem/protocol/MOESI_hammer-dma.sm
+++ b/src/mem/protocol/MOESI_hammer-dma.sm
@@ -66,8 +66,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("DMA Controller does not support getDataBlock function.\n");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(requestToDir_out, DMARequestMsg, requestToDir, desc="...");
diff --git a/src/mem/protocol/Network_test-cache.sm b/src/mem/protocol/Network_test-cache.sm
index e0307152d..6d81131f2 100644
--- a/src/mem/protocol/Network_test-cache.sm
+++ b/src/mem/protocol/Network_test-cache.sm
@@ -114,8 +114,12 @@ machine(L1Cache, "Network_test L1 Cache")
     return OOD;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("Network Test does not support get data block.");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("Network test does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("Network test does not support functional write.");
   }
 
   // NETWORK PORTS
diff --git a/src/mem/protocol/Network_test-dir.sm b/src/mem/protocol/Network_test-dir.sm
index 4d6472c54..81feffde0 100644
--- a/src/mem/protocol/Network_test-dir.sm
+++ b/src/mem/protocol/Network_test-dir.sm
@@ -76,8 +76,12 @@ machine(Directory, "Network_test Directory")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("Network Test does not support get data block.");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("Network test does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("Network test does not support functional write.");
   }
 
   // ** IN_PORTS **
diff --git a/src/mem/protocol/RubySlicc_Types.sm b/src/mem/protocol/RubySlicc_Types.sm
index 789595dbe..2d0658e68 100644
--- a/src/mem/protocol/RubySlicc_Types.sm
+++ b/src/mem/protocol/RubySlicc_Types.sm
@@ -163,6 +163,8 @@ structure (WireBuffer, inport="yes", outport="yes", external = "yes") {
 
 structure (MemoryControl, inport="yes", outport="yes", external = "yes") {
   void recordRequestType(CacheRequestType);
+  void functionalRead(Packet *pkt);
+  int functionalWrite(Packet *pkt);
 }
 
 structure (DMASequencer, external = "yes") {
diff --git a/src/mem/ruby/slicc_interface/AbstractCacheEntry.hh b/src/mem/ruby/slicc_interface/AbstractCacheEntry.hh
index ff5ff25ab..2ba128493 100644
--- a/src/mem/ruby/slicc_interface/AbstractCacheEntry.hh
+++ b/src/mem/ruby/slicc_interface/AbstractCacheEntry.hh
@@ -35,6 +35,7 @@
 
 #include <iostream>
 
+#include "base/misc.hh"
 #include "mem/protocol/AccessPermission.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/slicc_interface/AbstractEntry.hh"
@@ -50,6 +51,12 @@ class AbstractCacheEntry : public AbstractEntry
     // Get/Set permission of the entry
     void changePermission(AccessPermission new_perm);
 
+    // The methods below are those called by ruby runtime, add when it
+    // is absolutely necessary and should all be virtual function.
+    virtual DataBlock& getDataBlk()
+    { panic("getDataBlk() not implemented!"); }
+
+
     Address m_Address; // Address of this block, required by CacheMemory
     int m_locked; // Holds info whether the address is locked,
                   // required for implementing LL/SC
diff --git a/src/mem/ruby/slicc_interface/AbstractController.hh b/src/mem/ruby/slicc_interface/AbstractController.hh
index 42d158653..f30967e48 100644
--- a/src/mem/ruby/slicc_interface/AbstractController.hh
+++ b/src/mem/ruby/slicc_interface/AbstractController.hh
@@ -67,7 +67,6 @@ class AbstractController : public ClockedObject, public Consumer
 
     virtual MessageBuffer* getMandatoryQueue() const = 0;
     virtual AccessPermission getAccessPermission(const Address& addr) = 0;
-    virtual DataBlock& getDataBlock(const Address& addr) = 0;
 
     virtual void print(std::ostream & out) const = 0;
     virtual void wakeup() = 0;
@@ -82,9 +81,11 @@ class AbstractController : public ClockedObject, public Consumer
     //! The boolean return value indicates if the read was performed
     //! successfully.
     virtual bool functionalReadBuffers(PacketPtr&) = 0;
+    virtual void functionalRead(const Address &addr, PacketPtr) = 0;
     //! The return value indicates the number of messages written with the
     //! data from the packet.
     virtual uint32_t functionalWriteBuffers(PacketPtr&) = 0;
+    virtual int functionalWrite(const Address &addr, PacketPtr) = 0;
 
     //! Function for enqueuing a prefetch request
     virtual void enqueuePrefetch(const Address&, const RubyRequestType&)
diff --git a/src/mem/ruby/slicc_interface/AbstractEntry.hh b/src/mem/ruby/slicc_interface/AbstractEntry.hh
index b10306281..2cf1c4b5b 100644
--- a/src/mem/ruby/slicc_interface/AbstractEntry.hh
+++ b/src/mem/ruby/slicc_interface/AbstractEntry.hh
@@ -33,8 +33,6 @@
 
 #include "mem/protocol/AccessPermission.hh"
 
-class DataBlock;
-
 class AbstractEntry
 {
   public:
@@ -45,10 +43,6 @@ class AbstractEntry
     AccessPermission getPermission() const;
     void changePermission(AccessPermission new_perm);
 
-    // The methods below are those called by ruby runtime, add when it
-    // is absolutely necessary and should all be virtual function.
-    virtual DataBlock& getDataBlk() = 0;
-
     virtual void print(std::ostream& out) const = 0;
 
     AccessPermission m_Permission; // Access permission for this
diff --git a/src/mem/ruby/structures/DirectoryMemory.cc b/src/mem/ruby/structures/DirectoryMemory.cc
index 94775aa78..04849e31f 100644
--- a/src/mem/ruby/structures/DirectoryMemory.cc
+++ b/src/mem/ruby/structures/DirectoryMemory.cc
@@ -57,7 +57,6 @@ DirectoryMemory::init()
     m_entries = new AbstractEntry*[m_num_entries];
     for (int i = 0; i < m_num_entries; i++)
         m_entries[i] = NULL;
-    m_ram = g_system_ptr->getMemoryVector();
 
     m_num_directories++;
     m_num_directories_bits = ceilLog2(m_num_directories);
@@ -132,7 +131,6 @@ DirectoryMemory::allocate(const PhysAddress& address, AbstractEntry* entry)
 
     idx = mapAddressToLocalIdx(address);
     assert(idx < m_num_entries);
-    entry->getDataBlk().assign(m_ram->getBlockPtr(address));
     entry->changePermission(AccessPermission_Read_Only);
     m_entries[idx] = entry;
 
diff --git a/src/mem/ruby/structures/DirectoryMemory.hh b/src/mem/ruby/structures/DirectoryMemory.hh
index 523f16531..b75e6ab72 100644
--- a/src/mem/ruby/structures/DirectoryMemory.hh
+++ b/src/mem/ruby/structures/DirectoryMemory.hh
@@ -35,7 +35,6 @@
 #include "mem/protocol/DirectoryRequestType.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/slicc_interface/AbstractEntry.hh"
-#include "mem/ruby/structures/MemoryVector.hh"
 #include "params/RubyDirectoryMemory.hh"
 #include "sim/sim_object.hh"
 
@@ -80,8 +79,6 @@ class DirectoryMemory : public SimObject
     static int m_num_directories_bits;
     static uint64_t m_total_size_bytes;
     static int m_numa_high_bit;
-
-    MemoryVector* m_ram;
 };
 
 inline std::ostream&
diff --git a/src/mem/ruby/structures/MemoryControl.hh b/src/mem/ruby/structures/MemoryControl.hh
index 52064adf1..b1cbe982f 100644
--- a/src/mem/ruby/structures/MemoryControl.hh
+++ b/src/mem/ruby/structures/MemoryControl.hh
@@ -89,9 +89,9 @@ class MemoryControl : public ClockedObject, public Consumer
 
     virtual void recordRequestType(MemoryControlRequestType requestType);
 
-    virtual bool functionalReadBuffers(Packet *pkt)
+    virtual bool functionalRead(Packet *pkt)
     { fatal("Functional read access not implemented!");}
-    virtual uint32_t functionalWriteBuffers(Packet *pkt)
+    virtual uint32_t functionalWrite(Packet *pkt)
     { fatal("Functional read access not implemented!");}
 
 protected:
diff --git a/src/mem/ruby/structures/RubyMemoryControl.cc b/src/mem/ruby/structures/RubyMemoryControl.cc
index 69fd45fe4..2e71c0c2f 100644
--- a/src/mem/ruby/structures/RubyMemoryControl.cc
+++ b/src/mem/ruby/structures/RubyMemoryControl.cc
@@ -173,6 +173,7 @@ RubyMemoryControl::RubyMemoryControl(const Params *p)
 void
 RubyMemoryControl::init()
 {
+    m_ram = g_system_ptr->getMemoryVector();
     m_msg_counter = 0;
 
     assert(m_tFaw <= 62); // must fit in a uint64 shift register
@@ -282,6 +283,19 @@ RubyMemoryControl::enqueue(const MsgPtr& message, Cycles latency)
     physical_address_t addr = memMess->getAddr().getAddress();
     MemoryRequestType type = memMess->getType();
     bool is_mem_read = (type == MemoryRequestType_MEMORY_READ);
+
+    if (is_mem_read) {
+        m_ram->read(memMess->getAddr(), const_cast<uint8_t *>(
+                    memMess->getDataBlk().getData(0,
+                        RubySystem::getBlockSizeBytes())),
+                    RubySystem::getBlockSizeBytes());
+    }  else {
+        m_ram->write(memMess->getAddr(), const_cast<uint8_t *>(
+                     memMess->getDataBlk().getData(0,
+                         RubySystem::getBlockSizeBytes())),
+                     RubySystem::getBlockSizeBytes());
+    }
+
     MemoryNode *thisReq = new MemoryNode(arrival_time, message, addr,
                                          is_mem_read, !is_mem_read);
     enqueueMemRef(thisReq);
@@ -706,7 +720,7 @@ RubyMemoryControl::wakeup()
  * being lists.
  */
 bool
-RubyMemoryControl::functionalReadBuffers(Packet *pkt)
+RubyMemoryControl::functionalRead(Packet *pkt)
 {
     for (std::list<MemoryNode *>::iterator it = m_input_queue.begin();
          it != m_input_queue.end(); ++it) {
@@ -734,7 +748,10 @@ RubyMemoryControl::functionalReadBuffers(Packet *pkt)
         }
     }
 
-    return false;
+    m_ram->read(Address(pkt->getAddr()), pkt->getPtr<uint8_t>(true),
+                pkt->getSize());
+
+    return true;
 }
 
 /**
@@ -746,7 +763,7 @@ RubyMemoryControl::functionalReadBuffers(Packet *pkt)
  * for debugging purposes.
  */
 uint32_t
-RubyMemoryControl::functionalWriteBuffers(Packet *pkt)
+RubyMemoryControl::functionalWrite(Packet *pkt)
 {
     uint32_t num_functional_writes = 0;
 
@@ -776,6 +793,10 @@ RubyMemoryControl::functionalWriteBuffers(Packet *pkt)
         }
     }
 
+    m_ram->write(Address(pkt->getAddr()), pkt->getPtr<uint8_t>(true),
+                 pkt->getSize());
+    num_functional_writes++;
+
     return num_functional_writes;
 }
 
diff --git a/src/mem/ruby/structures/RubyMemoryControl.hh b/src/mem/ruby/structures/RubyMemoryControl.hh
index e7f1c54cc..dde6143c4 100644
--- a/src/mem/ruby/structures/RubyMemoryControl.hh
+++ b/src/mem/ruby/structures/RubyMemoryControl.hh
@@ -36,15 +36,12 @@
 
 #include "mem/protocol/MemoryMsg.hh"
 #include "mem/ruby/common/Address.hh"
-#include "mem/ruby/common/Consumer.hh"
 #include "mem/ruby/common/Global.hh"
 #include "mem/ruby/profiler/MemCntrlProfiler.hh"
-#include "mem/ruby/slicc_interface/Message.hh"
 #include "mem/ruby/structures/MemoryControl.hh"
-#include "mem/ruby/structures/MemoryNode.hh"
+#include "mem/ruby/structures/MemoryVector.hh"
 #include "mem/ruby/system/System.hh"
 #include "params/RubyMemoryControl.hh"
-#include "sim/sim_object.hh"
 
 // This constant is part of the definition of tFAW; see
 // the comments in header to RubyMemoryControl.cc
@@ -95,8 +92,8 @@ class RubyMemoryControl : public MemoryControl
     int getRanksPerDimm() { return m_ranks_per_dimm; };
     int getDimmsPerChannel() { return m_dimms_per_channel; }
 
-    bool functionalReadBuffers(Packet *pkt);
-    uint32_t functionalWriteBuffers(Packet *pkt);
+    bool functionalRead(Packet *pkt);
+    uint32_t functionalWrite(Packet *pkt);
 
   private:
     void enqueueToDirectory(MemoryNode *req, Cycles latency);
@@ -165,6 +162,9 @@ class RubyMemoryControl : public MemoryControl
     int m_idleCount;          // watchdog timer for shutting down
 
     MemCntrlProfiler* m_profiler_ptr;
+
+    // Actual physical memory.
+    MemoryVector* m_ram;
 };
 
 std::ostream& operator<<(std::ostream& out, const RubyMemoryControl& obj);
diff --git a/src/mem/ruby/system/System.cc b/src/mem/ruby/system/System.cc
index edc739b85..8bcc87474 100644
--- a/src/mem/ruby/system/System.cc
+++ b/src/mem/ruby/system/System.cc
@@ -427,10 +427,6 @@ RubySystem::functionalRead(PacketPtr pkt)
     }
     assert(num_rw <= 1);
 
-    uint8_t *data = pkt->getPtr<uint8_t>(true);
-    unsigned int size_in_bytes = pkt->getSize();
-    unsigned startByte = address.getAddress() - line_address.getAddress();
-
     // This if case is meant to capture what happens in a Broadcast/Snoop
     // protocol where the block does not exist in the cache hierarchy. You
     // only want to read from the Backing_Store memory if there is no copy in
@@ -439,20 +435,12 @@ RubySystem::functionalRead(PacketPtr pkt)
     // The reason is because the Backing_Store memory could easily be stale, if
     // there are copies floating around the cache hierarchy, so you want to read
     // it only if it's not in the cache hierarchy at all.
-    if (num_invalid == (num_controllers - 1) &&
-            num_backing_store == 1) {
+    if (num_invalid == (num_controllers - 1) && num_backing_store == 1) {
         DPRINTF(RubySystem, "only copy in Backing_Store memory, read from it\n");
         for (unsigned int i = 0; i < num_controllers; ++i) {
             access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_address);
             if (access_perm == AccessPermission_Backing_Store) {
-                DataBlock& block = m_abs_cntrl_vec[i]->
-                    getDataBlock(line_address);
-
-                DPRINTF(RubySystem, "reading from %s block %s\n",
-                        m_abs_cntrl_vec[i]->name(), block);
-                for (unsigned j = 0; j < size_in_bytes; ++j) {
-                    data[j] = block.getByte(j + startByte);
-                }
+                m_abs_cntrl_vec[i]->functionalRead(line_address, pkt);
                 return true;
             }
         }
@@ -470,14 +458,7 @@ RubySystem::functionalRead(PacketPtr pkt)
             access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_address);
             if (access_perm == AccessPermission_Read_Only ||
                 access_perm == AccessPermission_Read_Write) {
-                DataBlock& block = m_abs_cntrl_vec[i]->
-                    getDataBlock(line_address);
-
-                DPRINTF(RubySystem, "reading from %s block %s\n",
-                        m_abs_cntrl_vec[i]->name(), block);
-                for (unsigned j = 0; j < size_in_bytes; ++j) {
-                    data[j] = block.getByte(j + startByte);
-                }
+                m_abs_cntrl_vec[i]->functionalRead(line_address, pkt);
                 return true;
             }
         }
@@ -500,10 +481,6 @@ RubySystem::functionalWrite(PacketPtr pkt)
 
     DPRINTF(RubySystem, "Functional Write request for %s\n",addr);
 
-    uint8_t *data = pkt->getPtr<uint8_t>(true);
-    unsigned int size_in_bytes = pkt->getSize();
-    unsigned startByte = addr.getAddress() - line_addr.getAddress();
-
     uint32_t M5_VAR_USED num_functional_writes = 0;
 
     for (unsigned int i = 0; i < num_controllers;++i) {
@@ -513,23 +490,11 @@ RubySystem::functionalWrite(PacketPtr pkt)
         access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_addr);
         if (access_perm != AccessPermission_Invalid &&
             access_perm != AccessPermission_NotPresent) {
-
-            num_functional_writes++;
-
-            DataBlock& block = m_abs_cntrl_vec[i]->getDataBlock(line_addr);
-            DPRINTF(RubySystem, "%s\n",block);
-            for (unsigned j = 0; j < size_in_bytes; ++j) {
-              block.setByte(j + startByte, data[j]);
-            }
-            DPRINTF(RubySystem, "%s\n",block);
+            num_functional_writes +=
+                m_abs_cntrl_vec[i]->functionalWrite(line_addr, pkt);
         }
     }
 
-    for (unsigned int i = 0; i < m_memory_controller_vec.size() ;++i) {
-        num_functional_writes +=
-            m_memory_controller_vec[i]->functionalWriteBuffers(pkt);
-    }
-
     num_functional_writes += m_network->functionalWrite(pkt);
     DPRINTF(RubySystem, "Messages written = %u\n", num_functional_writes);