ruby: coherence protocols: remove data block from dirctory entry

This patch removes the data block present in the directory entry structure
of each protocol in gem5's mainline.  Firstly, this is required for moving
towards common set of memory controllers for classic and ruby memory systems.
Secondly, the data block was being misused in several places.  It was being
used for having free access to the physical memory instead of calling on the
memory controller.

From now on, the directory controller will not have a direct visibility into
the physical memory.  The Memory Vector object now resides in the
Memory Controller class.  This also means that some significant changes are
being made to the functional accesses in ruby.

23 files changed, 368 insertions, 335 deletions

diff --git a/src/mem/protocol/MESI_Three_Level-L0cache.sm b/src/mem/protocol/MESI_Three_Level-L0cache.sm
index 49b6aa7a9..c5802d776 100644
--- a/src/mem/protocol/MESI_Three_Level-L0cache.sm
+++ b/src/mem/protocol/MESI_Three_Level-L0cache.sm
@@ -205,13 +205,28 @@ machine(L0Cache, "MESI Directory L0 Cache")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   void setAccessPermission(Entry cache_entry, Address addr, State state) {
diff --git a/src/mem/protocol/MESI_Three_Level-L1cache.sm b/src/mem/protocol/MESI_Three_Level-L1cache.sm
index 59249d822..024f8f6da 100644
--- a/src/mem/protocol/MESI_Three_Level-L1cache.sm
+++ b/src/mem/protocol/MESI_Three_Level-L1cache.sm
@@ -205,13 +205,28 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   void setAccessPermission(Entry cache_entry, Address addr, State state) {
diff --git a/src/mem/protocol/MESI_Two_Level-L1cache.sm b/src/mem/protocol/MESI_Two_Level-L1cache.sm
index 080b0c0bb..b449c4f2b 100644
--- a/src/mem/protocol/MESI_Two_Level-L1cache.sm
+++ b/src/mem/protocol/MESI_Two_Level-L1cache.sm
@@ -224,13 +224,28 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   void setAccessPermission(Entry cache_entry, Address addr, State state) {
diff --git a/src/mem/protocol/MESI_Two_Level-L2cache.sm b/src/mem/protocol/MESI_Two_Level-L2cache.sm
index f4809959d..ede420626 100644
--- a/src/mem/protocol/MESI_Two_Level-L2cache.sm
+++ b/src/mem/protocol/MESI_Two_Level-L2cache.sm
@@ -212,13 +212,28 @@ machine(L2Cache, "MESI Directory L2 Cache CMP")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   void setAccessPermission(Entry cache_entry, Address addr, State state) {
diff --git a/src/mem/protocol/MESI_Two_Level-dir.sm b/src/mem/protocol/MESI_Two_Level-dir.sm
index dd0ecf49e..939ae2a36 100644
--- a/src/mem/protocol/MESI_Two_Level-dir.sm
+++ b/src/mem/protocol/MESI_Two_Level-dir.sm
@@ -73,7 +73,6 @@ machine(Directory, "MESI Two Level directory protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
     MachineID Owner;
   }
 
@@ -90,6 +89,8 @@ machine(Directory, "MESI Two Level directory protocol")
     void allocate(Address);
     void deallocate(Address);
     bool isPresent(Address);
+    bool functionalRead(Packet *pkt);
+    int functionalWrite(Packet *pkt);
   }
 
 
@@ -148,13 +149,22 @@ machine(Directory, "MESI Two Level directory protocol")
     return AccessPermission:NotPresent;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      memBuffer.functionalRead(pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndWrite(addr, tbe.DataBlk, pkt);
     }
 
-    return getDirectoryEntry(addr).DataBlk;
+    return memBuffer.functionalWrite(pkt);
   }
 
   void setAccessPermission(Address addr, State state) {
@@ -297,7 +307,6 @@ machine(Directory, "MESI Two Level directory protocol")
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
         out_msg.Prefetch := in_msg.Prefetch;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
 
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
@@ -320,13 +329,6 @@ machine(Directory, "MESI Two Level directory protocol")
     }
   }
 
-  action(m_writeDataToMemory, "m", desc="Write dirty writeback to memory") {
-    peek(responseNetwork_in, ResponseMsg) {
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
 //added by SS for dma
   action(qf_queueMemoryFetchRequestDMA, "qfd", desc="Queue off-chip fetch request") {
     peek(requestNetwork_in, RequestMsg) {
@@ -336,7 +338,6 @@ machine(Directory, "MESI Two Level directory protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := machineID;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -358,25 +359,14 @@ machine(Directory, "MESI Two Level directory protocol")
     }
   }
 
-  action(dw_writeDMAData, "dw", desc="DMA Write data to memory") {
-    peek(requestNetwork_in, RequestMsg) {
-      getDirectoryEntry(address).DataBlk.copyPartial(in_msg.DataBlk, addressOffset(in_msg.Addr), in_msg.Len);
-    }
-  }
-
   action(qw_queueMemoryWBRequest_partial, "qwp", desc="Queue off-chip writeback request") {
      peek(requestNetwork_in, RequestMsg) {
       enqueue(memQueue_out, MemoryMsg, to_mem_ctrl_latency) {
         out_msg.Addr := address;
         out_msg.Type := MemoryRequestType:MEMORY_WB;
         out_msg.OriginalRequestorMachId := machineID;
-        //out_msg.DataBlk := in_msg.DataBlk;
         out_msg.DataBlk.copyPartial(in_msg.DataBlk, addressOffset(address), in_msg.Len);
-
-
         out_msg.MessageSize := in_msg.MessageSize;
-        //out_msg.Prefetch := in_msg.Prefetch;
-
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -434,15 +424,6 @@ machine(Directory, "MESI Two Level directory protocol")
     }
   }
 
-  action(dwt_writeDMADataFromTBE, "dwt", desc="DMA Write data to memory from TBE") {
-    assert(is_valid(tbe));
-    //getDirectoryEntry(address).DataBlk.copyPartial(tbe.DataBlk, tbe.Offset, tbe.Len);
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
-
-
-  }
-
-
   action(qw_queueMemoryWBRequest_partialTBE, "qwt", desc="Queue off-chip writeback request") {
     peek(responseNetwork_in, ResponseMsg) {
       enqueue(memQueue_out, MemoryMsg, to_mem_ctrl_latency) {
@@ -493,7 +474,6 @@ machine(Directory, "MESI Two Level directory protocol")
   }
 
   transition(M, Data, MI) {
-    m_writeDataToMemory;
     qw_queueMemoryWBRequest;
     k_popIncomingResponseQueue;
   }
@@ -518,7 +498,6 @@ machine(Directory, "MESI Two Level directory protocol")
   }
 
   transition(I, DMA_WRITE, ID_W) {
-    dw_writeDMAData;
     qw_queueMemoryWBRequest_partial;
     j_popIncomingRequestQueue;
   }
@@ -545,7 +524,6 @@ machine(Directory, "MESI Two Level directory protocol")
 
   transition(M_DRD, Data, M_DRDI) {
     drp_sendDMAData;
-    m_writeDataToMemory;
     qw_queueMemoryWBRequest;
     k_popIncomingResponseQueue;
   }
@@ -563,13 +541,11 @@ machine(Directory, "MESI Two Level directory protocol")
   }
 
   transition(M_DWR, Data, M_DWRI) {
-    m_writeDataToMemory;
     qw_queueMemoryWBRequest_partialTBE;
     k_popIncomingResponseQueue;
   }
 
   transition(M_DWRI, Memory_Ack, I) {
-    dwt_writeDMADataFromTBE;
     aa_sendAck;
     da_sendDMAAck;
     w_deallocateTBE;
diff --git a/src/mem/protocol/MESI_Two_Level-dma.sm b/src/mem/protocol/MESI_Two_Level-dma.sm
index 845d4df2f..3d9f2336f 100644
--- a/src/mem/protocol/MESI_Two_Level-dma.sm
+++ b/src/mem/protocol/MESI_Two_Level-dma.sm
@@ -55,8 +55,9 @@ machine(DMA, "DMA Controller")
   State getState(Address addr) {
     return cur_state;
   }
+
   void setState(Address addr, State state) {
-  cur_state := state;
+    cur_state := state;
   }
 
   AccessPermission getAccessPermission(Address addr) {
@@ -66,8 +67,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("DMA does not support get data block.");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(requestToDir_out, RequestMsg, requestToDir, desc="...");
diff --git a/src/mem/protocol/MI_example-cache.sm b/src/mem/protocol/MI_example-cache.sm
index ee774f4c2..b0217ffea 100644
--- a/src/mem/protocol/MI_example-cache.sm
+++ b/src/mem/protocol/MI_example-cache.sm
@@ -171,13 +171,28 @@ machine(L1Cache, "MI Example L1 Cache")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-      return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   // NETWORK PORTS
diff --git a/src/mem/protocol/MI_example-dir.sm b/src/mem/protocol/MI_example-dir.sm
index cd12e3eb7..60662080a 100644
--- a/src/mem/protocol/MI_example-dir.sm
+++ b/src/mem/protocol/MI_example-dir.sm
@@ -84,7 +84,6 @@ machine(Directory, "Directory protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
     NetDest Sharers,                   desc="Sharers for this block";
     NetDest Owner,                     desc="Owner of this block";
   }
@@ -151,7 +150,6 @@ machine(Directory, "Directory protocol")
       if (state == State:I)  {
         assert(getDirectoryEntry(addr).Owner.count() == 0);
         assert(getDirectoryEntry(addr).Sharers.count() == 0);
-        directory.invalidateBlock(addr);
       }
     }
   }
@@ -175,13 +173,22 @@ machine(Directory, "Directory protocol")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-      return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      memBuffer.functionalRead(pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndWrite(addr, tbe.DataBlk, pkt);
     }
 
-    return getDirectoryEntry(addr).DataBlk;
+    return memBuffer.functionalWrite(pkt);
   }
 
   // ** OUT_PORTS **
@@ -326,7 +333,10 @@ machine(Directory, "Directory protocol")
         out_msg.PhysicalAddress := address;
         out_msg.LineAddress := address;
         out_msg.Type := DMAResponseType:DATA;
-        out_msg.DataBlk := in_msg.DataBlk;   // we send the entire data block and rely on the dma controller to split it up if need be
+
+        // we send the entire data block and rely on the dma controller
+        // to split it up if need be
+        out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Destination.add(tbe.DmaRequestor);
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
@@ -386,21 +396,7 @@ machine(Directory, "Directory protocol")
   action(p_popIncomingDMARequestQueue, "p", desc="Pop incoming DMA queue") {
     dmaRequestQueue_in.dequeue();
   }
-
-  action(l_writeDataToMemory, "pl", desc="Write PUTX data to memory") {
-    peek(requestQueue_in, RequestMsg) {
-      // assert(in_msg.Dirty);
-      // assert(in_msg.MessageSize == MessageSizeType:Writeback_Data);
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      //getDirectoryEntry(in_msg.Addr).DataBlk.copyPartial(in_msg.DataBlk, addressOffset(in_msg.Addr), in_msg.Len);
-    }
-  }
   
-  action(dwt_writeDMADataFromTBE, "dwt", desc="DMA Write data to memory from TBE") {
-    assert(is_valid(tbe));
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
-  }
-
   action(v_allocateTBE, "v", desc="Allocate TBE") {
     peek(dmaRequestQueue_in, DMARequestMsg) {
       TBEs.allocate(address);
@@ -450,7 +446,6 @@ machine(Directory, "Directory protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         DPRINTF(RubySlicc,"%s\n", out_msg);
       }
     }
@@ -464,7 +459,6 @@ machine(Directory, "Directory protocol")
         out_msg.Sender := machineID;
         //out_msg.OriginalRequestorMachId := machineID;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc,"%s\n", out_msg);
       }
     }
@@ -475,12 +469,9 @@ machine(Directory, "Directory protocol")
       enqueue(memQueue_out, MemoryMsg, 1) {
         out_msg.Addr := address;
         out_msg.Type := MemoryRequestType:MEMORY_WB;
-        //out_msg.OriginalRequestorMachId := machineID;
-        //out_msg.DataBlk := in_msg.DataBlk;
-        out_msg.DataBlk.copyPartial(in_msg.DataBlk, addressOffset(in_msg.PhysicalAddress), in_msg.Len);
+        out_msg.DataBlk.copyPartial(
+            in_msg.DataBlk, addressOffset(in_msg.PhysicalAddress), in_msg.Len);
         out_msg.MessageSize := in_msg.MessageSize;
-        //out_msg.Prefetch := in_msg.Prefetch;
-
         DPRINTF(RubySlicc,"%s\n", out_msg);
       }
     }
@@ -493,19 +484,17 @@ machine(Directory, "Directory protocol")
         out_msg.Addr := address;
         out_msg.Type := MemoryRequestType:MEMORY_WB;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
+
         // get incoming data
-        // out_msg.DataBlk := in_msg.DataBlk;
-        out_msg.DataBlk.copyPartial(tbe.DataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
+        out_msg.DataBlk.copyPartial(
+            tbe.DataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
         out_msg.MessageSize := in_msg.MessageSize;
-        //out_msg.Prefetch := in_msg.Prefetch;
-
         DPRINTF(RubySlicc,"%s\n", out_msg);
       }
     }
   }
 
 
-
   action(l_queueMemoryWBRequest, "lq", desc="Write PUTX data to memory") {
     peek(requestQueue_in, RequestMsg) {
       enqueue(memQueue_out, MemoryMsg, 1) {
@@ -525,13 +514,7 @@ machine(Directory, "Directory protocol")
     memQueue_in.dequeue();
   }
 
-  action(w_writeDataToMemoryFromTBE, "\w", desc="Write date to directory memory from TBE") {
-    assert(is_valid(tbe));
-    getDirectoryEntry(address).DataBlk := TBEs[address].DataBlk;
-  }
-
   // TRANSITIONS
-
   transition({M_DRD, M_DWR, M_DWRI, M_DRDI}, GETX) {
     z_recycleRequestQueue;
   }
@@ -582,7 +565,6 @@ machine(Directory, "Directory protocol")
   }
 
   transition(ID_W, Memory_Ack, I) {
-    dwt_writeDMADataFromTBE;
     da_sendDMAAck;
     w_deallocateTBE;
     l_popMemQueue;
@@ -595,7 +577,6 @@ machine(Directory, "Directory protocol")
   }
 
   transition(M_DRD, PUTX, M_DRDI) {     
-    l_writeDataToMemory;
     drp_sendDMAData;
     c_clearOwner;
     l_queueMemoryWBRequest;
@@ -616,14 +597,12 @@ machine(Directory, "Directory protocol")
   }
 
   transition(M_DWR, PUTX, M_DWRI) {
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest_partialTBE;
     c_clearOwner;
     i_popIncomingRequestQueue;
   }
 
   transition(M_DWRI, Memory_Ack, I) {
-    w_writeDataToMemoryFromTBE;
     l_sendWriteBackAck;
     da_sendDMAAck;
     w_deallocateTBE;
@@ -644,7 +623,6 @@ machine(Directory, "Directory protocol")
   }
 
   transition(MI, Memory_Ack, I) {
-    w_writeDataToMemoryFromTBE;
     l_sendWriteBackAck;
     w_deallocateTBE;
     l_popMemQueue;
@@ -659,5 +637,4 @@ machine(Directory, "Directory protocol")
     b_sendWriteBackNack;
     i_popIncomingRequestQueue;
   }
-
 }
diff --git a/src/mem/protocol/MI_example-dma.sm b/src/mem/protocol/MI_example-dma.sm
index e328d9e20..c3cc29ba2 100644
--- a/src/mem/protocol/MI_example-dma.sm
+++ b/src/mem/protocol/MI_example-dma.sm
@@ -66,8 +66,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("DMA Controller does not support getDataBlock function.\n");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(requestToDir_out, DMARequestMsg, requestToDir, desc="...");
diff --git a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
index 3cd87616f..e9b05a0c8 100644
--- a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
@@ -212,18 +212,34 @@ machine(L1Cache, "Directory protocol")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     Entry cache_entry := getCacheEntry(addr);
     if(is_valid(cache_entry)) {
-        return cache_entry.DataBlk;
+      testAndRead(addr, cache_entry.DataBlk, pkt);
+    } else {
+      TBE tbe := TBEs[addr];
+      if(is_valid(tbe)) {
+        testAndRead(addr, tbe.DataBlk, pkt);
+      } else {
+        error("Data block missing!");
+      }
     }
+  }
 
-    TBE tbe := TBEs[addr];
-    if(is_valid(tbe)) {
-      return tbe.DataBlk;
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    Entry cache_entry := getCacheEntry(addr);
+    if(is_valid(cache_entry)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, cache_entry.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    error("Data block missing!");
+    TBE tbe := TBEs[addr];
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+    return num_functional_writes;
   }
 
   Event mandatory_request_type_to_event(RubyRequestType type) {
diff --git a/src/mem/protocol/MOESI_CMP_directory-L2cache.sm b/src/mem/protocol/MOESI_CMP_directory-L2cache.sm
index 46fd12a3a..c01b9765d 100644
--- a/src/mem/protocol/MOESI_CMP_directory-L2cache.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-L2cache.sm
@@ -520,13 +520,28 @@ machine(L2Cache, "Token protocol")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     TBE tbe := TBEs[addr];
     if(is_valid(tbe)) {
-        return tbe.DataBlk;
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, tbe.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    return getCacheEntry(addr).DataBlk;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   MessageBuffer triggerQueue, ordered="true";
diff --git a/src/mem/protocol/MOESI_CMP_directory-dir.sm b/src/mem/protocol/MOESI_CMP_directory-dir.sm
index 272a8c9ab..a6b93fa54 100644
--- a/src/mem/protocol/MOESI_CMP_directory-dir.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-dir.sm
@@ -96,7 +96,6 @@ machine(Directory, "Directory protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface='AbstractEntry') {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
     NetDest Sharers,                   desc="Sharers for this block";
     NetDest Owner,                     desc="Owner of this block";
     int WaitingUnblocks,           desc="Number of acks we're waiting for";
@@ -191,8 +190,12 @@ machine(Directory, "Directory protocol")
     }
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    return getDirectoryEntry(addr).DataBlk;
+  void functionalRead(Address addr, Packet *pkt) {
+    memBuffer.functionalRead(pkt);
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    return memBuffer.functionalWrite(pkt);
   }
 
   // if no sharers, then directory can be considered
@@ -346,7 +349,6 @@ machine(Directory, "Directory protocol")
         out_msg.Sender := machineID;
         out_msg.SenderMachine := MachineType:Directory;
         out_msg.Destination.add(in_msg.OriginalRequestorMachId);
-        //out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Dirty := false; // By definition, the block is now clean
         out_msg.Acks := in_msg.Acks;
@@ -367,7 +369,6 @@ machine(Directory, "Directory protocol")
         out_msg.Sender := machineID;
         out_msg.SenderMachine := MachineType:Directory;
         out_msg.Destination.add(in_msg.Requestor);
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         out_msg.Dirty := false; // By definition, the block is now clean
         out_msg.Type := CoherenceResponseType:DATA_EXCLUSIVE;
         out_msg.MessageSize := MessageSizeType:Response_Data;
@@ -375,8 +376,6 @@ machine(Directory, "Directory protocol")
     }
   }
 
-
-
   action(e_ownerIsUnblocker, "e", desc="The owner is now the unblocker") {
     peek(unblockNetwork_in, ResponseMsg) {
       getDirectoryEntry(address).Owner.clear();
@@ -445,40 +444,6 @@ machine(Directory, "Directory protocol")
     unblockNetwork_in.dequeue();
   }
 
-  action(l_writeDataToMemory, "l", desc="Write PUTX/PUTO data to memory") {
-    peek(unblockNetwork_in, ResponseMsg) {
-      assert(in_msg.Dirty);
-      assert(in_msg.MessageSize == MessageSizeType:Writeback_Data);
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
-
-  action(p_writeFwdDataToMemory, "p", desc="Write Response data to memory") {
-     peek(unblockNetwork_in, ResponseMsg) {
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
- }
-
-  action(ll_checkDataInMemory, "\ld", desc="Check PUTX/PUTO data is same as in the memory") {
-    peek(unblockNetwork_in, ResponseMsg) {
-      assert(in_msg.Dirty == false);
-      assert(in_msg.MessageSize == MessageSizeType:Writeback_Control);
-
-      // NOTE: The following check would not be valid in a real
-      // implementation.  We include the data in the "dataless"
-      // message so we can assert the clean data matches the datablock
-      // in memory
-      DPRINTF(RubySlicc, "Address: %s, MsgDataBlock: %s MemoryDataBlock: %s\n",
-              in_msg.Addr, in_msg.DataBlk,
-              getDirectoryEntry(in_msg.Addr).DataBlk);
-      assert(getDirectoryEntry(in_msg.Addr).DataBlk == in_msg.DataBlk);
-    }
-  }
-
   action(m_addUnlockerToSharers, "m", desc="Add the unlocker to the sharer list") {
     peek(unblockNetwork_in, ResponseMsg) {
       getDirectoryEntry(address).Sharers.add(in_msg.Sender);
@@ -505,7 +470,6 @@ machine(Directory, "Directory protocol")
         out_msg.Type := MemoryRequestType:MEMORY_READ;
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         out_msg.MessageSize := in_msg.MessageSize;
         //out_msg.Prefetch := false;
         // These are not used by memory but are passed back here with the read data:
@@ -540,6 +504,29 @@ machine(Directory, "Directory protocol")
     }
   }
 
+  action(qw_queueMemoryWBRequestFromMessageAndTBE, "qwmt",
+    desc="Queue off-chip writeback request") {
+    peek(unblockNetwork_in, ResponseMsg) {
+      enqueue(memQueue_out, MemoryMsg, 1) {
+        out_msg.Addr := address;
+        out_msg.Type := MemoryRequestType:MEMORY_WB;
+        out_msg.Sender := machineID;
+        if (is_valid(tbe)) {
+          out_msg.OriginalRequestorMachId := tbe.Requestor;
+        }
+        out_msg.DataBlk := in_msg.DataBlk;
+        out_msg.DataBlk.copyPartial(tbe.DataBlk,
+            addressOffset(tbe.PhysicalAddress), tbe.Len);
+
+        out_msg.MessageSize := in_msg.MessageSize;
+        // Not used:
+        out_msg.ReadX := false;
+        out_msg.Acks := getDirectoryEntry(address).Sharers.count();  // for dma requests
+        DPRINTF(RubySlicc, "%s\n", out_msg);
+      }
+    }
+  }
+
   action(qw_queueMemoryWBRequest2, "/qw", desc="Queue off-chip writeback request") {
     peek(requestQueue_in, RequestMsg) {
       enqueue(memQueue_out, MemoryMsg, 1) {
@@ -594,18 +581,6 @@ machine(Directory, "Directory protocol")
     }
   }
 
-  action(l_writeDMADataToMemory, "\l", desc="Write data from a DMA_WRITE to memory") {
-    peek(requestQueue_in, RequestMsg) {
-      getDirectoryEntry(address).DataBlk.copyPartial(in_msg.DataBlk, addressOffset(in_msg.Addr), in_msg.Len);
-    }
-  }
-
-  action(l_writeDMADataToMemoryFromTBE, "\ll", desc="Write data from a DMA_WRITE to memory") {
-    assert(is_valid(tbe));
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DataBlk,
-                      addressOffset(tbe.PhysicalAddress), tbe.Len);
-  }
-
   action(v_allocateTBE, "v", desc="Allocate TBE entry") {
     peek (requestQueue_in, RequestMsg) {
       TBEs.allocate(address);
@@ -623,9 +598,7 @@ machine(Directory, "Directory protocol")
   }
 
 
-
   // TRANSITIONS
-
   transition(I, GETX, MM) {
     qf_queueMemoryFetchRequest;
     i_popIncomingRequestQueue;
@@ -639,7 +612,6 @@ machine(Directory, "Directory protocol")
   transition(I, DMA_WRITE, XI_U) {
     qw_queueMemoryWBRequest2;
     a_sendDMAAck;  // ack count may be zero
-    l_writeDMADataToMemory;
     i_popIncomingRequestQueue;
   }
 
@@ -670,7 +642,6 @@ machine(Directory, "Directory protocol")
   transition(S, DMA_WRITE, XI_U) {
     qw_queueMemoryWBRequest2;
     a_sendDMAAck;  // ack count may be zero
-    l_writeDMADataToMemory;
     g_sendInvalidations;  // the DMA will collect invalidations
     i_popIncomingRequestQueue;
   }
@@ -720,10 +691,8 @@ machine(Directory, "Directory protocol")
   }
 
   transition(OI_D, Data, XI_U) {
-    qw_queueMemoryWBRequest;
+    qw_queueMemoryWBRequestFromMessageAndTBE;
     a_sendDMAAck2;  // ack count may be zero
-    p_writeFwdDataToMemory;
-    l_writeDMADataToMemoryFromTBE;
     w_deallocateTBE;
     j_popIncomingUnblockQueue;
   }
@@ -842,14 +811,12 @@ machine(Directory, "Directory protocol")
   transition(MI, Dirty_Writeback, I) {
     c_clearOwner;
     cc_clearSharers;
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest;
     j_popIncomingUnblockQueue;
   }
 
   transition(MIS, Dirty_Writeback, S) {
     c_moveOwnerToSharer;
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest;
     j_popIncomingUnblockQueue;
   }
@@ -861,14 +828,12 @@ machine(Directory, "Directory protocol")
 
   transition(OS, Dirty_Writeback, S) {
     c_clearOwner;
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest;
     j_popIncomingUnblockQueue;
   }
 
   transition(OSS, Dirty_Writeback, S) {
     c_moveOwnerToSharer;
-    l_writeDataToMemory;
     qw_queueMemoryWBRequest;
     j_popIncomingUnblockQueue;
   }
@@ -881,13 +846,11 @@ machine(Directory, "Directory protocol")
   transition(MI, Clean_Writeback, I) {
     c_clearOwner;
     cc_clearSharers;
-    ll_checkDataInMemory;
     j_popIncomingUnblockQueue;
   }
 
   transition(OS, Clean_Writeback, S) {
     c_clearOwner;
-    ll_checkDataInMemory;
     j_popIncomingUnblockQueue;
   }
 
diff --git a/src/mem/protocol/MOESI_CMP_directory-dma.sm b/src/mem/protocol/MOESI_CMP_directory-dma.sm
index 9bdd791e1..d7e3a02d9 100644
--- a/src/mem/protocol/MOESI_CMP_directory-dma.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-dma.sm
@@ -91,8 +91,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-     error("DMA Controller does not support getDataBlock().\n");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(reqToDirectory_out, RequestMsg, reqToDir, desc="...");
diff --git a/src/mem/protocol/MOESI_CMP_token-L1cache.sm b/src/mem/protocol/MOESI_CMP_token-L1cache.sm
index 860744384..ebfa970ff 100644
--- a/src/mem/protocol/MOESI_CMP_token-L1cache.sm
+++ b/src/mem/protocol/MOESI_CMP_token-L1cache.sm
@@ -240,8 +240,15 @@ machine(L1Cache, "Token protocol")
     return L1Icache_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    return getCacheEntry(addr).DataBlk;
+  void functionalRead(Address addr, Packet *pkt) {
+    testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   Entry getL1DCacheEntry(Address addr), return_by_pointer="yes" {
diff --git a/src/mem/protocol/MOESI_CMP_token-L2cache.sm b/src/mem/protocol/MOESI_CMP_token-L2cache.sm
index a2488066a..6542ede49 100644
--- a/src/mem/protocol/MOESI_CMP_token-L2cache.sm
+++ b/src/mem/protocol/MOESI_CMP_token-L2cache.sm
@@ -157,8 +157,15 @@ machine(L2Cache, "Token protocol")
     return cache_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    return getCacheEntry(addr).DataBlk;
+  void functionalRead(Address addr, Packet *pkt) {
+    testAndRead(addr, getCacheEntry(addr).DataBlk, pkt);
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+    num_functional_writes := num_functional_writes +
+        testAndWrite(addr, getCacheEntry(addr).DataBlk, pkt);
+    return num_functional_writes;
   }
 
   int getTokens(Entry cache_entry) {
diff --git a/src/mem/protocol/MOESI_CMP_token-dir.sm b/src/mem/protocol/MOESI_CMP_token-dir.sm
index be5df02e0..8d6abd93c 100644
--- a/src/mem/protocol/MOESI_CMP_token-dir.sm
+++ b/src/mem/protocol/MOESI_CMP_token-dir.sm
@@ -121,7 +121,6 @@ machine(Directory, "Token protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
     int Tokens, default="max_tokens()", desc="Number of tokens for the line we're holding";
 
     // The following state is provided to allow for bandwidth
@@ -188,10 +187,6 @@ machine(Directory, "Token protocol")
     return dir_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    return getDirectoryEntry(addr).DataBlk;
-  }
-
   State getState(TBE tbe, Address addr) {
     if (is_valid(tbe)) {
       return tbe.TBEState;
@@ -250,6 +245,24 @@ machine(Directory, "Token protocol")
     persistentTable.markEntries(addr);
   }
 
+  void functionalRead(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      memBuffer.functionalRead(pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndWrite(addr, tbe.DataBlk, pkt);
+    }
+
+    return memBuffer.functionalWrite(pkt);
+  }
+
   // ** OUT_PORTS **
   out_port(responseNetwork_out, ResponseMsg, responseFromDir);
   out_port(persistentNetwork_out, PersistentMsg, persistentFromDir);
@@ -598,7 +611,7 @@ machine(Directory, "Token protocol")
         out_msg.Destination.add(in_msg.OriginalRequestorMachId);
         assert(getDirectoryEntry(address).Tokens > 0);
         out_msg.Tokens := getDirectoryEntry(in_msg.Addr).Tokens;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
+        out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Dirty := false;
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
@@ -615,7 +628,7 @@ machine(Directory, "Token protocol")
         out_msg.Destination.add(persistentTable.findSmallest(address));
         assert(getDirectoryEntry(address).Tokens > 0);
         out_msg.Tokens := getDirectoryEntry(address).Tokens;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
+        out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Dirty := false;
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
@@ -646,7 +659,6 @@ machine(Directory, "Token protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -659,7 +671,6 @@ machine(Directory, "Token protocol")
       out_msg.Sender := machineID;
       out_msg.OriginalRequestorMachId := persistentTable.findSmallest(address);
       out_msg.MessageSize := MessageSizeType:Request_Control;
-      out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
       DPRINTF(RubySlicc, "%s\n", out_msg);
     }
   }
@@ -672,18 +683,20 @@ machine(Directory, "Token protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
   }
 
   action(lq_queueMemoryWbRequest, "lq", desc="Write data to memory") {
-    enqueue(memQueue_out, MemoryMsg, 1) {
-      out_msg.Addr := address;
-      out_msg.Type := MemoryRequestType:MEMORY_WB;
-      out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
-      DPRINTF(RubySlicc, "%s\n", out_msg);
+    peek(responseNetwork_in, ResponseMsg) {
+      enqueue(memQueue_out, MemoryMsg, 1) {
+        out_msg.Addr := address;
+        out_msg.Type := MemoryRequestType:MEMORY_WB;
+        out_msg.MessageSize := in_msg.MessageSize;
+        out_msg.DataBlk := in_msg.DataBlk;
+        DPRINTF(RubySlicc, "%s\n", out_msg);
+      }
     }
   }
 
@@ -694,7 +707,8 @@ machine(Directory, "Token protocol")
       // first, initialize the data blk to the current version of system memory
       out_msg.DataBlk := tbe.DataBlk;
       // then add the dma write data
-      out_msg.DataBlk.copyPartial(tbe.DmaDataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
+      out_msg.DataBlk.copyPartial(
+        tbe.DmaDataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
       DPRINTF(RubySlicc, "%s\n", out_msg);
     }
   }
@@ -759,15 +773,6 @@ machine(Directory, "Token protocol")
     }
   }
 
-  action(cd_writeCleanDataToTbe, "cd", desc="Write clean memory data to TBE") {
-    tbe.DataBlk := getDirectoryEntry(address).DataBlk;
-  }
-
-  action(dwt_writeDmaDataFromTBE, "dwt", desc="DMA Write data to memory from TBE") {
-    getDirectoryEntry(address).DataBlk := tbe.DataBlk;
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DmaDataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
-  }
-
   action(f_incrementTokens, "f", desc="Increment the number of tokens we're tracking") {
     peek(responseNetwork_in, ResponseMsg) {
       assert(in_msg.Tokens >= 1);
@@ -811,20 +816,6 @@ machine(Directory, "Token protocol")
     memQueue_in.dequeue();
   }
 
-  action(m_writeDataToMemory, "m", desc="Write dirty writeback to memory") {
-    peek(responseNetwork_in, ResponseMsg) {
-      getDirectoryEntry(in_msg.Addr).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
-
-  action(n_checkData, "n", desc="Check incoming clean data message") {
-    peek(responseNetwork_in, ResponseMsg) {
-      assert(getDirectoryEntry(in_msg.Addr).DataBlk == in_msg.DataBlk);
-    }
-  }
-
   action(r_bounceResponse, "r", desc="Bounce response to starving processor") {
     peek(responseNetwork_in, ResponseMsg) {
       enqueue(responseNetwork_out, ResponseMsg, 1) {
@@ -869,12 +860,6 @@ machine(Directory, "Token protocol")
       assert(in_msg.Dirty == false);
       assert(in_msg.MessageSize == MessageSizeType:Writeback_Control);
 
-      // NOTE: The following check would not be valid in a real
-      // implementation.  We include the data in the "dataless"
-      // message so we can assert the clean data matches the datablock
-      // in memory
-      assert(getDirectoryEntry(in_msg.Addr).DataBlk == in_msg.DataBlk);
-
       // Bounce the message, but "re-associate" the data and the owner
       // token.  In essence we're converting an ACK_OWNER message to a
       // DATA_OWNER message, keeping the number of tokens the same.
@@ -884,7 +869,6 @@ machine(Directory, "Token protocol")
         out_msg.Sender := machineID;
         out_msg.Destination.add(persistentTable.findSmallest(address));
         out_msg.Tokens := in_msg.Tokens;
-        out_msg.DataBlk := getDirectoryEntry(in_msg.Addr).DataBlk;
         out_msg.Dirty := in_msg.Dirty;
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
@@ -948,7 +932,6 @@ machine(Directory, "Token protocol")
 
   transition(O, DMA_WRITE, O_DW) {
     vd_allocateDmaRequestInTBE;
-    cd_writeCleanDataToTbe;
     bw_broadcastWrite;
     st_scheduleTimeout;
     p_popDmaRequestQueue;
@@ -956,8 +939,6 @@ machine(Directory, "Token protocol")
 
   transition(O, DMA_WRITE_All_Tokens, O_DW_W) {
     vd_allocateDmaRequestInTBE;
-    cd_writeCleanDataToTbe;
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWriteFromTbe;
     p_popDmaRequestQueue;
   }
@@ -985,7 +966,6 @@ machine(Directory, "Token protocol")
   }
 
   transition(O, {Data_Owner, Data_All_Tokens}) {
-    n_checkData;
     f_incrementTokens;
     k_popIncomingResponseQueue;
   }
@@ -1026,7 +1006,6 @@ machine(Directory, "Token protocol")
 
   transition(O_DW, Ack_Owner) {
     f_incrementTokens;
-    cd_writeCleanDataToTbe;
     k_popIncomingResponseQueue;
   }
 
@@ -1038,7 +1017,6 @@ machine(Directory, "Token protocol")
   transition({NO_DW, O_DW}, Data_All_Tokens, O_DW_W) {
     f_incrementTokens;
     rd_recordDataInTbe;
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWriteFromTbe;
     ut_unsetReissueTimer;
     k_popIncomingResponseQueue;
@@ -1046,7 +1024,6 @@ machine(Directory, "Token protocol")
 
   transition(O_DW, Ack_All_Tokens, O_DW_W) {
     f_incrementTokens;
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWriteFromTbe;
     ut_unsetReissueTimer;
     k_popIncomingResponseQueue;
@@ -1054,8 +1031,6 @@ machine(Directory, "Token protocol")
 
   transition(O_DW, Ack_Owner_All_Tokens, O_DW_W) {
     f_incrementTokens;
-    cd_writeCleanDataToTbe;
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWriteFromTbe;
     ut_unsetReissueTimer;
     k_popIncomingResponseQueue;
@@ -1100,14 +1075,12 @@ machine(Directory, "Token protocol")
   }
 
   transition(NO, {Data_Owner, Data_All_Tokens}, O_W) {
-    m_writeDataToMemory;
     f_incrementTokens;
     lq_queueMemoryWbRequest;
     k_popIncomingResponseQueue;
   }
 
   transition(NO, {Ack_Owner, Ack_Owner_All_Tokens}, O) {
-    n_checkData;
     f_incrementTokens;
     k_popIncomingResponseQueue;
   }
@@ -1160,7 +1133,6 @@ machine(Directory, "Token protocol")
   }
 
   transition(NO_DR, {Data_Owner, Data_All_Tokens}, O_W) {
-    m_writeDataToMemory;
     f_incrementTokens;
     dd_sendDmaData;
     lr_queueMemoryDmaReadWriteback;
@@ -1195,11 +1167,17 @@ machine(Directory, "Token protocol")
     k_popIncomingResponseQueue;
   }
 
-  transition({DW_L, DR_L, L}, {Ack_Owner_All_Tokens, Ack_Owner}) {
+  transition({DW_L, DR_L}, {Ack_Owner_All_Tokens, Ack_Owner}) {
     bd_bounceDatalessOwnerToken;
     k_popIncomingResponseQueue;
   }
 
+  transition(L, {Ack_Owner_All_Tokens, Ack_Owner}, L_O_W) {
+    f_incrementTokens;
+    qp_queueMemoryForPersistent;
+    k_popIncomingResponseQueue;
+  }
+
   transition(L, {Unlockdown, Own_Lock_or_Unlock}, NO) {
     l_popIncomingPersistentQueue;
   }
diff --git a/src/mem/protocol/MOESI_CMP_token-dma.sm b/src/mem/protocol/MOESI_CMP_token-dma.sm
index 1c28971a1..f11e471b4 100644
--- a/src/mem/protocol/MOESI_CMP_token-dma.sm
+++ b/src/mem/protocol/MOESI_CMP_token-dma.sm
@@ -68,8 +68,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("DMA Controller does not support getDataBlock function.\n");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(reqToDirectory_out, DMARequestMsg, reqToDirectory, desc="...");
diff --git a/src/mem/protocol/MOESI_hammer-cache.sm b/src/mem/protocol/MOESI_hammer-cache.sm
index de502e118..badbe1d8b 100644
--- a/src/mem/protocol/MOESI_hammer-cache.sm
+++ b/src/mem/protocol/MOESI_hammer-cache.sm
@@ -205,18 +205,34 @@ machine({L1Cache, L2Cache}, "AMD Hammer-like protocol")
     return L1Icache_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
+  void functionalRead(Address addr, Packet *pkt) {
     Entry cache_entry := getCacheEntry(addr);
     if(is_valid(cache_entry)) {
-        return cache_entry.DataBlk;
+      testAndRead(addr, cache_entry.DataBlk, pkt);
+    } else {
+      TBE tbe := TBEs[addr];
+      if(is_valid(tbe)) {
+        testAndRead(addr, tbe.DataBlk, pkt);
+      } else {
+        error("Missing data block");
+      }
     }
+  }
 
-    TBE tbe := TBEs[addr];
-    if(is_valid(tbe)) {
-      return tbe.DataBlk;
+  int functionalWrite(Address addr, Packet *pkt) {
+    int num_functional_writes := 0;
+
+    Entry cache_entry := getCacheEntry(addr);
+    if(is_valid(cache_entry)) {
+      num_functional_writes := num_functional_writes +
+        testAndWrite(addr, cache_entry.DataBlk, pkt);
+      return num_functional_writes;
     }
 
-    error("Missing data block");
+    TBE tbe := TBEs[addr];
+    num_functional_writes := num_functional_writes +
+      testAndWrite(addr, tbe.DataBlk, pkt);
+    return num_functional_writes;
   }
 
   Entry getL2CacheEntry(Address address), return_by_pointer="yes" {
diff --git a/src/mem/protocol/MOESI_hammer-dir.sm b/src/mem/protocol/MOESI_hammer-dir.sm
index db11b290f..43d48c6d2 100644
--- a/src/mem/protocol/MOESI_hammer-dir.sm
+++ b/src/mem/protocol/MOESI_hammer-dir.sm
@@ -147,14 +147,12 @@ machine(Directory, "AMD Hammer-like protocol")
   // DirectoryEntry
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
-    DataBlock DataBlk,             desc="data for the block";
   }
 
   // ProbeFilterEntry
   structure(PfEntry, desc="...", interface="AbstractCacheEntry") {
     State PfState,                 desc="Directory state";
     MachineID Owner,               desc="Owner node";
-    DataBlock DataBlk,             desc="data for the block";
     Set Sharers,                   desc="sharing vector for full bit directory";
   }
 
@@ -208,20 +206,6 @@ machine(Directory, "AMD Hammer-like protocol")
     return dir_entry;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    Entry dir_entry := getDirectoryEntry(addr);
-    if(is_valid(dir_entry)) {
-        return dir_entry.DataBlk;
-    }
-
-    TBE tbe := TBEs[addr];
-    if(is_valid(tbe)) {
-      return tbe.DataBlk;
-    }
-
-    error("Data block missing!");
-  }
-
   PfEntry getProbeFilterEntry(Address addr), return_by_pointer="yes" {
     if (probe_filter_enabled || full_bit_dir_enabled) {
       PfEntry pfEntry := static_cast(PfEntry, "pointer", probeFilter.lookup(addr));
@@ -282,6 +266,24 @@ machine(Directory, "AMD Hammer-like protocol")
     getDirectoryEntry(addr).changePermission(Directory_State_to_permission(state));
   }
 
+  void functionalRead(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndRead(addr, tbe.DataBlk, pkt);
+    } else {
+      memBuffer.functionalRead(pkt);
+    }
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    TBE tbe := TBEs[addr];
+    if(is_valid(tbe)) {
+      testAndWrite(addr, tbe.DataBlk, pkt);
+    }
+
+    return memBuffer.functionalWrite(pkt);
+  }
+
   Event cache_request_to_event(CoherenceRequestType type) {
     if (type == CoherenceRequestType:GETS) {
       return Event:GETS;
@@ -851,7 +853,6 @@ machine(Directory, "AMD Hammer-like protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -865,7 +866,6 @@ machine(Directory, "AMD Hammer-like protocol")
         out_msg.Sender := machineID;
         out_msg.OriginalRequestorMachId := in_msg.Requestor;
         out_msg.MessageSize := in_msg.MessageSize;
-        out_msg.DataBlk := getDirectoryEntry(address).DataBlk;
         DPRINTF(RubySlicc, "%s\n", out_msg);
       }
     }
@@ -1179,38 +1179,6 @@ machine(Directory, "AMD Hammer-like protocol")
     }
   }
 
-  action(wr_writeResponseDataToMemory, "wr", desc="Write response data to memory") {
-    peek(responseToDir_in, ResponseMsg) {
-      getDirectoryEntry(address).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
-
-  action(l_writeDataToMemory, "l", desc="Write PUTX/PUTO data to memory") {
-    peek(memQueue_in, MemoryMsg) {
-      getDirectoryEntry(address).DataBlk := in_msg.DataBlk;
-      DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
-              in_msg.Addr, in_msg.DataBlk);
-    }
-  }
-
-  action(dwt_writeDmaDataFromTBE, "dwt", desc="DMA Write data to memory from TBE") {
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-    assert(is_valid(tbe));
-    getDirectoryEntry(address).DataBlk := tbe.DataBlk;
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-    getDirectoryEntry(address).DataBlk.copyPartial(tbe.DmaDataBlk, addressOffset(tbe.PhysicalAddress), tbe.Len);
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-  }
-
-  action(wdt_writeDataFromTBE, "wdt", desc="DMA Write data to memory from TBE") {
-    assert(is_valid(tbe));
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-    getDirectoryEntry(address).DataBlk := tbe.DataBlk;
-    DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-  }
-
   action(a_assertCacheData, "ac", desc="Assert that a cache provided the data") {
     assert(is_valid(tbe));
     assert(tbe.CacheDirty);
@@ -1277,18 +1245,21 @@ machine(Directory, "AMD Hammer-like protocol")
     }
   }
 
+  action(ly_queueMemoryWriteFromTBE, "ly", desc="Write data to memory from TBE") {
+    enqueue(memQueue_out, MemoryMsg, 1) {
+      assert(is_valid(tbe));
+      out_msg.Addr := address;
+      out_msg.Type := MemoryRequestType:MEMORY_WB;
+      out_msg.DataBlk := tbe.DataBlk;
+      DPRINTF(RubySlicc, "%s\n", out_msg);
+    }
+  }
+
   action(ll_checkIncomingWriteback, "\l", desc="Check PUTX/PUTO response message") {
     peek(unblockNetwork_in, ResponseMsg) {
       assert(in_msg.Dirty == false);
       assert(in_msg.MessageSize == MessageSizeType:Writeback_Control);
       DPRINTF(RubySlicc, "%s\n", in_msg.DataBlk);
-      DPRINTF(RubySlicc, "%s\n", getDirectoryEntry(address).DataBlk);
-
-      // NOTE: The following check would not be valid in a real
-      // implementation.  We include the data in the "dataless"
-      // message so we can assert the clean data matches the datablock
-      // in memory
-      assert(getDirectoryEntry(address).DataBlk == in_msg.DataBlk);
     }
   }
 
@@ -1651,20 +1622,26 @@ machine(Directory, "AMD Hammer-like protocol")
   }
 
   transition(S_R, Data) {
-    wr_writeResponseDataToMemory;
     m_decrementNumberOfMessages;
     o_checkForCompletion;
     n_popResponseQueue;
   }
 
   transition(NO_R, {Data, Exclusive_Data}) {
-    wr_writeResponseDataToMemory;
+    r_recordCacheData;
     m_decrementNumberOfMessages;
     o_checkForCompletion;
     n_popResponseQueue;
   }
 
-  transition({O_R, S_R, NO_R}, All_acks_and_data_no_sharers, E) {
+  transition({O_R, S_R}, All_acks_and_data_no_sharers, E) {
+    w_deallocateTBE;
+    k_wakeUpDependents;
+    g_popTriggerQueue;
+  }
+
+  transition(NO_R, All_acks_and_data_no_sharers, WB_E_W) {
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
@@ -1730,14 +1707,14 @@ machine(Directory, "AMD Hammer-like protocol")
     n_popResponseQueue;
   }
 
-  transition(NO_DR_B, All_acks_and_owner_data, O) {
+  transition(NO_DR_B, All_acks_and_owner_data, WB_O_W) {
     //
     // Note that the DMA consistency model allows us to send the DMA device
     // a response as soon as we receive valid data and prior to receiving
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
@@ -1750,20 +1727,19 @@ machine(Directory, "AMD Hammer-like protocol")
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(NO_DR_B_D, All_acks_and_owner_data, O) {
+  transition(NO_DR_B_D, All_acks_and_owner_data, WB_O_W) {
     //
     // Note that the DMA consistency model allows us to send the DMA device
     // a response as soon as we receive valid data and prior to receiving
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
@@ -1776,42 +1752,41 @@ machine(Directory, "AMD Hammer-like protocol")
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(O_DR_B, All_acks_and_owner_data, O) {
-    wdt_writeDataFromTBE;
+  transition(O_DR_B, All_acks_and_owner_data, WB_O_W) {
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(O_DR_B, All_acks_and_data_no_sharers, E) {
-    wdt_writeDataFromTBE;
+  transition(O_DR_B, All_acks_and_data_no_sharers, WB_E_W) {
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     pfd_probeFilterDeallocate;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(NO_DR_B, All_acks_and_data_no_sharers, E) {
+  transition(NO_DR_B, All_acks_and_data_no_sharers, WB_E_W) {
     //
     // Note that the DMA consistency model allows us to send the DMA device
     // a response as soon as we receive valid data and prior to receiving
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     ppfd_possibleProbeFilterDeallocate;
     k_wakeUpDependents;
     g_popTriggerQueue;
   }
 
-  transition(NO_DR_B_D, All_acks_and_data_no_sharers, E) {
+  transition(NO_DR_B_D, All_acks_and_data_no_sharers, WB_E_W) {
     a_assertCacheData;
     //
     // Note that the DMA consistency model allows us to send the DMA device
@@ -1819,7 +1794,7 @@ machine(Directory, "AMD Hammer-like protocol")
     // all acks.  However, to simplify the protocol we wait for all acks.
     //
     dt_sendDmaDataFromTbe;
-    wdt_writeDataFromTBE;
+    ly_queueMemoryWriteFromTBE;
     w_deallocateTBE;
     ppfd_possibleProbeFilterDeallocate;
     k_wakeUpDependents;
@@ -1827,7 +1802,6 @@ machine(Directory, "AMD Hammer-like protocol")
   }
 
   transition(NO_DW_B_W, All_acks_and_data_no_sharers, NO_DW_W) {
-    dwt_writeDmaDataFromTBE;
     ld_queueMemoryDmaWrite;
     g_popTriggerQueue;
   }
@@ -1883,18 +1857,16 @@ machine(Directory, "AMD Hammer-like protocol")
   transition(WB, Writeback_Exclusive_Dirty, WB_E_W) {
     rs_removeSharer;
     l_queueMemoryWBRequest;
+    pfd_probeFilterDeallocate;
     j_popIncomingUnblockQueue;
   }
 
   transition(WB_E_W, Memory_Ack, E) {
-    l_writeDataToMemory;
-    pfd_probeFilterDeallocate;
     k_wakeUpDependents;
     l_popMemQueue;
   }
 
   transition(WB_O_W, Memory_Ack, O) {
-    l_writeDataToMemory;
     k_wakeUpDependents;
     l_popMemQueue;
   }
diff --git a/src/mem/protocol/MOESI_hammer-dma.sm b/src/mem/protocol/MOESI_hammer-dma.sm
index ab41adb4d..067ded0ca 100644
--- a/src/mem/protocol/MOESI_hammer-dma.sm
+++ b/src/mem/protocol/MOESI_hammer-dma.sm
@@ -66,8 +66,12 @@ machine(DMA, "DMA Controller")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("DMA Controller does not support getDataBlock function.\n");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("DMA does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("DMA does not support functional write.");
   }
 
   out_port(requestToDir_out, DMARequestMsg, requestToDir, desc="...");
diff --git a/src/mem/protocol/Network_test-cache.sm b/src/mem/protocol/Network_test-cache.sm
index e0307152d..6d81131f2 100644
--- a/src/mem/protocol/Network_test-cache.sm
+++ b/src/mem/protocol/Network_test-cache.sm
@@ -114,8 +114,12 @@ machine(L1Cache, "Network_test L1 Cache")
     return OOD;
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("Network Test does not support get data block.");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("Network test does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("Network test does not support functional write.");
   }
 
   // NETWORK PORTS
diff --git a/src/mem/protocol/Network_test-dir.sm b/src/mem/protocol/Network_test-dir.sm
index 4d6472c54..81feffde0 100644
--- a/src/mem/protocol/Network_test-dir.sm
+++ b/src/mem/protocol/Network_test-dir.sm
@@ -76,8 +76,12 @@ machine(Directory, "Network_test Directory")
   void setAccessPermission(Address addr, State state) {
   }
 
-  DataBlock getDataBlock(Address addr), return_by_ref="yes" {
-    error("Network Test does not support get data block.");
+  void functionalRead(Address addr, Packet *pkt) {
+    error("Network test does not support functional read.");
+  }
+
+  int functionalWrite(Address addr, Packet *pkt) {
+    error("Network test does not support functional write.");
   }
 
   // ** IN_PORTS **
diff --git a/src/mem/protocol/RubySlicc_Types.sm b/src/mem/protocol/RubySlicc_Types.sm
index 789595dbe..2d0658e68 100644
--- a/src/mem/protocol/RubySlicc_Types.sm
+++ b/src/mem/protocol/RubySlicc_Types.sm
@@ -163,6 +163,8 @@ structure (WireBuffer, inport="yes", outport="yes", external = "yes") {
 
 structure (MemoryControl, inport="yes", outport="yes", external = "yes") {
   void recordRequestType(CacheRequestType);
+  void functionalRead(Packet *pkt);
+  int functionalWrite(Packet *pkt);
 }
 
 structure (DMASequencer, external = "yes") {