ruby: coherence protocols: remove data block from dirctory entry

This patch removes the data block present in the directory entry structure
of each protocol in gem5's mainline.  Firstly, this is required for moving
towards common set of memory controllers for classic and ruby memory systems.
Secondly, the data block was being misused in several places.  It was being
used for having free access to the physical memory instead of calling on the
memory controller.

From now on, the directory controller will not have a direct visibility into
the physical memory.  The Memory Vector object now resides in the
Memory Controller class.  This also means that some significant changes are
being made to the functional accesses in ruby.

9 files changed, 46 insertions, 63 deletions

diff --git a/src/mem/ruby/slicc_interface/AbstractCacheEntry.hh b/src/mem/ruby/slicc_interface/AbstractCacheEntry.hh
index ff5ff25ab..2ba128493 100644
--- a/src/mem/ruby/slicc_interface/AbstractCacheEntry.hh
+++ b/src/mem/ruby/slicc_interface/AbstractCacheEntry.hh
@@ -35,6 +35,7 @@
 
 #include <iostream>
 
+#include "base/misc.hh"
 #include "mem/protocol/AccessPermission.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/slicc_interface/AbstractEntry.hh"
@@ -50,6 +51,12 @@ class AbstractCacheEntry : public AbstractEntry
     // Get/Set permission of the entry
     void changePermission(AccessPermission new_perm);
 
+    // The methods below are those called by ruby runtime, add when it
+    // is absolutely necessary and should all be virtual function.
+    virtual DataBlock& getDataBlk()
+    { panic("getDataBlk() not implemented!"); }
+
+
     Address m_Address; // Address of this block, required by CacheMemory
     int m_locked; // Holds info whether the address is locked,
                   // required for implementing LL/SC
diff --git a/src/mem/ruby/slicc_interface/AbstractController.hh b/src/mem/ruby/slicc_interface/AbstractController.hh
index 42d158653..f30967e48 100644
--- a/src/mem/ruby/slicc_interface/AbstractController.hh
+++ b/src/mem/ruby/slicc_interface/AbstractController.hh
@@ -67,7 +67,6 @@ class AbstractController : public ClockedObject, public Consumer
 
     virtual MessageBuffer* getMandatoryQueue() const = 0;
     virtual AccessPermission getAccessPermission(const Address& addr) = 0;
-    virtual DataBlock& getDataBlock(const Address& addr) = 0;
 
     virtual void print(std::ostream & out) const = 0;
     virtual void wakeup() = 0;
@@ -82,9 +81,11 @@ class AbstractController : public ClockedObject, public Consumer
     //! The boolean return value indicates if the read was performed
     //! successfully.
     virtual bool functionalReadBuffers(PacketPtr&) = 0;
+    virtual void functionalRead(const Address &addr, PacketPtr) = 0;
     //! The return value indicates the number of messages written with the
     //! data from the packet.
     virtual uint32_t functionalWriteBuffers(PacketPtr&) = 0;
+    virtual int functionalWrite(const Address &addr, PacketPtr) = 0;
 
     //! Function for enqueuing a prefetch request
     virtual void enqueuePrefetch(const Address&, const RubyRequestType&)
diff --git a/src/mem/ruby/slicc_interface/AbstractEntry.hh b/src/mem/ruby/slicc_interface/AbstractEntry.hh
index b10306281..2cf1c4b5b 100644
--- a/src/mem/ruby/slicc_interface/AbstractEntry.hh
+++ b/src/mem/ruby/slicc_interface/AbstractEntry.hh
@@ -33,8 +33,6 @@
 
 #include "mem/protocol/AccessPermission.hh"
 
-class DataBlock;
-
 class AbstractEntry
 {
   public:
@@ -45,10 +43,6 @@ class AbstractEntry
     AccessPermission getPermission() const;
     void changePermission(AccessPermission new_perm);
 
-    // The methods below are those called by ruby runtime, add when it
-    // is absolutely necessary and should all be virtual function.
-    virtual DataBlock& getDataBlk() = 0;
-
     virtual void print(std::ostream& out) const = 0;
 
     AccessPermission m_Permission; // Access permission for this
diff --git a/src/mem/ruby/structures/DirectoryMemory.cc b/src/mem/ruby/structures/DirectoryMemory.cc
index 94775aa78..04849e31f 100644
--- a/src/mem/ruby/structures/DirectoryMemory.cc
+++ b/src/mem/ruby/structures/DirectoryMemory.cc
@@ -57,7 +57,6 @@ DirectoryMemory::init()
     m_entries = new AbstractEntry*[m_num_entries];
     for (int i = 0; i < m_num_entries; i++)
         m_entries[i] = NULL;
-    m_ram = g_system_ptr->getMemoryVector();
 
     m_num_directories++;
     m_num_directories_bits = ceilLog2(m_num_directories);
@@ -132,7 +131,6 @@ DirectoryMemory::allocate(const PhysAddress& address, AbstractEntry* entry)
 
     idx = mapAddressToLocalIdx(address);
     assert(idx < m_num_entries);
-    entry->getDataBlk().assign(m_ram->getBlockPtr(address));
     entry->changePermission(AccessPermission_Read_Only);
     m_entries[idx] = entry;
 
diff --git a/src/mem/ruby/structures/DirectoryMemory.hh b/src/mem/ruby/structures/DirectoryMemory.hh
index 523f16531..b75e6ab72 100644
--- a/src/mem/ruby/structures/DirectoryMemory.hh
+++ b/src/mem/ruby/structures/DirectoryMemory.hh
@@ -35,7 +35,6 @@
 #include "mem/protocol/DirectoryRequestType.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/slicc_interface/AbstractEntry.hh"
-#include "mem/ruby/structures/MemoryVector.hh"
 #include "params/RubyDirectoryMemory.hh"
 #include "sim/sim_object.hh"
 
@@ -80,8 +79,6 @@ class DirectoryMemory : public SimObject
     static int m_num_directories_bits;
     static uint64_t m_total_size_bytes;
     static int m_numa_high_bit;
-
-    MemoryVector* m_ram;
 };
 
 inline std::ostream&
diff --git a/src/mem/ruby/structures/MemoryControl.hh b/src/mem/ruby/structures/MemoryControl.hh
index 52064adf1..b1cbe982f 100644
--- a/src/mem/ruby/structures/MemoryControl.hh
+++ b/src/mem/ruby/structures/MemoryControl.hh
@@ -89,9 +89,9 @@ class MemoryControl : public ClockedObject, public Consumer
 
     virtual void recordRequestType(MemoryControlRequestType requestType);
 
-    virtual bool functionalReadBuffers(Packet *pkt)
+    virtual bool functionalRead(Packet *pkt)
     { fatal("Functional read access not implemented!");}
-    virtual uint32_t functionalWriteBuffers(Packet *pkt)
+    virtual uint32_t functionalWrite(Packet *pkt)
     { fatal("Functional read access not implemented!");}
 
 protected:
diff --git a/src/mem/ruby/structures/RubyMemoryControl.cc b/src/mem/ruby/structures/RubyMemoryControl.cc
index 69fd45fe4..2e71c0c2f 100644
--- a/src/mem/ruby/structures/RubyMemoryControl.cc
+++ b/src/mem/ruby/structures/RubyMemoryControl.cc
@@ -173,6 +173,7 @@ RubyMemoryControl::RubyMemoryControl(const Params *p)
 void
 RubyMemoryControl::init()
 {
+    m_ram = g_system_ptr->getMemoryVector();
     m_msg_counter = 0;
 
     assert(m_tFaw <= 62); // must fit in a uint64 shift register
@@ -282,6 +283,19 @@ RubyMemoryControl::enqueue(const MsgPtr& message, Cycles latency)
     physical_address_t addr = memMess->getAddr().getAddress();
     MemoryRequestType type = memMess->getType();
     bool is_mem_read = (type == MemoryRequestType_MEMORY_READ);
+
+    if (is_mem_read) {
+        m_ram->read(memMess->getAddr(), const_cast<uint8_t *>(
+                    memMess->getDataBlk().getData(0,
+                        RubySystem::getBlockSizeBytes())),
+                    RubySystem::getBlockSizeBytes());
+    }  else {
+        m_ram->write(memMess->getAddr(), const_cast<uint8_t *>(
+                     memMess->getDataBlk().getData(0,
+                         RubySystem::getBlockSizeBytes())),
+                     RubySystem::getBlockSizeBytes());
+    }
+
     MemoryNode *thisReq = new MemoryNode(arrival_time, message, addr,
                                          is_mem_read, !is_mem_read);
     enqueueMemRef(thisReq);
@@ -706,7 +720,7 @@ RubyMemoryControl::wakeup()
  * being lists.
  */
 bool
-RubyMemoryControl::functionalReadBuffers(Packet *pkt)
+RubyMemoryControl::functionalRead(Packet *pkt)
 {
     for (std::list<MemoryNode *>::iterator it = m_input_queue.begin();
          it != m_input_queue.end(); ++it) {
@@ -734,7 +748,10 @@ RubyMemoryControl::functionalReadBuffers(Packet *pkt)
         }
     }
 
-    return false;
+    m_ram->read(Address(pkt->getAddr()), pkt->getPtr<uint8_t>(true),
+                pkt->getSize());
+
+    return true;
 }
 
 /**
@@ -746,7 +763,7 @@ RubyMemoryControl::functionalReadBuffers(Packet *pkt)
  * for debugging purposes.
  */
 uint32_t
-RubyMemoryControl::functionalWriteBuffers(Packet *pkt)
+RubyMemoryControl::functionalWrite(Packet *pkt)
 {
     uint32_t num_functional_writes = 0;
 
@@ -776,6 +793,10 @@ RubyMemoryControl::functionalWriteBuffers(Packet *pkt)
         }
     }
 
+    m_ram->write(Address(pkt->getAddr()), pkt->getPtr<uint8_t>(true),
+                 pkt->getSize());
+    num_functional_writes++;
+
     return num_functional_writes;
 }
 
diff --git a/src/mem/ruby/structures/RubyMemoryControl.hh b/src/mem/ruby/structures/RubyMemoryControl.hh
index e7f1c54cc..dde6143c4 100644
--- a/src/mem/ruby/structures/RubyMemoryControl.hh
+++ b/src/mem/ruby/structures/RubyMemoryControl.hh
@@ -36,15 +36,12 @@
 
 #include "mem/protocol/MemoryMsg.hh"
 #include "mem/ruby/common/Address.hh"
-#include "mem/ruby/common/Consumer.hh"
 #include "mem/ruby/common/Global.hh"
 #include "mem/ruby/profiler/MemCntrlProfiler.hh"
-#include "mem/ruby/slicc_interface/Message.hh"
 #include "mem/ruby/structures/MemoryControl.hh"
-#include "mem/ruby/structures/MemoryNode.hh"
+#include "mem/ruby/structures/MemoryVector.hh"
 #include "mem/ruby/system/System.hh"
 #include "params/RubyMemoryControl.hh"
-#include "sim/sim_object.hh"
 
 // This constant is part of the definition of tFAW; see
 // the comments in header to RubyMemoryControl.cc
@@ -95,8 +92,8 @@ class RubyMemoryControl : public MemoryControl
     int getRanksPerDimm() { return m_ranks_per_dimm; };
     int getDimmsPerChannel() { return m_dimms_per_channel; }
 
-    bool functionalReadBuffers(Packet *pkt);
-    uint32_t functionalWriteBuffers(Packet *pkt);
+    bool functionalRead(Packet *pkt);
+    uint32_t functionalWrite(Packet *pkt);
 
   private:
     void enqueueToDirectory(MemoryNode *req, Cycles latency);
@@ -165,6 +162,9 @@ class RubyMemoryControl : public MemoryControl
     int m_idleCount;          // watchdog timer for shutting down
 
     MemCntrlProfiler* m_profiler_ptr;
+
+    // Actual physical memory.
+    MemoryVector* m_ram;
 };
 
 std::ostream& operator<<(std::ostream& out, const RubyMemoryControl& obj);
diff --git a/src/mem/ruby/system/System.cc b/src/mem/ruby/system/System.cc
index edc739b85..8bcc87474 100644
--- a/src/mem/ruby/system/System.cc
+++ b/src/mem/ruby/system/System.cc
@@ -427,10 +427,6 @@ RubySystem::functionalRead(PacketPtr pkt)
     }
     assert(num_rw <= 1);
 
-    uint8_t *data = pkt->getPtr<uint8_t>(true);
-    unsigned int size_in_bytes = pkt->getSize();
-    unsigned startByte = address.getAddress() - line_address.getAddress();
-
     // This if case is meant to capture what happens in a Broadcast/Snoop
     // protocol where the block does not exist in the cache hierarchy. You
     // only want to read from the Backing_Store memory if there is no copy in
@@ -439,20 +435,12 @@ RubySystem::functionalRead(PacketPtr pkt)
     // The reason is because the Backing_Store memory could easily be stale, if
     // there are copies floating around the cache hierarchy, so you want to read
     // it only if it's not in the cache hierarchy at all.
-    if (num_invalid == (num_controllers - 1) &&
-            num_backing_store == 1) {
+    if (num_invalid == (num_controllers - 1) && num_backing_store == 1) {
         DPRINTF(RubySystem, "only copy in Backing_Store memory, read from it\n");
         for (unsigned int i = 0; i < num_controllers; ++i) {
             access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_address);
             if (access_perm == AccessPermission_Backing_Store) {
-                DataBlock& block = m_abs_cntrl_vec[i]->
-                    getDataBlock(line_address);
-
-                DPRINTF(RubySystem, "reading from %s block %s\n",
-                        m_abs_cntrl_vec[i]->name(), block);
-                for (unsigned j = 0; j < size_in_bytes; ++j) {
-                    data[j] = block.getByte(j + startByte);
-                }
+                m_abs_cntrl_vec[i]->functionalRead(line_address, pkt);
                 return true;
             }
         }
@@ -470,14 +458,7 @@ RubySystem::functionalRead(PacketPtr pkt)
             access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_address);
             if (access_perm == AccessPermission_Read_Only ||
                 access_perm == AccessPermission_Read_Write) {
-                DataBlock& block = m_abs_cntrl_vec[i]->
-                    getDataBlock(line_address);
-
-                DPRINTF(RubySystem, "reading from %s block %s\n",
-                        m_abs_cntrl_vec[i]->name(), block);
-                for (unsigned j = 0; j < size_in_bytes; ++j) {
-                    data[j] = block.getByte(j + startByte);
-                }
+                m_abs_cntrl_vec[i]->functionalRead(line_address, pkt);
                 return true;
             }
         }
@@ -500,10 +481,6 @@ RubySystem::functionalWrite(PacketPtr pkt)
 
     DPRINTF(RubySystem, "Functional Write request for %s\n",addr);
 
-    uint8_t *data = pkt->getPtr<uint8_t>(true);
-    unsigned int size_in_bytes = pkt->getSize();
-    unsigned startByte = addr.getAddress() - line_addr.getAddress();
-
     uint32_t M5_VAR_USED num_functional_writes = 0;
 
     for (unsigned int i = 0; i < num_controllers;++i) {
@@ -513,23 +490,11 @@ RubySystem::functionalWrite(PacketPtr pkt)
         access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_addr);
         if (access_perm != AccessPermission_Invalid &&
             access_perm != AccessPermission_NotPresent) {
-
-            num_functional_writes++;
-
-            DataBlock& block = m_abs_cntrl_vec[i]->getDataBlock(line_addr);
-            DPRINTF(RubySystem, "%s\n",block);
-            for (unsigned j = 0; j < size_in_bytes; ++j) {
-              block.setByte(j + startByte, data[j]);
-            }
-            DPRINTF(RubySystem, "%s\n",block);
+            num_functional_writes +=
+                m_abs_cntrl_vec[i]->functionalWrite(line_addr, pkt);
         }
     }
 
-    for (unsigned int i = 0; i < m_memory_controller_vec.size() ;++i) {
-        num_functional_writes +=
-            m_memory_controller_vec[i]->functionalWriteBuffers(pkt);
-    }
-
     num_functional_writes += m_network->functionalWrite(pkt);
     DPRINTF(RubySystem, "Messages written = %u\n", num_functional_writes);