sim-ruby: checkpointing fixes and dependent eventq improvements

Fixes checkpointing with respect to lost events after swapping event queues.
Also adds DPRINTFs to better understand what's going on when Ruby serializes
and unserializes.

6 files changed, 68 insertions, 23 deletions

diff --git a/src/mem/ruby/system/CacheMemory.cc b/src/mem/ruby/system/CacheMemory.cc
index 393612355..9144a8dff 100644
--- a/src/mem/ruby/system/CacheMemory.cc
+++ b/src/mem/ruby/system/CacheMemory.cc
@@ -28,6 +28,7 @@
 
 #include "base/intmath.hh"
 #include "debug/RubyCache.hh"
+#include "debug/RubyCacheTrace.hh"
 #include "mem/protocol/AccessPermission.hh"
 #include "mem/ruby/system/CacheMemory.hh"
 #include "mem/ruby/system/System.hh"
@@ -398,7 +399,7 @@ CacheMemory::recordCacheContents(int cntrl, CacheRecorder* tr) const
         }
     }
 
-    DPRINTF(RubyCache, "%s: %lli blocks of %lli total blocks"
+    DPRINTF(RubyCacheTrace, "%s: %lli blocks of %lli total blocks"
             "recorded %.2f%% \n", name().c_str(), warmedUpBlocks,
             (uint64)m_cache_num_sets * (uint64)m_cache_assoc,
             (float(warmedUpBlocks)/float(totalBlocks))*100.0);
diff --git a/src/mem/ruby/system/MemoryControl.cc b/src/mem/ruby/system/MemoryControl.cc
index 2ab0736e5..4e5ebdbe9 100644
--- a/src/mem/ruby/system/MemoryControl.cc
+++ b/src/mem/ruby/system/MemoryControl.cc
@@ -144,7 +144,7 @@ operator<<(ostream& out, const MemoryControl& obj)
 
 // CONSTRUCTOR
 MemoryControl::MemoryControl(const Params *p)
-    : SimObject(p)
+    : SimObject(p), m_event(this)
 {
     m_mem_bus_cycle_multiplier = p->mem_bus_cycle_multiplier;
     m_banks_per_rank = p->banks_per_rank;
@@ -204,7 +204,6 @@ MemoryControl::init()
     m_refresh_count = 1;
     m_need_refresh = 0;
     m_refresh_bank = 0;
-    m_awakened = 0;
     m_idleCount = 0;
     m_ageCounter = 0;
 
@@ -253,16 +252,17 @@ MemoryControl::enqueueMemRef(MemoryNode& memRef)
     physical_address_t addr = memRef.m_addr;
     int bank = getBank(addr);
 
-    DPRINTF(RubyMemory, "New memory request%7d: %#08x %c arrived at %10d bank = %3x\n",
+    DPRINTF(RubyMemory,
+            "New memory request%7d: %#08x %c arrived at %10d bank = %3x sched %c\n",
             m_msg_counter, addr, memRef.m_is_mem_read ? 'R':'W',
             memRef.m_time * g_eventQueue_ptr->getClock(),
-            bank);
+            bank, m_event.scheduled() ? 'Y':'N');
 
     m_profiler_ptr->profileMemReq(bank);
     m_input_queue.push_back(memRef);
-    if (!m_awakened) {
-        g_eventQueue_ptr->scheduleEvent(this, 1);
-        m_awakened = 1;
+
+    if (!m_event.scheduled()) {
+        schedule(m_event, curTick() + 1);
     }
 }
 
@@ -289,8 +289,9 @@ MemoryControl::peekNode()
 {
     assert(isReady());
     MemoryNode req = m_response_queue.front();
-    DPRINTF(RubyMemory, "Peek: memory request%7d: %#08x %c\n",
-            req.m_msg_counter, req.m_addr, req.m_is_mem_read ? 'R':'W');
+    DPRINTF(RubyMemory, "Peek: memory request%7d: %#08x %c sched %c\n",
+            req.m_msg_counter, req.m_addr, req.m_is_mem_read ? 'R':'W',
+            m_event.scheduled() ? 'Y':'N');
 
     return req;
 }
@@ -513,9 +514,9 @@ MemoryControl::issueRequest(int bank)
     m_bankQueues[bank].pop_front();
 
     DPRINTF(RubyMemory, "Mem issue request%7d: %#08x %c "
-            "bank=%3x\n", req.m_msg_counter, req.m_addr,
+            "bank=%3x sched %c\n", req.m_msg_counter, req.m_addr,
             req.m_is_mem_read? 'R':'W',
-            bank);
+            bank, m_event.scheduled() ? 'Y':'N');
 
     if (req.m_msgptr) {  // don't enqueue L3 writebacks
         enqueueToDirectory(req, m_mem_ctl_latency + m_mem_fixed_delay);
@@ -642,19 +643,28 @@ MemoryControl::executeCycle()
     }
 }
 
+unsigned int
+MemoryControl::drain(Event *de)
+{
+    DPRINTF(RubyMemory, "MemoryController drain\n");
+    if(m_event.scheduled()) {
+        deschedule(m_event);
+    }
+    return 0;
+}
+
 // wakeup:  This function is called once per memory controller clock cycle.
 void
 MemoryControl::wakeup()
 {
+    DPRINTF(RubyMemory, "MemoryController wakeup\n");
     // execute everything
     executeCycle();
 
     m_idleCount--;
-    if (m_idleCount <= 0) {
-        m_awakened = 0;
-    } else {
-        // Reschedule ourselves so that we run every memory cycle:
-        g_eventQueue_ptr->scheduleEvent(this, m_mem_bus_cycle_multiplier);
+    if (m_idleCount > 0) {
+        assert(!m_event.scheduled());
+        schedule(m_event, curTick() + m_mem_bus_cycle_multiplier);
     }
 }
 
diff --git a/src/mem/ruby/system/MemoryControl.hh b/src/mem/ruby/system/MemoryControl.hh
index 1534851d5..48ce8a8e0 100644
--- a/src/mem/ruby/system/MemoryControl.hh
+++ b/src/mem/ruby/system/MemoryControl.hh
@@ -54,12 +54,15 @@ class MemoryControl :
     public SimObject, public Consumer, public AbstractMemOrCache
 {
   public:
+
     typedef RubyMemoryControlParams Params;
     MemoryControl(const Params *p);
     void init();
 
     ~MemoryControl();
 
+    unsigned int drain(Event *de);
+
     void wakeup();
 
     void setConsumer(Consumer* consumer_ptr);
@@ -90,6 +93,19 @@ class MemoryControl :
     int getDimmsPerChannel() { return m_dimms_per_channel; }
 
   private:
+    class MemCntrlEvent : public Event
+    {
+      public:
+        MemCntrlEvent(MemoryControl* _mem_cntrl)
+        {
+            mem_cntrl = _mem_cntrl;
+        }
+      private:
+        void process() { mem_cntrl->wakeup(); }
+
+        MemoryControl* mem_cntrl;
+    };
+
     void enqueueToDirectory(MemoryNode req, int latency);
     int getBank(physical_address_t addr);
     int getRank(int bank);
@@ -107,7 +123,6 @@ class MemoryControl :
     Consumer* m_consumer_ptr;  // Consumer to signal a wakeup()
     std::string m_description;
     int m_msg_counter;
-    int m_awakened;
 
     int m_mem_bus_cycle_multiplier;
     int m_banks_per_rank;
@@ -159,6 +174,8 @@ class MemoryControl :
     int m_idleCount;          // watchdog timer for shutting down
 
     MemCntrlProfiler* m_profiler_ptr;
+
+    MemCntrlEvent m_event;
 };
 
 #endif // __MEM_RUBY_SYSTEM_MEMORY_CONTROL_HH__
diff --git a/src/mem/ruby/system/MemoryVector.hh b/src/mem/ruby/system/MemoryVector.hh
index 9bd3516c2..315e00942 100644
--- a/src/mem/ruby/system/MemoryVector.hh
+++ b/src/mem/ruby/system/MemoryVector.hh
@@ -30,6 +30,7 @@
 #define __MEM_RUBY_SYSTEM_MEMORYVECTOR_HH__
 
 #include "base/trace.hh"
+#include "debug/RubyCacheTrace.hh"
 #include "mem/ruby/common/Address.hh"
 
 class DirectoryMemory;
@@ -184,6 +185,8 @@ MemoryVector::collatePages(uint8* &raw_data)
     memcpy(raw_data, &m_num_pages, sizeof(uint32));
     data_size = sizeof(uint32);
 
+    DPRINTF(RubyCacheTrace, "collating %d pages\n", m_num_pages);
+
     for (uint32 i = 0;i < m_num_pages; ++i)
     {
         if (m_pages[i] == 0) {
@@ -217,6 +220,8 @@ MemoryVector::populatePages(uint8* raw_data)
     data_size = sizeof(uint32);
     assert(num_pages == m_num_pages);
 
+    DPRINTF(RubyCacheTrace, "Populating %d pages\n", num_pages);
+
     for (uint32 i = 0;i < m_num_pages; ++i)
     {
         assert(m_pages[i] == 0);
diff --git a/src/mem/ruby/system/System.cc b/src/mem/ruby/system/System.cc
index 6f191819b..078e35492 100644
--- a/src/mem/ruby/system/System.cc
+++ b/src/mem/ruby/system/System.cc
@@ -33,11 +33,12 @@
 
 #include "base/intmath.hh"
 #include "base/output.hh"
-#include "debug/RubySystem.hh"
+#include "debug/RubyCacheTrace.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/network/Network.hh"
 #include "mem/ruby/profiler/Profiler.hh"
 #include "mem/ruby/system/System.hh"
+#include "sim/eventq.hh"
 #include "sim/simulate.hh"
 
 using namespace std;
@@ -224,6 +225,7 @@ RubySystem::serialize(std::ostream &os)
         }
     }
 
+    DPRINTF(RubyCacheTrace, "Recording Cache Trace\n");
     // Create the CacheRecorder and record the cache trace
     m_cache_recorder = new CacheRecorder(NULL, 0, sequencer_map);
 
@@ -231,15 +233,19 @@ RubySystem::serialize(std::ostream &os)
         m_abs_cntrl_vec[cntrl]->recordCacheTrace(cntrl, m_cache_recorder);
     }
 
+    DPRINTF(RubyCacheTrace, "Cache Trace Complete\n");
     // save the current tick value
     Tick curtick_original = curTick();
     // save the event queue head
     Event* eventq_head = eventq->replaceHead(NULL);
+    DPRINTF(RubyCacheTrace, "Recording current tick %ld and event queue\n",
+            curtick_original);
 
     // Schedule an event to start cache cooldown
-    RubyEvent* e = new RubyEvent(this);
-    schedule(e,curTick());
+    DPRINTF(RubyCacheTrace, "Starting cache flush\n");
+    enqueueRubyEvent(curTick());
     simulate();
+    DPRINTF(RubyCacheTrace, "Cache flush complete\n");
 
     // Restore eventq head
     eventq_head = eventq->replaceHead(eventq_head);
@@ -378,8 +384,7 @@ RubySystem::startup()
         curTick(0);
 
         // Schedule an event to start cache warmup
-        RubyEvent* e = new RubyEvent(this);
-        schedule(e,curTick());
+        enqueueRubyEvent(curTick());
         simulate();
 
         delete m_cache_recorder;
diff --git a/src/mem/ruby/system/System.hh b/src/mem/ruby/system/System.hh
index 461abffe2..d7d01bcac 100644
--- a/src/mem/ruby/system/System.hh
+++ b/src/mem/ruby/system/System.hh
@@ -129,6 +129,13 @@ class RubySystem : public SimObject
     void registerAbstractController(AbstractController*);
     void registerSparseMemory(SparseMemory*);
 
+    bool eventQueueEmpty() { return eventq->empty(); }
+    void enqueueRubyEvent(Tick tick)
+    {
+        RubyEvent* e = new RubyEvent(this);
+        schedule(e, tick);
+    }
+
   private:
     // Private copy constructor and assignment operator
     RubySystem(const RubySystem& obj);