Add in support for quiescing the system, taking checkpoints, restoring from checkpoints, changing memory modes, and switching CPUs.

Key new functions that can be called on the m5 object at the python interpreter:
doQuiesce(root) - A helper function that quiesces the object passed in and all of its children.
resume(root) - Another helper function that tells the object and all of its children that the quiesce is over.
checkpoint(root) - Takes a checkpoint of the system.  Checkpoint directory must be set before hand.
setCheckpointDir(name) - Sets the checkpoint directory.
restoreCheckpoint(root) - Restores the values from the checkpoint located in the checkpoint directory.
changeToAtomic(system) - Changes the system and all of its children to atomic memory mode.
changeToTiming(system) - Changes the system and all of its children to timing memory mode.
switchCpus(list) - Takes in a list of tuples, where each tuple is a pair of (old CPU, new CPU).  Quiesces the old CPUs, and then switches over to the new CPUs.

src/SConscript:
    Remove serializer, replaced by python code.
src/python/m5/__init__.py:
    Updates to support quiescing, checkpointing, changing memory modes, and switching CPUs.
src/python/m5/config.py:
    Several functions defined on the SimObject for quiescing, changing timing modes, and switching CPUs
src/sim/main.cc:
    Add some extra functions that are exported to python through SWIG.
src/sim/serialize.cc:
    Change serialization around a bit.  Now it is controlled through Python, so there's no need for SerializeEvents or SerializeParams.

    Also add in a new unserializeAll() function that loads a checkpoint and handles unserializing all objects.
src/sim/serialize.hh:
    Add unserializeAll function and a setCheckpointName function.
src/sim/sim_events.cc:
    Add process() function for CountedQuiesceEvent, which calls exitSimLoop() once its counter reaches 0.
src/sim/sim_events.hh:
    Add in a CountedQuiesceEvent, which is used when the system is preparing to quiesce.  Any objects that can't be quiesced immediately are given a pointer to a CountedQuiesceEvent.  The event has its counter set via Python, and as objects finish quiescing they call process() on the event.  Eventually the event causes the simulation to stop once all objects have quiesced.
src/sim/sim_object.cc:
    Add a few functions for quiescing, checkpointing, and changing memory modes.
src/sim/sim_object.hh:
    Add a state variable to all SimObjects that tracks both the timing mode of the object and the quiesce state of the object.  Currently this isn't serialized, and I'm not sure it needs to be so long as the timing mode starts up the same after a checkpoint.

--HG--
extra : convert_revision : a8c738d3911c68d5a7caf7de24d732dcc62cfb61

10 files changed, 284 insertions, 113 deletions

diff --git a/src/SConscript b/src/SConscript
index 124f88708..584643535 100644
--- a/src/SConscript
+++ b/src/SConscript
@@ -62,7 +62,6 @@ base_sources = Split('''
 	base/range.cc
 	base/random.cc
 	base/sat_counter.cc
-        base/serializer.cc
 	base/socket.cc
 	base/statistics.cc
 	base/str.cc
diff --git a/src/python/m5/__init__.py b/src/python/m5/__init__.py
index a7e653fc2..828165d15 100644
--- a/src/python/m5/__init__.py
+++ b/src/python/m5/__init__.py
@@ -34,7 +34,7 @@ import cc_main
 # import a few SWIG-wrapped items (those that are likely to be used
 # directly by user scripts) completely into this module for
 # convenience
-from cc_main import simulate, SimLoopExitEvent
+from cc_main import simulate, SimLoopExitEvent, setCheckpointDir
 
 # import the m5 compile options
 import defines
@@ -117,10 +117,6 @@ def debugBreak(option, opt_str, value, parser):
 def statsTextFile(option, opt_str, value, parser):
     objects.Statistics.text_file = value
 
-# Extra list to help for options that are true or false
-TrueOrFalse = ['True', 'False']
-TorF = "True | False"
-
 # Standard optparse options.  Need to be explicitly included by the
 # user script when it calls optparse.OptionParser().
 standardOptions = [
@@ -216,3 +212,81 @@ atexit.register(cc_main.doExitCleanup)
 # just doing an 'import m5' (without an 'import m5.objects').  May not
 # matter since most scripts will probably 'from m5.objects import *'.
 import objects
+
+def doQuiesce(root):
+    quiesce = cc_main.createCountedQuiesce()
+    unready_objects = root.startQuiesce(quiesce, True)
+    # If we've got some objects that can't quiesce immediately, then simulate
+    if unready_objects > 0:
+        quiesce.setCount(unready_objects)
+        simulate()
+    cc_main.cleanupCountedQuiesce(quiesce)
+
+def resume(root):
+    root.resume()
+
+def checkpoint(root):
+    if not isinstance(root, objects.Root):
+        raise TypeError, "Object is not a root object. Checkpoint must be called on a root object."
+    doQuiesce(root)
+    print "Writing checkpoint"
+    cc_main.serializeAll()
+    resume(root)
+
+def restoreCheckpoint(root):
+    print "Restoring from checkpoint"
+    cc_main.unserializeAll()
+
+def changeToAtomic(system):
+    if not isinstance(system, objects.Root) and not isinstance(system, System):
+        raise TypeError, "Object is not a root or system object.  Checkpoint must be "
+        "called on a root object."
+    doQuiesce(system)
+    print "Changing memory mode to atomic"
+    system.changeTiming(cc_main.SimObject.Atomic)
+    resume(system)
+
+def changeToTiming(system):
+    if not isinstance(system, objects.Root) and not isinstance(system, System):
+        raise TypeError, "Object is not a root or system object.  Checkpoint must be "
+        "called on a root object."
+    doQuiesce(system)
+    print "Changing memory mode to timing"
+    system.changeTiming(cc_main.SimObject.Timing)
+    resume(system)
+
+def switchCpus(cpuList):
+    if not isinstance(cpuList, list):
+        raise RuntimeError, "Must pass a list to this function"
+    for i in cpuList:
+        if not isinstance(i, tuple):
+            raise RuntimeError, "List must have tuples of (oldCPU,newCPU)"
+
+    [old_cpus, new_cpus] = zip(*cpuList)
+
+    for cpu in old_cpus:
+        if not isinstance(cpu, objects.BaseCPU):
+            raise TypeError, "%s is not of type BaseCPU", cpu
+    for cpu in new_cpus:
+        if not isinstance(cpu, objects.BaseCPU):
+            raise TypeError, "%s is not of type BaseCPU", cpu
+
+    # Quiesce all of the individual CPUs
+    quiesce = cc_main.createCountedQuiesce()
+    unready_cpus = 0
+    for old_cpu in old_cpus:
+        unready_cpus += old_cpu.startQuiesce(quiesce, False)
+    # If we've got some objects that can't quiesce immediately, then simulate
+    if unready_cpus > 0:
+        quiesce.setCount(unready_cpus)
+        simulate()
+    cc_main.cleanupCountedQuiesce(quiesce)
+    # Now all of the CPUs are ready to be switched out
+    for old_cpu in old_cpus:
+        old_cpu._ccObject.switchOut()
+    index = 0
+    print "Switching CPUs"
+    for new_cpu in new_cpus:
+        new_cpu.takeOverFrom(old_cpus[index])
+        new_cpu._ccObject.resume()
+        index += 1
diff --git a/src/python/m5/config.py b/src/python/m5/config.py
index c29477465..adabe0743 100644
--- a/src/python/m5/config.py
+++ b/src/python/m5/config.py
@@ -543,6 +543,33 @@ class SimObject(object):
         for child in self._children.itervalues():
             child.connectPorts()
 
+    def startQuiesce(self, quiesce_event, recursive):
+        count = 0
+        # ParamContexts don't serialize
+        if isinstance(self, SimObject) and not isinstance(self, ParamContext):
+            if self._ccObject.quiesce(quiesce_event):
+                count = 1
+        if recursive:
+            for child in self._children.itervalues():
+                count += child.startQuiesce(quiesce_event, True)
+        return count
+
+    def resume(self):
+        if isinstance(self, SimObject) and not isinstance(self, ParamContext):
+            self._ccObject.resume()
+        for child in self._children.itervalues():
+            child.resume()
+
+    def changeTiming(self, mode):
+        if isinstance(self, SimObject) and not isinstance(self, ParamContext):
+            self._ccObject.setMemoryMode(mode)
+        for child in self._children.itervalues():
+            child.changeTiming(mode)
+
+    def takeOverFrom(self, old_cpu):
+        cpu_ptr = cc_main.convertToBaseCPUPtr(old_cpu._ccObject)
+        self._ccObject.takeOverFrom(cpu_ptr)
+
     # generate output file for 'dot' to display as a pretty graph.
     # this code is currently broken.
     def outputDot(self, dot):
diff --git a/src/sim/main.cc b/src/sim/main.cc
index bf844da7f..3eb7fa95d 100644
--- a/src/sim/main.cc
+++ b/src/sim/main.cc
@@ -62,6 +62,7 @@
 #include "sim/async.hh"
 #include "sim/builder.hh"
 #include "sim/host.hh"
+#include "sim/serialize.hh"
 #include "sim/sim_events.hh"
 #include "sim/sim_exit.hh"
 #include "sim/sim_object.hh"
@@ -521,6 +522,37 @@ simulate(Tick num_cycles = -1)
     // not reached... only exit is return on SimLoopExitEvent
 }
 
+Event *
+createCountedQuiesce()
+{
+    return new CountedQuiesceEvent();
+}
+
+void
+cleanupCountedQuiesce(Event *counted_quiesce)
+{
+    CountedQuiesceEvent *event =
+        dynamic_cast<CountedQuiesceEvent *>(counted_quiesce);
+    if (event == NULL) {
+        fatal("Called cleanupCountedQuiesce() on an event that was not "
+              "a CountedQuiesceEvent.");
+    }
+    assert(event->getCount() == 0);
+    delete event;
+}
+
+void
+serializeAll()
+{
+    Serializable::serializeAll();
+}
+
+void
+unserializeAll()
+{
+    Serializable::unserializeAll();
+}
+
 /**
  * Queue of C++ callbacks to invoke on simulator exit.
  */
@@ -535,6 +567,16 @@ registerExitCallback(Callback *callback)
     exitCallbacks.add(callback);
 }
 
+BaseCPU *
+convertToBaseCPUPtr(SimObject *obj)
+{
+    BaseCPU *ptr = dynamic_cast<BaseCPU *>(obj);
+
+    if (ptr == NULL)
+        warn("Casting to BaseCPU pointer failed");
+    return ptr;
+}
+
 /**
  * Do C++ simulator exit processing.  Exported to SWIG to be invoked
  * when simulator terminates via Python's atexit mechanism.
diff --git a/src/sim/serialize.cc b/src/sim/serialize.cc
index 0e3139116..7450d7b7e 100644
--- a/src/sim/serialize.cc
+++ b/src/sim/serialize.cc
@@ -244,56 +244,41 @@ Serializable::serializeAll()
 
     globals.serialize(outstream);
     SimObject::serializeAll(outstream);
-
-    assert(Serializable::ckptPrevCount + 1 == Serializable::ckptCount);
-    Serializable::ckptPrevCount++;
-    if (ckptMaxCount && ++ckptCount >= ckptMaxCount)
-        exitSimLoop(curTick + 1, "Maximum number of checkpoints dropped");
-
 }
 
-
 void
-Serializable::unserializeGlobals(Checkpoint *cp)
-{
-    globals.unserialize(cp);
-}
-
-
-class SerializeEvent : public Event
+Serializable::unserializeAll()
 {
-  protected:
-    Tick repeat;
-
-  public:
-    SerializeEvent(Tick _when, Tick _repeat);
-    virtual void process();
-    virtual void serialize(std::ostream &os)
-    {
-        panic("Cannot serialize the SerializeEvent");
-    }
+    string dir = Checkpoint::dir();
+    string cpt_file = dir + Checkpoint::baseFilename;
+    string section = "";
 
-};
+    DPRINTFR(Config, "Loading checkpoint dir '%s'\n",
+             dir);
+    Checkpoint *cp = new Checkpoint(dir, section);
+    unserializeGlobals(cp);
 
-SerializeEvent::SerializeEvent(Tick _when, Tick _repeat)
-    : Event(&mainEventQueue, Serialize_Pri), repeat(_repeat)
-{
-    setFlags(AutoDelete);
-    schedule(_when);
+    SimObject::unserializeAll(cp);
 }
 
 void
-SerializeEvent::process()
+Serializable::unserializeGlobals(Checkpoint *cp)
 {
-    Serializable::serializeAll();
-    if (repeat)
-        schedule(curTick + repeat);
+    globals.unserialize(cp);
 }
 
 const char *Checkpoint::baseFilename = "m5.cpt";
 
 static string checkpointDirBase;
 
+void
+setCheckpointDir(const std::string &name)
+{
+    checkpointDirBase = name;
+    if (checkpointDirBase[checkpointDirBase.size() - 1] != '/')
+        checkpointDirBase += "/";
+}
+
 string
 Checkpoint::dir()
 {
@@ -304,75 +289,11 @@ Checkpoint::dir()
 }
 
 void
-Checkpoint::setup(Tick when, Tick period)
-{
-    new SerializeEvent(when, period);
-}
-
-class SerializeParamContext : public ParamContext
-{
-  private:
-    SerializeEvent *event;
-
-  public:
-    SerializeParamContext(const string &section);
-    ~SerializeParamContext();
-    void checkParams();
-};
-
-SerializeParamContext serialParams("serialize");
-
-Param<string> serialize_dir(&serialParams, "dir",
-                            "dir to stick checkpoint in "
-                            "(sprintf format with cycle #)");
-
-Param<Counter> serialize_cycle(&serialParams,
-                                "cycle",
-                                "cycle to serialize",
-                                0);
-
-Param<Counter> serialize_period(&serialParams,
-                                "period",
-                                "period to repeat serializations",
-                                0);
-
-Param<int> serialize_count(&serialParams, "count",
-                           "maximum number of checkpoints to drop");
-
-SerializeParamContext::SerializeParamContext(const string &section)
-    : ParamContext(section), event(NULL)
-{ }
-
-SerializeParamContext::~SerializeParamContext()
-{
-}
-
-void
-SerializeParamContext::checkParams()
-{
-    checkpointDirBase = simout.resolve(serialize_dir);
-
-    // guarantee that directory ends with a '/'
-    if (checkpointDirBase[checkpointDirBase.size() - 1] != '/')
-        checkpointDirBase += "/";
-
-    if (serialize_cycle > 0)
-        Checkpoint::setup(serialize_cycle, serialize_period);
-
-    Serializable::ckptMaxCount = serialize_count;
-}
-
-void
 debug_serialize()
 {
     Serializable::serializeAll();
 }
 
-void
-debug_serialize(Tick when)
-{
-    new SerializeEvent(when, 0);
-}
 
 ////////////////////////////////////////////////////////////////////////
 //
diff --git a/src/sim/serialize.hh b/src/sim/serialize.hh
index 64ed6142f..5a820b27e 100644
--- a/src/sim/serialize.hh
+++ b/src/sim/serialize.hh
@@ -127,6 +127,7 @@ class Serializable
     static int ckptMaxCount;
     static int ckptPrevCount;
     static void serializeAll();
+    static void unserializeAll();
     static void unserializeGlobals(Checkpoint *cp);
 };
 
@@ -204,6 +205,9 @@ class SerializableClass
 SerializableClass the##OBJ_CLASS##Class(CLASS_NAME,			   \
                                          OBJ_CLASS::createForUnserialize);
 
+void
+setCheckpointName(const std::string &name);
+
 class Checkpoint
 {
   private:
diff --git a/src/sim/sim_events.cc b/src/sim/sim_events.cc
index b7901832d..97f7ae03c 100644
--- a/src/sim/sim_events.cc
+++ b/src/sim/sim_events.cc
@@ -78,6 +78,14 @@ exitSimLoop(const std::string &message, int exit_code)
     exitSimLoop(curTick, message, exit_code);
 }
 
+void
+CountedQuiesceEvent::process()
+{
+    if (--count == 0) {
+        exitSimLoop("Finished quiesce");
+    }
+}
+
 //
 // constructor: automatically schedules at specified time
 //
diff --git a/src/sim/sim_events.hh b/src/sim/sim_events.hh
index 4f305ad38..50368f258 100644
--- a/src/sim/sim_events.hh
+++ b/src/sim/sim_events.hh
@@ -44,6 +44,11 @@ class SimLoopExitEvent : public Event
     int code;
 
   public:
+    // Default constructor.  Only really used for derived classes.
+    SimLoopExitEvent()
+        : Event(&mainEventQueue, Sim_Exit_Pri)
+    { }
+
     SimLoopExitEvent(Tick _when, const std::string &_cause, int c = 0)
         : Event(&mainEventQueue, Sim_Exit_Pri), cause(_cause),
           code(c)
@@ -62,6 +67,22 @@ class SimLoopExitEvent : public Event
     virtual const char *description();
 };
 
+class CountedQuiesceEvent : public SimLoopExitEvent
+{
+  private:
+    // Count down to quiescing
+    int count;
+  public:
+    CountedQuiesceEvent()
+        : count(0)
+    { }
+    void process();
+
+    void setCount(int _count) { count = _count; }
+
+    int getCount() { return count; }
+};
+
 //
 // Event class to terminate simulation after 'n' related events have
 // occurred using a shared counter: used to terminate when *all*
diff --git a/src/sim/sim_object.cc b/src/sim/sim_object.cc
index 97e6de439..5752e4ef1 100644
--- a/src/sim/sim_object.cc
+++ b/src/sim/sim_object.cc
@@ -73,6 +73,7 @@ SimObject::SimObject(Params *p)
 
     doRecordEvent = !Stats::event_ignore.match(name());
     simObjectList.push_back(this);
+    state = Atomic;
 }
 
 //
@@ -88,6 +89,7 @@ SimObject::SimObject(const string &_name)
 
     doRecordEvent = !Stats::event_ignore.match(name());
     simObjectList.push_back(this);
+    state = Atomic;
 }
 
 void
@@ -219,6 +221,24 @@ SimObject::serializeAll(ostream &os)
    }
 }
 
+void
+SimObject::unserializeAll(Checkpoint *cp)
+{
+    SimObjectList::reverse_iterator ri = simObjectList.rbegin();
+    SimObjectList::reverse_iterator rend = simObjectList.rend();
+
+    for (; ri != rend; ++ri) {
+        SimObject *obj = *ri;
+        DPRINTFR(Config, "Unserializing '%s'\n",
+                 obj->name());
+        if(cp->sectionExists(obj->name()))
+            obj->unserialize(cp, obj->name());
+        else
+            warn("Not unserializing '%s': no section found in checkpoint.\n",
+                 obj->name());
+   }
+}
+
 #ifdef DEBUG
 //
 // static function: flag which objects should have the debugger break
@@ -251,10 +271,50 @@ SimObject::recordEvent(const std::string &stat)
         Stats::recordEvent(stat);
 }
 
+bool
+SimObject::quiesce(Event *quiesce_event)
+{
+    if (state != QuiescedAtomic && state != Atomic) {
+        panic("Must implement your own quiesce function if it is to be used "
+              "in timing mode!");
+    }
+    state = QuiescedAtomic;
+    return false;
+}
+
+void
+SimObject::resume()
+{
+    if (state == QuiescedAtomic) {
+        state = Atomic;
+    } else if (state == QuiescedTiming) {
+        state = Timing;
+    }
+}
+
+void
+SimObject::setMemoryMode(State new_mode)
+{
+    assert(new_mode == Timing || new_mode == Atomic);
+    if (state == QuiescedAtomic && new_mode == Timing) {
+        state = QuiescedTiming;
+    } else if (state == QuiescedTiming && new_mode == Atomic) {
+        state = QuiescedAtomic;
+    } else {
+        state = new_mode;
+    }
+}
+
+void
+SimObject::switchOut()
+{
+    panic("Unimplemented!");
+}
+
 void
-SimObject::drain(Serializer *serializer)
+SimObject::takeOverFrom(BaseCPU *cpu)
 {
-    serializer->signalDrained();
+    panic("Unimplemented!");
 }
 
 DEFINE_SIM_OBJECT_CLASS_NAME("SimObject", SimObject)
diff --git a/src/sim/sim_object.hh b/src/sim/sim_object.hh
index 84e9376a0..e0b21782f 100644
--- a/src/sim/sim_object.hh
+++ b/src/sim/sim_object.hh
@@ -44,7 +44,8 @@
 #include "sim/serialize.hh"
 #include "sim/startup.hh"
 
-class Serializer;
+class BaseCPU;
+class Event;
 
 /*
  * Abstract superclass for simulation objects.  Represents things that
@@ -58,15 +59,26 @@ class SimObject : public Serializable, protected StartupCallback
         std::string name;
     };
 
+    enum State {
+        Atomic,
+        Timing,
+        Quiescing,
+        QuiescedAtomic,
+        QuiescedTiming
+    };
+
   protected:
     Params *_params;
+    State state;
+
+    void changeState(State new_state) { state = new_state; }
 
   public:
     const Params *params() const { return _params; }
 
-  private:
-    friend class Serializer;
+    State getState() { return state; }
 
+  private:
     typedef std::vector<SimObject *> SimObjectList;
 
     // list of all instantiated simulation objects
@@ -100,13 +112,16 @@ class SimObject : public Serializable, protected StartupCallback
 
     // static: call nameOut() & serialize() on all SimObjects
     static void serializeAll(std::ostream &);
+    static void unserializeAll(Checkpoint *cp);
 
     // Methods to drain objects in order to take checkpoints
     // Or switch from timing -> atomic memory model
-    virtual void drain(Serializer *serializer);
-    virtual void resume() { return;} ;
-    virtual void serializationComplete()
-    { assert(0 && "Unimplemented"); };
+    // Quiesce returns true if the SimObject cannot quiesce immediately.
+    virtual bool quiesce(Event *quiesce_event);
+    virtual void resume();
+    virtual void setMemoryMode(State new_mode);
+    virtual void switchOut();
+    virtual void takeOverFrom(BaseCPU *cpu);
 
 #ifdef DEBUG
   public: