sim: simulate with multiple threads and event queues

This patch adds support for simulating with multiple threads, each of
which operates on an event queue.  Each sim object specifies which eventq
is would like to be on.  A custom barrier implementation is being added
using which eventqs synchronize.

The patch was tested in two different configurations:
1. ruby_network_test.py: in this simulation L1 cache controllers receive
   requests from the cpu. The requests are replied to immediately without
   any communication taking place with any other level.
2. twosys-tsunami-simple-atomic: this configuration simulates a client-server
   system which are connected by an ethernet link.

We still lack the ability to communicate using message buffers or ports. But
other things like simulation start and end, synchronizing after every quantum
are working.

Committed by: Nilay Vaish

1 files changed, 152 insertions, 73 deletions

diff --git a/src/sim/eventq.hh b/src/sim/eventq.hh
index 223b4941c..66b324c4f 100644
--- a/src/sim/eventq.hh
+++ b/src/sim/eventq.hh
@@ -1,5 +1,7 @@
 /*
  * Copyright (c) 2000-2005 The Regents of The University of Michigan
+ * Copyright (c) 2013 Advanced Micro Devices, Inc.
+ * Copyright (c) 2013 Mark D. Hill and David A. Wood
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -40,6 +42,7 @@
 #include <cassert>
 #include <climits>
 #include <iosfwd>
+#include <mutex>
 #include <string>
 
 #include "base/flags.hh"
@@ -49,20 +52,48 @@
 #include "sim/serialize.hh"
 
 class EventQueue;       // forward declaration
+class BaseGlobalEvent;
 
-extern EventQueue mainEventQueue;
+//! Simulation Quantum for multiple eventq simulation.
+//! The quantum value is the period length after which the queues
+//! synchronize themselves with each other. This means that any
+//! event to scheduled on Queue A which is generated by an event on
+//! Queue B should be at least simQuantum ticks away in future.
+extern Tick simQuantum;
 
-/*
- * An item on an event queue.  The action caused by a given
- * event is specified by deriving a subclass and overriding the
- * process() member function.
- *
- * Caution, the order of members is chosen to maximize data packing.
+//! Current number of allocated main event queues.
+extern uint32_t numMainEventQueues;
+
+//! Array for main event queues.
+extern std::vector<EventQueue *> mainEventQueue;
+
+#ifndef SWIG
+//! The current event queue for the running thread. Access to this queue
+//! does not require any locking from the thread.
+
+extern __thread EventQueue *_curEventQueue;
+
+#endif
+
+//! Current mode of execution: parallel / serial
+extern bool inParallelMode;
+
+//! Function for returning eventq queue for the provided
+//! index. The function allocates a new queue in case one
+//! does not exist for the index, provided that the index
+//! is with in bounds.
+EventQueue *getEventQueue(uint32_t index);
+
+inline EventQueue *curEventQueue() { return _curEventQueue; }
+inline void curEventQueue(EventQueue *q) { _curEventQueue = q; }
+
+/**
+ * Common base class for Event and GlobalEvent, so they can share flag
+ * and priority definitions and accessor functions.  This class should
+ * not be used directly.
  */
-class Event : public Serializable
+class EventBase
 {
-    friend class EventQueue;
-
   protected:   
     typedef unsigned short FlagsType;
     typedef ::Flags<FlagsType> Flags;
@@ -78,15 +109,76 @@ class Event : public Serializable
     static const FlagsType Initialized   = 0x7a40; // somewhat random bits
     static const FlagsType InitMask      = 0xffc0; // mask for init bits
 
-    bool
-    initialized() const
-    {
-        return this && (flags & InitMask) == Initialized;
-    }
-
   public:
     typedef int8_t Priority;
 
+    /// Event priorities, to provide tie-breakers for events scheduled
+    /// at the same cycle.  Most events are scheduled at the default
+    /// priority; these values are used to control events that need to
+    /// be ordered within a cycle.
+
+    /// Minimum priority
+    static const Priority Minimum_Pri =          SCHAR_MIN;
+
+    /// If we enable tracing on a particular cycle, do that as the
+    /// very first thing so we don't miss any of the events on
+    /// that cycle (even if we enter the debugger).
+    static const Priority Debug_Enable_Pri =          -101;
+
+    /// Breakpoints should happen before anything else (except
+    /// enabling trace output), so we don't miss any action when
+    /// debugging.
+    static const Priority Debug_Break_Pri =           -100;
+
+    /// CPU switches schedule the new CPU's tick event for the
+    /// same cycle (after unscheduling the old CPU's tick event).
+    /// The switch needs to come before any tick events to make
+    /// sure we don't tick both CPUs in the same cycle.
+    static const Priority CPU_Switch_Pri =             -31;
+
+    /// For some reason "delayed" inter-cluster writebacks are
+    /// scheduled before regular writebacks (which have default
+    /// priority).  Steve?
+    static const Priority Delayed_Writeback_Pri =       -1;
+
+    /// Default is zero for historical reasons.
+    static const Priority Default_Pri =                  0;
+
+    /// Serailization needs to occur before tick events also, so
+    /// that a serialize/unserialize is identical to an on-line
+    /// CPU switch.
+    static const Priority Serialize_Pri =               32;
+
+    /// CPU ticks must come after other associated CPU events
+    /// (such as writebacks).
+    static const Priority CPU_Tick_Pri =                50;
+
+    /// Statistics events (dump, reset, etc.) come after
+    /// everything else, but before exit.
+    static const Priority Stat_Event_Pri =              90;
+
+    /// Progress events come at the end.
+    static const Priority Progress_Event_Pri =          95;
+
+    /// If we want to exit on this cycle, it's the very last thing
+    /// we do.
+    static const Priority Sim_Exit_Pri =               100;
+
+    /// Maximum priority
+    static const Priority Maximum_Pri =          SCHAR_MAX;
+};
+
+/*
+ * An item on an event queue.  The action caused by a given
+ * event is specified by deriving a subclass and overriding the
+ * process() member function.
+ *
+ * Caution, the order of members is chosen to maximize data packing.
+ */
+class Event : public EventBase, public Serializable
+{
+    friend class EventQueue;
+
   private:
     // The event queue is now a linked list of linked lists.  The
     // 'nextBin' pointer is to find the bin, where a bin is defined as
@@ -139,6 +231,12 @@ class Event : public Serializable
 #endif
     }
 
+    bool
+    initialized() const
+    {
+        return this && (flags & InitMask) == Initialized;
+    }
+
   protected:
     /// Accessor for flags.
     Flags
@@ -179,60 +277,6 @@ class Event : public Serializable
     virtual void trace(const char *action);     //!< trace event activity
 
   public:
-    /// Event priorities, to provide tie-breakers for events scheduled
-    /// at the same cycle.  Most events are scheduled at the default
-    /// priority; these values are used to control events that need to
-    /// be ordered within a cycle.
-
-    /// Minimum priority
-    static const Priority Minimum_Pri =          SCHAR_MIN;
-
-    /// If we enable tracing on a particular cycle, do that as the
-    /// very first thing so we don't miss any of the events on
-    /// that cycle (even if we enter the debugger).
-    static const Priority Debug_Enable_Pri =          -101;
-
-    /// Breakpoints should happen before anything else (except
-    /// enabling trace output), so we don't miss any action when
-    /// debugging.
-    static const Priority Debug_Break_Pri =           -100;
-
-    /// CPU switches schedule the new CPU's tick event for the
-    /// same cycle (after unscheduling the old CPU's tick event).
-    /// The switch needs to come before any tick events to make
-    /// sure we don't tick both CPUs in the same cycle.
-    static const Priority CPU_Switch_Pri =             -31;
-
-    /// For some reason "delayed" inter-cluster writebacks are
-    /// scheduled before regular writebacks (which have default
-    /// priority).  Steve?
-    static const Priority Delayed_Writeback_Pri =       -1;
-
-    /// Default is zero for historical reasons.
-    static const Priority Default_Pri =                  0;
-
-    /// Serailization needs to occur before tick events also, so
-    /// that a serialize/unserialize is identical to an on-line
-    /// CPU switch.
-    static const Priority Serialize_Pri =               32;
-
-    /// CPU ticks must come after other associated CPU events
-    /// (such as writebacks).
-    static const Priority CPU_Tick_Pri =                50;
-
-    /// Statistics events (dump, reset, etc.) come after
-    /// everything else, but before exit.
-    static const Priority Stat_Event_Pri =              90;
-
-    /// Progress events come at the end.
-    static const Priority Progress_Event_Pri =          95;
-
-    /// If we want to exit on this cycle, it's the very last thing
-    /// we do.
-    static const Priority Sim_Exit_Pri =               100;
-
-    /// Maximum priority
-    static const Priority Maximum_Pri =          SCHAR_MAX;
 
     /*
      * Event constructor
@@ -295,9 +339,21 @@ class Event : public Serializable
     /// Get the event priority
     Priority priority() const { return _priority; }
 
+    //! If this is part of a GlobalEvent, return the pointer to the
+    //! Global Event.  By default, there is no GlobalEvent, so return
+    //! NULL.  (Overridden in GlobalEvent::BarrierEvent.)
+    virtual BaseGlobalEvent *globalEvent() { return NULL; }
+
 #ifndef SWIG
     virtual void serialize(std::ostream &os);
     virtual void unserialize(Checkpoint *cp, const std::string &section);
+
+    //! This function is required to support restoring from checkpoints
+    //! when running with multiple queues. Since we still have not thrashed
+    //! out all the details on checkpointing, this function is most likely
+    //! to be revisited in future.
+    virtual void unserialize(Checkpoint *cp, const std::string &section,
+                     EventQueue *eventq);
 #endif
 };
 
@@ -352,20 +408,40 @@ class EventQueue : public Serializable
     Event *head;
     Tick _curTick;
 
+    //! Mutex to protect async queue.
+    std::mutex *async_queue_mutex;
+
+    //! List of events added by other threads to this event queue.
+    std::list<Event*> async_queue;
+
+    //! Insert / remove event from the queue. Should only be called
+    //! by thread operating this queue.
     void insert(Event *event);
     void remove(Event *event);
 
+    //! Function for adding events to the async queue. The added events
+    //! are added to main event queue later. Threads, other than the
+    //! owning thread, should call this function instead of insert().
+    void asyncInsert(Event *event);
+
     EventQueue(const EventQueue &);
-    const EventQueue &operator=(const EventQueue &);
 
   public:
     EventQueue(const std::string &n);
 
     virtual const std::string name() const { return objName; }
+    void name(const std::string &st) { objName = st; }
+
+    //! Schedule the given event on this queue. Safe to call from any
+    //! thread.
+    void schedule(Event *event, Tick when, bool global = false);
 
-    // schedule the given event on this queue
-    void schedule(Event *event, Tick when);
+    //! Deschedule the specified event. Should be called only from the
+    //! owning thread.
     void deschedule(Event *event);
+
+    //! Reschedule the specified event. Should be called only from
+    //! the owning thread.
     void reschedule(Event *event, Tick when, bool always = false);
 
     Tick nextTick() const { return head->when(); }
@@ -402,6 +478,9 @@ class EventQueue : public Serializable
 
     bool debugVerify() const;
 
+    //! Function for moving events from the async_queue to the main queue.
+    void handleAsyncInsertions();
+
     /**
      *  function for replacing the head of the event queue, so that a
      *  different set of events can run without disturbing events that have