kvm: Add support for multi-system simulation

The introduction of parallel event queues added most of the support
needed to run multiple VMs (systems) within the same gem5
instance. This changeset fixes up signal delivery so that KVM's
control signals are delivered to the thread that executes the CPU's
event queue. Specifically:

  * Timers and counters are now initialized from a separate method
    (startupThread) that is scheduled as the first event in the
    thread-specific event queue. This ensures that they are
    initialized from the thread that is going to execute the CPUs
    event queue and enables signal delivery to the right thread when
    exiting from KVM.

  * The POSIX-timer-based KVM timer (used to force exits from KVM) has
    been updated to deliver signals to the thread that's executing KVM
    instead of the process (thread is undefined in that case). This
    assumes that the timer is instantiated from the thread that is
    going to execute the KVM vCPU.

  * Signal masking is now done using pthread_sigmask instead of
    sigprocmask. The behavior of the latter is undefined in threaded
    applications.

  * Since signal masks can be inherited, make sure to actively unmask
    the control signals when setting up the KVM signal mask.

There are currently no facilities to multiplex between multiple KVM
CPUs in the same event queue, we are therefore limited to
configurations where there is only one KVM CPU per event queue. In
practice, this means that multi-system configurations can be
simulated, but not multiple CPUs in a shared-memory configuration.

3 files changed, 78 insertions, 22 deletions

diff --git a/src/cpu/kvm/base.cc b/src/cpu/kvm/base.cc
index d25e145a5..64c1a5e81 100644
--- a/src/cpu/kvm/base.cc
+++ b/src/cpu/kvm/base.cc
@@ -63,7 +63,7 @@
 /* Used by some KVM macros */
 #define PAGE_SIZE pageSize
 
-volatile bool timerOverflowed = false;
+static volatile __thread bool timerOverflowed = false;
 
 BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
     : BaseCPU(params),
@@ -92,18 +92,6 @@ BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams *params)
     thread->setStatus(ThreadContext::Halted);
     tc = thread->getTC();
     threadContexts.push_back(tc);
-
-    setupCounters();
-
-    if (params->usePerfOverflow)
-        runTimer.reset(new PerfKvmTimer(hwCycles,
-                                        KVM_TIMER_SIGNAL,
-                                        params->hostFactor,
-                                        params->hostFreq));
-    else
-        runTimer.reset(new PosixKvmTimer(KVM_TIMER_SIGNAL, CLOCK_MONOTONIC,
-                                         params->hostFactor,
-                                         params->hostFreq));
 }
 
 BaseKvmCPU::~BaseKvmCPU()
@@ -177,6 +165,35 @@ BaseKvmCPU::startup()
     }
 
     thread->startup();
+
+    Event *startupEvent(
+        new EventWrapper<BaseKvmCPU,
+                         &BaseKvmCPU::startupThread>(this, true));
+    schedule(startupEvent, curTick());
+}
+
+void
+BaseKvmCPU::startupThread()
+{
+    // Do thread-specific initialization. We need to setup signal
+    // delivery for counters and timers from within the thread that
+    // will execute the event queue to ensure that signals are
+    // delivered to the right threads.
+    const BaseKvmCPUParams * const p(
+        dynamic_cast<const BaseKvmCPUParams *>(params()));
+
+    setupCounters();
+
+    if (p->usePerfOverflow)
+        runTimer.reset(new PerfKvmTimer(hwCycles,
+                                        KVM_TIMER_SIGNAL,
+                                        p->hostFactor,
+                                        p->hostFreq));
+    else
+        runTimer.reset(new PosixKvmTimer(KVM_TIMER_SIGNAL, CLOCK_MONOTONIC,
+                                         p->hostFactor,
+                                         p->hostFreq));
+
 }
 
 void
@@ -1044,6 +1061,13 @@ BaseKvmCPU::flushCoalescedMMIO()
 /**
  * Cycle timer overflow when running in KVM. Forces the KVM syscall to
  * exit with EINTR and allows us to run the event queue.
+ *
+ * @warn This function might not be called since some kernels don't
+ * seem to deliver signals when the signal is only unmasked when
+ * running in KVM. This doesn't matter though since we are only
+ * interested in getting KVM to exit, which happens as expected. See
+ * setupSignalHandler() and kvmRun() for details about KVM signal
+ * handling.
  */
 static void
 onTimerOverflow(int signo, siginfo_t *si, void *data)
@@ -1079,20 +1103,22 @@ BaseKvmCPU::setupSignalHandler()
         panic("KVM: Failed to setup vCPU instruction signal handler\n");
 
     sigset_t sigset;
-    if (sigprocmask(SIG_BLOCK, NULL, &sigset) == -1)
+    if (pthread_sigmask(SIG_BLOCK, NULL, &sigset) == -1)
         panic("KVM: Failed get signal mask\n");
 
     // Request KVM to setup the same signal mask as we're currently
-    // running with. We'll sometimes need to mask the KVM_TIMER_SIGNAL
-    // to cause immediate exits from KVM after servicing IO
-    // requests. See kvmRun().
+    // running with except for the KVM control signals. We'll
+    // sometimes need to raise the KVM_TIMER_SIGNAL to cause immediate
+    // exits from KVM after servicing IO requests. See kvmRun().
+    sigdelset(&sigset, KVM_TIMER_SIGNAL);
+    sigdelset(&sigset, KVM_INST_SIGNAL);
     setSignalMask(&sigset);
 
     // Mask our control signals so they aren't delivered unless we're
     // actually executing inside KVM.
     sigaddset(&sigset, KVM_TIMER_SIGNAL);
     sigaddset(&sigset, KVM_INST_SIGNAL);
-    if (sigprocmask(SIG_SETMASK, &sigset, NULL) == -1)
+    if (pthread_sigmask(SIG_SETMASK, &sigset, NULL) == -1)
         panic("KVM: Failed mask the KVM control signals\n");
 }
 
diff --git a/src/cpu/kvm/base.hh b/src/cpu/kvm/base.hh
index 8788d6c8a..5a0b80b15 100644
--- a/src/cpu/kvm/base.hh
+++ b/src/cpu/kvm/base.hh
@@ -618,6 +618,20 @@ class BaseKvmCPU : public BaseCPU
      */
     bool discardPendingSignal(int signum) const;
 
+    /**
+     * Thread-specific initialization.
+     *
+     * Some KVM-related initialization requires us to know the TID of
+     * the thread that is going to execute our event queue. For
+     * example, when setting up timers, we need to know the TID of the
+     * thread executing in KVM in order to deliver the timer signal to
+     * that thread. This method is called as the first event in this
+     * SimObject's event queue.
+     *
+     * @see startup
+     */
+    void startupThread();
+
     /** Try to drain the CPU if a drain is pending */
     bool tryDrain();
 
diff --git a/src/cpu/kvm/timer.cc b/src/cpu/kvm/timer.cc
index 6882063e4..cb48046f7 100644
--- a/src/cpu/kvm/timer.cc
+++ b/src/cpu/kvm/timer.cc
@@ -40,12 +40,30 @@
 #include <algorithm>
 #include <csignal>
 #include <ctime>
+#include <unistd.h>
+#include <sys/syscall.h>
 
 #include "base/misc.hh"
 #include "base/trace.hh"
 #include "cpu/kvm/timer.hh"
 #include "debug/KvmTimer.hh"
 
+/* According to timer_create(2), the value SIGEV_THREAD_ID can be used
+ * to specify which thread a timer signal gets delivered to. According
+ * to the man page, the member sigev_notify_thread is used to specify
+ * the TID. This member is currently not defined by default in
+ * siginfo.h on x86, so we define it here as a workaround.
+ */
+#ifndef sigev_notify_thread_id
+#define sigev_notify_thread_id     _sigev_un._tid
+#endif
+
+static pid_t
+gettid()
+{
+    return syscall(__NR_gettid);
+}
+
 /**
  * Minimum number of cycles that a host can spend in a KVM call (used
  * to calculate the resolution of some timers).
@@ -62,11 +80,9 @@ PosixKvmTimer::PosixKvmTimer(int signo, clockid_t clockID,
 {
     struct sigevent sev;
 
-    // TODO: We should request signal delivery to thread instead of
-    // the process here. Unfortunately this seems to be broken, or at
-    // least not work as specified in the man page.
-    sev.sigev_notify = SIGEV_SIGNAL;
+    sev.sigev_notify = SIGEV_THREAD_ID;
     sev.sigev_signo = signo;
+    sev.sigev_notify_thread_id = gettid();
     sev.sigev_value.sival_ptr = NULL;
 
     while (timer_create(clockID, &sev, &timer) == -1) {