x86 isa: This patch attempts an implementation at mwait.

Mwait works as follows:
1. A cpu monitors an address of interest (monitor instruction)
2. A cpu calls mwait - this loads the cache line into that cpu's cache.
3. The cpu goes to sleep.
4. When another processor requests write permission for the line, it is
   evicted from the sleeping cpu's cache. This eviction is forwarded to the
   sleeping cpu, which then wakes up.

Committed by: Nilay Vaish <nilay@cs.wisc.edu>

16 files changed, 213 insertions, 3 deletions

diff --git a/src/cpu/SConscript b/src/cpu/SConscript
index df29f6c73..570f5e2f1 100644
--- a/src/cpu/SConscript
+++ b/src/cpu/SConscript
@@ -102,6 +102,7 @@ DebugFlag('IntrControl')
 DebugFlag('O3PipeView')
 DebugFlag('PCEvent')
 DebugFlag('Quiesce')
+DebugFlag('Mwait')
 
 CompoundFlag('ExecAll', [ 'ExecEnable', 'ExecCPSeq', 'ExecEffAddr',
     'ExecFaulting', 'ExecFetchSeq', 'ExecOpClass', 'ExecRegDelta',
diff --git a/src/cpu/base.cc b/src/cpu/base.cc
index ea4df2aa8..2f4745ee3 100644
--- a/src/cpu/base.cc
+++ b/src/cpu/base.cc
@@ -55,12 +55,14 @@
 #include "base/misc.hh"
 #include "base/output.hh"
 #include "base/trace.hh"
-#include "cpu/base.hh"
 #include "cpu/checker/cpu.hh"
+#include "cpu/base.hh"
 #include "cpu/cpuevent.hh"
 #include "cpu/profile.hh"
 #include "cpu/thread_context.hh"
+#include "debug/Mwait.hh"
 #include "debug/SyscallVerbose.hh"
+#include "mem/page_table.hh"
 #include "params/BaseCPU.hh"
 #include "sim/full_system.hh"
 #include "sim/process.hh"
@@ -123,7 +125,8 @@ BaseCPU::BaseCPU(Params *p, bool is_checker)
       _taskId(ContextSwitchTaskId::Unknown), _pid(Request::invldPid),
       _switchedOut(p->switched_out), _cacheLineSize(p->system->cacheLineSize()),
       interrupts(p->interrupts), profileEvent(NULL),
-      numThreads(p->numThreads), system(p->system)
+      numThreads(p->numThreads), system(p->system),
+      addressMonitor()
 {
     // if Python did not provide a valid ID, do it here
     if (_cpuId == -1 ) {
@@ -261,6 +264,63 @@ BaseCPU::~BaseCPU()
 }
 
 void
+BaseCPU::armMonitor(Addr address)
+{
+    addressMonitor.armed = true;
+    addressMonitor.vAddr = address;
+    addressMonitor.pAddr = 0x0;
+    DPRINTF(Mwait,"Armed monitor (vAddr=0x%lx)\n", address);
+}
+
+bool
+BaseCPU::mwait(PacketPtr pkt)
+{
+    if(addressMonitor.gotWakeup == false) {
+        int block_size = cacheLineSize();
+        uint64_t mask = ~((uint64_t)(block_size - 1));
+
+        assert(pkt->req->hasPaddr());
+        addressMonitor.pAddr = pkt->getAddr() & mask;
+        addressMonitor.waiting = true;
+
+        DPRINTF(Mwait,"mwait called (vAddr=0x%lx, line's paddr=0x%lx)\n",
+                addressMonitor.vAddr, addressMonitor.pAddr);
+        return true;
+    } else {
+        addressMonitor.gotWakeup = false;
+        return false;
+    }
+}
+
+void
+BaseCPU::mwaitAtomic(ThreadContext *tc, TheISA::TLB *dtb)
+{
+    Request req;
+    Addr addr = addressMonitor.vAddr;
+    int block_size = cacheLineSize();
+    uint64_t mask = ~((uint64_t)(block_size - 1));
+    int size = block_size;
+
+    //The address of the next line if it crosses a cache line boundary.
+    Addr secondAddr = roundDown(addr + size - 1, block_size);
+
+    if (secondAddr > addr)
+        size = secondAddr - addr;
+
+    req.setVirt(0, addr, size, 0x0, dataMasterId(), tc->instAddr());
+
+    // translate to physical address
+    Fault fault = dtb->translateAtomic(&req, tc, BaseTLB::Read);
+    assert(fault == NoFault);
+
+    addressMonitor.pAddr = req.getPaddr() & mask;
+    addressMonitor.waiting = true;
+
+    DPRINTF(Mwait,"mwait called (vAddr=0x%lx, line's paddr=0x%lx)\n",
+            addressMonitor.vAddr, addressMonitor.pAddr);
+}
+
+void
 BaseCPU::init()
 {
     if (!params()->switched_out) {
@@ -618,6 +678,25 @@ BaseCPU::scheduleInstStop(ThreadID tid, Counter insts, const char *cause)
     comInstEventQueue[tid]->schedule(event, now + insts);
 }
 
+AddressMonitor::AddressMonitor() {
+    armed = false;
+    waiting = false;
+    gotWakeup = false;
+}
+
+bool AddressMonitor::doMonitor(PacketPtr pkt) {
+    assert(pkt->req->hasPaddr());
+    if(armed && waiting) {
+        if(pAddr == pkt->getAddr()) {
+            DPRINTF(Mwait,"pAddr=0x%lx invalidated: waking up core\n",
+                    pkt->getAddr());
+            waiting = false;
+            return true;
+        }
+    }
+    return false;
+}
+
 void
 BaseCPU::scheduleLoadStop(ThreadID tid, Counter loads, const char *cause)
 {
diff --git a/src/cpu/base.hh b/src/cpu/base.hh
index 75c8f7263..3673a5f18 100644
--- a/src/cpu/base.hh
+++ b/src/cpu/base.hh
@@ -64,11 +64,26 @@
 #include "sim/insttracer.hh"
 #include "sim/probe/pmu.hh"
 #include "sim/system.hh"
+#include "debug/Mwait.hh"
 
+class BaseCPU;
 struct BaseCPUParams;
 class CheckerCPU;
 class ThreadContext;
 
+struct AddressMonitor
+{
+    AddressMonitor();
+    bool doMonitor(PacketPtr pkt);
+
+    bool armed;
+    Addr vAddr;
+    Addr pAddr;
+    uint64_t val;
+    bool waiting;   // 0=normal, 1=mwaiting
+    bool gotWakeup;
+};
+
 class CPUProgressEvent : public Event
 {
   protected:
@@ -536,6 +551,16 @@ class BaseCPU : public MemObject
     Stats::Scalar numCycles;
     Stats::Scalar numWorkItemsStarted;
     Stats::Scalar numWorkItemsCompleted;
+
+  private:
+    AddressMonitor addressMonitor;
+
+  public:
+    void armMonitor(Addr address);
+    bool mwait(PacketPtr pkt);
+    void mwaitAtomic(ThreadContext *tc, TheISA::TLB *dtb);
+    AddressMonitor *getCpuAddrMonitor() { return &addressMonitor; }
+    void atomicNotify(Addr address);
 };
 
 #endif // THE_ISA == NULL_ISA
diff --git a/src/cpu/base_dyn_inst.hh b/src/cpu/base_dyn_inst.hh
index 289627c9a..af4d238e2 100644
--- a/src/cpu/base_dyn_inst.hh
+++ b/src/cpu/base_dyn_inst.hh
@@ -853,6 +853,14 @@ class BaseDynInst : public ExecContext, public RefCounted
     /** Sets the number of consecutive store conditional failures. */
     void setStCondFailures(unsigned int sc_failures)
     { thread->storeCondFailures = sc_failures; }
+
+  public:
+    // monitor/mwait funtions
+    void armMonitor(Addr address) { cpu->armMonitor(address); }
+    bool mwait(PacketPtr pkt) { return cpu->mwait(pkt); }
+    void mwaitAtomic(ThreadContext *tc)
+    { return cpu->mwaitAtomic(tc, cpu->dtb); }
+    AddressMonitor *getAddrMonitor() { return cpu->getCpuAddrMonitor(); }
 };
 
 template<class Impl>
diff --git a/src/cpu/checker/cpu.hh b/src/cpu/checker/cpu.hh
index d684b142b..49f44ff00 100644
--- a/src/cpu/checker/cpu.hh
+++ b/src/cpu/checker/cpu.hh
@@ -349,6 +349,13 @@ class CheckerCPU : public BaseCPU, public ExecContext
         this->dtb->demapPage(vaddr, asn);
     }
 
+    // monitor/mwait funtions
+    virtual void armMonitor(Addr address) { BaseCPU::armMonitor(address); }
+    bool mwait(PacketPtr pkt) { return BaseCPU::mwait(pkt); }
+    void mwaitAtomic(ThreadContext *tc)
+    { return BaseCPU::mwaitAtomic(tc, thread->dtb); }
+    AddressMonitor *getAddrMonitor() { return BaseCPU::getCpuAddrMonitor(); }
+
     void demapInstPage(Addr vaddr, uint64_t asn)
     {
         this->itb->demapPage(vaddr, asn);
diff --git a/src/cpu/exec_context.hh b/src/cpu/exec_context.hh
index c85a746ac..c65841db2 100644
--- a/src/cpu/exec_context.hh
+++ b/src/cpu/exec_context.hh
@@ -47,6 +47,7 @@
 #include "arch/registers.hh"
 #include "base/types.hh"
 #include "config/the_isa.hh"
+#include "cpu/base.hh"
 #include "cpu/static_inst_fwd.hh"
 #include "cpu/translation.hh"
 
@@ -243,6 +244,10 @@ class ExecContext {
      * Invalidate a page in the DTLB <i>and</i> ITLB.
      */
     virtual void demapPage(Addr vaddr, uint64_t asn) = 0;
+    virtual void armMonitor(Addr address) = 0;
+    virtual bool mwait(PacketPtr pkt) = 0;
+    virtual void mwaitAtomic(ThreadContext *tc) = 0;
+    virtual AddressMonitor *getAddrMonitor() = 0;
 
     /** @} */
 
diff --git a/src/cpu/inorder/inorder_dyn_inst.cc b/src/cpu/inorder/inorder_dyn_inst.cc
index 18281e636..c64cf9da4 100644
--- a/src/cpu/inorder/inorder_dyn_inst.cc
+++ b/src/cpu/inorder/inorder_dyn_inst.cc
@@ -602,3 +602,25 @@ InOrderDynInst::dump(std::string &outstring)
 
     outstring = s.str();
 }
+
+void
+InOrderDynInst::armMonitor(Addr address) {
+    cpu->armMonitor(address);
+}
+
+bool
+InOrderDynInst::mwait(PacketPtr pkt) {
+    return cpu->mwait(pkt);
+}
+
+void
+InOrderDynInst::mwaitAtomic(ThreadContext *tc)
+{
+    return cpu->mwaitAtomic(tc, cpu->getDTBPtr());
+}
+
+AddressMonitor *
+InOrderDynInst::getAddrMonitor()
+{
+    return cpu->getCpuAddrMonitor();
+}
diff --git a/src/cpu/inorder/inorder_dyn_inst.hh b/src/cpu/inorder/inorder_dyn_inst.hh
index 369ebe2f4..ebb7bf912 100644
--- a/src/cpu/inorder/inorder_dyn_inst.hh
+++ b/src/cpu/inorder/inorder_dyn_inst.hh
@@ -1077,6 +1077,13 @@ class InOrderDynInst : public ExecContext, public RefCounted
     void demapPage(Addr vaddr, uint64_t asn) {
         panic("demapPage unimplemented");
     }
+
+  public:
+    // monitor/mwait funtions
+    void armMonitor(Addr address);
+    bool mwait(PacketPtr pkt);
+    void mwaitAtomic(ThreadContext *tc);
+    AddressMonitor *getAddrMonitor();
 };
 
 
diff --git a/src/cpu/minor/exec_context.hh b/src/cpu/minor/exec_context.hh
index f1143498e..41345d3bd 100644
--- a/src/cpu/minor/exec_context.hh
+++ b/src/cpu/minor/exec_context.hh
@@ -340,6 +340,15 @@ class ExecContext : public ::ExecContext
                 - TheISA::Misc_Reg_Base, val);
         }
     }
+
+  public:
+    // monitor/mwait funtions
+    void armMonitor(Addr address) { getCpuPtr()->armMonitor(address); }
+    bool mwait(PacketPtr pkt) { return getCpuPtr()->mwait(pkt); }
+    void mwaitAtomic(ThreadContext *tc)
+    { return getCpuPtr()->mwaitAtomic(tc, thread.dtb); }
+    AddressMonitor *getAddrMonitor()
+    { return getCpuPtr()->getCpuAddrMonitor(); }
 };
 
 }
diff --git a/src/cpu/o3/cpu.cc b/src/cpu/o3/cpu.cc
index fd51cd123..55ef04ffc 100644
--- a/src/cpu/o3/cpu.cc
+++ b/src/cpu/o3/cpu.cc
@@ -117,6 +117,10 @@ template <class Impl>
 void
 FullO3CPU<Impl>::DcachePort::recvTimingSnoopReq(PacketPtr pkt)
 {
+    // X86 ISA: Snooping an invalidation for monitor/mwait
+    if(cpu->getCpuAddrMonitor()->doMonitor(pkt)) {
+        cpu->wakeup();
+    }
     lsq->recvTimingSnoopReq(pkt);
 }
 
diff --git a/src/cpu/o3/cpu.hh b/src/cpu/o3/cpu.hh
index 96cd071e4..09b7db867 100644
--- a/src/cpu/o3/cpu.hh
+++ b/src/cpu/o3/cpu.hh
@@ -162,11 +162,13 @@ class FullO3CPU : public BaseO3CPU
 
         /** Pointer to LSQ. */
         LSQ<Impl> *lsq;
+        FullO3CPU<Impl> *cpu;
 
       public:
         /** Default constructor. */
         DcachePort(LSQ<Impl> *_lsq, FullO3CPU<Impl>* _cpu)
-            : MasterPort(_cpu->name() + ".dcache_port", _cpu), lsq(_lsq)
+            : MasterPort(_cpu->name() + ".dcache_port", _cpu), lsq(_lsq),
+              cpu(_cpu)
         { }
 
       protected:
diff --git a/src/cpu/simple/atomic.cc b/src/cpu/simple/atomic.cc
index d6dbb9292..e98da3ea7 100644
--- a/src/cpu/simple/atomic.cc
+++ b/src/cpu/simple/atomic.cc
@@ -272,6 +272,12 @@ AtomicSimpleCPU::AtomicCPUDPort::recvAtomicSnoop(PacketPtr pkt)
     DPRINTF(SimpleCPU, "received snoop pkt for addr:%#x %s\n", pkt->getAddr(),
             pkt->cmdString());
 
+    // X86 ISA: Snooping an invalidation for monitor/mwait
+    AtomicSimpleCPU *cpu = (AtomicSimpleCPU *)(&owner);
+    if(cpu->getAddrMonitor()->doMonitor(pkt)) {
+        cpu->wakeup();
+    }
+
     // if snoop invalidates, release any associated locks
     if (pkt->isInvalidate()) {
         DPRINTF(SimpleCPU, "received invalidation for addr:%#x\n",
@@ -288,6 +294,12 @@ AtomicSimpleCPU::AtomicCPUDPort::recvFunctionalSnoop(PacketPtr pkt)
     DPRINTF(SimpleCPU, "received snoop pkt for addr:%#x %s\n", pkt->getAddr(),
             pkt->cmdString());
 
+    // X86 ISA: Snooping an invalidation for monitor/mwait
+    AtomicSimpleCPU *cpu = (AtomicSimpleCPU *)(&owner);
+    if(cpu->getAddrMonitor()->doMonitor(pkt)) {
+        cpu->wakeup();
+    }
+
     // if snoop invalidates, release any associated locks
     if (pkt->isInvalidate()) {
         DPRINTF(SimpleCPU, "received invalidation for addr:%#x\n",
diff --git a/src/cpu/simple/base.cc b/src/cpu/simple/base.cc
index 60ab53999..636e08899 100644
--- a/src/cpu/simple/base.cc
+++ b/src/cpu/simple/base.cc
@@ -347,6 +347,8 @@ BaseSimpleCPU::dbg_vtophys(Addr addr)
 void
 BaseSimpleCPU::wakeup()
 {
+    getAddrMonitor()->gotWakeup = true;
+
     if (thread->status() != ThreadContext::Suspended)
         return;
 
diff --git a/src/cpu/simple/base.hh b/src/cpu/simple/base.hh
index 8f38a33c8..523bc9776 100644
--- a/src/cpu/simple/base.hh
+++ b/src/cpu/simple/base.hh
@@ -462,6 +462,14 @@ class BaseSimpleCPU : public BaseCPU, public ExecContext
 
   private:
     TheISA::PCState pred_pc;
+
+  public:
+    // monitor/mwait funtions
+    void armMonitor(Addr address) { BaseCPU::armMonitor(address); }
+    bool mwait(PacketPtr pkt) { return BaseCPU::mwait(pkt); }
+    void mwaitAtomic(ThreadContext *tc)
+    { return BaseCPU::mwaitAtomic(tc, thread->dtb); }
+    AddressMonitor *getAddrMonitor() { return BaseCPU::getCpuAddrMonitor(); }
 };
 
 #endif // __CPU_SIMPLE_BASE_HH__
diff --git a/src/cpu/simple/timing.cc b/src/cpu/simple/timing.cc
index 84a2c09fd..5bfc9799d 100644
--- a/src/cpu/simple/timing.cc
+++ b/src/cpu/simple/timing.cc
@@ -58,6 +58,8 @@
 #include "sim/full_system.hh"
 #include "sim/system.hh"
 
+#include "debug/Mwait.hh"
+
 using namespace std;
 using namespace TheISA;
 
@@ -818,9 +820,21 @@ TimingSimpleCPU::updateCycleCounts()
 void
 TimingSimpleCPU::DcachePort::recvTimingSnoopReq(PacketPtr pkt)
 {
+    // X86 ISA: Snooping an invalidation for monitor/mwait
+    if(cpu->getAddrMonitor()->doMonitor(pkt)) {
+        cpu->wakeup();
+    }
     TheISA::handleLockedSnoop(cpu->thread, pkt, cacheBlockMask);
 }
 
+void
+TimingSimpleCPU::DcachePort::recvFunctionalSnoop(PacketPtr pkt)
+{
+    // X86 ISA: Snooping an invalidation for monitor/mwait
+    if(cpu->getAddrMonitor()->doMonitor(pkt)) {
+        cpu->wakeup();
+    }
+}
 
 bool
 TimingSimpleCPU::DcachePort::recvTimingResp(PacketPtr pkt)
diff --git a/src/cpu/simple/timing.hh b/src/cpu/simple/timing.hh
index 84c8f7418..52eb6b1ba 100644
--- a/src/cpu/simple/timing.hh
+++ b/src/cpu/simple/timing.hh
@@ -228,11 +228,16 @@ class TimingSimpleCPU : public BaseSimpleCPU
          * a wakeup event on a cpu that is monitoring an address
          */
         virtual void recvTimingSnoopReq(PacketPtr pkt);
+        virtual void recvFunctionalSnoop(PacketPtr pkt);
 
         virtual bool recvTimingResp(PacketPtr pkt);
 
         virtual void recvRetry();
 
+        virtual bool isSnooping() const {
+            return true;
+        }
+
         struct DTickEvent : public TickEvent
         {
             DTickEvent(TimingSimpleCPU *_cpu)