CPA: Add new object for gathering critical path annotations.

8 files changed, 1953 insertions, 196 deletions

diff --git a/src/base/CPA.py b/src/base/CPA.py
new file mode 100644
index 000000000..c0beaedef
--- /dev/null
+++ b/src/base/CPA.py
@@ -0,0 +1,8 @@
+from m5.SimObject import SimObject
+from m5.params import *
+
+class CPA(SimObject):
+    type = 'CPA'
+
+    enabled = Param.Bool(False, "Is Annotation enabled?")
+    user_apps = VectorParam.String([], "List of apps to get symbols for")
diff --git a/src/base/SConscript b/src/base/SConscript
index ffad1929a..58c453184 100644
--- a/src/base/SConscript
+++ b/src/base/SConscript
@@ -30,7 +30,9 @@
 
 Import('*')
 
-Source('annotate.cc')
+if env['CP_ANNOTATE']:
+    SimObject('CPA.py')
+    Source('cp_annotate.cc')
 Source('atomicio.cc')
 Source('bigint.cc')
 Source('circlebuf.cc')
@@ -82,6 +84,8 @@ if env['USE_MYSQL']:
     Source('stats/mysql.cc')
 
 TraceFlag('Annotate', "State machine annotation debugging")
+TraceFlag('AnnotateQ', "State machine annotation queue debugging")
+TraceFlag('AnnotateVerbose', "Dump all state machine annotation details")
 TraceFlag('GDBAcc', "Remote debugger accesses")
 TraceFlag('GDBExtra', "Dump extra information on reads and writes")
 TraceFlag('GDBMisc', "Breakpoints, traps, watchpoints, etc.")
@@ -96,3 +100,6 @@ CompoundFlag('GDBAll',
     [ 'GDBMisc', 'GDBAcc', 'GDBRead', 'GDBWrite', 'GDBSend', 'GDBRecv',
       'GDBExtra' ],
     desc="All Remote debugging flags")
+CompoundFlag('AnnotateAll', ['Annotate', 'AnnotateQ', 'AnnotateVerbose'],
+    desc="All Annotation flags")
+
diff --git a/src/base/annotate.cc b/src/base/annotate.cc
deleted file mode 100644
index de7eeed51..000000000
--- a/src/base/annotate.cc
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright (c) 2006 The Regents of The University of Michigan
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met: redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer;
- * redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution;
- * neither the name of the copyright holders nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Authors: Ali Saidi
- */
-
-#include "base/annotate.hh"
-#include "base/callback.hh"
-#include "base/output.hh"
-#include "base/trace.hh"
-#include "sim/core.hh"
-#include "sim/sim_exit.hh"
-#include "sim/system.hh"
-
-
-
-class AnnotateDumpCallback : public Callback
-{
-  public:
-    virtual void process();
-};
-
-void
-AnnotateDumpCallback::process()
-{
-    Annotate::annotations.dump();
-}
-
-namespace Annotate {
-
-
-Annotate annotations;
-
-Annotate::Annotate()
-{
-    registerExitCallback(new AnnotateDumpCallback);
-}
-
-void
-Annotate::add(System *sys, Addr stack, uint32_t sm, uint32_t st,
-        uint32_t wm, uint32_t ws)
-{
-    AnnotateData *an;
-
-    an = new AnnotateData;
-    an->time = curTick;
-
-    std::map<System*, std::string>::iterator i = nameCache.find(sys);
-    if (i == nameCache.end()) {
-        nameCache[sys] = sys->name();
-    }
-
-    an->system = nameCache[sys];
-    an->stack = stack;
-    an->stateMachine = sm;
-    an->curState = st;
-    an->waitMachine = wm;
-    an->waitState = ws;
-
-    data.push_back(an);
-    if (an->waitMachine)
-        DPRINTF(Annotate, "Annotating: %s(%#llX) %d:%d waiting on %d:%d\n",
-                an->system, an->stack, an->stateMachine, an->curState,
-                an->waitMachine, an->waitState);
-    else
-        DPRINTF(Annotate, "Annotating: %s(%#llX) %d:%d beginning\n", an->system,
-                an->stack, an->stateMachine, an->curState);
-
-    DPRINTF(Annotate, "Now %d events on list\n", data.size());
-
-}
-
-void
-Annotate::dump()
-{
-
-    std::list<AnnotateData*>::iterator i;
-
-    i = data.begin();
-
-    if (i == data.end())
-        return;
-
-    std::ostream *os = simout.create("annotate.dat");
-
-    AnnotateData *an;
-
-    while (i != data.end()) {
-        DPRINTF(Annotate, "Writing\n", data.size());
-        an = *i;
-        ccprintf(*os, "%d %s(%#llX) %d %d %d %d\n", an->time, an->system,
-                an->stack, an->stateMachine, an->curState, an->waitMachine,
-                an->waitState);
-        i++;
-    }
-}
-
-} //namespace Annotate
diff --git a/src/base/annotate.hh b/src/base/annotate.hh
deleted file mode 100644
index 36607bf90..000000000
--- a/src/base/annotate.hh
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2006 The Regents of The University of Michigan
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met: redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer;
- * redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution;
- * neither the name of the copyright holders nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Authors: Ali Saidi
- */
-
-#ifndef __BASE__ANNOTATE_HH__
-#define __BASE__ANNOTATE_HH__
-
-#include "sim/host.hh"
-
-#include <string>
-#include <list>
-#include <map>
-
-
-class System;
-
-namespace Annotate {
-
-
-class Annotate {
-
-  protected:
-    struct AnnotateData {
-        Tick time;
-        std::string system;
-        Addr stack;
-        uint32_t stateMachine;
-        uint32_t curState;
-        uint32_t waitMachine;
-        uint32_t waitState;
-    };
-
-    std::list<AnnotateData*> data;
-    std::map<System*, std::string> nameCache;
-
-  public:
-   Annotate();
-   void add(System *sys, Addr stack, uint32_t sm, uint32_t st, uint32_t
-           wm, uint32_t ws);
-   void dump();
-};
-
-extern Annotate annotations;
-} //namespace Annotate
-
-#endif //__BASE__ANNOTATE_HH__
-
diff --git a/src/base/cp_annotate.cc b/src/base/cp_annotate.cc
new file mode 100644
index 000000000..0aba2d999
--- /dev/null
+++ b/src/base/cp_annotate.cc
@@ -0,0 +1,1404 @@
+/*
+ * Copyright (c) 2006-2009 The Regents of The University of Michigan
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Ali Saidi
+ */
+
+#include "arch/utility.hh"
+#include "arch/alpha/linux/threadinfo.hh"
+#include "base/cp_annotate.hh"
+#include "base/callback.hh"
+#include "base/loader/object_file.hh"
+#include "base/output.hh"
+#include "base/trace.hh"
+#include "cpu/thread_context.hh"
+#include "sim/arguments.hh"
+#include "sim/core.hh"
+#include "sim/sim_exit.hh"
+#include "sim/system.hh"
+
+struct CPAIgnoreSymbol
+{
+    const char *symbol;
+    size_t len;
+};
+#define CPA_IGNORE_SYMBOL(sym) { #sym, sizeof(#sym) }
+
+CPAIgnoreSymbol ignoreSymbols[] = {
+    CPA_IGNORE_SYMBOL("m5a_"),
+    CPA_IGNORE_SYMBOL("ret_from_sys_call"),
+    CPA_IGNORE_SYMBOL("ret_from_reschedule"),
+    CPA_IGNORE_SYMBOL("_spin_"),
+    CPA_IGNORE_SYMBOL("local_bh_"),
+    CPA_IGNORE_SYMBOL("restore_all"),
+    CPA_IGNORE_SYMBOL("Call_Pal_"),
+    CPA_IGNORE_SYMBOL("pal_post_interrupt"),
+    CPA_IGNORE_SYMBOL("rti_to_"),
+    CPA_IGNORE_SYMBOL("sys_int_2"),
+    CPA_IGNORE_SYMBOL("sys_interrupt"),
+    CPA_IGNORE_SYMBOL("normal_int"),
+    CPA_IGNORE_SYMBOL("TRAP_INTERRUPT_10_"),
+    CPA_IGNORE_SYMBOL("Trap_Interrupt"),
+    CPA_IGNORE_SYMBOL("do_entInt"),
+    CPA_IGNORE_SYMBOL("__do_softirq"),
+    CPA_IGNORE_SYMBOL("_end"),
+    CPA_IGNORE_SYMBOL("entInt"),
+    CPA_IGNORE_SYMBOL("entSys"),
+    {0,0}
+};
+#undef CPA_IGNORE_SYMBOL
+
+using namespace std;
+using namespace TheISA;
+
+bool CPA::exists;
+CPA *CPA::_cpa;
+
+class AnnotateDumpCallback : public Callback
+{
+
+  private:
+    CPA *cpa;
+  public:
+    virtual void process();
+    AnnotateDumpCallback(CPA *_cpa)
+        : cpa(_cpa)
+    {}
+};
+
+void
+AnnotateDumpCallback::process()
+{
+    cpa->dump(true);
+    cpa->dumpKey();
+}
+
+
+CPA::CPA(Params *p)
+    : SimObject(p), numSm(0), numSmt(0), numSys(0), numQs(0), conId(0)
+{
+    if (exists)
+        fatal("Multiple annotation objects found in system");
+    exists = true;
+
+    _enabled = p->enabled;
+    _cpa = this;
+
+    vector<string>::iterator i;
+    i = p->user_apps.begin();
+
+    while (i != p->user_apps.end()) {
+        ObjectFile *of = createObjectFile(*i);
+        string sf;
+        if (!of)
+            fatal("Couldn't load symbols from file: %s\n", *i);
+        sf = *i;
+        sf.erase(0, sf.rfind('/') + 1);;
+        DPRINTFN("file %s short: %s\n", *i, sf);
+        userApp[sf] = new SymbolTable;
+        bool result1 = of->loadGlobalSymbols(userApp[sf]);
+        bool result2 = of->loadLocalSymbols(userApp[sf]);
+        if (!result1 || !result2)
+            panic("blah");
+        assert(result1 && result2);
+        i++;
+    }
+}
+
+void
+CPA::startup()
+{
+    osbin = simout.create("annotate.bin", true);
+    // MAGIC version number 'M''5''A'N' + version/capabilities
+    ah.version = 0x4D35414E00000101ULL;
+    ah.num_recs = 0;
+    ah.key_off = 0;
+    osbin->write((char*)&ah, sizeof(AnnotateHeader));
+
+    registerExitCallback(new AnnotateDumpCallback(this));
+}
+void
+CPA::swSmBegin(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    Arguments args(tc);
+    std::string st;
+    Addr junk;
+    char sm[50];
+    if (!TheISA::inUserMode(tc))
+        debugSymbolTable->findNearestSymbol(
+            tc->readIntReg(ReturnAddressReg), st, junk);
+
+    CopyStringOut(tc, sm, args[0], 50);
+    System *sys = tc->getSystemPtr();
+    StringWrap name(sys->name());
+
+    if (!sm[0])
+        warn("Got null SM at tick %d\n", curTick);
+
+    int sysi = getSys(sys);
+    int smi = getSm(sysi, sm, args[1]);
+    DPRINTF(Annotate,  "Starting machine: %s(%d) sysi: %d id: %#x\n", sm,
+            smi, sysi, args[1]);
+    DPRINTF(Annotate, "smMap[%d] = %d, %s, %#x\n", smi, 
+            smMap[smi-1].first, smMap[smi-1].second.first, 
+            smMap[smi-1].second.second);
+
+    uint64_t frame = getFrame(tc);
+    StackId sid = StackId(sysi, frame);
+
+    // check if we need to link to the previous state machine
+    int flags = args[2];
+    if (flags & FL_LINK) {
+        if (smStack[sid].size()) {
+            int prev_smi = smStack[sid].back();
+            DPRINTF(Annotate, "Linking from %d to state machine %s(%d) [%#x]\n",
+                    prev_smi, sm, smi, args[1]);
+
+            if (lnMap[smi])
+                DPRINTF(Annotate, "LnMap already contains entry for %d of %d\n",
+                        smi, lnMap[smi]);
+            assert(lnMap[smi] == 0);
+            lnMap[smi] =  prev_smi;
+
+            add(OP_LINK, FL_NONE, tc->contextId(), prev_smi, smi);
+        } else {
+            DPRINTF(Annotate, "Not Linking to state machine %s(%d) [%#x]\n",
+                    sm, smi, args[1]);
+        }
+    }
+
+
+    smStack[sid].push_back(smi);
+
+    DPRINTF(Annotate, "Stack Now (%#X):\n", frame);
+    for (int x = smStack[sid].size()-1; x >= 0; x--)
+        DPRINTF(Annotate, "-- %d\n", smStack[sid][x]);
+
+    // reset the sw state exculsion to false
+    if (swExpl[sid])
+        swExpl[sid] = false;
+
+
+    Id id = Id(sm, frame);
+    if (scLinks[sysi-1][id]) {
+        AnnDataPtr an = scLinks[sysi-1][id];
+        scLinks[sysi-1].erase(id);
+        an->stq = smi;
+        an->dump = true;
+        DPRINTF(Annotate, 
+                "Found prev unknown linking from %d to state machine %s(%d)\n",
+                an->sm, sm, smi);
+
+        if (lnMap[smi])
+            DPRINTF(Annotate, "LnMap already contains entry for %d of %d\n",
+                    smi, lnMap[smi]);
+        assert(lnMap[smi] == 0);
+        lnMap[smi] =  an->sm;
+    }
+
+    // add a new begin ifwe have that info
+    if (st != "") {
+        DPRINTF(Annotate, "st: %s smi: %d stCache.size %d\n", st,
+                smi, stCache.size());
+        int sti = getSt(sm, st);
+        lastState[smi] = sti;
+        add(OP_BEGIN, FL_NONE, tc->contextId(), smi, sti);
+    }
+}
+
+void
+CPA::swSmEnd(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    Arguments args(tc);
+    char sm[50];
+    CopyStringOut(tc, sm, args[0], 50);
+    System *sys = tc->getSystemPtr();
+    doSwSmEnd(sys, tc->contextId(), sm, getFrame(tc));
+}
+
+void
+CPA::doSwSmEnd(System *sys, int cpuid, string sm, uint64_t frame)
+{
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, frame);
+
+
+    // reset the sw state exculsion to false
+    if (swExpl[sid])
+        swExpl[sid] = false;
+
+
+    int smib = smStack[sid].back();
+    StringWrap name(sys->name());
+    DPRINTF(Annotate, "Ending machine: %s[%d, %#x] (%d?)\n", sm, sysi,
+            frame, smib);
+
+    if (!smStack[sid].size() || smMap[smib-1].second.first != sm) {
+        DPRINTF(Annotate, "State Machine not unwinding correctly. sid: %d, %#x"
+                " top of stack: %s Current Stack:\n",
+                sysi, frame, smMap[smib-1].second.first);
+        for (int x = smStack[sid].size()-1; x >= 0; x--)
+            DPRINTF(Annotate, "-- %d\n", smStack[sid][x]);
+        DPRINTF(Annotate, "Ending machine: %s; end stack: %s\n", sm,
+                smMap[smib-1].second.first);
+
+        warn("State machine stack not unwinding correctly at %d\n", curTick);
+    } else {
+        DPRINTF(Annotate, 
+                "State machine ending:%s sysi:%d id:%#x back:%d getSm:%d\n",
+                sm, sysi, smMap[smib-1].second.second, smStack[sid].back(),
+                getSm(sysi, sm, smMap[smib-1].second.second));
+        assert(getSm(sysi, sm, smMap[smib-1].second.second) == 
+                smStack[sid].back());
+
+        int smi = smStack[sid].back();
+        smStack[sid].pop_back();
+
+        if (lnMap[smi]) {
+            DPRINTF(Annotate, "Linking %d back to %d\n", smi, lnMap[smi]);
+            add(OP_LINK, FL_NONE, cpuid, smi, lnMap[smi]);
+            lnMap.erase(smi);
+        }
+
+        if (smStack[sid].size()) {
+            add(OP_BEGIN, FL_NONE, cpuid, smi, lastState[smi]);
+        }
+
+        DPRINTF(Annotate, "Stack Now:\n");
+        for (int x = smStack[sid].size()-1; x >= 0; x--)
+            DPRINTF(Annotate, "-- %d\n", smStack[sid][x]);
+    }
+}
+
+
+void
+CPA::swExplictBegin(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    Arguments args(tc);
+    char st[50];
+    CopyStringOut(tc, st, args[1], 50);
+
+    StringWrap name(tc->getSystemPtr()->name());
+    DPRINTF(Annotate, "Explict begin of state %s\n", st);
+    uint32_t flags = args[0];
+    if (flags & FL_BAD)
+        warn("BAD state encountered: at cycle %d: %s\n", curTick, st);
+    swBegin(tc->getSystemPtr(), tc->contextId(), st, getFrame(tc), true, args[0]);
+}
+
+void
+CPA::swAutoBegin(ThreadContext *tc, Addr next_pc)
+{
+    if (!enabled())
+        return;
+
+    string sym;
+    Addr sym_addr = 0;
+    SymbolTable *symtab = NULL;
+
+
+    if (!TheISA::inUserMode(tc)) {
+        debugSymbolTable->findNearestSymbol(next_pc, sym, sym_addr);
+        symtab = debugSymbolTable;
+    } else {
+        Linux::ThreadInfo ti(tc);
+        string app = ti.curTaskName();
+        if (userApp.count(app))
+            userApp[app]->findNearestSymbol(next_pc, sym, sym_addr);
+    }
+
+    if (sym_addr)
+        swBegin(tc->getSystemPtr(), tc->contextId(), sym, getFrame(tc));
+}
+
+void
+CPA::swBegin(System *sys, int cpuid, std::string st, uint64_t frame, bool expl,
+        int flags)
+{
+    int x = 0;
+    int len;
+    while (ignoreSymbols[x].len)
+    {
+        len = ignoreSymbols[x].len;
+        if (!st.compare(0,len, ignoreSymbols[x].symbol, len))
+            return;
+        x++;
+    }
+
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, frame);
+    // if expl is true suspend symbol table based states
+    if (!smStack[sid].size())
+        return;
+    if (!expl && swExpl[sid])
+        return;
+    if (expl)
+        swExpl[sid] = true;
+    DPRINTFS(AnnotateVerbose, sys, "SwBegin: %s sysi: %d\n", st, sysi);
+    int smi = smStack[sid].back();
+    int sti = getSt(smMap[smi-1].second.first, st);
+    if (lastState[smi] != sti) {
+        lastState[smi] = sti;
+        add(OP_BEGIN, flags, cpuid, smi, sti);
+    }
+}
+
+void
+CPA::swEnd(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    std::string st;
+    Addr junk;
+    if (!TheISA::inUserMode(tc))
+        debugSymbolTable->findNearestSymbol(
+            tc->readIntReg(ReturnAddressReg), st, junk);
+    System *sys = tc->getSystemPtr();
+    StringWrap name(sys->name());
+ 
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size()) {
+        DPRINTF(Annotate, "Explict end of State: %s IGNORED\n",  st);
+        return;
+    }
+    DPRINTF(Annotate, "Explict end of State: %s\n",  st);
+    // return back to symbol table based states
+    swExpl[sid] = false;
+    int smi = smStack[sid].back();
+    if (st != "") {
+        int sti = getSt(smMap[smi-1].second.first, st);
+        lastState[smi] = sti;
+        add(OP_BEGIN, FL_NONE, tc->contextId(), smi, sti);
+    }
+}
+
+void
+CPA::swQ(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    char q[50];
+    Arguments args(tc);
+    uint64_t id = args[0];
+    CopyStringOut(tc, q, args[1], 50);
+    int32_t count = args[2];
+    System *sys = tc->getSystemPtr();
+
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        return;
+    int smi = smStack[sid].back();
+    if (swExpl[sid])
+        swExpl[sid] = false;
+    int qi = getQ(sysi, q, id);
+    if (count == 0) {
+        //warn("Tried to queue 0 bytes in %s, ignoring\n", q);
+        return;
+    }
+    DPRINTFS(AnnotateQ, sys, 
+            "swQ: %s[%#x] cur size %d %d bytes: %d adding: %d\n",
+            q, id, qSize[qi-1], qData[qi-1].size(), qBytes[qi-1], count);
+    doQ(sys, FL_NONE, tc->contextId(), smi, q, qi, count);
+}
+
+void
+CPA::swDq(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    char q[50];
+    Arguments args(tc);
+    uint64_t id = args[0];
+    CopyStringOut(tc, q, args[1], 50);
+    int32_t count = args[2];
+    System *sys = tc->getSystemPtr();
+
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        return;
+    int smi = smStack[sid].back();
+    int qi = getQ(sysi, q, id);
+    if (swExpl[sid])
+        swExpl[sid] = false;
+    DPRINTFS(AnnotateQ, sys, 
+            "swDq: %s[%#x] cur size %d %d bytes: %d removing: %d\n",
+            q, id, qSize[qi-1], qData[qi-1].size(), qBytes[qi-1], count);
+    assert(count != 0);
+
+    doDq(sys, FL_NONE, tc->contextId(), smi, q, qi, count);
+}
+
+void
+CPA::swPq(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    char q[50];
+    Arguments args(tc);
+    uint64_t id = args[0];
+    CopyStringOut(tc, q, args[1], 50);
+    System *sys = tc->getSystemPtr();
+    int32_t count = args[2];
+
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        return;
+    int smi = smStack[sid].back();
+    int qi = getQ(sysi, q, id);
+    if (swExpl[sid])
+        swExpl[sid] = false;
+    DPRINTFS(AnnotateQ, sys, 
+            "swPq: %s [%#x] cur size %d %d bytes: %d peeking: %d\n",
+            q, id, qSize[qi-1], qData[qi-1].size(), qBytes[qi-1], count);
+
+    assert(count != 0);
+    if (qBytes[qi-1] < count) {
+        dump(true);
+        dumpKey();
+        fatal("Queue %s peeking with not enough bytes available in queue!\n", q);
+    }
+
+    add(OP_PEEK, FL_NONE, tc->contextId(), smi, qi, count);
+}
+
+void
+CPA::swRq(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    char q[50];
+    Arguments args(tc);
+    uint64_t id = args[0];
+    CopyStringOut(tc, q, args[1], 50);
+    System *sys = tc->getSystemPtr();
+    int32_t count = args[2];
+
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        return;
+    int smi = smStack[sid].back();
+    int qi = getQ(sysi, q, id);
+    if (swExpl[sid])
+        swExpl[sid] = false;
+    DPRINTFS(AnnotateQ, sys, 
+            "swRq: %s [%#x] cur size %d %d bytes: %d reserve: %d\n",
+            q, id, qSize[qi-1], qData[qi-1].size(), qBytes[qi-1], count);
+
+    assert(count != 0);
+
+    add(OP_RESERVE, FL_NONE, tc->contextId(), smi, qi, count);
+}
+
+
+void
+CPA::swWf(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    char q[50];
+    Arguments args(tc);
+    uint64_t id = args[0];
+    CopyStringOut(tc, q, args[1], 50);
+    System *sys = tc->getSystemPtr();
+    int32_t count = args[3];
+
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        return;
+    int smi = smStack[sid].back();
+    int qi = getQ(sysi, q, id);
+    add(OP_WAIT_FULL, FL_NONE, tc->contextId(), smi, qi, count);
+
+    if (!!args[2]) {
+        char sm[50];
+        CopyStringOut(tc, sm, args[2], 50);
+        doSwSmEnd(tc->getSystemPtr(), tc->contextId(), sm, getFrame(tc));
+    }
+}
+
+void
+CPA::swWe(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    char q[50];
+    Arguments args(tc);
+    uint64_t id = args[0];
+    CopyStringOut(tc, q, args[1], 50);
+    System *sys = tc->getSystemPtr();
+    int32_t count = args[3];
+
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        return;
+    int smi = smStack[sid].back();
+    int qi = getQ(sysi, q, id);
+    add(OP_WAIT_EMPTY, FL_NONE, tc->contextId(), smi, qi, count);
+
+    if (!!args[2]) {
+        char sm[50];
+        CopyStringOut(tc, sm, args[2], 50);
+        doSwSmEnd(tc->getSystemPtr(), tc->contextId(), sm, getFrame(tc));
+    }
+}
+
+void
+CPA::swSq(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    char q[50];
+    Arguments args(tc);
+    uint64_t id = args[0];
+    CopyStringOut(tc, q, args[1], 50);
+    System *sys = tc->getSystemPtr();
+    StringWrap name(sys->name());
+    int32_t size = args[2];
+    int flags = args[3];
+
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        return;
+    int smi = smStack[sid].back();
+    int qi = getQ(sysi, q, id);
+    DPRINTF(AnnotateQ, "swSq: %s [%#x] cur size: %d bytes: %d, new size: %d\n",
+             q, id, qSize[qi-1], qBytes[qi-1], size);
+
+    if (FL_RESET & flags) {
+        DPRINTF(AnnotateQ, "Resetting Queue %s\n", q);
+        add(OP_SIZE_QUEUE, FL_NONE, tc->contextId(), smi, qi, 0);
+        qData[qi-1].clear();
+        qSize[qi-1] = 0;
+        qBytes[qi-1] = 0;
+    }
+
+    if (qBytes[qi-1] < size)
+        doQ(sys, FL_NONE, tc->contextId(), smi, q, qi, size - qBytes[qi-1]);
+    else if (qBytes[qi-1] > size) {
+        DPRINTF(AnnotateQ, "removing for resize of queue %s\n", q);
+        add(OP_SIZE_QUEUE, FL_NONE, tc->contextId(), smi, qi, size);
+        if (size <= 0) {
+            qData[qi-1].clear();
+            qSize[qi-1] = 0;
+            qBytes[qi-1] = 0;
+            return;
+        }
+        int need = qBytes[qi-1] - size;
+        qBytes[qi-1] = size;
+        while (need > 0) {
+            int32_t tail_bytes = qData[qi-1].back()->data;
+            if (qSize[qi-1] <= 0 || qBytes[qi-1] < 0) {
+                dump(true);
+                dumpKey();
+                fatal("Queue %s had inconsistancy when doing size queue!\n", q);
+            }
+            if (tail_bytes > need) {
+                qData[qi-1].back()->data -= need;
+                need = 0;
+            } else if (tail_bytes == need) {
+                qData[qi-1].pop_back();
+                qSize[qi-1]--;
+                need = 0;
+            } else {
+                qData[qi-1].pop_back();
+                qSize[qi-1]--;
+                need -= tail_bytes;
+            }
+        }
+    }
+}
+
+void
+CPA::swAq(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    char q[50];
+    Arguments args(tc);
+    uint64_t id = args[0];
+    CopyStringOut(tc, q, args[1], 50);
+    System *sys = tc->getSystemPtr();
+    StringWrap name(sys->name());
+    int32_t size = args[2];
+
+    int sysi = getSys(sys);
+    int qi = getQ(sysi, q, id);
+    if (qBytes[qi-1] != size) {
+        DPRINTF(AnnotateQ, "Queue %s [%#x] has inconsintant size\n", q, id);
+        //dump(true);
+        //dumpKey();
+        std::list<AnnDataPtr>::iterator ai = qData[qi-1].begin();
+        int x = 0;
+        while (ai != qData[qi-1].end()) {
+            DPRINTF(AnnotateQ, "--Element %d size %d\n", x, (*ai)->data);
+            ai++;
+            x++;
+        }
+
+        warn("%d: Queue Assert: SW said there should be %d byte(s) in %s," 
+                "however there are %d byte(s)\n",
+            curTick, size, q, qBytes[qi-1]);
+        DPRINTF(AnnotateQ, "%d: Queue Assert: SW said there should be %d"
+                " byte(s) in %s, however there are %d byte(s)\n",
+            curTick, size, q, qBytes[qi-1]);
+    }
+}
+
+void
+CPA::swLink(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    char lsm[50];
+    Arguments args(tc);
+    CopyStringOut(tc, lsm, args[0], 50);
+    System *sys = tc->getSystemPtr();
+    StringWrap name(sys->name());
+
+    int sysi = getSys(sys);
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        return;
+    int smi = smStack[sid].back();
+    int lsmi = getSm(sysi, lsm, args[1]);
+
+    DPRINTF(Annotate, "Linking from %d to state machine %s(%d) [%#x]\n",
+            smi, lsm, lsmi, args[1]);
+
+    if (lnMap[lsmi])
+        DPRINTF(Annotate, "LnMap already contains entry for %d of %d\n",
+                lsmi, lnMap[lsmi]);
+    assert(lnMap[lsmi] == 0);
+    lnMap[lsmi] =  smi;
+
+    add(OP_LINK, FL_NONE, tc->contextId(), smi, lsmi);
+
+    if (!!args[2]) {
+        char sm[50];
+        CopyStringOut(tc, sm, args[2], 50);
+        doSwSmEnd(tc->getSystemPtr(), tc->contextId(), sm, getFrame(tc));
+    }
+}
+
+void
+CPA::swIdentify(ThreadContext *tc)
+{
+    if (!enabled())
+        return;
+
+    Arguments args(tc);
+    int sysi = getSys(tc->getSystemPtr());
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        return;
+    int smi = smStack[sid].back();
+
+    DPRINTFS(Annotate, tc->getSystemPtr(), "swIdentify: id %#X\n", args[0]);
+
+    add(OP_IDENT, FL_NONE, tc->contextId(), smi, 0, args[0]);
+}
+
+uint64_t
+CPA::swGetId(ThreadContext *tc)
+{
+    if (!enabled())
+        return 0;
+
+    uint64_t id = ++conId;
+    int sysi = getSys(tc->getSystemPtr());
+    StackId sid = StackId(sysi, getFrame(tc));
+    if (!smStack[sid].size())
+        panic("swGetId called without a state machine stack!");
+    int smi = smStack[sid].back();
+
+    DPRINTFS(Annotate, tc->getSystemPtr(), "swGetId: id %#X\n", id);
+
+    add(OP_IDENT, FL_NONE, tc->contextId(), smi, 0, id);
+    return id;
+}
+
+
+void
+CPA::swSyscallLink(ThreadContext  *tc)
+{
+    if (!enabled())
+        return;
+
+    char lsm[50];
+    Arguments args(tc);
+    CopyStringOut(tc, lsm, args[0], 50);
+    System *sys = tc->getSystemPtr();
+    StringWrap name(sys->name());
+    int sysi = getSys(sys);
+
+    Id id = Id(lsm, getFrame(tc));
+    StackId sid = StackId(sysi, getFrame(tc));
+
+    if (!smStack[sid].size())
+        return;
+
+    int smi = smStack[sid].back();
+
+    DPRINTF(Annotate, "Linking from %d to state machine %s(UNKNOWN)\n",
+            smi, lsm);
+
+    if (scLinks[sysi-1][id])
+        DPRINTF(Annotate, 
+                "scLinks already contains entry for system %d %s[%x] of %d\n",
+                sysi, lsm, getFrame(tc), scLinks[sysi-1][id]);
+    assert(scLinks[sysi-1][id] == 0);
+    scLinks[sysi-1][id] = add(OP_LINK, FL_NONE, tc->contextId(), smi, 0xFFFF);
+    scLinks[sysi-1][id]->dump = false;
+
+    if (!!args[1]) {
+        char sm[50];
+        CopyStringOut(tc, sm, args[1], 50);
+        doSwSmEnd(tc->getSystemPtr(), tc->contextId(), sm, getFrame(tc));
+    }
+}
+
+CPA::AnnDataPtr
+CPA::add(int t, int f, int c, int sm, int stq, int32_t d)
+{
+    AnnDataPtr an = new AnnotateData;
+    an->time = curTick;
+    an->data = d;
+    an->orig_data = d;
+    an->op = t;
+    an->flag = f;
+    an->sm = sm;
+    an->stq = stq;
+    an->cpu = c;
+    an->dump = true;
+
+    data.push_back(an);
+
+    DPRINTF(AnnotateVerbose, "Annotate: op: %d flags: 0x%x sm: %d state: %d time: %d, data: %d\n",
+            an->op, an->flag, an->sm, an->stq, an->time, an->data);
+    
+    // Don't dump Links because we might be setting no-dump on it
+    if (an->op != OP_LINK)
+        dump(false);
+
+    return an;
+}
+
+void
+CPA::dumpKey()
+{
+    std::streampos curpos = osbin->tellp();
+    ah.key_off = curpos;
+
+    // Output the various state machines and their corresponding states
+    *osbin << "# Automatically generated state machine descriptor file" << endl;
+
+    *osbin << "sms = {}" << endl << endl;
+    vector<string> state_machines;
+    state_machines.resize(numSmt+1);
+
+    // State machines, id -> states
+    SCache::iterator i = smtCache.begin();
+    while (i != smtCache.end()) {
+        state_machines[i->second] = i->first;
+        i++;
+    }
+
+    for (int x = 1; x < state_machines.size(); x++) {
+        vector<string> states;
+        states.resize(numSt[x-1]+1);
+        assert(x-1 < stCache.size());
+        SCache::iterator i = stCache[x-1].begin();
+        while (i != stCache[x-1].end()) {
+            states[i->second] = i->first;
+            i++;
+        }
+        *osbin << "sms[\"" << state_machines[x] << "\"] = [\"NULL\"";
+        for (int y = 1; y < states.size(); y++)
+            *osbin << ", \"" << states[y] << "\"";
+        *osbin << "]" << endl;
+    }
+
+    *osbin << endl << endl << endl;
+
+    // state machine number -> system, name, id
+    *osbin << "smNum = [\"NULL\"";
+    for (int x = 0; x < smMap.size(); x++)
+        *osbin << ", (" << smMap[x].first << ", \"" << smMap[x].second.first <<
+            "\", " << smMap[x].second.second << ")";
+    *osbin << "]" << endl;
+
+    *osbin << endl << endl << endl;
+
+    // Output the systems
+    vector<string> systems;
+    systems.resize(numSys+1);
+    NameCache::iterator i2 = nameCache.begin();
+    while (i2 != nameCache.end()) {
+        systems[i2->second.second] = i2->second.first;
+        i2++;
+    }
+
+    *osbin << "sysNum = [\"NULL\"";
+    for (int x = 1; x < systems.size(); x++) {
+        *osbin << ", \"" << systems[x] << "\"";
+    }
+    *osbin << "]" << endl;
+
+    // queue number -> system, qname, qid
+    *osbin << "queues = [\"NULL\"";
+    for (int x = 0; x < qMap.size(); x++)
+        *osbin << ", (" << qMap[x].first << ", \"" << qMap[x].second.first <<
+            "\", " << qMap[x].second.second << ")";
+    *osbin << "]" << endl;
+
+    *osbin << "smComb = [s for s in [(i,r) for i in xrange(1,len(sysNum)) "
+           << "for r in xrange (1,len(smNum))]]" << endl;
+    ah.key_len = osbin->tellp() - curpos;
+
+    // output index
+    curpos = osbin->tellp();
+    ah.idx_off = curpos;
+
+    for (int x = 0; x < annotateIdx.size(); x++)
+        osbin->write((char*)&annotateIdx[x], sizeof(uint64_t));
+    ah.idx_len = osbin->tellp() - curpos;
+
+    osbin->seekp(0);
+    osbin->write((char*)&ah, sizeof(AnnotateHeader));
+    osbin->flush();
+
+}
+
+void
+CPA::dump(bool all)
+{
+
+    list<AnnDataPtr>::iterator i;
+
+    i = data.begin();
+
+    if (i == data.end())
+        return;
+
+    // Dump the data every
+    if (!all && data.size() < 10000)
+        return;
+
+    DPRINTF(Annotate, "Writing %d\n", data.size());
+    while (i != data.end()) {
+        AnnDataPtr an = *i;
+
+        // If we can't dump this record, hold here
+        if (!an->dump && !all)
+            break;
+
+        ah.num_recs++;
+        if (ah.num_recs % 100000 == 0)
+            annotateIdx.push_back(osbin->tellp());
+
+
+        osbin->write((char*)&(an->time), sizeof(an->time));
+        osbin->write((char*)&(an->orig_data), sizeof(an->orig_data));
+        osbin->write((char*)&(an->sm), sizeof(an->sm));
+        osbin->write((char*)&(an->stq), sizeof(an->stq));
+        osbin->write((char*)&(an->op), sizeof(an->op));
+        osbin->write((char*)&(an->flag), sizeof(an->flag));
+        osbin->write((char*)&(an->cpu), sizeof(an->cpu));
+        i++;
+    }
+    if (data.begin() != i)
+        data.erase(data.begin(), i);
+
+    if (all)
+        osbin->flush();
+}
+
+void
+CPA::doQ(System *sys, int flags, int cpuid, int sm,
+              string q, int qi, int count)
+{
+    qSize[qi-1]++;
+    qBytes[qi-1] += count;
+    if (qSize[qi-1] > 2501 || qBytes[qi-1] > 2000000000)
+        warn("Queue %s is %d elements/%d bytes, "
+                "maybe things aren't being removed?\n",
+                q, qSize[qi-1], qBytes[qi-1]);
+    if (flags & FL_QOPP)
+        qData[qi-1].push_front(add(OP_QUEUE, flags, cpuid, sm, qi, count));
+    else
+        qData[qi-1].push_back(add(OP_QUEUE, flags, cpuid, sm, qi, count));
+    DPRINTFS(AnnotateQ, sys, "Queing in queue %s size now %d/%d\n",
+            q, qSize[qi-1], qBytes[qi-1]);
+    assert(qSize[qi-1] >= 0);
+    assert(qBytes[qi-1] >= 0);
+}
+
+
+void
+CPA::doDq(System *sys, int flags, int cpuid, int sm,
+               string q, int qi, int count)
+{
+
+    StringWrap name(sys->name());
+    if (count == -1) {
+        add(OP_DEQUEUE, flags, cpuid, sm, qi, count);
+        qData[qi-1].clear();
+        qSize[qi-1] = 0;
+        qBytes[qi-1] = 0;
+        DPRINTF(AnnotateQ, "Dequeing all data in queue %s size now %d/%d\n",
+                q, qSize[qi-1], qBytes[qi-1]);
+        return;
+    }
+
+    assert(count > 0);
+    if (qSize[qi-1] <= 0 || qBytes[qi-1] <= 0 || !qData[qi-1].size()) {
+        dump(true);
+        dumpKey();
+        fatal("Queue %s dequing with no data available in queue!\n",
+                q);
+    }
+    assert(qSize[qi-1] >= 0);
+    assert(qBytes[qi-1] >= 0);
+    assert(qData[qi-1].size());
+
+    int32_t need = count;
+    qBytes[qi-1] -= count;
+    if (qBytes[qi-1] < 0) {
+        dump(true);
+        dumpKey();
+        fatal("Queue %s dequing with no bytes available in queue!\n",
+                q);
+    }
+
+    while (need > 0) {
+        int32_t head_bytes = qData[qi-1].front()->data;
+        if (qSize[qi-1] <= 0 || qBytes[qi-1] < 0) {
+            dump(true);
+            dumpKey();
+            fatal("Queue %s dequing with nothing in queue!\n",
+                q);
+        }
+
+        if (head_bytes > need) {
+            qData[qi-1].front()->data -= need;
+            need = 0;
+        } else if (head_bytes == need) {
+            qData[qi-1].pop_front();
+            qSize[qi-1]--;
+            need = 0;
+        } else {
+            qData[qi-1].pop_front();
+            qSize[qi-1]--;
+            need -= head_bytes;
+        }
+    }
+
+    add(OP_DEQUEUE, flags, cpuid, sm, qi, count);
+    DPRINTF(AnnotateQ, "Dequeing in queue %s size now %d/%d\n",
+            q, qSize[qi-1], qBytes[qi-1]);
+}
+
+
+
+void
+CPA::serialize(std::ostream &os)
+{
+
+    SERIALIZE_SCALAR(numSm);
+    SERIALIZE_SCALAR(numSmt);
+    arrayParamOut(os, "numSt", numSt);
+    arrayParamOut(os, "numQ", numQ);
+    SERIALIZE_SCALAR(numSys);
+    SERIALIZE_SCALAR(numQs);
+    SERIALIZE_SCALAR(conId);
+    arrayParamOut(os, "qSize", qSize);
+    arrayParamOut(os, "qSize", qSize);
+    arrayParamOut(os, "qBytes", qBytes);
+
+    std::list<AnnDataPtr>::iterator ai;
+
+    SCache::iterator i;
+    int x = 0, y = 0;
+
+    // smtCache (SCache)
+    x = 0;
+    y = 0;
+    i = smtCache.begin();
+    while (i != smtCache.end()) {
+        paramOut(os, csprintf("smtCache%d.str", x), i->first);
+        paramOut(os, csprintf("smtCache%d.int", x), i->second);
+        x++; i++;
+    }
+
+    // stCache  (StCache)
+    for (x = 0; x < stCache.size(); x++) {
+        i = stCache[x].begin();
+        y = 0;
+        while (i != stCache[x].end()) {
+            paramOut(os, csprintf("stCache%d_%d.str", x, y), i->first);
+            paramOut(os, csprintf("stCache%d_%d.int", x, y), i->second);
+            y++; i++;
+        }
+    }
+
+    // qCache (IdCache)
+    IdHCache::iterator idi;
+    for (x = 0; x < qCache.size(); x++) {
+        idi = qCache[x].begin();
+        y = 0;
+        while (idi != qCache[x].end()) {
+            paramOut(os, csprintf("qCache%d_%d.str", x, y), idi->first.first);
+            paramOut(os, csprintf("qCache%d_%d.id", x, y), idi->first.second);
+            paramOut(os, csprintf("qCache%d_%d.int", x, y), idi->second);
+            y++; idi++;
+        }
+    }
+
+    // smCache (IdCache)
+    for (x = 0; x < smCache.size(); x++) {
+        idi = smCache[x].begin();
+        y = 0;
+        paramOut(os, csprintf("smCache%d", x), smCache[x].size());
+        while (idi != smCache[x].end()) {
+            paramOut(os, csprintf("smCache%d_%d.str", x, y), idi->first.first);
+            paramOut(os, csprintf("smCache%d_%d.id", x, y), idi->first.second);
+            paramOut(os, csprintf("smCache%d_%d.int", x, y), idi->second);
+            y++; idi++;
+        }
+    }
+
+    // scLinks (ScCache) -- data not serialize
+
+
+    // namecache (NameCache)
+    NameCache::iterator ni;
+
+    ni = nameCache.begin();
+    x = 0;
+    while (ni != nameCache.end()) {
+        paramOut(os, csprintf("nameCache%d.name", x), ni->first->name());
+        paramOut(os, csprintf("nameCache%d.str", x), ni->second.first);
+        paramOut(os, csprintf("nameCache%d.int", x), ni->second.second);
+        x++; ni++;
+    }
+
+    // smStack (SmStack)
+    SmStack::iterator si;
+    si = smStack.begin();
+    x = 0;
+    paramOut(os, "smStackIdCount", smStack.size());
+    while (si != smStack.end()) {
+        paramOut(os, csprintf("smStackId%d.sys", x), si->first.first);
+        paramOut(os, csprintf("smStackId%d.frame", x), si->first.second);
+        paramOut(os, csprintf("smStackId%d.count", x), si->second.size());
+        for (y = 0; y < si->second.size(); y++)
+            paramOut(os, csprintf("smStackId%d_%d", x, y), si->second[y]);
+        x++; si++;
+    }
+
+    // lnMap (LinkMap)
+    x = 0;
+    LinkMap::iterator li;
+    li = lnMap.begin();
+    paramOut(os, "lnMapSize", lnMap.size());
+    while (li != lnMap.end()) {
+        paramOut(os, csprintf("lnMap%d.smi", x), li->first);
+        paramOut(os, csprintf("lnMap%d.lsmi", x), li->second);
+        x++; li++;
+    }
+
+    // swExpl (vector)
+    SwExpl::iterator swexpli;
+    swexpli = swExpl.begin();
+    x = 0;
+    paramOut(os, "swExplCount", swExpl.size());
+    while (swexpli != swExpl.end()) {
+        paramOut(os, csprintf("swExpl%d.sys", x), swexpli->first.first);
+        paramOut(os, csprintf("swExpl%d.frame", x), swexpli->first.second);
+        paramOut(os, csprintf("swExpl%d.swexpl", x), swexpli->second);
+        x++; swexpli++;
+    }
+
+    // lastState (IMap)
+    x = 0;
+    IMap::iterator ii;
+    ii = lastState.begin();
+    paramOut(os, "lastStateSize", lastState.size());
+    while (ii != lastState.end()) {
+        paramOut(os, csprintf("lastState%d.smi", x), ii->first);
+        paramOut(os, csprintf("lastState%d.sti", x), ii->second);
+        x++; ii++;
+    }
+
+    // smMap (IdMap)
+    for (x = 0; x < smMap.size(); x++) {
+        paramOut(os, csprintf("smMap%d.sys", x), smMap[x].first);
+        paramOut(os, csprintf("smMap%d.smname", x), smMap[x].second.first);
+        paramOut(os, csprintf("smMap%d.id", x), smMap[x].second.second);
+    }
+
+    // qMap (IdMap)
+    for (x = 0; x < qMap.size(); x++) {
+        paramOut(os, csprintf("qMap%d.sys", x), qMap[x].first);
+        paramOut(os, csprintf("qMap%d.qname", x), qMap[x].second.first);
+        paramOut(os, csprintf("qMap%d.id", x), qMap[x].second.second);
+    }
+
+    // qData (vector<AnnotateList>)
+    for(x = 0; x < qData.size(); x++) {
+        if (!qData[x].size())
+            continue;
+        y = 0;
+        ai = qData[x].begin();
+        while (ai != qData[x].end()) {
+            nameOut(os, csprintf("%s.Q%d_%d", name(), x, y));
+            (*ai)->serialize(os);
+            ai++;
+            y++;
+        }
+    }
+}
+
+void
+CPA::unserialize(Checkpoint *cp, const std::string &section)
+{
+    UNSERIALIZE_SCALAR(numSm);
+    UNSERIALIZE_SCALAR(numSmt);
+    arrayParamIn(cp, section, "numSt", numSt);
+    arrayParamIn(cp, section, "numQ", numQ);
+    UNSERIALIZE_SCALAR(numSys);
+    UNSERIALIZE_SCALAR(numQs);
+    UNSERIALIZE_SCALAR(conId);
+    arrayParamIn(cp, section, "qSize", qSize);
+    arrayParamIn(cp, section, "qBytes", qBytes);
+
+
+    // smtCache (SCache
+    string str;
+    int smi;
+    for (int x = 0;  x < numSmt; x++) {
+        paramIn(cp, section, csprintf("smtCache%d.str", x), str);
+        paramIn(cp, section, csprintf("smtCache%d.int", x), smi);
+        smtCache[str] = smi;
+    }
+
+    // stCache  (StCache)
+    stCache.resize(numSmt);
+    for (int x = 0;  x < numSmt; x++) {
+        for (int y = 0; y < numSt[x]; y++) {
+            paramIn(cp, section, csprintf("stCache%d_%d.str", x,y), str);
+            paramIn(cp, section, csprintf("stCache%d_%d.int", x,y), smi);
+            stCache[x][str] = smi;
+        }
+    }
+
+    // qCache (IdCache)
+    uint64_t id;
+    qCache.resize(numSys);
+    for (int x = 0;  x < numSys; x++) {
+        for (int y = 0; y < numQ[x]; y++) {
+            paramIn(cp, section, csprintf("qCache%d_%d.str", x,y), str);
+            paramIn(cp, section, csprintf("qCache%d_%d.id", x,y), id);
+            paramIn(cp, section, csprintf("qCache%d_%d.int", x,y), smi);
+            qCache[x][Id(str,id)] = smi;
+        }
+    }
+
+    // smCache (IdCache)
+    smCache.resize(numSys);
+    for (int x = 0;  x < numSys; x++) {
+        int size;
+        paramIn(cp, section, csprintf("smCache%d", x), size);
+        for (int y = 0; y < size; y++) {
+            paramIn(cp, section, csprintf("smCache%d_%d.str", x,y), str);
+            paramIn(cp, section, csprintf("smCache%d_%d.id", x,y), id);
+            paramIn(cp, section, csprintf("smCache%d_%d.int", x,y), smi);
+            smCache[x][Id(str,id)] = smi;
+        }
+    }
+
+    // scLinks (ScCache) -- data not serialized, just creating one per sys
+    for (int x = 0; x < numSys; x++)
+        scLinks.push_back(ScHCache());
+
+    // nameCache (NameCache)
+    for (int x = 0; x < numSys; x++) {
+        System *sys;
+        SimObject *sptr;
+        string str;
+        int sysi;
+
+        objParamIn(cp, section, csprintf("nameCache%d.name", x), sptr);
+        sys = dynamic_cast<System*>(sptr);
+
+        paramIn(cp, section, csprintf("nameCache%d.str", x), str);
+        paramIn(cp, section, csprintf("nameCache%d.int", x), sysi);
+        nameCache[sys] = std::make_pair<std::string,int>(str, sysi);
+    }
+
+    //smStack (SmStack)
+    int smStack_size;
+    paramIn(cp, section, "smStackIdCount", smStack_size);
+    for (int x = 0; x < smStack_size; x++) {
+        int sysi;
+        uint64_t frame;
+        int count;
+        paramIn(cp, section, csprintf("smStackId%d.sys", x), sysi);
+        paramIn(cp, section, csprintf("smStackId%d.frame", x), frame);
+        paramIn(cp, section, csprintf("smStackId%d.count", x), count);
+        StackId sid = StackId(sysi, frame);
+        for (int y = 0; y < count; y++) {
+            paramIn(cp, section, csprintf("smStackId%d_%d", x, y), smi);
+            smStack[sid].push_back(smi);
+        }
+    }
+
+    // lnMap (LinkMap)
+    int lsmi;
+    int lnMap_size;
+    paramIn(cp, section, "lnMapSize", lnMap_size);
+    for (int x = 0;  x < lnMap_size; x++) {
+        paramIn(cp, section, csprintf("lnMap%d.smi", x), smi);
+        paramIn(cp, section, csprintf("lnMap%d.lsmi", x), lsmi);
+        lnMap[smi] = lsmi;
+    }
+
+    // swExpl (vector)
+    int swExpl_size;
+    paramIn(cp, section, "swExplCount", swExpl_size);
+    for (int x = 0; x < swExpl_size; x++) {
+        int sysi;
+        uint64_t frame;
+        bool b;
+        paramIn(cp, section, csprintf("swExpl%d.sys", x), sysi);
+        paramIn(cp, section, csprintf("swExpl%d.frame", x), frame);
+        paramIn(cp, section, csprintf("swExpl%d.swexpl", x), b);
+        StackId sid = StackId(sysi, frame);
+        swExpl[sid] = b;
+    }
+
+    // lastState (IMap)
+    int sti;
+    int lastState_size;
+    paramIn(cp, section, "lastStateSize", lastState_size);
+    for (int x = 0;  x < lastState_size; x++) {
+        paramIn(cp, section, csprintf("lastState%d.smi", x), smi);
+        paramIn(cp, section, csprintf("lastState%d.sti", x), sti);
+        lastState[smi] = sti;
+    }
+
+
+    //smMap (IdMap)
+    smMap.resize(numSm);
+    for (int x = 0; x < smMap.size(); x++) {
+        paramIn(cp, section, csprintf("smMap%d.sys", x), smMap[x].first);
+        paramIn(cp, section, csprintf("smMap%d.smname", x), smMap[x].second.first);
+        paramIn(cp, section, csprintf("smMap%d.id", x), smMap[x].second.second);
+    }
+
+    //qMap (IdMap)
+    qMap.resize(numQs);
+    for (int x = 0; x < qMap.size(); x++) {
+        paramIn(cp, section, csprintf("qMap%d.sys", x), qMap[x].first);
+        paramIn(cp, section, csprintf("qMap%d.qname", x), qMap[x].second.first);
+        paramIn(cp, section, csprintf("qMap%d.id", x), qMap[x].second.second);
+    }
+
+
+    // qData (vector<AnnotateList>)
+    qData.resize(qSize.size());
+    for (int x = 0; x < qSize.size(); x++) {
+        if (!qSize[x])
+            continue;
+        for (int y = 0; y < qSize[x]; y++) {
+            AnnDataPtr a = new AnnotateData;
+            a->unserialize(cp, csprintf("%s.Q%d_%d", section, x, y));
+            data.push_back(a);
+            qData[x].push_back(a);
+        }
+    }
+}
+
+void
+CPA::AnnotateData::serialize(std::ostream &os)
+{
+    SERIALIZE_SCALAR(time);
+    SERIALIZE_SCALAR(data);
+    SERIALIZE_SCALAR(sm);
+    SERIALIZE_SCALAR(stq);
+    SERIALIZE_SCALAR(op);
+    SERIALIZE_SCALAR(flag);
+    SERIALIZE_SCALAR(cpu);
+}
+
+void
+CPA::AnnotateData::unserialize(Checkpoint *cp, const std::string &section)
+{
+    UNSERIALIZE_SCALAR(time);
+    UNSERIALIZE_SCALAR(data);
+    orig_data = data;
+    UNSERIALIZE_SCALAR(sm);
+    UNSERIALIZE_SCALAR(stq);
+    UNSERIALIZE_SCALAR(op);
+    UNSERIALIZE_SCALAR(flag);
+    UNSERIALIZE_SCALAR(cpu);
+    dump = true;
+}
+
+CPA*
+CPAParams::create()
+{
+    return new CPA(this);
+}
+
diff --git a/src/base/cp_annotate.hh b/src/base/cp_annotate.hh
new file mode 100644
index 000000000..9ec39b6cb
--- /dev/null
+++ b/src/base/cp_annotate.hh
@@ -0,0 +1,522 @@
+/*
+ * Copyright (c) 2006-2009 The Regents of The University of Michigan
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Ali Saidi
+ */
+
+#ifndef __BASE__CP_ANNOTATE_HH__
+#define __BASE__CP_ANNOTATE_HH__
+
+#include "base/loader/symtab.hh"
+#include "config/cp_annotate.hh"
+#include "sim/host.hh"
+#include "sim/serialize.hh"
+#include "sim/startup.hh"
+#include "sim/system.hh"
+
+#include <string>
+#include <list>
+#include <vector>
+#include <map>
+#include "base/hashmap.hh"
+#include "base/trace.hh"
+#if CP_ANNOTATE
+#include "params/CPA.hh"
+#endif
+
+class System;
+class ThreadContext;
+
+
+#if !CP_ANNOTATE
+class CPA : SimObject
+{
+  public:
+    enum flags {
+        FL_NONE     = 0x00,
+        FL_HW       = 0x01,
+        FL_BAD      = 0x02,
+        FL_QOPP     = 0x04,
+        FL_WAIT     = 0x08,
+        FL_LINK     = 0x10,
+        FL_RESET    = 0x20
+    };
+
+    static CPA *cpa()                                        { return NULL; }
+    static bool available()                                 { return false; }
+    bool enabled()                                          { return false; }
+    void swSmBegin(ThreadContext *tc)                             { return; }
+    void swSmEnd(ThreadContext *tc)                               { return; }
+    void swExplictBegin(ThreadContext *tc)                        { return; }
+    void swAutoBegin(ThreadContext *tc, Addr next_pc)             { return; }
+    void swEnd(ThreadContext *tc)                                 { return; }
+    void swQ(ThreadContext *tc)                                   { return; }
+    void swDq(ThreadContext *tc)                                  { return; }
+    void swPq(ThreadContext *tc)                                  { return; }
+    void swRq(ThreadContext *tc)                                  { return; }
+    void swWf(ThreadContext *tc)                                  { return; }
+    void swWe(ThreadContext *tc)                                  { return; }
+    void swSq(ThreadContext *tc)                                  { return; }
+    void swAq(ThreadContext *tc)                                  { return; }
+    void swLink(ThreadContext *tc)                                { return; }
+    void swIdentify(ThreadContext *tc)                            { return; }
+    uint64_t swGetId(ThreadContext *tc)                         { return 0; }
+    void swSyscallLink(ThreadContext *tc)                         { return; }
+    void hwBegin(flags f, System *sys, uint64_t frame, std::string sm, 
+                 std::string st)                                  { return; }
+    void hwQ(flags f, System *sys, uint64_t frame, std::string sm, 
+             std::string q, uint64_t qid, System *q_sys = NULL, 
+             int32_t count = 1)                                   { return; }
+    void hwDq(flags f, System *sys, uint64_t frame, std::string sm, 
+              std::string q, uint64_t qid, System *q_sys = NULL, 
+              int32_t count = 1)                                  { return; }
+    void hwPq(flags f, System *sys, uint64_t frame, std::string sm, 
+              std::string q, uint64_t qid, System *q_sys = NULL, 
+              int32_t count = 1)                                  { return; }
+    void hwRq(flags f, System *sys, uint64_t frame, std::string sm, 
+              std::string q, uint64_t qid, System *q_sys = NULL, 
+              int32_t count = 1)                                  { return; }
+    void hwWf(flags f, System *sys, uint64_t frame, std::string sm, 
+              std::string q, uint64_t qid, System *q_sys = NULL, 
+              int32_t count = 1)                                  { return; }
+    void hwWe(flags f, System *sys, uint64_t frame, std::string sm, 
+              std::string q, uint64_t qid, System *q_sys = NULL, 
+              int32_t count = 1)                                  { return; }
+};
+#else
+class CPA : SimObject
+{
+  public:
+    typedef CPAParams Params;
+    
+    /** The known operations that are written to the annotation output file. */
+    enum ops {
+        OP_BEGIN           = 0x01,
+        OP_WAIT_EMPTY      = 0x02,
+        OP_WAIT_FULL       = 0x03,
+        OP_QUEUE           = 0x04,
+        OP_DEQUEUE         = 0x05,
+        OP_SIZE_QUEUE      = 0x08,
+        OP_PEEK            = 0x09,
+        OP_LINK            = 0x0A,
+        OP_IDENT           = 0x0B,
+        OP_RESERVE         = 0x0C
+    };
+
+    /** Flags for the various options.*/
+    enum flags {
+        /* no flags */
+        FL_NONE     = 0x00,
+        /* operation was done on hardware */
+        FL_HW       = 0x01,
+        /* operation should cause a warning when encountered */
+        FL_BAD      = 0x02,
+        /* Queue like a stack, not a queue */
+        FL_QOPP     = 0x04,
+        /* Mark HW state as waiting for some non-resource constraint
+         * (e.g. wait because SM only starts after 10 items are queued) */ 
+        FL_WAIT     = 0x08,
+        /* operation is linking to another state machine */
+        FL_LINK     = 0x10,
+        /* queue should be completely cleared/reset before executing this
+         * operation */
+        FL_RESET    = 0x20
+    };
+
+
+
+  protected:
+    const Params *
+        params() const
+        {
+            return dynamic_cast<const Params *>(_params);
+        }
+
+    /* struct that is written to the annotation output file */
+    struct AnnotateData : public RefCounted {
+
+        Tick time;
+        uint32_t data;
+        uint32_t orig_data;
+        uint16_t sm;
+        uint16_t stq;
+        uint8_t  op;
+        uint8_t  flag;
+        uint8_t  cpu;
+        bool dump;
+
+        void serialize(std::ostream &os);
+        void unserialize(Checkpoint *cp, const std::string &section);
+
+    };
+
+    typedef RefCountingPtr<AnnotateData> AnnDataPtr;
+
+    /* header for the annotation file */
+    struct AnnotateHeader {
+        uint64_t version;
+        uint64_t num_recs;
+        uint64_t key_off;
+        uint64_t idx_off;
+        uint32_t key_len;
+        uint32_t idx_len;
+    };
+
+    AnnotateHeader ah;
+
+    std::vector<uint64_t> annotateIdx;
+
+    // number of state machines encountered in the simulation 
+    int numSm;
+    // number of states encountered in the simulation 
+    int numSmt;
+    // number of states/queues for a given state machine/system respectively
+    std::vector<int> numSt, numQ;
+    // number of systems in the simulation
+    int numSys;
+    // number of queues in the state machine
+    int numQs;
+    // maximum connection id assigned so far
+    uint64_t conId;
+
+    // Convert state strings into state ids
+    typedef m5::hash_map<std::string, int> SCache;
+    typedef std::vector<SCache> StCache;
+
+    // Convert sm and queue name,id into queue id
+    typedef std::pair<std::string, uint64_t> Id;
+    typedef m5::hash_map<Id, int> IdHCache;
+    typedef std::vector<IdHCache> IdCache;
+
+    // Hold mapping of sm and queues to output python
+    typedef std::vector<std::pair<int, Id> > IdMap;
+
+    // System pointer to name,id
+    typedef std::map<System*, std::pair<std::string, int> > NameCache;
+
+    // array of systems each of which is a stack of running sm
+    typedef std::pair<int, uint64_t> StackId;
+    typedef std::map<StackId, std::vector<int> > SmStack;
+
+    // map of each context and if it's currently in explict state mode
+    // states are not automatically updated until it leaves
+    typedef std::map<StackId, bool> SwExpl;
+
+    typedef std::map<int,int> IMap;
+    // List of annotate records have not been written/completed yet
+    typedef std::list<AnnDataPtr> AnnotateList;
+
+    // Maintain link state information
+    typedef std::map<int, int> LinkMap;
+
+    // SC Links
+    typedef m5::hash_map<Id, AnnDataPtr> ScHCache;
+    typedef std::vector<ScHCache> ScCache;
+
+
+    AnnotateList data;
+
+    // vector indexed by queueid to find current number of elements and bytes
+    std::vector<int> qSize;
+    std::vector<int32_t> qBytes;
+
+
+    // Turn state machine string into state machine id (small int)
+    // Used for outputting key to convert id back into string
+    SCache smtCache;
+    // Turn state machine id, state name into state id (small int)
+    StCache stCache;
+    // turn system, queue, and queue identify into qid (small int)
+    // turn system, state, and context into state machine id (small int)
+    IdCache qCache, smCache;
+    //Link state machines accross system calls
+    ScCache scLinks;
+    // System pointer to name,id
+    NameCache nameCache;
+    // Stack of state machines currently nested (should unwind correctly)
+    SmStack smStack;
+    // Map of currently outstanding links 
+    LinkMap lnMap;
+    // If the state machine is currently exculding automatic changes
+    SwExpl swExpl;
+    // Last state that a given state machine was in
+    IMap lastState;
+    // Hold mapping of sm and queues to output python
+    IdMap smMap, qMap;
+    // Items still in queue, used for sanity checking 
+    std::vector<AnnotateList> qData;
+
+    void doDq(System *sys, int flags, int cpu, int sm, std::string q, int qi,
+            int count);
+    void doQ(System *sys, int flags, int cpu, int sm, std::string q, int qi,
+            int count);
+
+    void doSwSmEnd(System *sys, int cpuid, std::string sm, uint64_t frame);
+
+    // Turn a system id, state machine string, state machine id into a small int
+    // for annotation output
+    int 
+    getSm(int sysi, std::string si, uint64_t id)
+    {
+        int smi;
+        Id smid = Id(si, id);
+
+        smi = smCache[sysi-1][smid];
+        if (smi == 0) {
+            smCache[sysi-1][smid] = smi = ++numSm;
+            assert(smi < 65535);
+            smMap.push_back(std::make_pair<int, Id>(sysi, smid));
+        }
+        return smi;
+    }
+
+    // Turn a state machine string, state string into a small int
+    // for annotation output
+    int 
+    getSt(std::string sm, std::string s)
+    {
+        int sti, smi;
+
+        smi = smtCache[sm];
+        if (smi == 0)
+           smi = smtCache[sm] = ++numSmt;
+
+        while (stCache.size() < smi) {
+            //stCache.resize(sm);
+            stCache.push_back(SCache());
+            numSt.push_back(0);
+        }
+        //assert(stCache.size() == sm);
+        //assert(numSt.size() == sm);
+        sti = stCache[smi-1][s];
+        if (sti == 0)
+            stCache[smi-1][s] = sti = ++numSt[smi-1];
+        return sti;
+    }
+
+    // Turn state machine pointer into a smal int for annotation output
+    int 
+    getSys(System *s)
+    {
+        NameCache::iterator i = nameCache.find(s);
+        if (i == nameCache.end()) {
+            nameCache[s] = std::make_pair<std::string,int>(s->name(), ++numSys);
+            i = nameCache.find(s);
+            // might need to put smstackid into map here, but perhaps not
+            //smStack.push_back(std::vector<int>());
+            //swExpl.push_back(false);
+            numQ.push_back(0);
+            qCache.push_back(IdHCache());
+            smCache.push_back(IdHCache());
+            scLinks.push_back(ScHCache());
+        }
+        return i->second.second;
+    }
+
+    // Turn queue name, and queue context into small int for 
+    // annotation output
+    int 
+    getQ(int sys, std::string q, uint64_t id)
+    {
+        int qi;
+        Id qid = Id(q, id);
+
+        qi = qCache[sys-1][qid];
+        if (qi == 0) {
+            qi = qCache[sys-1][qid] = ++numQs;
+            assert(qi < 65535);
+            qSize.push_back(0);
+            qBytes.push_back(0);
+            qData.push_back(AnnotateList());
+            numQ[sys-1]++;
+            qMap.push_back(std::make_pair<int, Id>(sys, qid));
+        }
+        return qi;
+    }
+
+    void swBegin(System *sys, int cpuid, std::string st, uint64_t frame, 
+            bool expl = false, int flags = FL_NONE);
+
+    AnnDataPtr add(int t, int f, int c, int sm, int stq, int32_t data=0);
+
+    std::ostream *osbin;
+
+    bool _enabled;
+
+    /** Only allow one CPA object in a system. It doesn't make sense to have 
+     * more that one per simulation because if a part of the system was
+     * important it would have annotations and queues, and with more than one
+     * object none of the sanity checking for queues will work. */
+    static bool exists;
+    static CPA *_cpa;
+
+
+    std::map<std::string, SymbolTable*> userApp;
+
+  public:
+    static CPA *cpa() { return _cpa; }
+    void swSmBegin(ThreadContext *tc);
+    void swSmEnd(ThreadContext *tc);
+    void swExplictBegin(ThreadContext *tc);
+    void swAutoBegin(ThreadContext *tc, Addr next_pc);
+    void swEnd(ThreadContext *tc);
+    void swQ(ThreadContext *tc);
+    void swDq(ThreadContext *tc);
+    void swPq(ThreadContext *tc);
+    void swRq(ThreadContext *tc);
+    void swWf(ThreadContext *tc);
+    void swWe(ThreadContext *tc);
+    void swSq(ThreadContext *tc);
+    void swAq(ThreadContext *tc);
+    void swLink(ThreadContext *tc);
+    void swIdentify(ThreadContext *tc);
+    uint64_t swGetId(ThreadContext *tc);
+    void swSyscallLink(ThreadContext *tc);
+
+    inline void hwBegin(flags f, System *sys, uint64_t frame, std::string sm, 
+            std::string st)
+    {
+        if (!enabled())
+            return;
+
+        int sysi = getSys(sys);
+        int smi = getSm(sysi, sm, frame);
+        add(OP_BEGIN, FL_HW | f, 0, smi, getSt(sm, st));
+        if (f & FL_BAD)
+            warn("BAD state encountered: at cycle %d: %s\n", curTick, st);
+    }
+
+    inline void hwQ(flags f, System *sys, uint64_t frame, std::string sm, 
+            std::string q, uint64_t qid, System *q_sys = NULL, int32_t count = 1)
+    {
+        if (!enabled())
+            return;
+
+        int sysi = getSys(sys);
+        int qi = getQ(q_sys ?  getSys(q_sys) : sysi, q, qid);
+        DPRINTFS(AnnotateQ, sys, 
+                "hwQ: %s[%#x] cur size %d %d bytes: %d adding: %d\n",
+                q, qid, qSize[qi-1], qData[qi-1].size(), qBytes[qi-1], count);
+        doQ(sys, FL_HW | f, 0, getSm(sysi, sm, frame), q, qi, count);
+
+    }
+
+    inline void hwDq(flags f, System *sys, uint64_t frame, std::string sm, 
+            std::string q, uint64_t qid, System *q_sys = NULL, int32_t count = 1)
+    {
+        if (!enabled())
+            return;
+
+        int sysi = getSys(sys);
+        int qi = getQ(q_sys ?  getSys(q_sys) : sysi, q, qid);
+        DPRINTFS(AnnotateQ, sys, 
+                "hwDQ: %s[%#x] cur size %d %d bytes: %d removing: %d\n",
+                q, qid, qSize[qi-1], qData[qi-1].size(), qBytes[qi-1], count);
+        doDq(sys, FL_HW | f, 0, getSm(sysi,sm, frame), q, qi, count);
+    }
+
+    inline void hwPq(flags f, System *sys, uint64_t frame, std::string sm, 
+            std::string q, uint64_t qid, System *q_sys = NULL, int32_t count = 1)
+    {
+        if (!enabled())
+            return;
+
+        int sysi = getSys(sys);
+        int qi = getQ(q_sys ?  getSys(q_sys) : sysi, q, qid);
+        DPRINTFS(AnnotateQ, sys, 
+                "hwPQ: %s[%#x] cur size %d %d bytes: %d peeking: %d\n",
+                q, qid, qSize[qi-1], qData[qi-1].size(), qBytes[qi-1], count);
+        add(OP_PEEK, FL_HW | f, 0, getSm(sysi, sm, frame), qi, count);
+    }
+
+    inline void hwRq(flags f, System *sys, uint64_t frame, std::string sm, 
+            std::string q, uint64_t qid, System *q_sys = NULL, int32_t count = 1)
+    {
+        if (!enabled())
+            return;
+
+        int sysi = getSys(sys);
+        int qi = getQ(q_sys ?  getSys(q_sys) : sysi, q, qid);
+        DPRINTFS(AnnotateQ, sys, 
+                "hwRQ: %s[%#x] cur size %d %d bytes: %d reserving: %d\n",
+                q, qid, qSize[qi-1], qData[qi-1].size(), qBytes[qi-1], count);
+        add(OP_RESERVE, FL_HW | f, 0, getSm(sysi, sm, frame), qi, count);
+    }
+
+    inline void hwWf(flags f, System *sys, uint64_t frame, std::string sm, 
+            std::string q, uint64_t qid, System *q_sys = NULL, int32_t count = 1)
+    {
+        if (!enabled())
+            return;
+
+        int sysi = getSys(sys);
+        int qi = getQ(q_sys ?  getSys(q_sys) : sysi, q, qid);
+        add(OP_WAIT_FULL, FL_HW | f, 0, getSm(sysi, sm, frame), qi, count);
+    }
+
+    inline void hwWe(flags f, System *sys, uint64_t frame, std::string sm, 
+            std::string q, uint64_t qid, System *q_sys = NULL, int32_t count = 1) 
+    {
+        if (!enabled())
+            return;
+
+        int sysi = getSys(sys);
+        int qi = getQ(q_sys ?  getSys(q_sys) : sysi, q, qid);
+        add(OP_WAIT_EMPTY, FL_HW | f, 0, getSm(sysi, sm, frame), qi, count);
+    }
+
+  public:
+    CPA(Params *p);
+    void startup();
+
+    // This code is ISA specific and will need to be changed
+    // if the annotation code is used for something other than Alpha
+    inline uint64_t getFrame(ThreadContext *tc)
+        { return (tc->readMiscRegNoEffect(TheISA::IPR_PALtemp23) & 
+                ~ULL(0x3FFF)); }
+
+    static bool available()  { return true; }
+
+    bool 
+    enabled() 
+    {   
+        if (!this)
+            return false;
+        return _enabled;
+    }
+        
+    void dump(bool all);
+    void dumpKey();
+
+    void serialize(std::ostream &os);
+    void unserialize(Checkpoint *cp, const std::string &section);
+
+};
+#endif // !CP_ANNOTATE
+
+#endif //__BASE__CP_ANNOTATE_HH__
+
diff --git a/src/base/hashmap.hh b/src/base/hashmap.hh
index f8d799780..ff2aa151f 100644
--- a/src/base/hashmap.hh
+++ b/src/base/hashmap.hh
@@ -81,6 +81,16 @@ namespace __hash_namespace {
             return(__stl_hash_string(s.c_str()));
         }
     };
+
+    template <>
+    struct hash<std::pair<std::string, uint64_t> > {
+        size_t operator() (std::pair<std::string, uint64_t> r) const {
+            return (__stl_hash_string(r.first.c_str())) ^ r.second;
+        }
+    };
+
+
+
 }
 
 
diff --git a/src/base/loader/symtab.hh b/src/base/loader/symtab.hh
index 97a675140..cc1dc368f 100644
--- a/src/base/loader/symtab.hh
+++ b/src/base/loader/symtab.hh
@@ -34,6 +34,7 @@
 
 #include <iosfwd>
 #include <map>
+#include <string>
 
 #include "sim/host.hh"  // for Addr