CPA: Add annotations to IGbE and CopyEngine device models.

4 files changed, 227 insertions, 5 deletions

diff --git a/src/dev/copy_engine.cc b/src/dev/copy_engine.cc
index 46d3127e7..3c759ac1d 100644
--- a/src/dev/copy_engine.cc
+++ b/src/dev/copy_engine.cc
@@ -34,6 +34,7 @@
 
 #include <algorithm>
 
+#include "base/cp_annotate.hh"
 #include "base/trace.hh"
 #include "dev/copy_engine.hh"
 #include "mem/packet.hh"
@@ -427,6 +428,8 @@ CopyEngine::regStats()
 void
 CopyEngine::CopyEngineChannel::fetchDescriptor(Addr address)
 {
+    anDq();
+    anBegin("FetchDescriptor");
     DPRINTF(DMACopyEngine, "Reading descriptor from at memory location %#x(%#x)\n",
            address, ce->platform->pciToDma(address));
     assert(address);
@@ -455,6 +458,8 @@ CopyEngine::CopyEngineChannel::fetchDescComplete()
             if (inDrain()) return;
             writeCompletionStatus();
         } else {
+            anBegin("Idle");
+            anWait();
             busy = false;
             nextState = Idle;
             inDrain();
@@ -473,6 +478,7 @@ CopyEngine::CopyEngineChannel::fetchDescComplete()
 void
 CopyEngine::CopyEngineChannel::readCopyBytes()
 {
+    anBegin("ReadCopyBytes");
     DPRINTF(DMACopyEngine, "Reading %d bytes from buffer to memory location %#x(%#x)\n",
            curDmaDesc->len, curDmaDesc->dest,
            ce->platform->pciToDma(curDmaDesc->src));
@@ -493,6 +499,7 @@ CopyEngine::CopyEngineChannel::readCopyBytesComplete()
 void
 CopyEngine::CopyEngineChannel::writeCopyBytes()
 {
+    anBegin("WriteCopyBytes");
     DPRINTF(DMACopyEngine, "Writing %d bytes from buffer to memory location %#x(%#x)\n",
            curDmaDesc->len, curDmaDesc->dest,
            ce->platform->pciToDma(curDmaDesc->dest));
@@ -513,6 +520,8 @@ CopyEngine::CopyEngineChannel::writeCopyBytesComplete()
     cr.status.compl_desc_addr(lastDescriptorAddr >> 6);
     completionDataReg = cr.status() | 1;
 
+    anQ("DMAUsedDescQ", channelId, 1);
+    anQ("AppRecvQ", curDmaDesc->user1, curDmaDesc->len);
     if (curDmaDesc->command & DESC_CTRL_CP_STS) {
         nextState = CompletionWrite;
         if (inDrain()) return;
@@ -529,6 +538,8 @@ CopyEngine::CopyEngineChannel::continueProcessing()
     busy = false;
 
     if (underReset) {
+        anBegin("Reset");
+        anWait();
         underReset = false;
         refreshNext = false;
         busy = false;
@@ -549,12 +560,15 @@ CopyEngine::CopyEngineChannel::continueProcessing()
     } else {
         inDrain();
         nextState = Idle;
+        anWait();
+        anBegin("Idle");
     }
 }
 
 void
 CopyEngine::CopyEngineChannel::writeCompletionStatus()
 {
+    anBegin("WriteCompletionStatus");
     DPRINTF(DMACopyEngine, "Writing completion status %#x to address %#x(%#x)\n",
             completionDataReg, cr.completionAddr,
             ce->platform->pciToDma(cr.completionAddr));
@@ -574,6 +588,7 @@ CopyEngine::CopyEngineChannel::writeStatusComplete()
 void
 CopyEngine::CopyEngineChannel::fetchNextAddr(Addr address)
 {
+    anBegin("FetchNextAddr");
     DPRINTF(DMACopyEngine, "Fetching next address...\n");
     busy = true;
     cePort->dmaAction(MemCmd::ReadReq, ce->platform->pciToDma(address +
@@ -590,6 +605,8 @@ CopyEngine::CopyEngineChannel::fetchAddrComplete()
         DPRINTF(DMACopyEngine, "Got NULL descriptor, nothing more to do\n");
         busy = false;
         nextState = Idle;
+        anWait();
+        anBegin("Idle");
         inDrain();
         return;
     }
diff --git a/src/dev/copy_engine.hh b/src/dev/copy_engine.hh
index 475c0a5bf..12531f68c 100644
--- a/src/dev/copy_engine.hh
+++ b/src/dev/copy_engine.hh
@@ -130,6 +130,36 @@ class CopyEngine : public PciDev
         void recvCommand();
         bool inDrain();
         void restartStateMachine();
+        inline void anBegin(const char *s)
+        {
+            CPA::cpa()->hwBegin(CPA::FL_NONE, ce->sys,
+                         channelId, "CopyEngine", s);
+        }
+
+        inline void anWait()
+        {
+            CPA::cpa()->hwWe(CPA::FL_NONE, ce->sys,
+                     channelId, "CopyEngine", "DMAUnusedDescQ", channelId);
+        }
+
+        inline void anDq()
+        {
+            CPA::cpa()->hwDq(CPA::FL_NONE, ce->sys,
+                      channelId, "CopyEngine", "DMAUnusedDescQ", channelId);
+        }
+
+        inline void anPq()
+        {
+            CPA::cpa()->hwDq(CPA::FL_NONE, ce->sys,
+                      channelId, "CopyEngine", "DMAUnusedDescQ", channelId);
+        }
+
+        inline void anQ(const char * s, uint64_t id, int size = 1)
+        {
+            CPA::cpa()->hwQ(CPA::FL_NONE, ce->sys, channelId,
+                    "CopyEngine", s, id, NULL, size);
+        }
+
     };
 
   private:
diff --git a/src/dev/i8254xGBe.cc b/src/dev/i8254xGBe.cc
index d6a888b21..274f60e39 100644
--- a/src/dev/i8254xGBe.cc
+++ b/src/dev/i8254xGBe.cc
@@ -113,10 +113,20 @@ IGbE::IGbE(const Params *p)
     // Magic happy checksum value
     flash[EEPROM_SIZE-1] = htobe((uint16_t)(EEPROM_CSUM - csum));
 
+    // Store the MAC address as queue ID
+    macAddr = p->hardware_address;
+
     rxFifo.clear();
     txFifo.clear();
 }
 
+void
+IGbE::init()
+{
+    cpa = CPA::cpa();
+    PciDev::init();
+}
+
 EtherInt*
 IGbE::getEthPort(const std::string &if_name, int idx)
 {
@@ -793,6 +803,13 @@ IGbE::RxDescCache::RxDescCache(IGbE *i, const std::string n, int s)
       pktEvent(this), pktHdrEvent(this), pktDataEvent(this)
 
 {
+    annSmFetch = "RX Desc Fetch";
+    annSmWb = "RX Desc Writeback";
+    annUnusedDescQ = "RX Unused Descriptors";
+    annUnusedCacheQ = "RX Unused Descriptor Cache";
+    annUsedCacheQ = "RX Used Descriptor Cache";
+    annUsedDescQ = "RX Used Descriptors";
+    annDescQ = "RX Descriptors";
 }
 
 void
@@ -910,6 +927,8 @@ IGbE::RxDescCache::pktComplete()
     RxDesc *desc;
     desc = unusedCache.front();
 
+    igbe->anBegin("RXS", "Update Desc");
+
     uint16_t crcfixup = igbe->regs.rctl.secrc() ? 0 : 4 ;
     DPRINTF(EthernetDesc, "pktPtr->length: %d bytesCopied: %d stripcrc offset: %d value written: %d %d\n",
             pktPtr->length, bytesCopied, crcfixup,
@@ -1052,8 +1071,11 @@ IGbE::RxDescCache::pktComplete()
     enableSm();
     pktDone = true;
 
+    igbe->anBegin("RXS", "Done Updating Desc");
     DPRINTF(EthernetDesc, "Processing of this descriptor complete\n");
+    igbe->anDq("RXS", annUnusedCacheQ);
     unusedCache.pop_front();
+    igbe->anQ("RXS", annUsedCacheQ);
     usedCache.push_back(desc);
 }
 
@@ -1112,6 +1134,13 @@ IGbE::TxDescCache::TxDescCache(IGbE *i, const std::string n, int s)
        useTso(false), pktEvent(this), headerEvent(this), nullEvent(this)
 
 {
+    annSmFetch = "TX Desc Fetch";
+    annSmWb = "TX Desc Writeback";
+    annUnusedDescQ = "TX Unused Descriptors";
+    annUnusedCacheQ = "TX Unused Descriptor Cache";
+    annUsedCacheQ = "TX Used Descriptor Cache";
+    annUsedDescQ = "TX Used Descriptors";
+    annDescQ = "TX Descriptors";
 }
 
 void
@@ -1154,7 +1183,9 @@ IGbE::TxDescCache::processContextDesc()
 
         TxdOp::setDd(desc);
         unusedCache.pop_front();
+        igbe->anDq("TXS", annUnusedCacheQ);
         usedCache.push_back(desc);
+        igbe->anQ("TXS", annUsedCacheQ);
     }
 
     if (!unusedCache.size())
@@ -1298,6 +1329,8 @@ IGbE::TxDescCache::pktComplete()
     assert(unusedCache.size());
     assert(pktPtr);
 
+    igbe->anBegin("TXS", "Update Desc");
+
     DPRINTF(EthernetDesc, "DMA of packet complete\n");
 
 
@@ -1323,7 +1356,9 @@ IGbE::TxDescCache::pktComplete()
             (pktPtr->length < ( tsoMss + tsoHeaderLen) &&
              tsoTotalLen != tsoUsedLen && useTso)) {
         assert(!useTso || (tsoDescBytesUsed == TxdOp::getLen(desc)));
+        igbe->anDq("TXS", annUnusedCacheQ);
         unusedCache.pop_front();
+        igbe->anQ("TXS", annUsedCacheQ);
         usedCache.push_back(desc);
 
         tsoDescBytesUsed = 0;
@@ -1441,7 +1476,9 @@ IGbE::TxDescCache::pktComplete()
 
     if (!useTso ||  TxdOp::getLen(desc) == tsoDescBytesUsed) {
         DPRINTF(EthernetDesc, "Descriptor Done\n");
+        igbe->anDq("TXS", annUnusedCacheQ);
         unusedCache.pop_front();
+        igbe->anQ("TXS", annUsedCacheQ);
         usedCache.push_back(desc);
         tsoDescBytesUsed = 0;
     }
@@ -1458,14 +1495,17 @@ IGbE::TxDescCache::pktComplete()
     tsoPktHasHeader = false;
 
     if (igbe->regs.txdctl.wthresh() == 0) {
+        igbe->anBegin("TXS", "Desc Writeback");
         DPRINTF(EthernetDesc, "WTHRESH == 0, writing back descriptor\n");
         writeback(0);
     } else if (igbe->regs.txdctl.gran() && igbe->regs.txdctl.wthresh() >=
             descInBlock(usedCache.size())) {
         DPRINTF(EthernetDesc, "used > WTHRESH, writing back descriptor\n");
+        igbe->anBegin("TXS", "Desc Writeback");
         writeback((igbe->cacheBlockSize()-1)>>4);
     } else if (igbe->regs.txdctl.wthresh() >= usedCache.size()) {
         DPRINTF(EthernetDesc, "used > WTHRESH, writing back descriptor\n");
+        igbe->anBegin("TXS", "Desc Writeback");
         writeback((igbe->cacheBlockSize()-1)>>4);
     }
 
@@ -1604,6 +1644,7 @@ IGbE::drain(Event *de)
     else
         changeState(Drained);
 
+    DPRINTF(EthernetSM, "got drain() returning %d", count);
     return count;
 }
 
@@ -1617,6 +1658,7 @@ IGbE::resume()
     rxTick = true;
 
     restartClock();
+    DPRINTF(EthernetSM, "resuming from drain");
 }
 
 void
@@ -1625,6 +1667,7 @@ IGbE::checkDrain()
     if (!drainEvent)
         return;
 
+    DPRINTF(EthernetSM, "checkDrain() in drain\n");
     txFifoTick = false;
     txTick = false;
     rxTick = false;
@@ -1651,11 +1694,13 @@ IGbE::txStateMachine()
                  && !txDescCache.packetMultiDesc() && txPacket->length) {
         bool success;
 
+        anQ("TXS", "TX FIFO Q");
         DPRINTF(EthernetSM, "TXS: packet placed in TX FIFO\n");
         success = txFifo.push(txPacket);
         txFifoTick = true && !drainEvent;
         assert(success);
         txPacket = NULL;
+        anBegin("TXS", "Desc Writeback");
         txDescCache.writeback((cacheBlockSize()-1)>>4);
         return;
     }
@@ -1673,7 +1718,10 @@ IGbE::txStateMachine()
     if (!txDescCache.packetWaiting()) {
         if (txDescCache.descLeft() == 0) {
             postInterrupt(IT_TXQE);
+            anBegin("TXS", "Desc Writeback");
             txDescCache.writeback(0);
+            anBegin("TXS", "Desc Fetch");
+            anWe("TXS", txDescCache.annUnusedCacheQ);
             txDescCache.fetchDescriptors();
             DPRINTF(EthernetSM, "TXS: No descriptors left in ring, forcing "
                     "writeback stopping ticking and posting TXQE\n");
@@ -1683,11 +1731,14 @@ IGbE::txStateMachine()
 
 
         if (!(txDescCache.descUnused())) {
+            anBegin("TXS", "Desc Fetch");
             txDescCache.fetchDescriptors();
+            anWe("TXS", txDescCache.annUnusedCacheQ);
             DPRINTF(EthernetSM, "TXS: No descriptors available in cache, fetching and stopping ticking\n");
             txTick = false;
             return;
         }
+        anPq("TXS", txDescCache.annUnusedCacheQ);
 
 
         txDescCache.processContextDesc();
@@ -1700,6 +1751,8 @@ IGbE::txStateMachine()
         int size;
         size = txDescCache.getPacketSize(txPacket);
         if (size > 0 && txFifo.avail() > size) {
+            anRq("TXS", "TX FIFO Q");
+            anBegin("TXS", "DMA Packet");
             DPRINTF(EthernetSM, "TXS: Reserving %d bytes in FIFO and begining "
                     "DMA of next packet\n", size);
             txFifo.reserve(size);
@@ -1707,8 +1760,10 @@ IGbE::txStateMachine()
         } else if (size <= 0) {
             DPRINTF(EthernetSM, "TXS: getPacketSize returned: %d\n", size);
             DPRINTF(EthernetSM, "TXS: No packets to get, writing back used descriptors\n");
+            anBegin("TXS", "Desc Writeback");
             txDescCache.writeback(0);
         } else {
+            anWf("TXS", "TX FIFO Q");
             DPRINTF(EthernetSM, "TXS: FIFO full, stopping ticking until space "
                     "available in FIFO\n");
             txTick = false;
@@ -1728,9 +1783,12 @@ IGbE::ethRxPkt(EthPacketPtr pkt)
     rxPackets++;
 
     DPRINTF(Ethernet, "RxFIFO: Receiving pcakte from wire\n");
+    anBegin("RXQ", "Wire Recv");
+
 
     if (!regs.rctl.en()) {
         DPRINTF(Ethernet, "RxFIFO: RX not enabled, dropping\n");
+        anBegin("RXQ", "FIFO Drop", CPA::FL_BAD);
         return true;
     }
 
@@ -1744,9 +1802,23 @@ IGbE::ethRxPkt(EthPacketPtr pkt)
     if (!rxFifo.push(pkt)) {
         DPRINTF(Ethernet, "RxFIFO: Packet won't fit in fifo... dropped\n");
         postInterrupt(IT_RXO, true);
+        anBegin("RXQ", "FIFO Drop", CPA::FL_BAD);
         return false;
     }
 
+    if (CPA::available() && cpa->enabled()) {
+        assert(sys->numSystemsRunning <= 2);
+        System *other_sys;
+        if (sys->systemList[0] == sys)
+            other_sys = sys->systemList[1];
+        else
+            other_sys = sys->systemList[0];
+
+        cpa->hwDq(CPA::FL_NONE, sys, macAddr, "RXQ", "WireQ", 0, other_sys);
+        anQ("RXQ", "RX FIFO Q");
+        cpa->hwWe(CPA::FL_NONE, sys, macAddr, "RXQ", "WireQ", 0, other_sys);
+    }
+
     return true;
 }
 
@@ -1780,6 +1852,7 @@ IGbE::rxStateMachine()
             rxDescCache.writeback(0);
 
         if (descLeft == 0) {
+            anBegin("RXS", "Writeback Descriptors");
             rxDescCache.writeback(0);
             DPRINTF(EthernetSM, "RXS: No descriptors left in ring, forcing"
                     " writeback and stopping ticking\n");
@@ -1791,6 +1864,7 @@ IGbE::rxStateMachine()
 
         if (regs.rxdctl.wthresh() >= rxDescCache.descUsed()) {
             DPRINTF(EthernetSM, "RXS: Writing back because WTHRESH >= descUsed\n");
+            anBegin("RXS", "Writeback Descriptors");
             if (regs.rxdctl.wthresh() < (cacheBlockSize()>>4))
                 rxDescCache.writeback(regs.rxdctl.wthresh()-1);
             else
@@ -1800,11 +1874,14 @@ IGbE::rxStateMachine()
         if ((rxDescCache.descUnused() < regs.rxdctl.pthresh()) &&
              ((rxDescCache.descLeft() - rxDescCache.descUnused()) > regs.rxdctl.hthresh())) {
             DPRINTF(EthernetSM, "RXS: Fetching descriptors because descUnused < PTHRESH\n");
+            anBegin("RXS", "Fetch Descriptors");
             rxDescCache.fetchDescriptors();
         }
 
         if (rxDescCache.descUnused() == 0) {
+            anBegin("RXS", "Fetch Descriptors");
             rxDescCache.fetchDescriptors();
+            anWe("RXS", rxDescCache.annUnusedCacheQ);
             DPRINTF(EthernetSM, "RXS: No descriptors available in cache, "
                     "fetching descriptors and stopping ticking\n");
             rxTick = false;
@@ -1819,18 +1896,24 @@ IGbE::rxStateMachine()
     }
 
     if (!rxDescCache.descUnused()) {
+        anBegin("RXS", "Fetch Descriptors");
         rxDescCache.fetchDescriptors();
+        anWe("RXS", rxDescCache.annUnusedCacheQ);
         DPRINTF(EthernetSM, "RXS: No descriptors available in cache, stopping ticking\n");
         rxTick = false;
         DPRINTF(EthernetSM, "RXS: No descriptors available, fetching\n");
         return;
     }
+    anPq("RXS", rxDescCache.annUnusedCacheQ);
 
     if (rxFifo.empty()) {
+        anWe("RXS", "RX FIFO Q");
         DPRINTF(EthernetSM, "RXS: RxFIFO empty, stopping ticking\n");
         rxTick = false;
         return;
     }
+    anPq("RXS", "RX FIFO Q");
+    anBegin("RXS", "Get Desc");
 
     EthPacketPtr pkt;
     pkt = rxFifo.front();
@@ -1839,26 +1922,32 @@ IGbE::rxStateMachine()
     pktOffset = rxDescCache.writePacket(pkt, pktOffset);
     DPRINTF(EthernetSM, "RXS: Writing packet into memory\n");
     if (pktOffset == pkt->length) {
+        anBegin( "RXS", "FIFO Dequeue");
         DPRINTF(EthernetSM, "RXS: Removing packet from FIFO\n");
         pktOffset = 0;
+        anDq("RXS", "RX FIFO Q");
         rxFifo.pop();
     }
 
     DPRINTF(EthernetSM, "RXS: stopping ticking until packet DMA completes\n");
     rxTick = false;
     rxDmaPacket = true;
+    anBegin("RXS", "DMA Packet");
 }
 
 void
 IGbE::txWire()
 {
     if (txFifo.empty()) {
+        anWe("TXQ", "TX FIFO Q");
         txFifoTick = false;
         return;
     }
 
 
+    anPq("TXQ", "TX FIFO Q");
     if (etherInt->sendPacket(txFifo.front())) {
+        cpa->hwQ(CPA::FL_NONE, sys, macAddr, "TXQ", "WireQ", 0);
         if (DTRACE(EthernetSM)) {
             IpPtr ip(txFifo.front());
             if (ip)
@@ -1867,6 +1956,8 @@ IGbE::txWire()
             else
                 DPRINTF(EthernetSM, "Transmitting Non-Ip packet\n");
         }
+        anDq("TXQ", "TX FIFO Q");
+        anBegin("TXQ", "Wire Send");
         DPRINTF(EthernetSM, "TxFIFO: Successful transmit, bytes available in fifo: %d\n",
                 txFifo.avail());
 
@@ -1903,6 +1994,7 @@ IGbE::tick()
 void
 IGbE::ethTxDone()
 {
+    anBegin("TXQ", "Send Done");
     // restart the tx state machines if they are stopped
     // fifo to send another packet
     // tx sm to put more data into the fifo
diff --git a/src/dev/i8254xGBe.hh b/src/dev/i8254xGBe.hh
index 6dd2141b0..0422abc33 100644
--- a/src/dev/i8254xGBe.hh
+++ b/src/dev/i8254xGBe.hh
@@ -38,6 +38,7 @@
 #include <deque>
 #include <string>
 
+#include "base/cp_annotate.hh"
 #include "base/inet.hh"
 #include "dev/etherdevice.hh"
 #include "dev/etherint.hh"
@@ -54,6 +55,7 @@ class IGbE : public EtherDevice
 {
   private:
     IGbEInt *etherInt;
+    CPA *cpa;
 
     // device registers
     iGbReg::Regs regs;
@@ -176,6 +178,35 @@ class IGbE : public EtherDevice
      */
     void checkDrain();
 
+    void anBegin(std::string sm, std::string st, int flags = CPA::FL_NONE) {
+        cpa->hwBegin((CPA::flags)flags, sys, macAddr, sm, st);
+    }
+
+    void anQ(std::string sm, std::string q) { 
+        cpa->hwQ(CPA::FL_NONE, sys, macAddr, sm, q, macAddr);
+    }
+
+    void anDq(std::string sm, std::string q) {
+        cpa->hwDq(CPA::FL_NONE, sys, macAddr, sm, q, macAddr);
+    }
+
+    void anPq(std::string sm, std::string q, int num = 1) {
+        cpa->hwPq(CPA::FL_NONE, sys, macAddr, sm, q, macAddr, NULL, num);
+    }
+
+    void anRq(std::string sm, std::string q, int num = 1) {
+        cpa->hwPq(CPA::FL_NONE, sys, macAddr, sm, q, macAddr, NULL, num);
+    }
+
+    void anWe(std::string sm, std::string q) {
+        cpa->hwWe(CPA::FL_NONE, sys, macAddr, sm, q, macAddr);
+    }
+
+    void anWf(std::string sm, std::string q) {
+        cpa->hwWf(CPA::FL_NONE, sys, macAddr, sm, q, macAddr);
+    }
+
+
     template<class T>
     class DescCache
     {
@@ -225,6 +256,10 @@ class IGbE : public EtherDevice
         EthPacketPtr pktPtr;
 
       public:
+        /** Annotate sm*/
+        std::string annSmFetch, annSmWb, annUnusedDescQ, annUsedCacheQ,
+            annUsedDescQ, annUnusedCacheQ, annDescQ;
+
         DescCache(IGbE *i, const std::string n, int s)
             : igbe(i), _name(n), cachePnt(0), size(s), curFetching(0), wbOut(0),
               pktPtr(NULL), wbDelayEvent(this), fetchDelayEvent(this), 
@@ -290,6 +325,10 @@ class IGbE : public EtherDevice
             DPRINTF(EthernetDesc, "Writing back %d descriptors\n", max_to_wb);
 
             if (max_to_wb <= 0) {
+                if (usedCache.size())
+                    igbe->anBegin(annSmWb, "Wait Alignment", CPA::FL_WAIT);
+                else
+                    igbe->anWe(annSmWb, annUsedCacheQ);
                 return;
             }
 
@@ -297,25 +336,30 @@ class IGbE : public EtherDevice
 
             assert(!wbDelayEvent.scheduled()); 
             igbe->schedule(wbDelayEvent, curTick + igbe->wbDelay);
+            igbe->anBegin(annSmWb, "Prepare Writeback Desc");
         }
             
         void writeback1()
         {
             // If we're draining delay issuing this DMA
-            if (igbe->drainEvent) {
+            if (igbe->getState() != SimObject::Running) {
                 igbe->schedule(wbDelayEvent, curTick + igbe->wbDelay);
                 return;
             }
 
-            DPRINTF(EthernetDesc, "Beining DMA of %d descriptors\n", wbOut);
+            DPRINTF(EthernetDesc, "Begining DMA of %d descriptors\n", wbOut);
             
             for (int x = 0; x < wbOut; x++) {
                 assert(usedCache.size());
                 memcpy(&wbBuf[x], usedCache[x], sizeof(T));
-                 //delete usedCache[0];
-                //usedCache.pop_front();
+                igbe->anPq(annSmWb, annUsedCacheQ);
+                igbe->anPq(annSmWb, annDescQ);
+                igbe->anQ(annSmWb, annUsedDescQ);
             }
 
+    
+            igbe->anBegin(annSmWb, "Writeback Desc DMA");
+
             assert(wbOut);
             igbe->dmaWrite(igbe->platform->pciToDma(descBase() + descHead() * sizeof(T)),
                     wbOut * sizeof(T), &wbEvent, (uint8_t*)wbBuf,
@@ -343,6 +387,18 @@ class IGbE : public EtherDevice
 
             size_t free_cache = size - usedCache.size() - unusedCache.size();
 
+            if (!max_to_fetch)
+                igbe->anWe(annSmFetch, annUnusedDescQ);
+            else
+                igbe->anPq(annSmFetch, annUnusedDescQ, max_to_fetch);
+
+            if (max_to_fetch) {
+                if (!free_cache)
+                    igbe->anWf(annSmFetch, annDescQ);
+                else
+                    igbe->anRq(annSmFetch, annDescQ, free_cache);
+            }
+
             max_to_fetch = std::min(max_to_fetch, free_cache);
             
 
@@ -360,16 +416,19 @@ class IGbE : public EtherDevice
 
             assert(!fetchDelayEvent.scheduled());
             igbe->schedule(fetchDelayEvent, curTick + igbe->fetchDelay);
+            igbe->anBegin(annSmFetch, "Prepare Fetch Desc");
         }
 
         void fetchDescriptors1()
         {
             // If we're draining delay issuing this DMA
-            if (igbe->drainEvent) {
+            if (igbe->getState() != SimObject::Running) {
                 igbe->schedule(fetchDelayEvent, curTick + igbe->fetchDelay);
                 return;
             }
 
+            igbe->anBegin(annSmFetch, "Fetch Desc");
+
             DPRINTF(EthernetDesc, "Fetching descriptors at %#x (%#x), size: %#x\n",
                     descBase() + cachePnt * sizeof(T),
                     igbe->platform->pciToDma(descBase() + cachePnt * sizeof(T)),
@@ -387,10 +446,14 @@ class IGbE : public EtherDevice
         void fetchComplete()
         {
             T *newDesc;
+            igbe->anBegin(annSmFetch, "Fetch Complete");
             for (int x = 0; x < curFetching; x++) {
                 newDesc = new T;
                 memcpy(newDesc, &fetchBuf[x], sizeof(T));
                 unusedCache.push_back(newDesc);
+                igbe->anDq(annSmFetch, annUnusedDescQ);
+                igbe->anQ(annSmFetch, annUnusedCacheQ);
+                igbe->anQ(annSmFetch, annDescQ);
             }
 
 
@@ -408,6 +471,16 @@ class IGbE : public EtherDevice
             DPRINTF(EthernetDesc, "Fetching complete cachePnt %d -> %d\n",
                     oldCp, cachePnt);
 
+            if ((descTail() >= cachePnt ? (descTail() - cachePnt) : (descLen() -
+                    cachePnt)) == 0)
+            {
+                igbe->anWe(annSmFetch, annUnusedDescQ);
+            } else if (!(size - usedCache.size() - unusedCache.size())) {
+                igbe->anWf(annSmFetch, annDescQ);
+            } else {
+                igbe->anBegin(annSmFetch, "Wait", CPA::FL_WAIT);
+            }
+
             enableSm();
             igbe->checkDrain();
         }
@@ -419,6 +492,8 @@ class IGbE : public EtherDevice
         void wbComplete()
         {
 
+            igbe->anBegin(annSmWb, "Finish Writeback");
+
             long  curHead = descHead();
 #ifndef NDEBUG
             long oldHead = curHead;
@@ -428,6 +503,9 @@ class IGbE : public EtherDevice
                 assert(usedCache.size());
                 delete usedCache[0];
                 usedCache.pop_front();
+
+                igbe->anDq(annSmWb, annUsedCacheQ);
+                igbe->anDq(annSmWb, annDescQ);
             }
 
             curHead += wbOut;
@@ -452,6 +530,10 @@ class IGbE : public EtherDevice
 
             if (!wbOut) {
                 igbe->checkDrain();
+                if (usedCache.size())
+                    igbe->anBegin(annSmWb, "Wait", CPA::FL_WAIT);
+                else
+                    igbe->anWe(annSmWb, annUsedCacheQ);
             }
             fetchAfterWb();
         }
@@ -747,6 +829,7 @@ class IGbE : public EtherDevice
     }
     IGbE(const Params *params);
     ~IGbE() {}
+    virtual void init();
 
     virtual EtherInt *getEthPort(const std::string &if_name, int idx);