Virtualize sinic

separate the rx thread and tx thread and get rid of the dedicated flag.

dev/ns_gige.cc:
dev/ns_gige.hh:
dev/ns_gige_reg.h:
python/m5/objects/Ethernet.py:
    dedicated flag goes away, we have new individual flags for
    rx thread and tx thread
dev/sinic.cc:
    Virtualize sinic
    - The io registers are replicated many times in memory, allowing the NIC to
    differentiate among several virtual interfaces.
    - On the TX side, this allows multiple CPUs to initiate transmits at the same
    time without locking in the software.  If a partial packet is transmitted,
    then the state machine blocks waiting for that virtual interface to complete
    its packet.  Then the state machine will move on to the next virtual
    interface.  The commands are kept in fifo order.
    - On the RX side, multiple partial transmits can be simultaneously done.
    Though a packet does not deallocate its fifo space until all preceeding
    packets in the fifo are deallocated.  To enable multiple receives, it
    is necessary for each virtual nic to keep its own information about its
    progress through the state machine.
dev/sinic.hh:
    Virtualize sinic
    Receive state must be virtualized since we allow the receipt of packets in
    parallel.
dev/sinicreg.hh:
    Virtualize sinic
    separate rx thread and tx thread
    create a soft interrupt and add a command to trigger it.
    pad out the reserved bits in the RxDone and TxDone regs

--HG--
extra : convert_revision : c10bb23a46a89ffd1e08866c1f1621cb98069205

6 files changed, 362 insertions, 163 deletions

diff --git a/dev/ns_gige.cc b/dev/ns_gige.cc
index 979bb6b7a..9010850ab 100644
--- a/dev/ns_gige.cc
+++ b/dev/ns_gige.cc
@@ -764,8 +764,10 @@ NSGigE::read(MemReqPtr &req, uint8_t *data)
 
               case M5REG:
                 reg = 0;
-                if (params()->dedicated)
-                    reg |= M5REG_DEDICATED;
+                if (params()->rx_thread)
+                    reg |= M5REG_RX_THREAD;
+                if (params()->tx_thread)
+                    reg |= M5REG_TX_THREAD;
                 break;
 
               default:
@@ -3047,7 +3049,8 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS(NSGigE)
 
     Param<bool> rx_filter;
     Param<string> hardware_address;
-    Param<bool> dedicated;
+    Param<bool> rx_thread;
+    Param<bool> tx_thread;
 
 END_DECLARE_SIM_OBJECT_PARAMS(NSGigE)
 
@@ -3087,7 +3090,8 @@ BEGIN_INIT_SIM_OBJECT_PARAMS(NSGigE)
 
     INIT_PARAM(rx_filter, "Enable Receive Filter"),
     INIT_PARAM(hardware_address, "Ethernet Hardware Address"),
-    INIT_PARAM(dedicated, "dedicate a kernel thread to the driver")
+    INIT_PARAM(rx_thread, ""),
+    INIT_PARAM(tx_thread, "")
 
 END_INIT_SIM_OBJECT_PARAMS(NSGigE)
 
@@ -3131,7 +3135,8 @@ CREATE_SIM_OBJECT(NSGigE)
 
     params->rx_filter = rx_filter;
     params->eaddr = hardware_address;
-    params->dedicated = dedicated;
+    params->rx_thread = rx_thread;
+    params->tx_thread = tx_thread;
 
     return new NSGigE(params);
 }
diff --git a/dev/ns_gige.hh b/dev/ns_gige.hh
index 7db833028..ade7e32e6 100644
--- a/dev/ns_gige.hh
+++ b/dev/ns_gige.hh
@@ -396,7 +396,8 @@ class NSGigE : public PciDev
         Net::EthAddr eaddr;
         uint32_t tx_fifo_size;
         uint32_t rx_fifo_size;
-        bool dedicated;
+        bool rx_thread;
+        bool tx_thread;
         bool dma_no_allocate;
     };
 
diff --git a/dev/ns_gige_reg.h b/dev/ns_gige_reg.h
index f919ff086..eadc60d03 100644
--- a/dev/ns_gige_reg.h
+++ b/dev/ns_gige_reg.h
@@ -305,8 +305,9 @@
 #define TANAR_UNUSED		0x00000E1F
 
 /* M5 control register */
-#define M5REG_RESERVED		0xfffffffe
-#define M5REG_DEDICATED		0x00000001
+#define M5REG_RESERVED		0xfffffffc
+#define M5REG_RX_THREAD		0x00000002
+#define M5REG_TX_THREAD		0x00000001
 
 struct ns_desc32 {
     uint32_t link;    /* link field to next descriptor in linked list */
diff --git a/dev/sinic.cc b/dev/sinic.cc
index 13410fb43..6a30f93d4 100644
--- a/dev/sinic.cc
+++ b/dev/sinic.cc
@@ -313,34 +313,49 @@ Device::writeConfig(int offset, int size, const uint8_t *data)
 }
 
 void
-Device::prepareIO(int cpu)
+Device::prepareIO(int cpu, int index)
 {
-    if (cpu >= writeQueue.size())
-        writeQueue.resize(cpu + 1);
+    int size = virtualRegs.size();
+    if (index < size)
+        return;
+
+    virtualRegs.resize(index + 1);
+    for (int i = size; i <= index; ++i)
+        virtualRegs[i].rxPacket = rxFifo.end();
 }
 
 void
-Device::prepareRead(int cpu)
+Device::prepareRead(int cpu, int index)
 {
     using namespace Regs;
+    prepareIO(cpu, index);
+
+    VirtualReg &vnic = virtualRegs[index];
 
     // update rx registers
-    regs.RxDone = set_RxDone_Packets(regs.RxDone, rxFifo.packets());
-    regs.RxWait = regs.RxDone;
+    uint64_t rxdone = vnic.RxDone;
+    rxdone = set_RxDone_Packets(rxdone, rxFifo.packets());
+    regs.RxData = vnic.RxData;
+    regs.RxDone = rxdone;
+    regs.RxWait = rxdone;
 
     // update tx regsiters
-    regs.TxDone = set_TxDone_Packets(regs.TxDone, txFifo.packets());
-    regs.TxDone = set_TxDone_Full(regs.TxDone,
-                                  txFifo.avail() < regs.TxMaxCopy);
-    regs.TxDone = set_TxDone_Low(regs.TxDone,
-                                 txFifo.size() < regs.TxFifoMark);
-    regs.TxWait = regs.TxDone;
+    uint64_t txdone = vnic.TxDone;
+    txdone = set_TxDone_Packets(txdone, txFifo.packets());
+    txdone = set_TxDone_Full(txdone, txFifo.avail() < regs.TxMaxCopy);
+    txdone = set_TxDone_Low(txdone, txFifo.size() < regs.TxFifoMark);
+    regs.TxData = vnic.TxData;
+    regs.TxDone = txdone;
+    regs.TxWait = txdone;
 }
 
 void
-Device::prepareWrite(int cpu)
+Device::prepareWrite(int cpu, int index)
 {
-    prepareIO(cpu);
+    if (cpu >= writeQueue.size())
+        writeQueue.resize(cpu + 1);
+
+    prepareIO(cpu, index);
 }
 
 /**
@@ -366,12 +381,14 @@ Fault
 Device::readBar0(MemReqPtr &req, Addr daddr, uint8_t *data)
 {
     int cpu = (req->xc->regs.ipr[TheISA::IPR_PALtemp16] >> 8) & 0xff;
+    Addr index = daddr >> Regs::VirtualShift;
+    Addr raddr = daddr & Regs::VirtualMask;
 
-    if (!regValid(daddr))
+    if (!regValid(raddr))
         panic("invalid register: cpu=%d, da=%#x pa=%#x va=%#x size=%d",
               cpu, daddr, req->paddr, req->vaddr, req->size);
 
-    const Regs::Info &info = regInfo(daddr);
+    const Regs::Info &info = regInfo(raddr);
     if (!info.read)
         panic("reading %s (write only): cpu=%d da=%#x pa=%#x va=%#x size=%d",
               info.name, cpu, daddr, req->paddr, req->vaddr, req->size);
@@ -380,18 +397,18 @@ Device::readBar0(MemReqPtr &req, Addr daddr, uint8_t *data)
         panic("invalid size for reg %s: cpu=%d da=%#x pa=%#x va=%#x size=%d",
               info.name, cpu, daddr, req->paddr, req->vaddr, req->size);
 
-    prepareRead(cpu);
+    prepareRead(cpu, index);
 
     uint64_t value = 0;
     if (req->size == 4) {
         uint32_t &reg = *(uint32_t *)data;
-        reg = regData32(daddr);
+        reg = regData32(raddr);
         value = reg;
     }
 
     if (req->size == 8) {
         uint64_t &reg = *(uint64_t *)data;
-        reg = regData64(daddr);
+        reg = regData64(raddr);
         value = reg;
     }
 
@@ -401,7 +418,7 @@ Device::readBar0(MemReqPtr &req, Addr daddr, uint8_t *data)
 
     // reading the interrupt status register has the side effect of
     // clearing it
-    if (daddr == Regs::IntrStatus)
+    if (raddr == Regs::IntrStatus)
         devIntrClear();
 
     return No_Fault;
@@ -423,7 +440,7 @@ Device::iprRead(Addr daddr, int cpu, uint64_t &result)
     DPRINTF(EthernetPIO, "IPR read %s: cpu=%d da=%#x\n",
             info.name, cpu, daddr);
 
-    prepareRead(cpu);
+    prepareRead(cpu, 0);
 
     if (info.size == 4)
         result = regData32(daddr);
@@ -460,14 +477,17 @@ Fault
 Device::writeBar0(MemReqPtr &req, Addr daddr, const uint8_t *data)
 {
     int cpu = (req->xc->regs.ipr[TheISA::IPR_PALtemp16] >> 8) & 0xff;
+    Addr index = daddr >> Regs::VirtualShift;
+    Addr raddr = daddr & Regs::VirtualMask;
 
-    if (!regValid(daddr))
+    if (!regValid(raddr))
         panic("invalid address: cpu=%d da=%#x pa=%#x va=%#x size=%d",
               cpu, daddr, req->paddr, req->vaddr, req->size);
 
-    const Regs::Info &info = regInfo(daddr);
+    const Regs::Info &info = regInfo(raddr);
     if (!info.write)
-        panic("writing %s (read only): cpu=%d da=%#x", info.name, cpu, daddr);
+        panic("writing %s (read only): cpu=%d da=%#x",
+              info.name, cpu, daddr);
 
     if (req->size != info.size)
         panic("invalid size for %s: cpu=%d da=%#x pa=%#x va=%#x size=%d",
@@ -480,6 +500,7 @@ Device::writeBar0(MemReqPtr &req, Addr daddr, const uint8_t *data)
             info.name, cpu, info.size == 4 ? reg32 : reg64, daddr,
             req->paddr, req->vaddr, req->size);
 
+    prepareWrite(cpu, index);
 
     if (pioDelayWrite)
         writeQueue[cpu].push_back(RegWriteData(daddr, reg64));
@@ -493,10 +514,14 @@ Device::writeBar0(MemReqPtr &req, Addr daddr, const uint8_t *data)
 void
 Device::regWrite(Addr daddr, int cpu, const uint8_t *data)
 {
+    Addr index = daddr >> Regs::VirtualShift;
+    Addr raddr = daddr & Regs::VirtualMask;
+
     uint32_t reg32 = *(uint32_t *)data;
     uint64_t reg64 = *(uint64_t *)data;
+    VirtualReg &vnic = virtualRegs[index];
 
-    switch (daddr) {
+    switch (raddr) {
       case Regs::Config:
         changeConfig(reg32);
         break;
@@ -514,26 +539,29 @@ Device::regWrite(Addr daddr, int cpu, const uint8_t *data)
         break;
 
       case Regs::RxData:
-        if (rxState != rxIdle)
+        if (Regs::get_RxDone_Busy(vnic.RxDone))
             panic("receive machine busy with another request! rxState=%s",
                   RxStateStrings[rxState]);
 
-        regs.RxDone = Regs::RxDone_Busy;
-        regs.RxData = reg64;
-        if (rxEnable) {
+        vnic.RxDone = Regs::RxDone_Busy;
+        vnic.RxData = reg64;
+        rxList.push_back(index);
+        if (rxEnable && rxState == rxIdle) {
             rxState = rxFifoBlock;
             rxKick();
         }
         break;
 
       case Regs::TxData:
-        if (txState != txIdle)
+        if (Regs::get_TxDone_Busy(vnic.TxDone))
             panic("transmit machine busy with another request! txState=%s",
                   TxStateStrings[txState]);
 
-        regs.TxDone = Regs::TxDone_Busy;
-        regs.TxData = reg64;
-        if (txEnable) {
+        vnic.TxDone = Regs::TxDone_Busy;
+        vnic.TxData = reg64;
+        if (txList.empty() || txList.front() != index)
+            txList.push_back(index);
+        if (txEnable && txState == txIdle) {
             txState = txFifoBlock;
             txKick();
         }
@@ -733,6 +761,9 @@ Device::changeConfig(uint32_t newconf)
 void
 Device::command(uint32_t command)
 {
+    if (command & Regs::Command_Intr)
+        devIntrPost(Regs::Intr_Soft);
+
     if (command & Regs::Command_Reset)
         reset();
 }
@@ -745,9 +776,11 @@ Device::reset()
     memset(&regs, 0, sizeof(regs));
 
     regs.Config = 0;
-    if (params()->dedicated)
-        regs.Config |= Config_Thread;
-    regs.IntrMask = Intr_RxHigh | Intr_RxDMA | Intr_TxLow;
+    if (params()->rx_thread)
+        regs.Config |= Config_RxThread;
+    if (params()->tx_thread)
+        regs.Config |= Config_TxThread;
+    regs.IntrMask = Intr_Soft | Intr_RxHigh | Intr_RxPacket | Intr_TxLow;
     regs.RxMaxCopy = params()->rx_max_copy;
     regs.TxMaxCopy = params()->tx_max_copy;
     regs.RxMaxIntr = params()->rx_max_intr;
@@ -757,13 +790,23 @@ Device::reset()
     regs.TxFifoMark = params()->tx_fifo_threshold;
     regs.HwAddr = params()->eaddr;
 
+    rxList.clear();
+    txList.clear();
+
     rxState = rxIdle;
     txState = txIdle;
 
     rxFifo.clear();
+    rxFifoPtr = rxFifo.end();
     txFifo.clear();
     rxEmpty = false;
     txFull = false;
+
+    int size = virtualRegs.size();
+    virtualRegs.clear();
+    virtualRegs.resize(size);
+    for (int i = 0; i < size; ++i)
+        virtualRegs[i].rxPacket = rxFifo.end();
 }
 
 void
@@ -792,6 +835,8 @@ Device::rxDmaDone()
 void
 Device::rxKick()
 {
+    VirtualReg *vnic;
+
     DPRINTF(EthernetSM, "receive kick rxState=%s (rxFifo.size=%d)\n",
             RxStateStrings[rxState], rxFifo.size());
 
@@ -802,52 +847,60 @@ Device::rxKick()
     }
 
   next:
-    switch (rxState) {
-      case rxIdle:
+    if (rxState == rxIdle)
         goto exit;
 
+    assert(!rxList.empty());
+    vnic = &virtualRegs[rxList.front()];
+
+    DPRINTF(EthernetSM, "processing rxState=%s for virtual nic %d\n",
+            RxStateStrings[rxState], rxList.front());
+
+    switch (rxState) {
       case rxFifoBlock:
-        if (rxPacket) {
+        if (vnic->rxPacket != rxFifo.end()) {
             rxState = rxBeginCopy;
             break;
         }
 
-        if (rxFifo.empty()) {
+        if (rxFifoPtr == rxFifo.end()) {
             DPRINTF(EthernetSM, "receive waiting for data.  Nothing to do.\n");
             goto exit;
         }
 
+        assert(!rxFifo.empty());
+
         // Grab a new packet from the fifo.
-        rxPacket = rxFifo.front();
-        rxPacketBufPtr = rxPacket->data;
-        rxPktBytes = rxPacket->length;
-        assert(rxPktBytes);
+        vnic->rxPacket = rxFifoPtr++;
+        vnic->rxPacketOffset = 0;
+        vnic->rxPacketBytes = (*vnic->rxPacket)->length;
+        assert(vnic->rxPacketBytes);
 
-        rxDoneData = 0;
+        vnic->rxDoneData = 0;
         /* scope for variables */ {
-            IpPtr ip(rxPacket);
+            IpPtr ip(*vnic->rxPacket);
             if (ip) {
-                rxDoneData |= Regs::RxDone_IpPacket;
+                vnic->rxDoneData |= Regs::RxDone_IpPacket;
                 rxIpChecksums++;
                 if (cksum(ip) != 0) {
                     DPRINTF(EthernetCksum, "Rx IP Checksum Error\n");
-                    rxDoneData |= Regs::RxDone_IpError;
+                    vnic->rxDoneData |= Regs::RxDone_IpError;
                 }
                 TcpPtr tcp(ip);
                 UdpPtr udp(ip);
                 if (tcp) {
-                    rxDoneData |= Regs::RxDone_TcpPacket;
+                    vnic->rxDoneData |= Regs::RxDone_TcpPacket;
                     rxTcpChecksums++;
                     if (cksum(tcp) != 0) {
                         DPRINTF(EthernetCksum, "Rx TCP Checksum Error\n");
-                        rxDoneData |= Regs::RxDone_TcpError;
+                        vnic->rxDoneData |= Regs::RxDone_TcpError;
                     }
                 } else if (udp) {
-                    rxDoneData |= Regs::RxDone_UdpPacket;
+                    vnic->rxDoneData |= Regs::RxDone_UdpPacket;
                     rxUdpChecksums++;
                     if (cksum(udp) != 0) {
                         DPRINTF(EthernetCksum, "Rx UDP Checksum Error\n");
-                        rxDoneData |= Regs::RxDone_UdpError;
+                        vnic->rxDoneData |= Regs::RxDone_UdpError;
                     }
                 }
             }
@@ -859,9 +912,10 @@ Device::rxKick()
         if (dmaInterface && dmaInterface->busy())
             goto exit;
 
-        rxDmaAddr = plat->pciToDma(Regs::get_RxData_Addr(regs.RxData));
-        rxDmaLen = min<int>(Regs::get_RxData_Len(regs.RxData), rxPktBytes);
-        rxDmaData = rxPacketBufPtr;
+        rxDmaAddr = plat->pciToDma(Regs::get_RxData_Addr(vnic->RxData));
+        rxDmaLen = min<int>(Regs::get_RxData_Len(vnic->RxData),
+                            vnic->rxPacketBytes);
+        rxDmaData = (*vnic->rxPacket)->data + vnic->rxPacketOffset;
         rxState = rxCopy;
 
         if (dmaInterface) {
@@ -885,20 +939,27 @@ Device::rxKick()
         goto exit;
 
       case rxCopyDone:
-        regs.RxDone = rxDoneData | rxDmaLen;
+        vnic->RxDone = vnic->rxDoneData | rxDmaLen;
+        vnic->RxDone |= Regs::RxDone_Complete;
 
-        if (rxPktBytes == rxDmaLen) {
-            rxPacket = NULL;
-            rxFifo.pop();
+        if (vnic->rxPacketBytes == rxDmaLen) {
+            rxFifo.remove(vnic->rxPacket);
+            vnic->rxPacket = rxFifo.end();
         } else {
-            regs.RxDone |= Regs::RxDone_More;
-            rxPktBytes -= rxDmaLen;
-            rxPacketBufPtr += rxDmaLen;
+            vnic->RxDone |= Regs::RxDone_More;
+            vnic->rxPacketBytes -= rxDmaLen;
+            vnic->rxPacketOffset += rxDmaLen;
+        }
+
+        rxList.pop_front();
+        rxState = rxList.empty() ? rxIdle : rxFifoBlock;
+
+        if (rxFifo.empty()) {
+            devIntrPost(Regs::Intr_RxEmpty);
+            rxEmpty = true;
         }
 
-        regs.RxDone |= Regs::RxDone_Complete;
         devIntrPost(Regs::Intr_RxDMA);
-        rxState = rxIdle;
         break;
 
       default:
@@ -994,6 +1055,7 @@ Device::transmit()
 void
 Device::txKick()
 {
+    VirtualReg *vnic;
     DPRINTF(EthernetSM, "transmit kick txState=%s (txFifo.size=%d)\n",
             TxStateStrings[txState], txFifo.size());
 
@@ -1004,19 +1066,22 @@ Device::txKick()
     }
 
   next:
-    switch (txState) {
-      case txIdle:
+    if (txState == txIdle)
         goto exit;
 
+    assert(!txList.empty());
+    vnic = &virtualRegs[txList.front()];
+
+    switch (txState) {
       case txFifoBlock:
         if (!txPacket) {
             // Grab a new packet from the fifo.
             txPacket = new PacketData(16384);
-            txPacketBufPtr = txPacket->data;
+            txPacketOffset = 0;
         }
 
         if (txFifo.avail() - txPacket->length <
-            Regs::get_TxData_Len(regs.TxData)) {
+            Regs::get_TxData_Len(vnic->TxData)) {
             DPRINTF(EthernetSM, "transmit fifo full.  Nothing to do.\n");
             goto exit;
         }
@@ -1028,9 +1093,9 @@ Device::txKick()
         if (dmaInterface && dmaInterface->busy())
             goto exit;
 
-        txDmaAddr = plat->pciToDma(Regs::get_TxData_Addr(regs.TxData));
-        txDmaLen = Regs::get_TxData_Len(regs.TxData);
-        txDmaData = txPacketBufPtr;
+        txDmaAddr = plat->pciToDma(Regs::get_TxData_Addr(vnic->TxData));
+        txDmaLen = Regs::get_TxData_Len(vnic->TxData);
+        txDmaData = txPacket->data + txPacketOffset;
         txState = txCopy;
 
         if (dmaInterface) {
@@ -1054,45 +1119,49 @@ Device::txKick()
         goto exit;
 
       case txCopyDone:
+        vnic->TxDone = txDmaLen | Regs::TxDone_Complete;
         txPacket->length += txDmaLen;
-        if ((regs.TxData & Regs::TxData_More)) {
-            txPacketBufPtr += txDmaLen;
-        } else {
-            assert(txPacket->length <= txFifo.avail());
-            if ((regs.TxData & Regs::TxData_Checksum)) {
-                IpPtr ip(txPacket);
-                if (ip) {
-                    TcpPtr tcp(ip);
-                    if (tcp) {
-                        tcp->sum(0);
-                        tcp->sum(cksum(tcp));
-                        txTcpChecksums++;
-                    }
+        if ((vnic->TxData & Regs::TxData_More)) {
+            txPacketOffset += txDmaLen;
+            txState = txIdle;
+            devIntrPost(Regs::Intr_TxDMA);
+            break;
+        }
 
-                    UdpPtr udp(ip);
-                    if (udp) {
-                        udp->sum(0);
-                        udp->sum(cksum(udp));
-                        txUdpChecksums++;
-                    }
+        assert(txPacket->length <= txFifo.avail());
+        if ((vnic->TxData & Regs::TxData_Checksum)) {
+            IpPtr ip(txPacket);
+            if (ip) {
+                TcpPtr tcp(ip);
+                if (tcp) {
+                    tcp->sum(0);
+                    tcp->sum(cksum(tcp));
+                    txTcpChecksums++;
+                }
 
-                    ip->sum(0);
-                    ip->sum(cksum(ip));
-                    txIpChecksums++;
+                UdpPtr udp(ip);
+                if (udp) {
+                    udp->sum(0);
+                    udp->sum(cksum(udp));
+                    txUdpChecksums++;
                 }
+
+                ip->sum(0);
+                ip->sum(cksum(ip));
+                txIpChecksums++;
             }
-            txFifo.push(txPacket);
-            if (txFifo.avail() < regs.TxMaxCopy) {
-                devIntrPost(Regs::Intr_TxFull);
-                txFull = true;
-            }
-            txPacket = 0;
-            transmit();
         }
 
-        regs.TxDone = txDmaLen | Regs::TxDone_Complete;
+        txFifo.push(txPacket);
+        if (txFifo.avail() < regs.TxMaxCopy) {
+            devIntrPost(Regs::Intr_TxFull);
+            txFull = true;
+        }
+        txPacket = 0;
+        transmit();
+        txList.pop_front();
+        txState = txList.empty() ? txIdle : txFifoBlock;
         devIntrPost(Regs::Intr_TxDMA);
-        txState = txIdle;
         break;
 
       default:
@@ -1201,6 +1270,11 @@ Device::recvPacket(PacketPtr packet)
         return false;
     }
 
+    // If we were at the last element, back up one ot go to the new
+    // last element of the list.
+    if (rxFifoPtr == rxFifo.end())
+        --rxFifoPtr;
+
     devIntrPost(Regs::Intr_RxPacket);
     rxKick();
     return true;
@@ -1282,21 +1356,59 @@ Device::serialize(ostream &os)
     SERIALIZE_SCALAR(regs.TxDone);
 
     /*
+     * Serialize the virtual nic state
+     */
+    int virtualRegsSize = virtualRegs.size();
+    SERIALIZE_SCALAR(virtualRegsSize);
+    for (int i = 0; i < virtualRegsSize; ++i) {
+        VirtualReg *vnic = &virtualRegs[i];
+
+        string reg = csprintf("vnic%d", i);
+        paramOut(os, reg + ".RxData", vnic->RxData);
+        paramOut(os, reg + ".RxDone", vnic->RxDone);
+        paramOut(os, reg + ".TxData", vnic->TxData);
+        paramOut(os, reg + ".TxDone", vnic->TxDone);
+
+        PacketFifo::iterator rxFifoPtr;
+
+        bool rxPacketExists = vnic->rxPacket != rxFifo.end();
+        paramOut(os, reg + ".rxPacketExists", rxPacketExists);
+        if (rxPacketExists) {
+            int rxPacket = 0;
+            PacketFifo::iterator i = rxFifo.begin();
+            while (i != vnic->rxPacket) {
+                assert(i != rxFifo.end());
+                ++i;
+                ++rxPacket;
+            }
+
+            paramOut(os, reg + ".rxPacket", rxPacket);
+            paramOut(os, reg + ".rxPacketOffset", vnic->rxPacketOffset);
+            paramOut(os, reg + ".rxPacketBytes", vnic->rxPacketBytes);
+        }
+        paramOut(os, reg + ".rxDoneData", vnic->rxDoneData);
+    }
+
+    VirtualList::iterator i, end;
+    int count;
+
+    int rxListSize = rxList.size();
+    SERIALIZE_SCALAR(rxListSize);
+    for (count = 0, i = rxList.begin(), end = rxList.end(); i != end; ++i)
+        paramOut(os, csprintf("rxList%d", count++), *i);
+
+    int txListSize = txList.size();
+    SERIALIZE_SCALAR(txListSize);
+    for (count = 0, i = txList.begin(), end = txList.end(); i != end; ++i)
+        paramOut(os, csprintf("txList%d", count++), *i);
+
+    /*
      * Serialize rx state machine
      */
     int rxState = this->rxState;
     SERIALIZE_SCALAR(rxState);
     SERIALIZE_SCALAR(rxEmpty);
     rxFifo.serialize("rxFifo", os);
-    bool rxPacketExists = rxPacket;
-    SERIALIZE_SCALAR(rxPacketExists);
-    if (rxPacketExists) {
-        rxPacket->serialize("rxPacket", os);
-        uint32_t rxPktBufPtr = (uint32_t) (rxPacketBufPtr - rxPacket->data);
-        SERIALIZE_SCALAR(rxPktBufPtr);
-        SERIALIZE_SCALAR(rxPktBytes);
-    }
-    SERIALIZE_SCALAR(rxDoneData);
 
     /*
      * Serialize tx state machine
@@ -1309,9 +1421,8 @@ Device::serialize(ostream &os)
     SERIALIZE_SCALAR(txPacketExists);
     if (txPacketExists) {
         txPacket->serialize("txPacket", os);
-        uint32_t txPktBufPtr = (uint32_t) (txPacketBufPtr - txPacket->data);
-        SERIALIZE_SCALAR(txPktBufPtr);
-        SERIALIZE_SCALAR(txPktBytes);
+        SERIALIZE_SCALAR(txPacketOffset);
+        SERIALIZE_SCALAR(txPacketBytes);
     }
 
     /*
@@ -1342,6 +1453,24 @@ Device::unserialize(Checkpoint *cp, const std::string &section)
     UNSERIALIZE_SCALAR(regs.TxData);
     UNSERIALIZE_SCALAR(regs.TxDone);
 
+    int rxListSize;
+    UNSERIALIZE_SCALAR(rxListSize);
+    rxList.clear();
+    for (int i = 0; i < rxListSize; ++i) {
+        int value;
+        paramIn(cp, section, csprintf("rxList%d", i), value);
+        rxList.push_back(value);
+    }
+
+    int txListSize;
+    UNSERIALIZE_SCALAR(txListSize);
+    txList.clear();
+    for (int i = 0; i < txListSize; ++i) {
+        int value;
+        paramIn(cp, section, csprintf("txList%d", i), value);
+        txList.push_back(value);
+    }
+
     /*
      * Unserialize rx state machine
      */
@@ -1350,18 +1479,6 @@ Device::unserialize(Checkpoint *cp, const std::string &section)
     UNSERIALIZE_SCALAR(rxEmpty);
     this->rxState = (RxState) rxState;
     rxFifo.unserialize("rxFifo", cp, section);
-    bool rxPacketExists;
-    UNSERIALIZE_SCALAR(rxPacketExists);
-    rxPacket = 0;
-    if (rxPacketExists) {
-        rxPacket = new PacketData(16384);
-        rxPacket->unserialize("rxPacket", cp, section);
-        uint32_t rxPktBufPtr;
-        UNSERIALIZE_SCALAR(rxPktBufPtr);
-        this->rxPacketBufPtr = (uint8_t *) rxPacket->data + rxPktBufPtr;
-        UNSERIALIZE_SCALAR(rxPktBytes);
-    }
-    UNSERIALIZE_SCALAR(rxDoneData);
 
     /*
      * Unserialize tx state machine
@@ -1377,10 +1494,45 @@ Device::unserialize(Checkpoint *cp, const std::string &section)
     if (txPacketExists) {
         txPacket = new PacketData(16384);
         txPacket->unserialize("txPacket", cp, section);
-        uint32_t txPktBufPtr;
-        UNSERIALIZE_SCALAR(txPktBufPtr);
-        this->txPacketBufPtr = (uint8_t *) txPacket->data + txPktBufPtr;
-        UNSERIALIZE_SCALAR(txPktBytes);
+        UNSERIALIZE_SCALAR(txPacketOffset);
+        UNSERIALIZE_SCALAR(txPacketBytes);
+    }
+
+    /*
+     * unserialize the virtual nic registers/state
+     *
+     * this must be done after the unserialization of the rxFifo
+     * because the packet iterators depend on the fifo being populated
+     */
+    int virtualRegsSize;
+    UNSERIALIZE_SCALAR(virtualRegsSize);
+    virtualRegs.clear();
+    virtualRegs.resize(virtualRegsSize);
+    for (int i = 0; i < virtualRegsSize; ++i) {
+        VirtualReg *vnic = &virtualRegs[i];
+        string reg = csprintf("vnic%d", i);
+
+        paramIn(cp, section, reg + ".RxData", vnic->RxData);
+        paramIn(cp, section, reg + ".RxDone", vnic->RxDone);
+        paramIn(cp, section, reg + ".TxData", vnic->TxData);
+        paramIn(cp, section, reg + ".TxDone", vnic->TxDone);
+
+        bool rxPacketExists;
+        paramIn(cp, section, reg + ".rxPacketExists", rxPacketExists);
+        if (rxPacketExists) {
+            int rxPacket;
+            paramIn(cp, section, reg + ".rxPacket", rxPacket);
+            vnic->rxPacket = rxFifo.begin();
+            while (rxPacket--)
+                ++vnic->rxPacket;
+
+            paramIn(cp, section, reg + ".rxPacketOffset",
+                    vnic->rxPacketOffset);
+            paramIn(cp, section, reg + ".rxPacketBytes", vnic->rxPacketBytes);
+        } else {
+            vnic->rxPacket = rxFifo.end();
+        }
+        paramIn(cp, section, reg + ".rxDoneData", vnic->rxDoneData);
     }
 
     /*
@@ -1505,7 +1657,8 @@ BEGIN_DECLARE_SIM_OBJECT_PARAMS(Device)
 
     Param<bool> rx_filter;
     Param<string> hardware_address;
-    Param<bool> dedicated;
+    Param<bool> rx_thread;
+    Param<bool> tx_thread;
 
 END_DECLARE_SIM_OBJECT_PARAMS(Device)
 
@@ -1548,7 +1701,8 @@ BEGIN_INIT_SIM_OBJECT_PARAMS(Device)
 
     INIT_PARAM(rx_filter, "Enable Receive Filter"),
     INIT_PARAM(hardware_address, "Ethernet Hardware Address"),
-    INIT_PARAM(dedicated, "dedicate a kernel thread to the driver")
+    INIT_PARAM(rx_thread, ""),
+    INIT_PARAM(tx_thread, "")
 
 END_INIT_SIM_OBJECT_PARAMS(Device)
 
@@ -1595,7 +1749,8 @@ CREATE_SIM_OBJECT(Device)
 
     params->rx_filter = rx_filter;
     params->eaddr = hardware_address;
-    params->dedicated = dedicated;
+    params->rx_thread = rx_thread;
+    params->tx_thread = tx_thread;
 
     return new Device(params);
 }
diff --git a/dev/sinic.hh b/dev/sinic.hh
index b3255b6c0..af2f109a4 100644
--- a/dev/sinic.hh
+++ b/dev/sinic.hh
@@ -136,6 +136,28 @@ class Device : public Base
         uint64_t HwAddr;       // 0x60
     } regs;
 
+    struct VirtualReg {
+        uint64_t RxData;
+        uint64_t RxDone;
+        uint64_t TxData;
+        uint64_t TxDone;
+
+        PacketFifo::iterator rxPacket;
+        int rxPacketOffset;
+        int rxPacketBytes;
+        uint64_t rxDoneData;
+
+        VirtualReg()
+            : RxData(0), RxDone(0), TxData(0), TxDone(0),
+              rxPacketOffset(0), rxPacketBytes(0), rxDoneData(0)
+        { }
+    };
+    typedef std::vector<VirtualReg> VirtualRegs;
+    typedef std::list<int> VirtualList;
+    VirtualRegs virtualRegs;
+    VirtualList rxList;
+    VirtualList txList;
+
     uint8_t  &regData8(Addr daddr) { return *((uint8_t *)&regs + daddr); }
     uint32_t &regData32(Addr daddr) { return *(uint32_t *)&regData8(daddr); }
     uint64_t &regData64(Addr daddr) { return *(uint64_t *)&regData8(daddr); }
@@ -147,11 +169,8 @@ class Device : public Base
   protected:
     RxState rxState;
     PacketFifo rxFifo;
+    PacketFifo::iterator rxFifoPtr;
     bool rxEmpty;
-    PacketPtr rxPacket;
-    uint8_t *rxPacketBufPtr;
-    int rxPktBytes;
-    uint64_t rxDoneData;
     Addr rxDmaAddr;
     uint8_t *rxDmaData;
     int rxDmaLen;
@@ -160,8 +179,8 @@ class Device : public Base
     PacketFifo txFifo;
     bool txFull;
     PacketPtr txPacket;
-    uint8_t *txPacketBufPtr;
-    int txPktBytes;
+    int txPacketOffset;
+    int txPacketBytes;
     Addr txDmaAddr;
     uint8_t *txDmaData;
     int txDmaLen;
@@ -255,9 +274,9 @@ class Device : public Base
     virtual Fault read(MemReqPtr &req, uint8_t *data);
     virtual Fault write(MemReqPtr &req, const uint8_t *data);
 
-    void prepareIO(int cpu);
-    void prepareRead(int cpu);
-    void prepareWrite(int cpu);
+    void prepareIO(int cpu, int index);
+    void prepareRead(int cpu, int index);
+    void prepareWrite(int cpu, int index);
     Fault iprRead(Addr daddr, int cpu, uint64_t &result);
     Fault readBar0(MemReqPtr &req, Addr daddr, uint8_t *data);
     Fault writeBar0(MemReqPtr &req, Addr daddr, const uint8_t *data);
@@ -347,7 +366,8 @@ class Device : public Base
         Tick dma_write_delay;
         Tick dma_write_factor;
         bool dma_no_allocate;
-        bool dedicated;
+        bool rx_thread;
+        bool tx_thread;
     };
 
   protected:
diff --git a/dev/sinicreg.hh b/dev/sinicreg.hh
index 30f5b3c95..b7008b4e1 100644
--- a/dev/sinicreg.hh
+++ b/dev/sinicreg.hh
@@ -55,6 +55,9 @@
 namespace Sinic {
 namespace Regs {
 
+static const int VirtualMask = 0xff;
+static const int VirtualShift = 8;
+
 // Registers
 __SINIC_REG32(Config,      0x00); // 32: configuration register
 __SINIC_REG32(Command,     0x04); // 32: command register
@@ -78,20 +81,23 @@ __SINIC_REG32(HwAddr,      0x60); // 64: mac address
 __SINIC_REG32(Size,        0x68); // register addres space size
 
 // Config register bits
-__SINIC_VAL32(Config_Thread,  8, 1); // enable receive filter
-__SINIC_VAL32(Config_Filter,  7, 1); // enable receive filter
-__SINIC_VAL32(Config_Vlan,    6, 1); // enable vlan tagging
-__SINIC_VAL32(Config_Virtual, 5, 1); // enable virtual addressing
-__SINIC_VAL32(Config_Desc,    4, 1); // enable tx/rx descriptors
-__SINIC_VAL32(Config_Poll,    3, 1); // enable polling
-__SINIC_VAL32(Config_IntEn,   2, 1); // enable interrupts
-__SINIC_VAL32(Config_TxEn,    1, 1); // enable transmit
-__SINIC_VAL32(Config_RxEn,    0, 1); // enable receive
+__SINIC_VAL32(Config_RxThread,  9, 1); // enable receive threads
+__SINIC_VAL32(Config_TxThread,  8, 1); // enable transmit thread
+__SINIC_VAL32(Config_Filter,    7, 1); // enable receive filter
+__SINIC_VAL32(Config_Vlan,      6, 1); // enable vlan tagging
+__SINIC_VAL32(Config_Virtual,   5, 1); // enable virtual addressing
+__SINIC_VAL32(Config_Desc,      4, 1); // enable tx/rx descriptors
+__SINIC_VAL32(Config_Poll,      3, 1); // enable polling
+__SINIC_VAL32(Config_IntEn,     2, 1); // enable interrupts
+__SINIC_VAL32(Config_TxEn,      1, 1); // enable transmit
+__SINIC_VAL32(Config_RxEn,      0, 1); // enable receive
 
 // Command register bits
+__SINIC_VAL32(Command_Intr,  1, 1); // software interrupt
 __SINIC_VAL32(Command_Reset, 0, 1); // reset chip
 
 // Interrupt register bits
+__SINIC_VAL32(Intr_Soft,      8, 1); // software interrupt
 __SINIC_VAL32(Intr_TxLow,     7, 1); // tx fifo dropped below watermark
 __SINIC_VAL32(Intr_TxFull,    6, 1); // tx fifo full
 __SINIC_VAL32(Intr_TxDMA,     5, 1); // tx dma completed w/ interrupt
@@ -100,9 +106,9 @@ __SINIC_VAL32(Intr_RxHigh,    3, 1); // rx fifo above high watermark
 __SINIC_VAL32(Intr_RxEmpty,   2, 1); // rx fifo empty
 __SINIC_VAL32(Intr_RxDMA,     1, 1); // rx dma completed w/ interrupt
 __SINIC_VAL32(Intr_RxPacket,  0, 1); // packet received
-__SINIC_REG32(Intr_All,       0xff); // all valid interrupts
-__SINIC_REG32(Intr_NoDelay,   0xcc); // interrupts that shouldn't be coalesced
-__SINIC_REG32(Intr_Res,      ~0xff); // reserved interrupt bits
+__SINIC_REG32(Intr_All,       0x01ff); // all valid interrupts
+__SINIC_REG32(Intr_NoDelay,   0x01cc); // interrupts that aren't coalesced
+__SINIC_REG32(Intr_Res,      ~0x01ff); // reserved interrupt bits
 
 // RX Data Description
 __SINIC_VAL64(RxData_Len, 40, 20); // 0 - 1M
@@ -119,6 +125,9 @@ __SINIC_VAL64(RxDone_Packets,   32, 16); // number of packets in rx fifo
 __SINIC_VAL64(RxDone_Busy,      31,  1); // receive dma busy copying
 __SINIC_VAL64(RxDone_Complete,  30,  1); // valid data (packet complete)
 __SINIC_VAL64(RxDone_More,      29,  1); // Packet has more data (dma again)
+__SINIC_VAL64(RxDone_Res0,      28,  1); // reserved
+__SINIC_VAL64(RxDone_Res1,      27,  1); // reserved
+__SINIC_VAL64(RxDone_Res2,      26,  1); // reserved
 __SINIC_VAL64(RxDone_TcpError,  25,  1); // TCP packet error (bad checksum)
 __SINIC_VAL64(RxDone_UdpError,  24,  1); // UDP packet error (bad checksum)
 __SINIC_VAL64(RxDone_IpError,   23,  1); // IP packet error (bad checksum)
@@ -133,6 +142,14 @@ __SINIC_VAL64(TxDone_Busy,      31,  1); // transmit dma busy copying
 __SINIC_VAL64(TxDone_Complete,  30,  1); // valid data (packet complete)
 __SINIC_VAL64(TxDone_Full,      29,  1); // tx fifo is full
 __SINIC_VAL64(TxDone_Low,       28,  1); // tx fifo is below the watermark
+__SINIC_VAL64(TxDone_Res0,      27,  1); // reserved
+__SINIC_VAL64(TxDone_Res1,      26,  1); // reserved
+__SINIC_VAL64(TxDone_Res2,      25,  1); // reserved
+__SINIC_VAL64(TxDone_Res3,      24,  1); // reserved
+__SINIC_VAL64(TxDone_Res4,      23,  1); // reserved
+__SINIC_VAL64(TxDone_Res5,      22,  1); // reserved
+__SINIC_VAL64(TxDone_Res6,      21,  1); // reserved
+__SINIC_VAL64(TxDone_Res7,      20,  1); // reserved
 __SINIC_VAL64(TxDone_CopyLen,    0, 20); // up to 256k
 
 struct Info