dev: Refactor the EtherTapStub to make room for using tap.

A lot of the implementation of EtherTapStub can be shared with a version
which uses a tap device directly. This change factors out those parts to
accommodate that.

Change-Id: I9c2e31f1be139ca73859a83f05457cef90101006
Reviewed-on: https://gem5-review.googlesource.com/3645
Reviewed-by: Nathan Binkert <nate@binkert.org>
Maintainer: Nathan Binkert <nate@binkert.org>

3 files changed, 303 insertions, 223 deletions

diff --git a/src/dev/net/Ethernet.py b/src/dev/net/Ethernet.py
index 1f8e75200..68867c00f 100644
--- a/src/dev/net/Ethernet.py
+++ b/src/dev/net/Ethernet.py
@@ -95,13 +95,18 @@ class EtherSwitch(EtherObject):
     delay_var = Param.Latency('0ns', "packet transmit delay variability")
     time_to_live = Param.Latency('10ms', "time to live of MAC address maping")
 
-class EtherTapStub(EtherObject):
-    type = 'EtherTapStub'
+class EtherTapBase(EtherObject):
+    type = 'EtherTapBase'
+    abstract = True
     cxx_header = "dev/net/ethertap.hh"
     bufsz = Param.Int(10000, "tap buffer size")
     dump = Param.EtherDump(NULL, "dump object")
+    tap = SlavePort("Ethernet interface to connect to gem5's network")
+
+class EtherTapStub(EtherTapBase):
+    type = 'EtherTapStub'
+    cxx_header = "dev/net/ethertap.hh"
     port = Param.UInt16(3500, "Port helper should send packets to")
-    tap = SlavePort("Ethernet interface to gem5's network")
 
 class EtherDump(SimObject):
     type = 'EtherDump'
diff --git a/src/dev/net/ethertap.cc b/src/dev/net/ethertap.cc
index e7b928972..f08de0ebf 100644
--- a/src/dev/net/ethertap.cc
+++ b/src/dev/net/ethertap.cc
@@ -56,39 +56,180 @@
 
 using namespace std;
 
-/**
- */
+class TapEvent : public PollEvent
+{
+  protected:
+    EtherTapBase *tap;
+
+  public:
+    TapEvent(EtherTapBase *_tap, int fd, int e)
+        : PollEvent(fd, e), tap(_tap) {}
+    virtual void process(int revent) { tap->recvReal(revent); }
+};
+
+EtherTapBase::EtherTapBase(const Params *p)
+    : EtherObject(p), buflen(p->bufsz), dump(p->dump), event(NULL),
+      interface(NULL), txEvent(this)
+{
+    buffer = new uint8_t[buflen];
+    interface = new EtherTapInt(name() + ".interface", this);
+}
+
+EtherTapBase::~EtherTapBase()
+{
+    delete buffer;
+    delete event;
+    delete interface;
+}
+
+void
+EtherTapBase::serialize(CheckpointOut &cp) const
+{
+    SERIALIZE_SCALAR(buflen);
+    uint8_t *buffer = (uint8_t *)this->buffer;
+    SERIALIZE_ARRAY(buffer, buflen);
+
+    bool tapevent_present = false;
+    if (event) {
+        tapevent_present = true;
+        SERIALIZE_SCALAR(tapevent_present);
+        event->serialize(cp);
+    } else {
+        SERIALIZE_SCALAR(tapevent_present);
+    }
+}
+
+void
+EtherTapBase::unserialize(CheckpointIn &cp)
+{
+    UNSERIALIZE_SCALAR(buflen);
+    uint8_t *buffer = (uint8_t *)this->buffer;
+    UNSERIALIZE_ARRAY(buffer, buflen);
+
+    bool tapevent_present;
+    UNSERIALIZE_SCALAR(tapevent_present);
+    if (tapevent_present) {
+        event = new TapEvent(this, 0, 0);
+        event->unserialize(cp);
+        if (event->queued())
+            pollQueue.schedule(event);
+    }
+}
+
+
+void
+EtherTapBase::pollFd(int fd)
+{
+    assert(!event);
+    event = new TapEvent(this, fd, POLLIN|POLLERR);
+    pollQueue.schedule(event);
+}
+
+void
+EtherTapBase::stopPolling()
+{
+    assert(event);
+    delete event;
+    event = NULL;
+}
+
+
+EtherInt*
+EtherTapBase::getEthPort(const std::string &if_name, int idx)
+{
+    if (if_name == "tap") {
+        if (interface->getPeer())
+            panic("Interface already connected to\n");
+        return interface;
+    }
+    return NULL;
+}
+
+bool
+EtherTapBase::recvSimulated(EthPacketPtr packet)
+{
+    if (dump)
+        dump->dump(packet);
+
+    DPRINTF(Ethernet, "EtherTap sim->real len=%d\n", packet->length);
+    DDUMP(EthernetData, packet->data, packet->length);
+
+    bool success = sendReal(packet->data, packet->length);
+
+    interface->recvDone();
+
+    return success;
+}
+
+void
+EtherTapBase::sendSimulated(void *data, size_t len)
+{
+    EthPacketPtr packet;
+    packet = make_shared<EthPacketData>(len);
+    packet->length = len;
+    packet->simLength = len;
+    memcpy(packet->data, data, len);
+
+    DPRINTF(Ethernet, "EtherTap real->sim len=%d\n", packet->length);
+    DDUMP(EthernetData, packet->data, packet->length);
+    if (!packetBuffer.empty() || !interface->sendPacket(packet)) {
+        DPRINTF(Ethernet, "bus busy...buffer for retransmission\n");
+        packetBuffer.push(packet);
+        if (!txEvent.scheduled())
+            schedule(txEvent, curTick() + retryTime);
+    } else if (dump) {
+        dump->dump(packet);
+    }
+}
+
+void
+EtherTapBase::retransmit()
+{
+    if (packetBuffer.empty())
+        return;
+
+    EthPacketPtr packet = packetBuffer.front();
+    if (interface->sendPacket(packet)) {
+        if (dump)
+            dump->dump(packet);
+        DPRINTF(Ethernet, "EtherTap retransmit\n");
+        packetBuffer.front() = NULL;
+        packetBuffer.pop();
+    }
+
+    if (!packetBuffer.empty() && !txEvent.scheduled())
+        schedule(txEvent, curTick() + retryTime);
+}
+
+
 class TapListener
 {
   protected:
-    /**
-     */
     class Event : public PollEvent
     {
       protected:
         TapListener *listener;
 
       public:
-        Event(TapListener *l, int fd, int e)
-            : PollEvent(fd, e), listener(l) {}
+        Event(TapListener *l, int fd, int e) : PollEvent(fd, e), listener(l) {}
 
-        virtual void process(int revent) { listener->accept(); }
+        void process(int revent) override { listener->accept(); }
     };
 
     friend class Event;
     Event *event;
 
+    void accept();
+
   protected:
     ListenSocket listener;
     EtherTapStub *tap;
     int port;
 
   public:
-    TapListener(EtherTapStub *t, int p)
-        : event(NULL), tap(t), port(p) {}
-    ~TapListener() { if (event) delete event; }
+    TapListener(EtherTapStub *t, int p) : event(NULL), tap(t), port(p) {}
+    ~TapListener() { delete event; }
 
-    void accept();
     void listen();
 };
 
@@ -121,226 +262,120 @@ TapListener::accept()
         tap->attach(sfd);
 }
 
-/**
- */
-class TapEvent : public PollEvent
-{
-  protected:
-    EtherTapStub *tap;
-
-  public:
-    TapEvent(EtherTapStub *_tap, int fd, int e)
-        : PollEvent(fd, e), tap(_tap) {}
-    virtual void process(int revent) { tap->process(revent); }
-};
 
-EtherTapStub::EtherTapStub(const Params *p)
-    : EtherObject(p), event(NULL), socket(-1), buflen(p->bufsz), dump(p->dump),
-      interface(NULL), txEvent(this)
+EtherTapStub::EtherTapStub(const Params *p) : EtherTapBase(p), socket(-1)
 {
     if (ListenSocket::allDisabled())
         fatal("All listeners are disabled! EtherTapStub can't work!");
 
-    buffer = new char[buflen];
     listener = new TapListener(this, p->port);
     listener->listen();
-    interface = new EtherTapInt(name() + ".interface", this);
 }
 
 EtherTapStub::~EtherTapStub()
 {
-    if (event)
-        delete event;
-    if (buffer)
-        delete [] buffer;
-
-    delete interface;
     delete listener;
 }
 
 void
+EtherTapStub::serialize(CheckpointOut &cp) const
+{
+    EtherTapBase::serialize(cp);
+
+    SERIALIZE_SCALAR(socket);
+    SERIALIZE_SCALAR(buffer_used);
+    SERIALIZE_SCALAR(frame_len);
+}
+
+void
+EtherTapStub::unserialize(CheckpointIn &cp)
+{
+    EtherTapBase::unserialize(cp);
+
+    UNSERIALIZE_SCALAR(socket);
+    UNSERIALIZE_SCALAR(buffer_used);
+    UNSERIALIZE_SCALAR(frame_len);
+}
+
+
+void
 EtherTapStub::attach(int fd)
 {
     if (socket != -1)
         close(fd);
 
-    buffer_offset = 0;
-    data_len = 0;
+    buffer_used = 0;
+    frame_len = 0;
     socket = fd;
     DPRINTF(Ethernet, "EtherTapStub attached\n");
-    event = new TapEvent(this, socket, POLLIN|POLLERR);
-    pollQueue.schedule(event);
+    pollFd(socket);
 }
 
 void
 EtherTapStub::detach()
 {
     DPRINTF(Ethernet, "EtherTapStub detached\n");
-    delete event;
-    event = 0;
+    stopPolling();
     close(socket);
     socket = -1;
 }
 
-bool
-EtherTapStub::recvPacket(EthPacketPtr packet)
-{
-    if (dump)
-        dump->dump(packet);
-
-    DPRINTF(Ethernet, "EtherTapStub output len=%d\n", packet->length);
-    DDUMP(EthernetData, packet->data, packet->length);
-    uint32_t len = htonl(packet->length);
-    ssize_t ret = write(socket, &len, sizeof(len));
-    if (ret != sizeof(len))
-        return false;
-    ret = write(socket, packet->data, packet->length);
-    if (ret != packet->length)
-        return false;
-
-    interface->recvDone();
-
-    return true;
-}
-
 void
-EtherTapStub::sendDone()
-{}
-
-void
-EtherTapStub::process(int revent)
+EtherTapStub::recvReal(int revent)
 {
     if (revent & POLLERR) {
         detach();
         return;
     }
 
-    char *data = buffer + sizeof(uint32_t);
     if (!(revent & POLLIN))
         return;
 
-    if (buffer_offset < data_len + sizeof(uint32_t)) {
-        int len = read(socket, buffer + buffer_offset, buflen - buffer_offset);
-        if (len == 0) {
-            detach();
-            return;
-        }
-
-        buffer_offset += len;
-
-        if (data_len == 0)
-            data_len = ntohl(*(uint32_t *)buffer);
-
-        DPRINTF(Ethernet, "Received data from peer: len=%d buffer_offset=%d "
-                "data_len=%d\n", len, buffer_offset, data_len);
-    }
-
-    while (data_len != 0 && buffer_offset >= data_len + sizeof(uint32_t)) {
-        EthPacketPtr packet;
-        packet = make_shared<EthPacketData>(data_len);
-        packet->length = data_len;
-        packet->simLength = data_len;
-        memcpy(packet->data, data, data_len);
-
-        assert(buffer_offset >= data_len + sizeof(uint32_t));
-        buffer_offset -= data_len + sizeof(uint32_t);
-        if (buffer_offset > 0) {
-            memmove(buffer, data + data_len, buffer_offset);
-            data_len = ntohl(*(uint32_t *)buffer);
-        } else
-            data_len = 0;
-
-        DPRINTF(Ethernet, "EtherTapStub input len=%d\n", packet->length);
-        DDUMP(EthernetData, packet->data, packet->length);
-        if (!interface->sendPacket(packet)) {
-            DPRINTF(Ethernet, "bus busy...buffer for retransmission\n");
-            packetBuffer.push(packet);
-            if (!txEvent.scheduled())
-                schedule(txEvent, curTick() + retryTime);
-        } else if (dump) {
-            dump->dump(packet);
-        }
-    }
-}
-
-void
-EtherTapStub::retransmit()
-{
-    if (packetBuffer.empty())
+    // Read in as much of the new data as we can.
+    int len = read(socket, buffer + buffer_used, buflen - buffer_used);
+    if (len == 0) {
+        detach();
         return;
-
-    EthPacketPtr packet = packetBuffer.front();
-    if (interface->sendPacket(packet)) {
-        if (dump)
-            dump->dump(packet);
-        DPRINTF(Ethernet, "EtherTapStub retransmit\n");
-        packetBuffer.front() = NULL;
-        packetBuffer.pop();
     }
+    buffer_used += len;
 
-    if (!packetBuffer.empty() && !txEvent.scheduled())
-        schedule(txEvent, curTick() + retryTime);
-}
+    // If there's not enough data for the frame length, wait for more.
+    if (buffer_used < sizeof(uint32_t))
+        return;
 
-EtherInt*
-EtherTapStub::getEthPort(const std::string &if_name, int idx)
-{
-    if (if_name == "tap") {
-        if (interface->getPeer())
-            panic("Interface already connected to\n");
-        return interface;
-    }
-    return NULL;
-}
+    if (frame_len == 0)
+        frame_len = ntohl(*(uint32_t *)buffer);
 
+    DPRINTF(Ethernet, "Received data from peer: len=%d buffer_used=%d "
+            "frame_len=%d\n", len, buffer_used, frame_len);
 
-//=====================================================================
+    uint8_t *frame_start = &buffer[sizeof(uint32_t)];
+    while (frame_len != 0 && buffer_used >= frame_len + sizeof(uint32_t)) {
+        sendSimulated(frame_start, frame_len);
 
-void
-EtherTapStub::serialize(CheckpointOut &cp) const
-{
-    SERIALIZE_SCALAR(socket);
-    SERIALIZE_SCALAR(buflen);
-    uint8_t *buffer = (uint8_t *)this->buffer;
-    SERIALIZE_ARRAY(buffer, buflen);
-    SERIALIZE_SCALAR(buffer_offset);
-    SERIALIZE_SCALAR(data_len);
+        // Bookkeeping.
+        buffer_used -= frame_len + sizeof(uint32_t);
+        if (buffer_used > 0) {
+            // If there's still any data left, move it into position.
+            memmove(buffer, frame_start + frame_len, buffer_used);
+        }
+        frame_len = 0;
 
-    bool tapevent_present = false;
-    if (event) {
-        tapevent_present = true;
-        SERIALIZE_SCALAR(tapevent_present);
-        event->serialize(cp);
-    }
-    else {
-        SERIALIZE_SCALAR(tapevent_present);
+        if (buffer_used >= sizeof(uint32_t))
+            frame_len = ntohl(*(uint32_t *)buffer);
     }
 }
 
-void
-EtherTapStub::unserialize(CheckpointIn &cp)
+bool
+EtherTapStub::sendReal(const void *data, size_t len)
 {
-    UNSERIALIZE_SCALAR(socket);
-    UNSERIALIZE_SCALAR(buflen);
-    uint8_t *buffer = (uint8_t *)this->buffer;
-    UNSERIALIZE_ARRAY(buffer, buflen);
-    UNSERIALIZE_SCALAR(buffer_offset);
-    UNSERIALIZE_SCALAR(data_len);
-
-    bool tapevent_present;
-    UNSERIALIZE_SCALAR(tapevent_present);
-    if (tapevent_present) {
-        event = new TapEvent(this, socket, POLLIN|POLLERR);
-
-        event->unserialize(cp);
-
-        if (event->queued()) {
-            pollQueue.schedule(event);
-        }
-    }
+    uint32_t frame_len = htonl(len);
+    ssize_t ret = write(socket, &frame_len, sizeof(frame_len));
+    if (ret != sizeof(frame_len))
+        return false;
+    return write(socket, data, len) == len;
 }
 
-//=====================================================================
 
 EtherTapStub *
 EtherTapStubParams::create()
diff --git a/src/dev/net/ethertap.hh b/src/dev/net/ethertap.hh
index b4af6979e..718af1808 100644
--- a/src/dev/net/ethertap.hh
+++ b/src/dev/net/ethertap.hh
@@ -47,65 +47,107 @@
 #include "sim/sim_object.hh"
 
 class TapEvent;
-class TapListener;
 class EtherTapInt;
 
-/*
- * Interface to connect a simulated ethernet device to the real world. An
- * external helper program bridges between this object's TCP port and a
- * source/sink for Ethernet frames. Each frame going in either direction is
- * prepended with the frame's length in a 32 bit integer in network byte order.
- */
-class EtherTapStub : public EtherObject
+class EtherTapBase : public EtherObject
 {
-  protected:
-    friend class TapEvent;
-    TapEvent *event;
+  public:
+    typedef EtherTapBaseParams Params;
+    EtherTapBase(const Params *p);
+    virtual ~EtherTapBase();
+
+    const Params *
+    params() const
+    {
+        return dynamic_cast<const Params *>(_params);
+    }
+
+    void serialize(CheckpointOut &cp) const override;
+    void unserialize(CheckpointIn &cp) override;
 
   protected:
-    friend class TapListener;
-    TapListener *listener;
-    int socket;
-    char *buffer;
+    uint8_t *buffer;
     int buflen;
-    uint32_t buffer_offset;
-    uint32_t data_len;
 
     EtherDump *dump;
 
-    void attach(int fd);
-    void detach();
 
+    /*
+     * Interface to the real network.
+     */
   protected:
-    std::string device;
-    std::queue<EthPacketPtr> packetBuffer;
-    EtherTapInt *interface;
+    friend class TapEvent;
+    TapEvent *event;
+    void pollFd(int fd);
+    void stopPolling();
+
+    // Receive data from the real network.
+    virtual void recvReal(int revent) = 0;
+    // Prepare and send data out to the real network.
+    virtual bool sendReal(const void *data, size_t len) = 0;
 
-    void process(int revent);
-    void enqueue(EthPacketData *packet);
-    void retransmit();
 
     /*
+     * Interface to the simulated network.
      */
+  protected:
+    EtherTapInt *interface;
+
+  public:
+    EtherInt *getEthPort(const std::string &if_name, int idx) override;
+
+    bool recvSimulated(EthPacketPtr packet);
+    void sendSimulated(void *data, size_t len);
+
+  protected:
+    std::queue<EthPacketPtr> packetBuffer;
+    void retransmit();
+
     class TxEvent : public Event
     {
       protected:
-        EtherTapStub *tap;
+        EtherTapBase *tap;
 
       public:
-        TxEvent(EtherTapStub *_tap) : tap(_tap) {}
+        TxEvent(EtherTapBase *_tap) : tap(_tap) {}
         void process() { tap->retransmit(); }
         virtual const char *description() const
-            { return "EtherTapStub retransmit"; }
+            { return "EtherTapBase retransmit"; }
     };
 
     friend class TxEvent;
     TxEvent txEvent;
+};
 
+class EtherTapInt : public EtherInt
+{
+  private:
+    EtherTapBase *tap;
+  public:
+    EtherTapInt(const std::string &name, EtherTapBase *t) :
+            EtherInt(name), tap(t)
+    { }
+
+    bool recvPacket(EthPacketPtr pkt) override
+        { return tap->recvSimulated(pkt); }
+    void sendDone() override {}
+};
+
+
+class TapListener;
+
+/*
+ * Interface to connect a simulated ethernet device to the real world. An
+ * external helper program bridges between this object's TCP port and a
+ * source/sink for Ethernet frames. Each frame going in either direction is
+ * prepended with the frame's length in a 32 bit integer in network byte order.
+ */
+class EtherTapStub : public EtherTapBase
+{
   public:
     typedef EtherTapStubParams Params;
     EtherTapStub(const Params *p);
-    virtual ~EtherTapStub();
+    ~EtherTapStub();
 
     const Params *
     params() const
@@ -113,26 +155,24 @@ class EtherTapStub : public EtherObject
         return dynamic_cast<const Params *>(_params);
     }
 
-    EtherInt *getEthPort(const std::string &if_name, int idx) override;
-
-    virtual bool recvPacket(EthPacketPtr packet);
-    virtual void sendDone();
-
     void serialize(CheckpointOut &cp) const override;
     void unserialize(CheckpointIn &cp) override;
-};
 
-class EtherTapInt : public EtherInt
-{
-  private:
-    EtherTapStub *tap;
-  public:
-    EtherTapInt(const std::string &name, EtherTapStub *t)
-            : EtherInt(name), tap(t)
-    { }
 
-    virtual bool recvPacket(EthPacketPtr pkt) { return tap->recvPacket(pkt); }
-    virtual void sendDone() { tap->sendDone(); }
+  protected:
+    friend class TapListener;
+    TapListener *listener;
+
+    int socket;
+
+    void attach(int fd);
+    void detach();
+
+    uint32_t buffer_used;
+    uint32_t frame_len;
+
+    void recvReal(int revent) override;
+    bool sendReal(const void *data, size_t len) override;
 };