ruby: route all packets through ruby port

Currently, the interrupt controller in x86 is connected to the io bus
directly.  Therefore the packets between the io devices and the interrupt
controller do not go through ruby.  This patch changes ruby port so that
these packets arrive at the ruby port first, which then routes them to their
destination.  Note that the patch does not make these packets go through the
ruby network.  That would happen in a subsequent patch.

1 files changed, 129 insertions, 60 deletions

diff --git a/src/mem/ruby/system/RubyPort.cc b/src/mem/ruby/system/RubyPort.cc
index 14c156883..fe6a33196 100644
--- a/src/mem/ruby/system/RubyPort.cc
+++ b/src/mem/ruby/system/RubyPort.cc
@@ -50,25 +50,28 @@
 
 RubyPort::RubyPort(const Params *p)
     : MemObject(p), m_version(p->version), m_controller(NULL),
-      m_mandatory_q_ptr(NULL),
-      pio_port(csprintf("%s-pio-port", name()), this),
-      m_usingRubyTester(p->using_ruby_tester),
-      drainManager(NULL), ruby_system(p->ruby_system), system(p->system),
+      m_mandatory_q_ptr(NULL), m_usingRubyTester(p->using_ruby_tester),
+      pioMasterPort(csprintf("%s.pio-master-port", name()), this),
+      pioSlavePort(csprintf("%s.pio-slave-port", name()), this),
+      memMasterPort(csprintf("%s.mem-master-port", name()), this),
+      memSlavePort(csprintf("%s-mem-slave-port", name()), this,
+          p->ruby_system, p->access_phys_mem, -1),
+      gotAddrRanges(p->port_master_connection_count), drainManager(NULL),
+      ruby_system(p->ruby_system), system(p->system),
       access_phys_mem(p->access_phys_mem)
 {
     assert(m_version != -1);
 
     // create the slave ports based on the number of connected ports
     for (size_t i = 0; i < p->port_slave_connection_count; ++i) {
-        slave_ports.push_back(new M5Port(csprintf("%s-slave%d", name(), i),
-                                         this, ruby_system,
-                                         access_phys_mem, i));
+        slave_ports.push_back(new MemSlavePort(csprintf("%s.slave%d", name(),
+            i), this, ruby_system, access_phys_mem, i));
     }
 
     // create the master ports based on the number of connected ports
     for (size_t i = 0; i < p->port_master_connection_count; ++i) {
-        master_ports.push_back(new PioPort(csprintf("%s-master%d", name(), i),
-                                           this));
+        master_ports.push_back(new PioMasterPort(csprintf("%s.master%d",
+            name(), i), this));
     }
 }
 
@@ -83,8 +86,12 @@ RubyPort::init()
 BaseMasterPort &
 RubyPort::getMasterPort(const std::string &if_name, PortID idx)
 {
-    if (if_name == "pio_port") {
-        return pio_port;
+    if (if_name == "mem_master_port") {
+        return memMasterPort;
+    }
+
+    if (if_name == "pio_master_port") {
+        return pioMasterPort;
     }
 
     // used by the x86 CPUs to connect the interrupt PIO and interrupt slave
@@ -104,6 +111,13 @@ RubyPort::getMasterPort(const std::string &if_name, PortID idx)
 BaseSlavePort &
 RubyPort::getSlavePort(const std::string &if_name, PortID idx)
 {
+    if (if_name == "mem_slave_port") {
+        return memSlavePort;
+    }
+
+    if (if_name == "pio_slave_port")
+        return pioSlavePort;
+
     // used by the CPUs to connect the caches to the interconnect, and
     // for the x86 case also the interrupt master
     if (if_name != "slave") {
@@ -118,32 +132,48 @@ RubyPort::getSlavePort(const std::string &if_name, PortID idx)
     }
 }
 
-RubyPort::PioPort::PioPort(const std::string &_name,
+RubyPort::PioMasterPort::PioMasterPort(const std::string &_name,
+                           RubyPort *_port)
+    : QueuedMasterPort(_name, _port, queue), queue(*_port, *this)
+{
+    DPRINTF(RubyPort, "Created master pioport on sequencer %s\n", _name);
+}
+
+RubyPort::PioSlavePort::PioSlavePort(const std::string &_name,
+                           RubyPort *_port)
+    : QueuedSlavePort(_name, _port, queue), queue(*_port, *this)
+{
+    DPRINTF(RubyPort, "Created slave pioport on sequencer %s\n", _name);
+}
+
+RubyPort::MemMasterPort::MemMasterPort(const std::string &_name,
                            RubyPort *_port)
-    : QueuedMasterPort(_name, _port, queue), queue(*_port, *this),
-      ruby_port(_port)
+    : QueuedMasterPort(_name, _port, queue), queue(*_port, *this)
 {
-    DPRINTF(RubyPort, "creating master port on ruby sequencer %s\n", _name);
+    DPRINTF(RubyPort, "Created master memport on ruby sequencer %s\n", _name);
 }
 
-RubyPort::M5Port::M5Port(const std::string &_name, RubyPort *_port,
+RubyPort::MemSlavePort::MemSlavePort(const std::string &_name, RubyPort *_port,
                          RubySystem *_system, bool _access_phys_mem, PortID id)
     : QueuedSlavePort(_name, _port, queue, id), queue(*_port, *this),
-      ruby_port(_port), ruby_system(_system),
-      access_phys_mem(_access_phys_mem)
+      ruby_system(_system), access_phys_mem(_access_phys_mem)
 {
-    DPRINTF(RubyPort, "creating slave port on ruby sequencer %s\n", _name);
+    DPRINTF(RubyPort, "Created slave memport on ruby sequencer %s\n", _name);
 }
 
-Tick
-RubyPort::M5Port::recvAtomic(PacketPtr pkt)
+bool
+RubyPort::PioMasterPort::recvTimingResp(PacketPtr pkt)
 {
-    panic("RubyPort::M5Port::recvAtomic() not implemented!\n");
-    return 0;
+    RubyPort *ruby_port = static_cast<RubyPort *>(&owner);
+    DPRINTF(RubyPort, "Response for address: 0x%#x\n", pkt->getAddr());
+
+    // send next cycle
+    ruby_port->pioSlavePort.schedTimingResp(
+            pkt, curTick() + g_system_ptr->clockPeriod());
+    return true;
 }
 
-bool
-RubyPort::recvTimingResp(PacketPtr pkt, PortID master_port_id)
+bool RubyPort::MemMasterPort::recvTimingResp(PacketPtr pkt)
 {
     // got a response from a device
     assert(pkt->isResponse());
@@ -153,43 +183,66 @@ RubyPort::recvTimingResp(PacketPtr pkt, PortID master_port_id)
     DPRINTF(RubyPort,  "Pio response for address %#x, going to %d\n",
             pkt->getAddr(), pkt->getDest());
 
-    // Retrieve the port from the destination field
-    assert(pkt->getDest() < slave_ports.size());
+    // First we must retrieve the request port from the sender State
+    RubyPort::SenderState *senderState =
+        safe_cast<RubyPort::SenderState *>(pkt->popSenderState());
+    MemSlavePort *port = senderState->port;
+    assert(port != NULL);
+    delete senderState;
 
     // attempt to send the response in the next cycle
-    slave_ports[pkt->getDest()]->schedTimingResp(pkt, curTick() +
-                                                 g_system_ptr->clockPeriod());
+    port->schedTimingResp(pkt, curTick() + g_system_ptr->clockPeriod());
 
     return true;
 }
 
 bool
-RubyPort::M5Port::recvTimingReq(PacketPtr pkt)
+RubyPort::PioSlavePort::recvTimingReq(PacketPtr pkt)
 {
-    DPRINTF(RubyPort,
-            "Timing access for address %#x on port %d\n", pkt->getAddr(),
-            id);
+    RubyPort *ruby_port = static_cast<RubyPort *>(&owner);
+
+    for (size_t i = 0; i < ruby_port->master_ports.size(); ++i) {
+        AddrRangeList l = ruby_port->master_ports[i]->getAddrRanges();
+        for (auto it = l.begin(); it != l.end(); ++it) {
+            if (it->contains(pkt->getAddr())) {
+                ruby_port->master_ports[i]->sendTimingReq(pkt);
+                return true;
+            }
+        }
+    }
+    panic("Should never reach here!\n");
+}
+
+bool
+RubyPort::MemSlavePort::recvTimingReq(PacketPtr pkt)
+{
+    DPRINTF(RubyPort, "Timing request for address %#x on port %d\n",
+            pkt->getAddr(), id);
+    RubyPort *ruby_port = static_cast<RubyPort *>(&owner);
 
     if (pkt->memInhibitAsserted())
         panic("RubyPort should never see an inhibited request\n");
 
-    // Save the port id to be used later to route the response
-    pkt->setSrc(id);
-
     // Check for pio requests and directly send them to the dedicated
     // pio port.
     if (!isPhysMemAddress(pkt->getAddr())) {
-        assert(ruby_port->pio_port.isConnected());
-        DPRINTF(RubyPort,
-                "Request for address 0x%#x is assumed to be a pio request\n",
+        assert(ruby_port->memMasterPort.isConnected());
+        DPRINTF(RubyPort, "Request address %#x assumed to be a pio address\n",
                 pkt->getAddr());
 
+        // Save the port in the sender state object to be used later to
+        // route the response
+        pkt->pushSenderState(new SenderState(this));
+
         // send next cycle
-        ruby_port->pio_port.schedTimingReq(pkt,
+        ruby_port->memMasterPort.schedTimingReq(pkt,
             curTick() + g_system_ptr->clockPeriod());
         return true;
     }
 
+    // Save the port id to be used later to route the response
+    pkt->setSrc(id);
+
     assert(Address(pkt->getAddr()).getOffset() + pkt->getSize() <=
            RubySystem::getBlockSizeBytes());
 
@@ -213,26 +266,24 @@ RubyPort::M5Port::recvTimingReq(PacketPtr pkt)
         ruby_port->addToRetryList(this);
     }
 
-    DPRINTF(RubyPort,
-            "Request for address %#x did not issue because %s\n",
+    DPRINTF(RubyPort, "Request for address %#x did not issued because %s\n",
             pkt->getAddr(), RequestStatus_to_string(requestStatus));
 
     return false;
 }
 
 void
-RubyPort::M5Port::recvFunctional(PacketPtr pkt)
+RubyPort::MemSlavePort::recvFunctional(PacketPtr pkt)
 {
-    DPRINTF(RubyPort, "Functional access caught for address %#x\n",
-                                                           pkt->getAddr());
+    DPRINTF(RubyPort, "Functional access for address: %#x\n", pkt->getAddr());
+    RubyPort *ruby_port = static_cast<RubyPort *>(&owner);
 
     // Check for pio requests and directly send them to the dedicated
     // pio port.
     if (!isPhysMemAddress(pkt->getAddr())) {
-        assert(ruby_port->pio_port.isConnected());
-        DPRINTF(RubyPort, "Request for address 0x%#x is a pio request\n",
-                                                           pkt->getAddr());
-        panic("RubyPort::PioPort::recvFunctional() not implemented!\n");
+        assert(ruby_port->memMasterPort.isConnected());
+        DPRINTF(RubyPort, "Pio Request for address: 0x%#x\n", pkt->getAddr());
+        panic("RubyPort::PioMasterPort::recvFunctional() not implemented!\n");
     }
 
     assert(pkt->getAddr() + pkt->getSize() <=
@@ -248,8 +299,7 @@ RubyPort::M5Port::recvFunctional(PacketPtr pkt)
     } else if (pkt->isWrite()) {
         accessSucceeded = ruby_system->functionalWrite(pkt);
     } else {
-        panic("RubyPort: unsupported functional command %s\n",
-              pkt->cmdString());
+        panic("Unsupported functional command %s\n", pkt->cmdString());
     }
 
     // Unless the requester explicitly said otherwise, generate an error if
@@ -298,7 +348,7 @@ RubyPort::ruby_hit_callback(PacketPtr pkt)
     slave_ports[pkt->getSrc()]->hitCallback(pkt);
 
     //
-    // If we had to stall the M5Ports, wake them up because the sequencer
+    // If we had to stall the MemSlavePorts, wake them up because the sequencer
     // likely has free resources now.
     //
     if (!retryList.empty()) {
@@ -309,7 +359,7 @@ RubyPort::ruby_hit_callback(PacketPtr pkt)
         // list. Therefore we want to clear the retryList before calling
         // sendRetry.
         //
-        std::vector<M5Port*> curRetryList(retryList);
+        std::vector<MemSlavePort *> curRetryList(retryList);
 
         retryList.clear();
 
@@ -345,8 +395,8 @@ RubyPort::getChildDrainCount(DrainManager *dm)
 {
     int count = 0;
 
-    if (pio_port.isConnected()) {
-        count += pio_port.drain(dm);
+    if (memMasterPort.isConnected()) {
+        count += memMasterPort.drain(dm);
         DPRINTF(Config, "count after pio check %d\n", count);
     }
 
@@ -355,14 +405,13 @@ RubyPort::getChildDrainCount(DrainManager *dm)
         DPRINTF(Config, "count after slave port check %d\n", count);
     }
 
-    for (std::vector<PioPort*>::iterator p = master_ports.begin();
+    for (std::vector<PioMasterPort *>::iterator p = master_ports.begin();
          p != master_ports.end(); ++p) {
         count += (*p)->drain(dm);
         DPRINTF(Config, "count after master port check %d\n", count);
     }
 
     DPRINTF(Config, "final count %d\n", count);
-
     return count;
 }
 
@@ -401,7 +450,7 @@ RubyPort::drain(DrainManager *dm)
 }
 
 void
-RubyPort::M5Port::hitCallback(PacketPtr pkt)
+RubyPort::MemSlavePort::hitCallback(PacketPtr pkt)
 {
     bool needsResponse = pkt->needsResponse();
 
@@ -444,6 +493,7 @@ RubyPort::M5Port::hitCallback(PacketPtr pkt)
     DPRINTF(RubyPort, "Hit callback needs response %d\n", needsResponse);
 
     if (accessPhysMem) {
+        RubyPort *ruby_port = static_cast<RubyPort *>(&owner);
         ruby_port->system->getPhysMem().access(pkt);
     } else if (needsResponse) {
         pkt->makeResponse();
@@ -461,16 +511,25 @@ RubyPort::M5Port::hitCallback(PacketPtr pkt)
 }
 
 AddrRangeList
-RubyPort::M5Port::getAddrRanges() const
+RubyPort::PioSlavePort::getAddrRanges() const
 {
     // at the moment the assumption is that the master does not care
     AddrRangeList ranges;
+    RubyPort *ruby_port = static_cast<RubyPort *>(&owner);
+
+    for (size_t i = 0; i < ruby_port->master_ports.size(); ++i) {
+        ranges.splice(ranges.begin(),
+                ruby_port->master_ports[i]->getAddrRanges());
+    }
+    for (AddrRangeConstIter r = ranges.begin(); r != ranges.end(); ++r)
+        DPRINTF(RubyPort, "%s\n", r->to_string());
     return ranges;
 }
 
 bool
-RubyPort::M5Port::isPhysMemAddress(Addr addr) const
+RubyPort::MemSlavePort::isPhysMemAddress(Addr addr) const
 {
+    RubyPort *ruby_port = static_cast<RubyPort *>(&owner);
     return ruby_port->system->isMemAddr(addr);
 }
 
@@ -494,3 +553,13 @@ RubyPort::ruby_eviction_callback(const Address& address)
         }
     }
 }
+
+void
+RubyPort::PioMasterPort::recvRangeChange()
+{
+    RubyPort &r = static_cast<RubyPort &>(owner);
+    r.gotAddrRanges--;
+    if (r.gotAddrRanges == 0) {
+        r.pioSlavePort.sendRangeChange();
+    }
+}