sim: Decouple draining from the SimObject hierarchy

Draining is currently done by traversing the SimObject graph and
calling drain()/drainResume() on the SimObjects. This is not ideal
when non-SimObjects (e.g., ports) need draining since this means that
SimObjects owning those objects need to be aware of this.

This changeset moves the responsibility for finding objects that need
draining from SimObjects and the Python-side of the simulator to the
DrainManager. The DrainManager now maintains a set of all objects that
need draining. To reduce the overhead in classes owning non-SimObjects
that need draining, objects inheriting from Drainable now
automatically register with the DrainManager. If such an object is
destroyed, it is automatically unregistered. This means that drain()
and drainResume() should never be called directly on a Drainable
object.

While implementing the new functionality, the DrainManager has now
been made thread safe. In practice, this means that it takes a lock
whenever it manipulates the set of Drainable objects since SimObjects
in different threads may create Drainable objects
dynamically. Similarly, the drain counter is now an atomic_uint, which
ensures that it is manipulated correctly when objects signal that they
are done draining.

A nice side effect of these changes is that it makes the drain state
changes stricter, which the simulation scripts can exploit to avoid
redundant drains.

10 files changed, 5 insertions, 88 deletions

diff --git a/src/dev/arm/ufs_device.cc b/src/dev/arm/ufs_device.cc
index ef6398216..696aeba6f 100644
--- a/src/dev/arm/ufs_device.cc
+++ b/src/dev/arm/ufs_device.cc
@@ -2319,26 +2319,16 @@ UFSHostDevice::unserialize(CheckpointIn &cp)
 unsigned int
 UFSHostDevice::drain(DrainManager *dm)
 {
-    unsigned int count = 0;
-
-    // check pio, dma port, and doorbells
-    count = pioPort.drain(dm) + dmaPort.drain(dm);
-
     if (UFSHCIMem.TRUTRLDBR) {
-        count += 1;
         drainManager = dm;
-    } else {
-        DPRINTF(UFSHostDevice, "UFSHostDevice in drained state\n");
-    }
-
-    if (count) {
         DPRINTF(UFSHostDevice, "UFSDevice is draining...\n");
         setDrainState(DrainState::Draining);
+        return 1;
     } else {
         DPRINTF(UFSHostDevice, "UFSDevice drained\n");
         setDrainState(DrainState::Drained);
+        return 0;
     }
-    return count;
 }
 
 /**
diff --git a/src/dev/copy_engine.cc b/src/dev/copy_engine.cc
index 035f824fa..ed177cf7f 100644
--- a/src/dev/copy_engine.cc
+++ b/src/dev/copy_engine.cc
@@ -650,29 +650,10 @@ CopyEngine::CopyEngineChannel::drain(DrainManager *dm)
 {
     if (nextState == Idle || ce->getDrainState() != DrainState::Running)
         return 0;
-    unsigned int count = 1;
-    count += cePort.drain(dm);
 
     DPRINTF(Drain, "CopyEngineChannel not drained\n");
     this->drainManager = dm;
-    return count;
-}
-
-unsigned int
-CopyEngine::drain(DrainManager *dm)
-{
-    unsigned int count;
-    count = pioPort.drain(dm) + dmaPort.drain(dm) + configPort.drain(dm);
-    for (int x = 0;x < chan.size(); x++)
-        count += chan[x]->drain(dm);
-
-    if (count)
-        setDrainState(DrainState::Draining);
-    else
-        setDrainState(DrainState::Drained);
-
-    DPRINTF(Drain, "CopyEngine not drained\n");
-    return count;
+    return 1;
 }
 
 void
@@ -756,15 +737,6 @@ CopyEngine::CopyEngineChannel::restartStateMachine()
 }
 
 void
-CopyEngine::drainResume()
-{
-    Drainable::drainResume();
-    for (int x = 0;x < chan.size(); x++)
-        chan[x]->drainResume();
-}
-
-
-void
 CopyEngine::CopyEngineChannel::drainResume()
 {
     DPRINTF(DMACopyEngine, "Restarting state machine at state %d\n", nextState);
diff --git a/src/dev/copy_engine.hh b/src/dev/copy_engine.hh
index 80c9798ee..d09a18dbd 100644
--- a/src/dev/copy_engine.hh
+++ b/src/dev/copy_engine.hh
@@ -207,9 +207,6 @@ class CopyEngine : public PciDevice
 
     void serialize(CheckpointOut &cp) const M5_ATTR_OVERRIDE;
     void unserialize(CheckpointIn &cp) M5_ATTR_OVERRIDE;
-
-    unsigned int drain(DrainManager *drainManger);
-    void drainResume();
 };
 
 #endif //__DEV_COPY_ENGINE_HH__
diff --git a/src/dev/dma_device.cc b/src/dev/dma_device.cc
index a778833ff..97b31e976 100644
--- a/src/dev/dma_device.cc
+++ b/src/dev/dma_device.cc
@@ -126,17 +126,6 @@ DmaDevice::init()
 }
 
 unsigned int
-DmaDevice::drain(DrainManager *dm)
-{
-    unsigned int count = pioPort.drain(dm) + dmaPort.drain(dm);
-    if (count)
-        setDrainState(DrainState::Draining);
-    else
-        setDrainState(DrainState::Drained);
-    return count;
-}
-
-unsigned int
 DmaPort::drain(DrainManager *dm)
 {
     if (pendingCount == 0)
diff --git a/src/dev/dma_device.hh b/src/dev/dma_device.hh
index 933cbeb00..92bf8f72c 100644
--- a/src/dev/dma_device.hh
+++ b/src/dev/dma_device.hh
@@ -51,7 +51,7 @@
 #include "sim/drain.hh"
 #include "sim/system.hh"
 
-class DmaPort : public MasterPort
+class DmaPort : public MasterPort, public Drainable
 {
   private:
 
@@ -176,8 +176,6 @@ class DmaDevice : public PioDevice
 
     virtual void init();
 
-    unsigned int drain(DrainManager *drainManger);
-
     unsigned int cacheBlockSize() const { return sys->cacheLineSize(); }
 
     virtual BaseMasterPort &getMasterPort(const std::string &if_name,
diff --git a/src/dev/i8254xGBe.cc b/src/dev/i8254xGBe.cc
index 3b49c8b5d..cd96c9eca 100644
--- a/src/dev/i8254xGBe.cc
+++ b/src/dev/i8254xGBe.cc
@@ -2058,8 +2058,7 @@ IGbE::restartClock()
 unsigned int
 IGbE::drain(DrainManager *dm)
 {
-    unsigned int count;
-    count = pioPort.drain(dm) + dmaPort.drain(dm);
+    unsigned int count(0);
     if (rxDescCache.hasOutstandingEvents() ||
         txDescCache.hasOutstandingEvents()) {
         count++;
diff --git a/src/dev/io_device.cc b/src/dev/io_device.cc
index 9e5855a0b..74ab8c3b3 100644
--- a/src/dev/io_device.cc
+++ b/src/dev/io_device.cc
@@ -93,18 +93,6 @@ PioDevice::getSlavePort(const std::string &if_name, PortID idx)
     return MemObject::getSlavePort(if_name, idx);
 }
 
-unsigned int
-PioDevice::drain(DrainManager *dm)
-{
-    unsigned int count;
-    count = pioPort.drain(dm);
-    if (count)
-        setDrainState(DrainState::Draining);
-    else
-        setDrainState(DrainState::Drained);
-    return count;
-}
-
 BasicPioDevice::BasicPioDevice(const Params *p, Addr size)
     : PioDevice(p), pioAddr(p->pio_addr), pioSize(size),
       pioDelay(p->pio_latency)
diff --git a/src/dev/io_device.hh b/src/dev/io_device.hh
index a528b250e..7e323b3bc 100644
--- a/src/dev/io_device.hh
+++ b/src/dev/io_device.hh
@@ -125,8 +125,6 @@ class PioDevice : public MemObject
 
     virtual void init();
 
-    unsigned int drain(DrainManager *drainManger);
-
     virtual BaseSlavePort &getSlavePort(const std::string &if_name,
                                         PortID idx = InvalidPortID);
 
diff --git a/src/dev/pcidev.cc b/src/dev/pcidev.cc
index 5788c94d2..4126141b9 100644
--- a/src/dev/pcidev.cc
+++ b/src/dev/pcidev.cc
@@ -255,18 +255,6 @@ PciDevice::init()
    DmaDevice::init();
 }
 
-unsigned int
-PciDevice::drain(DrainManager *dm)
-{
-    unsigned int count;
-    count = pioPort.drain(dm) + dmaPort.drain(dm) + configPort.drain(dm);
-    if (count)
-        setDrainState(DrainState::Draining);
-    else
-        setDrainState(DrainState::Drained);
-    return count;
-}
-
 Tick
 PciDevice::readConfig(PacketPtr pkt)
 {
diff --git a/src/dev/pcidev.hh b/src/dev/pcidev.hh
index 4f997932b..903d83c77 100644
--- a/src/dev/pcidev.hh
+++ b/src/dev/pcidev.hh
@@ -259,8 +259,6 @@ class PciDevice : public DmaDevice
     void unserialize(CheckpointIn &cp) M5_ATTR_OVERRIDE;
 
 
-    virtual unsigned int drain(DrainManager *dm);
-
     virtual BaseSlavePort &getSlavePort(const std::string &if_name,
                                         PortID idx = InvalidPortID)
     {