mem: Add tWR to DRAM activate and precharge constraints

This patch adds the write recovery time to the DRAM timing
constraints, and changes the current tRASDoneAt to a more generic
preAllowedAt, capturing when a precharge is allowed to take place.

The part of the DRAM access code that accounts for the precharge and
activate constraints is updated accordingly.

3 files changed, 101 insertions, 65 deletions

diff --git a/src/mem/DRAMCtrl.py b/src/mem/DRAMCtrl.py
index 895b9624d..62d237cfb 100644
--- a/src/mem/DRAMCtrl.py
+++ b/src/mem/DRAMCtrl.py
@@ -132,6 +132,9 @@ class DRAMCtrl(AbstractMemory):
     # minimum time between an activate and a precharge to the same row
     tRAS = Param.Latency("ACT to PRE delay")
 
+    # minimum time between a write data transfer and a precharge
+    tWR = Param.Latency("Write recovery time")
+
     # time to complete a burst transfer, typically the burst length
     # divided by two due to the DDR bus, but by making it a parameter
     # it is easier to also evaluate SDR memories like WideIO.
@@ -194,6 +197,7 @@ class DDR3_1600_x64(DRAMCtrl):
     tCL = '13.75ns'
     tRP = '13.75ns'
     tRAS = '35ns'
+    tWR = '15ns'
 
     # 8 beats across an x64 interface translates to 4 clocks @ 800 MHz.
     # Note this is a BL8 DDR device.
@@ -252,6 +256,7 @@ class DDR3_1333_x64_DRAMSim2(DRAMCtrl):
     tCL = '15ns'
     tRP = '15ns'
     tRAS = '36ns'
+    tWR = '15ns'
 
     # 8 beats across an x64 interface translates to 4 clocks @ 666.66 MHz.
     # Note this is a BL8 DDR device.
@@ -307,6 +312,7 @@ class LPDDR2_S4_1066_x32(DRAMCtrl):
     tRP = '15ns'
 
     tRAS = '42ns'
+    tWR = '15ns'
 
     # 8 beats across an x32 DDR interface translates to 4 clocks @ 533 MHz.
     # Note this is a BL8 DDR device.
@@ -358,6 +364,7 @@ class WideIO_200_x128(DRAMCtrl):
     tCL = '18ns'
     tRP = '18ns'
     tRAS = '42ns'
+    tWR = '15ns'
 
     # 4 beats across an x128 SDR interface translates to 4 clocks @ 200 MHz.
     # Note this is a BL4 SDR device.
@@ -411,6 +418,7 @@ class LPDDR3_1600_x32(DRAMCtrl):
     tCL = '15ns'
 
     tRAS = '42ns'
+    tWR = '15ns'
 
     # Pre-charge one bank 15 ns (all banks 18 ns)
     tRP = '15ns'
diff --git a/src/mem/dram_ctrl.cc b/src/mem/dram_ctrl.cc
index 7062390e8..a18702607 100644
--- a/src/mem/dram_ctrl.cc
+++ b/src/mem/dram_ctrl.cc
@@ -75,7 +75,7 @@ DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) :
     minWritesPerSwitch(p->min_writes_per_switch),
     writesThisTime(0), readsThisTime(0),
     tWTR(p->tWTR), tRTW(p->tRTW), tBURST(p->tBURST),
-    tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS),
+    tRCD(p->tRCD), tCL(p->tCL), tRP(p->tRP), tRAS(p->tRAS), tWR(p->tWR),
     tRFC(p->tRFC), tREFI(p->tREFI), tRRD(p->tRRD),
     tXAW(p->tXAW), activationLimit(p->activation_limit),
     memSchedPolicy(p->mem_sched_policy), addrMapping(p->addr_mapping),
@@ -559,9 +559,10 @@ DRAMCtrl::printParams() const
             "tREFI     %d ticks\n"                        \
             "tWTR      %d ticks\n"                        \
             "tRTW      %d ticks\n"                        \
+            "tWR       %d ticks\n"                        \
             "tXAW (%d) %d ticks\n",
             name(), tRCD, tCL, tRP, tBURST, tRFC, tREFI, tWTR,
-            tRTW, activationLimit, tXAW);
+            tRTW, tWR, activationLimit, tXAW);
 }
 
 void
@@ -807,7 +808,6 @@ DRAMCtrl::estimateLatency(DRAMPacket* dram_pkt, Tick inTime)
     Tick accLat = 0;
     Tick bankLat = 0;
     rowHitFlag = false;
-    Tick potentialActTick;
 
     const Bank& bank = dram_pkt->bankRef;
 
@@ -832,22 +832,27 @@ DRAMCtrl::estimateLatency(DRAMPacket* dram_pkt, Tick inTime)
             bankLat += tCL;
         }
     } else {
-        // Row-buffer miss, need to close existing row
-        // once tRAS has expired, then open the new one,
-        // then add cas latency.
-        Tick freeTime = std::max(bank.tRASDoneAt, bank.freeAt);
+        // Row-buffer miss, need to potentially close an existing row,
+        // then open the new one, then add CAS latency
+        Tick free_at = bank.freeAt;
+        Tick precharge_delay = 0;
 
-        if (freeTime > inTime)
-            accLat += freeTime - inTime;
+        // Check if we first need to precharge
+        if (bank.openRow != Bank::NO_ROW) {
+            free_at = std::max(bank.preAllowedAt, free_at);
+            precharge_delay = tRP;
+        }
 
-        // If the there is no open row, then there is no precharge
-        // delay, otherwise go with tRP
-        Tick precharge_delay = bank.openRow == Bank::NO_ROW ? 0 : tRP;
+        // If the earliest time to issue the command is in the future,
+        // add it to the access latency
+        if (free_at > inTime)
+            accLat += free_at - inTime;
 
-        //The bank is free, and you may be able to activate
-        potentialActTick = inTime + accLat + precharge_delay;
-        if (potentialActTick < bank.actAllowedAt)
-            accLat += bank.actAllowedAt - potentialActTick;
+        // We also need to account for the earliest activation time,
+        // and potentially add that as well to the access latency
+        Tick act_at = inTime + accLat + precharge_delay;
+        if (act_at < bank.actAllowedAt)
+            accLat += bank.actAllowedAt - act_at;
 
         accLat += precharge_delay + tRCD + tCL;
         bankLat += precharge_delay + tRCD + tCL;
@@ -860,8 +865,8 @@ DRAMCtrl::estimateLatency(DRAMPacket* dram_pkt, Tick inTime)
 }
 
 void
-DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank,
-                         uint16_t row)
+DRAMCtrl::activateBank(Tick act_tick, uint8_t rank, uint8_t bank,
+                       uint16_t row, Bank& bank_ref)
 {
     assert(0 <= rank && rank < ranksPerChannel);
     assert(actTicks[rank].size() == activationLimit);
@@ -869,14 +874,14 @@ DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank,
     DPRINTF(DRAM, "Activate at tick %d\n", act_tick);
 
     // update the open row
-    assert(banks[rank][bank].openRow == Bank::NO_ROW);
-    banks[rank][bank].openRow = row;
+    assert(bank_ref.openRow == Bank::NO_ROW);
+    bank_ref.openRow = row;
 
     // start counting anew, this covers both the case when we
     // auto-precharged, and when this access is forced to
     // precharge
-    banks[rank][bank].bytesAccessed = 0;
-    banks[rank][bank].rowAccesses = 0;
+    bank_ref.bytesAccessed = 0;
+    bank_ref.rowAccesses = 0;
 
     ++numBanksActive;
     assert(numBanksActive <= banksPerRank * ranksPerChannel);
@@ -886,15 +891,12 @@ DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank,
 
     // start by enforcing tRRD
     for(int i = 0; i < banksPerRank; i++) {
-        // next activate must not happen before tRRD
-        banks[rank][i].actAllowedAt = act_tick + tRRD;
+        // next activate to any bank in this rank must not happen
+        // before tRRD
+        banks[rank][i].actAllowedAt = std::max(act_tick + tRRD,
+                                               banks[rank][i].actAllowedAt);
     }
 
-    // tRC should be added to activation tick of the bank currently accessed,
-    // where tRC = tRAS + tRP, this is just for a check as actAllowedAt for same
-    // bank is already captured by bank.freeAt and bank.tRASDoneAt
-    banks[rank][bank].actAllowedAt = act_tick + tRAS + tRP;
-
     // next, we deal with tXAW, if the activation limit is disabled
     // then we are done
     if (actTicks[rank].empty())
@@ -902,10 +904,9 @@ DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank,
 
     // sanity check
     if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) {
-        // @todo For now, stick with a warning
-        warn("Got %d activates in window %d (%d - %d) which is smaller "
-             "than %d\n", activationLimit, act_tick - actTicks[rank].back(),
-             act_tick, actTicks[rank].back(), tXAW);
+        panic("Got %d activates in window %d (%llu - %llu) which is smaller "
+              "than %llu\n", activationLimit, act_tick - actTicks[rank].back(),
+              act_tick, actTicks[rank].back(), tXAW);
     }
 
     // shift the times used for the book keeping, the last element
@@ -920,10 +921,12 @@ DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank,
     // oldest in our window of X
     if (actTicks[rank].back() && (act_tick - actTicks[rank].back()) < tXAW) {
         DPRINTF(DRAM, "Enforcing tXAW with X = %d, next activate no earlier "
-                "than %d\n", activationLimit, actTicks[rank].back() + tXAW);
+                "than %llu\n", activationLimit, actTicks[rank].back() + tXAW);
             for(int j = 0; j < banksPerRank; j++)
                 // next activate must not happen before end of window
-                banks[rank][j].actAllowedAt = actTicks[rank].back() + tXAW;
+                banks[rank][j].actAllowedAt =
+                    std::max(actTicks[rank].back() + tXAW,
+                             banks[rank][j].actAllowedAt);
     }
 
     // at the point when this activate takes place, make sure we
@@ -1006,10 +1009,20 @@ DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt)
     // to estimateLatency(). However, between then and now, both the
     // accessLatency and/or busBusyUntil may have changed. We need
     // to correct for that.
-
     Tick addDelay = (curTick() + accessLat < busBusyUntil) ?
         busBusyUntil - (curTick() + accessLat) : 0;
 
+    // Update request parameters
+    dram_pkt->readyTime = curTick() + addDelay + accessLat + tBURST;
+
+    DPRINTF(DRAM, "Req %lld: curtick is %lld accessLat is %d " \
+                  "readytime is %lld busbusyuntil is %lld. " \
+                  "Scheduling at readyTime\n", dram_pkt->addr,
+                   curTick(), accessLat, dram_pkt->readyTime, busBusyUntil);
+
+    // Make sure requests are not overlapping on the databus
+    assert(dram_pkt->readyTime - busBusyUntil >= tBURST);
+
     Bank& bank = dram_pkt->bankRef;
 
     // Update bank state
@@ -1019,7 +1032,7 @@ DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt)
         // If there is a page open, precharge it.
         if (bank.openRow != Bank::NO_ROW) {
             prechargeBank(bank, std::max(std::max(bank.freeAt,
-                                                  bank.tRASDoneAt),
+                                                  bank.preAllowedAt),
                                          curTick()) + tRP);
         }
 
@@ -1030,14 +1043,30 @@ DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt)
         // any waiting for banks account for in freeAt
         actTick = bank.freeAt - tCL - tRCD;
 
-        // If you activated a new row do to this access, the next access
-        // will have to respect tRAS for this bank
-        bank.tRASDoneAt = actTick + tRAS;
+        // The next access has to respect tRAS for this bank
+        bank.preAllowedAt = actTick + tRAS;
 
-        recordActivate(actTick, dram_pkt->rank, dram_pkt->bank,
-                       dram_pkt->row);
+        // Record the activation and deal with all the global timing
+        // constraints caused be a new activation (tRRD and tXAW)
+        activateBank(actTick, dram_pkt->rank, dram_pkt->bank,
+                     dram_pkt->row, bank);
+
+    }
+
+    // If this is a write, we also need to respect the write
+    // recovery time before a precharge
+    if (!dram_pkt->isRead) {
+        bank.preAllowedAt = std::max(bank.preAllowedAt,
+                                     dram_pkt->readyTime + tWR);
     }
 
+    // We also have to respect tRP, and any constraints on when we may
+    // precharge the bank, in the case of reads this is really only
+    // going to cause any change if we did not have a row hit and are
+    // now forced to respect tRAS
+    bank.actAllowedAt = std::max(bank.actAllowedAt,
+                                 bank.preAllowedAt + tRP);
+
     // increment the bytes accessed and the accesses per row
     bank.bytesAccessed += burstSize;
     ++bank.rowAccesses;
@@ -1094,24 +1123,13 @@ DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt)
     // if this access should use auto-precharge, then we are
     // closing the row
     if (auto_precharge) {
-        prechargeBank(bank, std::max(bank.freeAt, bank.tRASDoneAt) + tRP);
+        prechargeBank(bank, std::max(bank.freeAt, bank.preAllowedAt) + tRP);
 
         DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId);
     }
 
     DPRINTF(DRAM, "doDRAMAccess::bank.freeAt is %lld\n", bank.freeAt);
 
-    // Update request parameters
-    dram_pkt->readyTime = curTick() + addDelay + accessLat + tBURST;
-
-    DPRINTF(DRAM, "Req %lld: curtick is %lld accessLat is %d " \
-                  "readytime is %lld busbusyuntil is %lld. " \
-                  "Scheduling at readyTime\n", dram_pkt->addr,
-                   curTick(), accessLat, dram_pkt->readyTime, busBusyUntil);
-
-    // Make sure requests are not overlapping on the databus
-    assert(dram_pkt->readyTime - busBusyUntil >= tBURST);
-
     // Update bus state
     busBusyUntil = dram_pkt->readyTime;
 
@@ -1412,7 +1430,7 @@ DRAMCtrl::processRefreshEvent()
                         // constraints
                         Tick free_at =
                             std::max(std::max(banks[i][j].freeAt,
-                                              banks[i][j].tRASDoneAt),
+                                              banks[i][j].preAllowedAt),
                                      curTick()) + tRP;
 
                         prechargeBank(banks[i][j], free_at);
diff --git a/src/mem/dram_ctrl.hh b/src/mem/dram_ctrl.hh
index 5f2a2c12b..bf52422fc 100644
--- a/src/mem/dram_ctrl.hh
+++ b/src/mem/dram_ctrl.hh
@@ -143,10 +143,13 @@ class DRAMCtrl : public AbstractMemory
     std::vector<std::deque<Tick>> actTicks;
 
     /**
-     * A basic class to track the bank state indirectly via times
-     * "freeAt" and "tRASDoneAt" and what page is currently open. The
-     * bank also keeps track of how many bytes have been accessed in
-     * the open row since it was opened.
+     * A basic class to track the bank state, i.e. what row is
+     * currently open (if any), when is the bank free to accept a new
+     * command, when can it be precharged, and when can it be
+     * activated.
+     *
+     * The bank also keeps track of how many bytes have been accessed
+     * in the open row since it was opened.
      */
     class Bank
     {
@@ -158,14 +161,14 @@ class DRAMCtrl : public AbstractMemory
         uint32_t openRow;
 
         Tick freeAt;
-        Tick tRASDoneAt;
+        Tick preAllowedAt;
         Tick actAllowedAt;
 
         uint32_t rowAccesses;
         uint32_t bytesAccessed;
 
         Bank() :
-            openRow(NO_ROW), freeAt(0), tRASDoneAt(0), actAllowedAt(0),
+            openRow(NO_ROW), freeAt(0), preAllowedAt(0), actAllowedAt(0),
             rowAccesses(0), bytesAccessed(0)
         { }
     };
@@ -410,9 +413,15 @@ class DRAMCtrl : public AbstractMemory
      * the maximum number of activations in the activation window. The
      * method updates the time that the banks become available based
      * on the current limits.
+     *
+     * @param act_tick Time when the activation takes place
+     * @param rank Index of the rank
+     * @param bank Index of the bank
+     * @param row Index of the row
+     * @param bank_ref Reference to the bank
      */
-    void recordActivate(Tick act_tick, uint8_t rank, uint8_t bank,
-                        uint16_t row);
+    void activateBank(Tick act_tick, uint8_t rank, uint8_t bank,
+                      uint16_t row, Bank& bank_ref);
 
     /**
      * Precharge a given bank and also update when the precharge is
@@ -420,9 +429,9 @@ class DRAMCtrl : public AbstractMemory
      * accesses to the open page.
      *
      * @param bank The bank to precharge
-     * @param free_at Time when the precharge is done
+     * @param pre_done_at Time when the precharge is done
      */
-    void prechargeBank(Bank& bank, Tick free_at);
+    void prechargeBank(Bank& bank, Tick pre_done_at);
 
     void printParams() const;
 
@@ -495,6 +504,7 @@ class DRAMCtrl : public AbstractMemory
     const Tick tCL;
     const Tick tRP;
     const Tick tRAS;
+    const Tick tWR;
     const Tick tRFC;
     const Tick tREFI;
     const Tick tRRD;