mem-cache: Promote targets that don't require writable

Until now, all deferred targets of an MSHR would be promoted together
as soon as the targets were serviced. Due to the way we handle cache
clean operations we might need to promote only deferred targets that
don't require writable, leaving some targets as deferred. This change
adds support for this selective promotion.

Change-Id: I502e523dc9adbaf394955cbacea8286ab6a9b6bc
Reviewed-on: https://gem5-review.googlesource.com/11017
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>

2 files changed, 70 insertions, 17 deletions

diff --git a/src/mem/cache/mshr.cc b/src/mem/cache/mshr.cc
index 8ef8a2ca2..8629b3377 100644
--- a/src/mem/cache/mshr.cc
+++ b/src/mem/cache/mshr.cc
@@ -548,6 +548,49 @@ MSHR::promoteDeferredTargets()
     return true;
 }
 
+void
+MSHR::promoteIf(const std::function<bool (Target &)>& pred)
+{
+    // if any of the deferred targets were upper-level cache
+    // requests marked downstreamPending, need to clear that
+    assert(!downstreamPending);  // not pending here anymore
+
+    // find the first target does not satisfy the condition
+    auto last_it = std::find_if_not(deferredTargets.begin(),
+                                    deferredTargets.end(),
+                                    pred);
+
+    // for the prefix of the deferredTargets [begin(), last_it) clear
+    // the downstreamPending flag and move them to the target list
+    deferredTargets.clearDownstreamPending(deferredTargets.begin(),
+                                           last_it);
+    targets.splice(targets.end(), deferredTargets,
+                   deferredTargets.begin(), last_it);
+    // We need to update the flags for the target lists after the
+    // modifications
+    deferredTargets.populateFlags();
+}
+
+void
+MSHR::promoteReadable()
+{
+    if (!deferredTargets.empty() && !hasPostInvalidate()) {
+        // We got a non invalidating response, and we have the block
+        // but we have deferred targets which are waiting and they do
+        // not need writable. This can happen if the original request
+        // was for a cache clean operation and we had a copy of the
+        // block. Since we serviced the cache clean operation and we
+        // have the block, there's no need to defer the targets, so
+        // move them up to the regular target list.
+
+        auto pred = [](Target &t) {
+            assert(t.source == Target::FromCPU);
+            return !t.pkt->req->isCacheInvalidate() &&
+                   !t.pkt->needsWritable();
+        };
+        promoteIf(pred);
+    }
+}
 
 void
 MSHR::promoteWritable()
@@ -563,23 +606,13 @@ MSHR::promoteWritable()
         // target list.
         assert(!targets.needsWritable);
         targets.needsWritable = true;
-        // if any of the deferred targets were upper-level cache
-        // requests marked downstreamPending, need to clear that
-        assert(!downstreamPending);  // not pending here anymore
-
-        auto last_it = std::find_if(
-            deferredTargets.begin(), deferredTargets.end(),
-            [](MSHR::Target &t) {
-                assert(t.source == Target::FromCPU);
-                return t.pkt->req->isCacheInvalidate();
-            });
-        deferredTargets.clearDownstreamPending(deferredTargets.begin(),
-                                               last_it);
-        targets.splice(targets.end(), deferredTargets,
-                       deferredTargets.begin(), last_it);
-        // We need to update the flags for the target lists after the
-        // modifications
-        deferredTargets.populateFlags();
+
+        auto pred = [](Target &t) {
+            assert(t.source == Target::FromCPU);
+            return !t.pkt->req->isCacheInvalidate();
+        };
+
+        promoteIf(pred);
     }
 }
 
diff --git a/src/mem/cache/mshr.hh b/src/mem/cache/mshr.hh
index 71c2da2c4..050dbd1bf 100644
--- a/src/mem/cache/mshr.hh
+++ b/src/mem/cache/mshr.hh
@@ -286,6 +286,16 @@ class MSHR : public QueueEntry, public Printable
     }
 
   private:
+    /**
+     * Promotes deferred targets that satisfy a predicate
+     *
+     * Deferred targets are promoted to the target list if they
+     * satisfy a given condition. The operation stops at the first
+     * deferred target that doesn't satisfy the condition.
+     *
+     * @param pred A condition on a Target
+     */
+    void promoteIf(const std::function<bool (Target &)>& pred);
 
     /**
      * Pointer to this MSHR on the ready list.
@@ -388,6 +398,16 @@ class MSHR : public QueueEntry, public Printable
     /**
      * Promotes deferred targets that do not require writable
      *
+     * Move targets from the deferred targets list to the target list
+     * starting from the first deferred target until the first target
+     * that is a cache maintenance operation or needs a writable copy
+     * of the block
+     */
+    void promoteReadable();
+
+    /**
+     * Promotes deferred targets that do not require writable
+     *
      * Requests in the deferred target list are moved to the target
      * list up until the first target that is a cache maintenance
      * operation or needs a writable copy of the block