dev: Add m5 op to toggle synchronization for dist-gem5.

This patch adds the ability for an application to request dist-gem5 to begin/
end synchronization using an m5 op. When toggling on sync, all nodes agree
on the next sync point based on the maximum of all nodes' ticks. CPUs are
suspended until the sync point to avoid sending network messages until sync has
been enabled. Toggling off sync acts like a global execution barrier, where
all CPUs are disabled until every node reaches the toggle off point. This
avoids tricky situations such as one node hitting a toggle off followed by a
toggle on before the other nodes hit the first toggle off.

2 files changed, 13 insertions, 0 deletions

diff --git a/src/sim/pseudo_inst.cc b/src/sim/pseudo_inst.cc
index 4f9bbff6e..eed332747 100644
--- a/src/sim/pseudo_inst.cc
+++ b/src/sim/pseudo_inst.cc
@@ -211,6 +211,11 @@ pseudoInst(ThreadContext *tc, uint8_t func, uint8_t subfunc)
         m5PageFault(tc);
         break;
 
+      /* dist-gem5 functions */
+      case 0x62: // distToggleSync_func
+        togglesync(tc);
+        break;
+
       default:
         warn("Unhandled m5 op: 0x%x\n", func);
         break;
@@ -574,6 +579,13 @@ switchcpu(ThreadContext *tc)
     exitSimLoop("switchcpu");
 }
 
+void
+togglesync(ThreadContext *tc)
+{
+    DPRINTF(PseudoInst, "PseudoInst::togglesync()\n");
+    DistIface::toggleSync(tc);
+}
+
 //
 // This function is executed when annotated work items begin.  Depending on
 // what the user specified at the command line, the simulation may exit and/or
diff --git a/src/sim/pseudo_inst.hh b/src/sim/pseudo_inst.hh
index 5f0b700c6..d9b981f94 100644
--- a/src/sim/pseudo_inst.hh
+++ b/src/sim/pseudo_inst.hh
@@ -88,6 +88,7 @@ void debugbreak(ThreadContext *tc);
 void switchcpu(ThreadContext *tc);
 void workbegin(ThreadContext *tc, uint64_t workid, uint64_t threadid);
 void workend(ThreadContext *tc, uint64_t workid, uint64_t threadid);
+void togglesync(ThreadContext *tc);
 
 } // namespace PseudoInst