ruby: add support for clusters

A cluster over here means a set of controllers that can be accessed only by a
certain set of cores.  For example,  consider a two level hierarchy. Assume
there are 4 L1 controllers (private) and 2 L2 controllers.  We can have two
different hierarchies here:

a. the address space is partitioned between the two L2 controllers.  Each L1
controller accesses both the L2 controllers.  In this case, each L1 controller
is a cluster initself.

b. both the L2 controllers can cache any address.  An L1 controller has access
to only one of the L2 controllers.  In this case, each L2 controller
along with the L1 controllers that access it, form a cluster.

This patch allows for each controller to have a cluster ID, which is 0 by
default.  By setting the cluster ID properly,  one can instantiate hierarchies
with clusters.  Note that the coherence protocol might have to be changed as
well.

7 files changed, 79 insertions, 91 deletions

diff --git a/src/mem/protocol/MESI_CMP_directory-L1cache.sm b/src/mem/protocol/MESI_CMP_directory-L1cache.sm
index 347e548bb..bd56b5660 100644
--- a/src/mem/protocol/MESI_CMP_directory-L1cache.sm
+++ b/src/mem/protocol/MESI_CMP_directory-L1cache.sm
@@ -500,7 +500,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:GETS;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -518,7 +518,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:GETS;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                      l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -535,7 +535,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:GET_INSTR;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -555,7 +555,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
               out_msg.Requestor := machineID;
               out_msg.Destination.add(
                   mapAddressToRange(address, MachineType:L2Cache,
-                                    l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
               out_msg.MessageSize := MessageSizeType:Control;
               out_msg.Prefetch := in_msg.Prefetch;
               out_msg.AccessMode := in_msg.AccessMode;
@@ -574,7 +574,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Requestor := machineID;
         DPRINTF(RubySlicc, "%s\n", machineID);
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -593,10 +593,8 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
               out_msg.Requestor := machineID;
               DPRINTF(RubySlicc, "%s\n", machineID);
 
-              out_msg.Destination.add(mapAddressToRange(address,
-                                                        MachineType:L2Cache,
-                                                        l2_select_low_bit,
-                                                        l2_select_num_bits));
+              out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
 
               DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                       address, out_msg.Destination);
@@ -614,7 +612,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:UPGRADE;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -648,7 +646,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Data;
     }
   }
@@ -677,7 +675,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := tbe.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Data;
     }
   }
@@ -703,7 +701,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Data;
     }
   }
@@ -717,7 +715,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := tbe.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Data;
     }
   }
@@ -751,7 +749,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Requestor:= machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       if (cache_entry.Dirty) {
         out_msg.MessageSize := MessageSizeType:Writeback_Data;
       } else {
@@ -766,7 +764,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Type := CoherenceResponseType:UNBLOCK;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Control;
       DPRINTF(RubySlicc, "%s\n", address);
     }
@@ -778,7 +776,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Type := CoherenceResponseType:EXCLUSIVE_UNBLOCK;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Control;
       DPRINTF(RubySlicc, "%s\n", address);
 
diff --git a/src/mem/protocol/MESI_CMP_directory-dir.sm b/src/mem/protocol/MESI_CMP_directory-dir.sm
index eb87b8c9b..9418b75ac 100644
--- a/src/mem/protocol/MESI_CMP_directory-dir.sm
+++ b/src/mem/protocol/MESI_CMP_directory-dir.sm
@@ -39,7 +39,6 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
    MemoryControl * memBuffer,
    Cycles to_mem_ctrl_latency = 1,
    Cycles directory_latency = 6,
-   int l2_select_num_bits
 {
   MessageBuffer requestToDir, network="From", virtual_network="0",
         ordered="false", vnet_type="request";
@@ -83,6 +82,7 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
     DataBlock DataBlk,             desc="data for the block";
+    MachineID Owner;
   }
 
   // TBE entries for DMA requests
@@ -102,8 +102,6 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
 
 
   // ** OBJECTS **
-
-  int l2_select_low_bit, default="RubySystem::getBlockSizeBits()";
   TBETable TBEs, template="<Directory_TBE>", constructor="m_number_of_TBEs";
 
   void set_tbe(TBE tbe);
@@ -262,6 +260,9 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
         out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Dirty := false;
         out_msg.MessageSize := MessageSizeType:Response_Data;
+
+        Entry e := getDirectoryEntry(in_msg.Addr);
+        e.Owner := in_msg.OriginalRequestorMachId;
       }
     }
   }
@@ -409,12 +410,11 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
   action(inv_sendCacheInvalidate, "inv", desc="Invalidate a cache block") {
     peek(requestNetwork_in, RequestMsg) {
       enqueue(responseNetwork_out, ResponseMsg, latency=directory_latency) {
-      out_msg.Addr := address;
-      out_msg.Type := CoherenceResponseType:INV;
-      out_msg.Sender := machineID;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                     l2_select_low_bit, l2_select_num_bits));
-      out_msg.MessageSize := MessageSizeType:Response_Control;
+        out_msg.Addr := address;
+        out_msg.Type := CoherenceResponseType:INV;
+        out_msg.Sender := machineID;
+        out_msg.Destination.add(getDirectoryEntry(address).Owner);
+        out_msg.MessageSize := MessageSizeType:Response_Control;
       }
     }
   }
@@ -483,6 +483,11 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
     j_popIncomingRequestQueue;
   }
 
+  transition(M, Fetch) {
+    inv_sendCacheInvalidate;
+    z_stallAndWaitRequest;
+  }
+
   transition(IM, Memory_Data, M) {
     d_sendData;
     l_popMemQueue;
@@ -492,6 +497,7 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
   transition(M, CleanReplacement, I) {
     a_sendAck;
     k_popIncomingResponseQueue;
+    kd_wakeUpDependents;
   }
 
   transition(M, Data, MI) {
@@ -578,5 +584,4 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
     l_popMemQueue;
     kd_wakeUpDependents;
   }
-
 }
diff --git a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
index ec6576693..40ac524d6 100644
--- a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
@@ -420,7 +420,7 @@ machine(L1Cache, "Directory protocol")
         out_msg.Requestor := machineID;
         out_msg.RequestorMachine := MachineType:L1Cache;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
         out_msg.MessageSize := MessageSizeType:Request_Control;
         out_msg.AccessMode := in_msg.AccessMode;
         out_msg.Prefetch := in_msg.Prefetch;
@@ -436,7 +436,7 @@ machine(L1Cache, "Directory protocol")
         out_msg.Requestor := machineID;
         out_msg.RequestorMachine := MachineType:L1Cache;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
         out_msg.MessageSize := MessageSizeType:Request_Control;
         out_msg.AccessMode := in_msg.AccessMode;
         out_msg.Prefetch := in_msg.Prefetch;
@@ -452,7 +452,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -465,7 +465,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -478,7 +478,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -493,7 +493,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := cache_entry.DataBlk;
           // out_msg.Dirty := cache_entry.Dirty;
           out_msg.Dirty := false;
@@ -528,7 +528,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.DataBlk := cache_entry.DataBlk;
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Acks := 0; // irrelevant
@@ -546,7 +546,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := cache_entry.DataBlk;
           out_msg.Dirty := cache_entry.Dirty;
           out_msg.Acks := in_msg.Acks;
@@ -591,7 +591,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.Acks := 0 - 1; // -1
           out_msg.MessageSize := MessageSizeType:Response_Control;
         }
@@ -606,7 +606,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Unblock_Control;
     }
   }
@@ -618,7 +618,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Unblock_Control;
     }
   }
@@ -715,7 +715,7 @@ machine(L1Cache, "Directory protocol")
         out_msg.Sender := machineID;
         out_msg.SenderMachine := MachineType:L1Cache;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
         out_msg.Dirty := false;
         out_msg.Acks := 1;
         out_msg.MessageSize := MessageSizeType:Response_Control;
@@ -748,7 +748,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := tbe.DataBlk;
           // out_msg.Dirty := tbe.Dirty;
           out_msg.Dirty := false;
@@ -782,7 +782,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := tbe.DataBlk;
           out_msg.Dirty := tbe.Dirty;
           out_msg.Acks := in_msg.Acks;
@@ -800,7 +800,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.Dirty := tbe.Dirty;
       if (tbe.Dirty) {
         out_msg.Type := CoherenceResponseType:WRITEBACK_DIRTY_DATA;
diff --git a/src/mem/protocol/MOESI_CMP_token-L1cache.sm b/src/mem/protocol/MOESI_CMP_token-L1cache.sm
index e472d0437..463ecf65f 100644
--- a/src/mem/protocol/MOESI_CMP_token-L1cache.sm
+++ b/src/mem/protocol/MOESI_CMP_token-L1cache.sm
@@ -406,7 +406,7 @@ machine(L1Cache, "Token protocol")
     } else if (machineIDToMachineType(sender) == MachineType:L2Cache) {
 
       if (sender == mapAddressToRange(addr, MachineType:L2Cache,
-                      l2_select_low_bit, l2_select_num_bits)) {
+                      l2_select_low_bit, l2_select_num_bits, intToID(0))) {
         return false;
       } else {
         return  true;
@@ -540,9 +540,8 @@ machine(L1Cache, "Token protocol")
         if ( machineIDToMachineType(in_msg.Sender) == MachineType:L2Cache ) {
 
           if (in_msg.Sender == mapAddressToRange(in_msg.Addr,
-                                                 MachineType:L2Cache,
-                                                 l2_select_low_bit,
-                                                 l2_select_num_bits)) {
+                                 MachineType:L2Cache, l2_select_low_bit,
+                                 l2_select_num_bits, intToID(0))) {
 
             // came from an off-chip L2 cache
             if (is_valid(tbe)) {
@@ -748,9 +747,8 @@ machine(L1Cache, "Token protocol")
             //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
             out_msg.Destination.add(mapAddressToRange(address,
-                                                      MachineType:L2Cache,
-                                                      l2_select_low_bit,
-                                                      l2_select_num_bits));
+                                      MachineType:L2Cache, l2_select_low_bit,
+                                      l2_select_num_bits, intToID(0)));
 
             out_msg.Destination.add(map_Address_to_Directory(address));
             out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -792,9 +790,8 @@ machine(L1Cache, "Token protocol")
           out_msg.Type := CoherenceRequestType:GETS;
           out_msg.Requestor := machineID;
           out_msg.Destination.add(mapAddressToRange(address,
-                                                    MachineType:L2Cache,
-                                                    l2_select_low_bit,
-                                                    l2_select_num_bits));
+                                    MachineType:L2Cache, l2_select_low_bit,
+                                    l2_select_num_bits, intToID(0)));
 
           out_msg.RetryNum := tbe.IssueCount;
           if (tbe.IssueCount == 0) {
@@ -869,9 +866,8 @@ machine(L1Cache, "Token protocol")
             //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
             out_msg.Destination.add(mapAddressToRange(address,
-                                                      MachineType:L2Cache,
-                                                      l2_select_low_bit,
-                                                      l2_select_num_bits));
+                                      MachineType:L2Cache, l2_select_low_bit,
+                                      l2_select_num_bits, intToID(0)));
 
             out_msg.Destination.add(map_Address_to_Directory(address));
             out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -914,9 +910,8 @@ machine(L1Cache, "Token protocol")
           out_msg.Requestor := machineID;
 
           out_msg.Destination.add(mapAddressToRange(address,
-                                                    MachineType:L2Cache,
-                                                    l2_select_low_bit,
-                                                    l2_select_num_bits));
+                                    MachineType:L2Cache, l2_select_low_bit,
+                                    l2_select_num_bits, intToID(0)));
 
           out_msg.RetryNum := tbe.IssueCount;
 
@@ -991,9 +986,8 @@ machine(L1Cache, "Token protocol")
       out_msg.Sender := machineID;
 
       out_msg.Destination.add(mapAddressToRange(address,
-                                                MachineType:L2Cache,
-                                                l2_select_low_bit,
-                                                l2_select_num_bits));
+                                MachineType:L2Cache, l2_select_low_bit,
+                                l2_select_num_bits, intToID(0)));
 
       out_msg.Tokens := cache_entry.Tokens;
       out_msg.DataBlk := cache_entry.DataBlk;
@@ -1016,9 +1010,8 @@ machine(L1Cache, "Token protocol")
         out_msg.Sender := machineID;
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Tokens := cache_entry.Tokens;
         out_msg.DataBlk := cache_entry.DataBlk;
@@ -1039,9 +1032,8 @@ machine(L1Cache, "Token protocol")
         out_msg.Sender := machineID;
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Tokens := cache_entry.Tokens;
         out_msg.DataBlk := cache_entry.DataBlk;
@@ -1384,10 +1376,8 @@ machine(L1Cache, "Token protocol")
        out_msg.Sender := machineID;
 
        out_msg.Destination.add(mapAddressToRange(address,
-                                                 MachineType:L2Cache,
-                                                 l2_select_low_bit,
-                                                 l2_select_num_bits));
-
+                                 MachineType:L2Cache, l2_select_low_bit,
+                                 l2_select_num_bits, intToID(0)));
        out_msg.MessageSize := MessageSizeType:Response_Control;
     }
   }
@@ -1427,9 +1417,8 @@ machine(L1Cache, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
diff --git a/src/mem/protocol/MOESI_CMP_token-dir.sm b/src/mem/protocol/MOESI_CMP_token-dir.sm
index 854e6e2af..ab2ae2365 100644
--- a/src/mem/protocol/MOESI_CMP_token-dir.sm
+++ b/src/mem/protocol/MOESI_CMP_token-dir.sm
@@ -443,9 +443,8 @@ machine(Directory, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -485,9 +484,8 @@ machine(Directory, "Token protocol")
         //
         out_msg.Destination.broadcast(MachineType:L1Cache);
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.RetryNum := 0;
         out_msg.MessageSize := MessageSizeType:Broadcast_Control;
@@ -513,9 +511,8 @@ machine(Directory, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -551,9 +548,8 @@ machine(Directory, "Token protocol")
         //
         out_msg.Destination.broadcast(MachineType:L1Cache);
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.RetryNum := 0;
         out_msg.MessageSize := MessageSizeType:Broadcast_Control;
@@ -730,9 +726,8 @@ machine(Directory, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
diff --git a/src/mem/protocol/RubySlicc_ComponentMapping.sm b/src/mem/protocol/RubySlicc_ComponentMapping.sm
index 7c40901d7..4744c9d4f 100644
--- a/src/mem/protocol/RubySlicc_ComponentMapping.sm
+++ b/src/mem/protocol/RubySlicc_ComponentMapping.sm
@@ -30,7 +30,8 @@
 // Mapping functions
 
 int machineCount(MachineType machType);
-MachineID mapAddressToRange(Address addr, MachineType type, int low, int high);
+MachineID mapAddressToRange(Address addr, MachineType type,
+                            int low, int high, NodeID n);
 NetDest broadcast(MachineType type);
 MachineID map_Address_to_DMA(Address addr);
 MachineID map_Address_to_Directory(Address addr);
diff --git a/src/mem/protocol/RubySlicc_Defines.sm b/src/mem/protocol/RubySlicc_Defines.sm
index 011cb7664..f3923eb9f 100644
--- a/src/mem/protocol/RubySlicc_Defines.sm
+++ b/src/mem/protocol/RubySlicc_Defines.sm
@@ -31,4 +31,4 @@
 NodeID id;
 NodeID version;
 MachineID machineID;
-
+NodeID clusterID;