ruby: add support for clusters

A cluster over here means a set of controllers that can be accessed only by a
certain set of cores.  For example,  consider a two level hierarchy. Assume
there are 4 L1 controllers (private) and 2 L2 controllers.  We can have two
different hierarchies here:

a. the address space is partitioned between the two L2 controllers.  Each L1
controller accesses both the L2 controllers.  In this case, each L1 controller
is a cluster initself.

b. both the L2 controllers can cache any address.  An L1 controller has access
to only one of the L2 controllers.  In this case, each L2 controller
along with the L1 controllers that access it, form a cluster.

This patch allows for each controller to have a cluster ID, which is 0 by
default.  By setting the cluster ID properly,  one can instantiate hierarchies
with clusters.  Note that the coherence protocol might have to be changed as
well.

16 files changed, 107 insertions, 110 deletions

diff --git a/src/mem/protocol/MESI_CMP_directory-L1cache.sm b/src/mem/protocol/MESI_CMP_directory-L1cache.sm
index 347e548bb..bd56b5660 100644
--- a/src/mem/protocol/MESI_CMP_directory-L1cache.sm
+++ b/src/mem/protocol/MESI_CMP_directory-L1cache.sm
@@ -500,7 +500,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:GETS;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -518,7 +518,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:GETS;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                      l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -535,7 +535,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:GET_INSTR;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -555,7 +555,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
               out_msg.Requestor := machineID;
               out_msg.Destination.add(
                   mapAddressToRange(address, MachineType:L2Cache,
-                                    l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
               out_msg.MessageSize := MessageSizeType:Control;
               out_msg.Prefetch := in_msg.Prefetch;
               out_msg.AccessMode := in_msg.AccessMode;
@@ -574,7 +574,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Requestor := machineID;
         DPRINTF(RubySlicc, "%s\n", machineID);
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -593,10 +593,8 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
               out_msg.Requestor := machineID;
               DPRINTF(RubySlicc, "%s\n", machineID);
 
-              out_msg.Destination.add(mapAddressToRange(address,
-                                                        MachineType:L2Cache,
-                                                        l2_select_low_bit,
-                                                        l2_select_num_bits));
+              out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
 
               DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                       address, out_msg.Destination);
@@ -614,7 +612,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
         out_msg.Type := CoherenceRequestType:UPGRADE;
         out_msg.Requestor := machineID;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
         DPRINTF(RubySlicc, "address: %s, destination: %s\n",
                 address, out_msg.Destination);
         out_msg.MessageSize := MessageSizeType:Control;
@@ -648,7 +646,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Data;
     }
   }
@@ -677,7 +675,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := tbe.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Data;
     }
   }
@@ -703,7 +701,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Data;
     }
   }
@@ -717,7 +715,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := tbe.Dirty;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Data;
     }
   }
@@ -751,7 +749,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Requestor:= machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       if (cache_entry.Dirty) {
         out_msg.MessageSize := MessageSizeType:Writeback_Data;
       } else {
@@ -766,7 +764,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Type := CoherenceResponseType:UNBLOCK;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Control;
       DPRINTF(RubySlicc, "%s\n", address);
     }
@@ -778,7 +776,7 @@ machine(L1Cache, "MESI Directory L1 Cache CMP")
       out_msg.Type := CoherenceResponseType:EXCLUSIVE_UNBLOCK;
       out_msg.Sender := machineID;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                                  l2_select_low_bit, l2_select_num_bits));
+                          l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Response_Control;
       DPRINTF(RubySlicc, "%s\n", address);
 
diff --git a/src/mem/protocol/MESI_CMP_directory-dir.sm b/src/mem/protocol/MESI_CMP_directory-dir.sm
index eb87b8c9b..9418b75ac 100644
--- a/src/mem/protocol/MESI_CMP_directory-dir.sm
+++ b/src/mem/protocol/MESI_CMP_directory-dir.sm
@@ -39,7 +39,6 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
    MemoryControl * memBuffer,
    Cycles to_mem_ctrl_latency = 1,
    Cycles directory_latency = 6,
-   int l2_select_num_bits
 {
   MessageBuffer requestToDir, network="From", virtual_network="0",
         ordered="false", vnet_type="request";
@@ -83,6 +82,7 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
   structure(Entry, desc="...", interface="AbstractEntry") {
     State DirectoryState,          desc="Directory state";
     DataBlock DataBlk,             desc="data for the block";
+    MachineID Owner;
   }
 
   // TBE entries for DMA requests
@@ -102,8 +102,6 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
 
 
   // ** OBJECTS **
-
-  int l2_select_low_bit, default="RubySystem::getBlockSizeBits()";
   TBETable TBEs, template="<Directory_TBE>", constructor="m_number_of_TBEs";
 
   void set_tbe(TBE tbe);
@@ -262,6 +260,9 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
         out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Dirty := false;
         out_msg.MessageSize := MessageSizeType:Response_Data;
+
+        Entry e := getDirectoryEntry(in_msg.Addr);
+        e.Owner := in_msg.OriginalRequestorMachId;
       }
     }
   }
@@ -409,12 +410,11 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
   action(inv_sendCacheInvalidate, "inv", desc="Invalidate a cache block") {
     peek(requestNetwork_in, RequestMsg) {
       enqueue(responseNetwork_out, ResponseMsg, latency=directory_latency) {
-      out_msg.Addr := address;
-      out_msg.Type := CoherenceResponseType:INV;
-      out_msg.Sender := machineID;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                                     l2_select_low_bit, l2_select_num_bits));
-      out_msg.MessageSize := MessageSizeType:Response_Control;
+        out_msg.Addr := address;
+        out_msg.Type := CoherenceResponseType:INV;
+        out_msg.Sender := machineID;
+        out_msg.Destination.add(getDirectoryEntry(address).Owner);
+        out_msg.MessageSize := MessageSizeType:Response_Control;
       }
     }
   }
@@ -483,6 +483,11 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
     j_popIncomingRequestQueue;
   }
 
+  transition(M, Fetch) {
+    inv_sendCacheInvalidate;
+    z_stallAndWaitRequest;
+  }
+
   transition(IM, Memory_Data, M) {
     d_sendData;
     l_popMemQueue;
@@ -492,6 +497,7 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
   transition(M, CleanReplacement, I) {
     a_sendAck;
     k_popIncomingResponseQueue;
+    kd_wakeUpDependents;
   }
 
   transition(M, Data, MI) {
@@ -578,5 +584,4 @@ machine(Directory, "MESI_CMP_filter_directory protocol")
     l_popMemQueue;
     kd_wakeUpDependents;
   }
-
 }
diff --git a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
index ec6576693..40ac524d6 100644
--- a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
@@ -420,7 +420,7 @@ machine(L1Cache, "Directory protocol")
         out_msg.Requestor := machineID;
         out_msg.RequestorMachine := MachineType:L1Cache;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
         out_msg.MessageSize := MessageSizeType:Request_Control;
         out_msg.AccessMode := in_msg.AccessMode;
         out_msg.Prefetch := in_msg.Prefetch;
@@ -436,7 +436,7 @@ machine(L1Cache, "Directory protocol")
         out_msg.Requestor := machineID;
         out_msg.RequestorMachine := MachineType:L1Cache;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
         out_msg.MessageSize := MessageSizeType:Request_Control;
         out_msg.AccessMode := in_msg.AccessMode;
         out_msg.Prefetch := in_msg.Prefetch;
@@ -452,7 +452,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -465,7 +465,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -478,7 +478,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -493,7 +493,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := cache_entry.DataBlk;
           // out_msg.Dirty := cache_entry.Dirty;
           out_msg.Dirty := false;
@@ -528,7 +528,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.DataBlk := cache_entry.DataBlk;
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Acks := 0; // irrelevant
@@ -546,7 +546,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := cache_entry.DataBlk;
           out_msg.Dirty := cache_entry.Dirty;
           out_msg.Acks := in_msg.Acks;
@@ -591,7 +591,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.Acks := 0 - 1; // -1
           out_msg.MessageSize := MessageSizeType:Response_Control;
         }
@@ -606,7 +606,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Unblock_Control;
     }
   }
@@ -618,7 +618,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.MessageSize := MessageSizeType:Unblock_Control;
     }
   }
@@ -715,7 +715,7 @@ machine(L1Cache, "Directory protocol")
         out_msg.Sender := machineID;
         out_msg.SenderMachine := MachineType:L1Cache;
         out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
         out_msg.Dirty := false;
         out_msg.Acks := 1;
         out_msg.MessageSize := MessageSizeType:Response_Control;
@@ -748,7 +748,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := tbe.DataBlk;
           // out_msg.Dirty := tbe.Dirty;
           out_msg.Dirty := false;
@@ -782,7 +782,7 @@ machine(L1Cache, "Directory protocol")
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
           out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-                l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
           out_msg.DataBlk := tbe.DataBlk;
           out_msg.Dirty := tbe.Dirty;
           out_msg.Acks := in_msg.Acks;
@@ -800,7 +800,7 @@ machine(L1Cache, "Directory protocol")
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
       out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-            l2_select_low_bit, l2_select_num_bits));
+              l2_select_low_bit, l2_select_num_bits, intToID(0)));
       out_msg.Dirty := tbe.Dirty;
       if (tbe.Dirty) {
         out_msg.Type := CoherenceResponseType:WRITEBACK_DIRTY_DATA;
diff --git a/src/mem/protocol/MOESI_CMP_token-L1cache.sm b/src/mem/protocol/MOESI_CMP_token-L1cache.sm
index e472d0437..463ecf65f 100644
--- a/src/mem/protocol/MOESI_CMP_token-L1cache.sm
+++ b/src/mem/protocol/MOESI_CMP_token-L1cache.sm
@@ -406,7 +406,7 @@ machine(L1Cache, "Token protocol")
     } else if (machineIDToMachineType(sender) == MachineType:L2Cache) {
 
       if (sender == mapAddressToRange(addr, MachineType:L2Cache,
-                      l2_select_low_bit, l2_select_num_bits)) {
+                      l2_select_low_bit, l2_select_num_bits, intToID(0))) {
         return false;
       } else {
         return  true;
@@ -540,9 +540,8 @@ machine(L1Cache, "Token protocol")
         if ( machineIDToMachineType(in_msg.Sender) == MachineType:L2Cache ) {
 
           if (in_msg.Sender == mapAddressToRange(in_msg.Addr,
-                                                 MachineType:L2Cache,
-                                                 l2_select_low_bit,
-                                                 l2_select_num_bits)) {
+                                 MachineType:L2Cache, l2_select_low_bit,
+                                 l2_select_num_bits, intToID(0))) {
 
             // came from an off-chip L2 cache
             if (is_valid(tbe)) {
@@ -748,9 +747,8 @@ machine(L1Cache, "Token protocol")
             //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
             out_msg.Destination.add(mapAddressToRange(address,
-                                                      MachineType:L2Cache,
-                                                      l2_select_low_bit,
-                                                      l2_select_num_bits));
+                                      MachineType:L2Cache, l2_select_low_bit,
+                                      l2_select_num_bits, intToID(0)));
 
             out_msg.Destination.add(map_Address_to_Directory(address));
             out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -792,9 +790,8 @@ machine(L1Cache, "Token protocol")
           out_msg.Type := CoherenceRequestType:GETS;
           out_msg.Requestor := machineID;
           out_msg.Destination.add(mapAddressToRange(address,
-                                                    MachineType:L2Cache,
-                                                    l2_select_low_bit,
-                                                    l2_select_num_bits));
+                                    MachineType:L2Cache, l2_select_low_bit,
+                                    l2_select_num_bits, intToID(0)));
 
           out_msg.RetryNum := tbe.IssueCount;
           if (tbe.IssueCount == 0) {
@@ -869,9 +866,8 @@ machine(L1Cache, "Token protocol")
             //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
             out_msg.Destination.add(mapAddressToRange(address,
-                                                      MachineType:L2Cache,
-                                                      l2_select_low_bit,
-                                                      l2_select_num_bits));
+                                      MachineType:L2Cache, l2_select_low_bit,
+                                      l2_select_num_bits, intToID(0)));
 
             out_msg.Destination.add(map_Address_to_Directory(address));
             out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -914,9 +910,8 @@ machine(L1Cache, "Token protocol")
           out_msg.Requestor := machineID;
 
           out_msg.Destination.add(mapAddressToRange(address,
-                                                    MachineType:L2Cache,
-                                                    l2_select_low_bit,
-                                                    l2_select_num_bits));
+                                    MachineType:L2Cache, l2_select_low_bit,
+                                    l2_select_num_bits, intToID(0)));
 
           out_msg.RetryNum := tbe.IssueCount;
 
@@ -991,9 +986,8 @@ machine(L1Cache, "Token protocol")
       out_msg.Sender := machineID;
 
       out_msg.Destination.add(mapAddressToRange(address,
-                                                MachineType:L2Cache,
-                                                l2_select_low_bit,
-                                                l2_select_num_bits));
+                                MachineType:L2Cache, l2_select_low_bit,
+                                l2_select_num_bits, intToID(0)));
 
       out_msg.Tokens := cache_entry.Tokens;
       out_msg.DataBlk := cache_entry.DataBlk;
@@ -1016,9 +1010,8 @@ machine(L1Cache, "Token protocol")
         out_msg.Sender := machineID;
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Tokens := cache_entry.Tokens;
         out_msg.DataBlk := cache_entry.DataBlk;
@@ -1039,9 +1032,8 @@ machine(L1Cache, "Token protocol")
         out_msg.Sender := machineID;
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Tokens := cache_entry.Tokens;
         out_msg.DataBlk := cache_entry.DataBlk;
@@ -1384,10 +1376,8 @@ machine(L1Cache, "Token protocol")
        out_msg.Sender := machineID;
 
        out_msg.Destination.add(mapAddressToRange(address,
-                                                 MachineType:L2Cache,
-                                                 l2_select_low_bit,
-                                                 l2_select_num_bits));
-
+                                 MachineType:L2Cache, l2_select_low_bit,
+                                 l2_select_num_bits, intToID(0)));
        out_msg.MessageSize := MessageSizeType:Response_Control;
     }
   }
@@ -1427,9 +1417,8 @@ machine(L1Cache, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
diff --git a/src/mem/protocol/MOESI_CMP_token-dir.sm b/src/mem/protocol/MOESI_CMP_token-dir.sm
index 854e6e2af..ab2ae2365 100644
--- a/src/mem/protocol/MOESI_CMP_token-dir.sm
+++ b/src/mem/protocol/MOESI_CMP_token-dir.sm
@@ -443,9 +443,8 @@ machine(Directory, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -485,9 +484,8 @@ machine(Directory, "Token protocol")
         //
         out_msg.Destination.broadcast(MachineType:L1Cache);
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.RetryNum := 0;
         out_msg.MessageSize := MessageSizeType:Broadcast_Control;
@@ -513,9 +511,8 @@ machine(Directory, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
@@ -551,9 +548,8 @@ machine(Directory, "Token protocol")
         //
         out_msg.Destination.broadcast(MachineType:L1Cache);
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.RetryNum := 0;
         out_msg.MessageSize := MessageSizeType:Broadcast_Control;
@@ -730,9 +726,8 @@ machine(Directory, "Token protocol")
         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 
         out_msg.Destination.add(mapAddressToRange(address,
-                                                  MachineType:L2Cache,
-                                                  l2_select_low_bit,
-                                                  l2_select_num_bits));
+                                  MachineType:L2Cache, l2_select_low_bit,
+                                  l2_select_num_bits, intToID(0)));
 
         out_msg.Destination.add(map_Address_to_Directory(address));
         out_msg.MessageSize := MessageSizeType:Persistent_Control;
diff --git a/src/mem/protocol/RubySlicc_ComponentMapping.sm b/src/mem/protocol/RubySlicc_ComponentMapping.sm
index 7c40901d7..4744c9d4f 100644
--- a/src/mem/protocol/RubySlicc_ComponentMapping.sm
+++ b/src/mem/protocol/RubySlicc_ComponentMapping.sm
@@ -30,7 +30,8 @@
 // Mapping functions
 
 int machineCount(MachineType machType);
-MachineID mapAddressToRange(Address addr, MachineType type, int low, int high);
+MachineID mapAddressToRange(Address addr, MachineType type,
+                            int low, int high, NodeID n);
 NetDest broadcast(MachineType type);
 MachineID map_Address_to_DMA(Address addr);
 MachineID map_Address_to_Directory(Address addr);
diff --git a/src/mem/protocol/RubySlicc_Defines.sm b/src/mem/protocol/RubySlicc_Defines.sm
index 011cb7664..f3923eb9f 100644
--- a/src/mem/protocol/RubySlicc_Defines.sm
+++ b/src/mem/protocol/RubySlicc_Defines.sm
@@ -31,4 +31,4 @@
 NodeID id;
 NodeID version;
 MachineID machineID;
-
+NodeID clusterID;
diff --git a/src/mem/ruby/common/NetDest.cc b/src/mem/ruby/common/NetDest.cc
index f7508f1da..b8c490ac5 100644
--- a/src/mem/ruby/common/NetDest.cc
+++ b/src/mem/ruby/common/NetDest.cc
@@ -102,7 +102,7 @@ NetDest::broadcast()
 void
 NetDest::broadcast(MachineType machineType)
 {
-    for (int i = 0; i < MachineType_base_count(machineType); i++) {
+    for (NodeID i = 0; i < MachineType_base_count(machineType); i++) {
         MachineID mach = {machineType, i};
         add(mach);
     }
@@ -146,7 +146,7 @@ NetDest::smallestElement() const
 {
     assert(count() > 0);
     for (int i = 0; i < m_bits.size(); i++) {
-        for (int j = 0; j < m_bits[i].getSize(); j++) {
+        for (NodeID j = 0; j < m_bits[i].getSize(); j++) {
             if (m_bits[i].isElement(j)) {
                 MachineID mach = {MachineType_from_base_level(i), j};
                 return mach;
@@ -160,7 +160,7 @@ MachineID
 NetDest::smallestElement(MachineType machine) const
 {
     int size = m_bits[MachineType_base_level(machine)].getSize();
-    for (int j = 0; j < size; j++) {
+    for (NodeID j = 0; j < size; j++) {
         if (m_bits[MachineType_base_level(machine)].isElement(j)) {
             MachineID mach = {machine, j};
             return mach;
diff --git a/src/mem/ruby/common/TypeDefines.hh b/src/mem/ruby/common/TypeDefines.hh
index af1a6ca4c..391c9365a 100644
--- a/src/mem/ruby/common/TypeDefines.hh
+++ b/src/mem/ruby/common/TypeDefines.hh
@@ -37,8 +37,8 @@ typedef int64 Time;
 typedef uint64 physical_address_t;
 
 typedef int64 Index;            // what the address bit ripper returns
-typedef int LinkID;
-typedef int NodeID;
-typedef int SwitchID;
+typedef unsigned int LinkID;
+typedef unsigned int NodeID;
+typedef unsigned int SwitchID;
 
 #endif
diff --git a/src/mem/ruby/network/Topology.cc b/src/mem/ruby/network/Topology.cc
index 4f71c6208..cb13d1530 100644
--- a/src/mem/ruby/network/Topology.cc
+++ b/src/mem/ruby/network/Topology.cc
@@ -129,7 +129,7 @@ Topology::createLinks(Network *net)
     SwitchID max_switch_id = 0;
     for (LinkMap::const_iterator i = m_link_map.begin();
          i != m_link_map.end(); ++i) {
-        std::pair<int, int> src_dest = (*i).first;
+        std::pair<SwitchID, SwitchID> src_dest = (*i).first;
         max_switch_id = max(max_switch_id, src_dest.first);
         max_switch_id = max(max_switch_id, src_dest.second);        
     }
@@ -310,7 +310,7 @@ shortest_path_to_node(SwitchID src, SwitchID next, const Matrix& weights,
     max_machines = MachineType_base_number(MachineType_NUM);
 
     for (int m = 0; m < machines; m++) {
-        for (int i = 0; i < MachineType_base_count((MachineType)m); i++) {
+        for (NodeID i = 0; i < MachineType_base_count((MachineType)m); i++) {
             // we use "d+max_machines" below since the "destination"
             // switches for the machines are numbered
             // [MachineType_base_number(MachineType_NUM)...
diff --git a/src/mem/ruby/network/Topology.hh b/src/mem/ruby/network/Topology.hh
index cd0e03d09..1a11156e7 100644
--- a/src/mem/ruby/network/Topology.hh
+++ b/src/mem/ruby/network/Topology.hh
@@ -59,7 +59,7 @@ struct LinkEntry
     LinkDirection direction;
 };
 
-typedef std::map<std::pair<int, int>, LinkEntry> LinkMap;
+typedef std::map<std::pair<SwitchID, SwitchID>, LinkEntry> LinkMap;
 
 class Topology
 {
diff --git a/src/mem/ruby/slicc_interface/AbstractController.cc b/src/mem/ruby/slicc_interface/AbstractController.cc
index 26cf91e9f..e46158ca0 100644
--- a/src/mem/ruby/slicc_interface/AbstractController.cc
+++ b/src/mem/ruby/slicc_interface/AbstractController.cc
@@ -35,6 +35,8 @@ AbstractController::AbstractController(const Params *p)
     m_request_count(0)
 {
     m_version = p->version;
+    m_clusterID = p->cluster_id;
+
     m_transitions_per_cycle = p->transitions_per_cycle;
     m_buffer_size = p->buffer_size;
     m_recycle_latency = p->recycle_latency;
diff --git a/src/mem/ruby/slicc_interface/AbstractController.hh b/src/mem/ruby/slicc_interface/AbstractController.hh
index 3bf331c62..345eefa0a 100644
--- a/src/mem/ruby/slicc_interface/AbstractController.hh
+++ b/src/mem/ruby/slicc_interface/AbstractController.hh
@@ -56,7 +56,7 @@ class AbstractController : public ClockedObject, public Consumer
     void init();
     const Params *params() const { return (const Params *)_params; }
 
-    const int & getVersion() const { return m_version; }
+    const NodeID getVersion() const { return m_version; }
     void initNetworkPtr(Network* net_ptr) { m_net_ptr = net_ptr; }
 
     // return instance name
@@ -133,13 +133,12 @@ class AbstractController : public ClockedObject, public Consumer
     void wakeUpAllBuffers();
 
   protected:
-    int m_transitions_per_cycle;
-    int m_buffer_size;
-    Cycles m_recycle_latency;
     std::string m_name;
     NodeID m_version;
-    Network* m_net_ptr;
     MachineID m_machineID;
+    NodeID m_clusterID;
+
+    Network* m_net_ptr;
     bool m_is_blocking;
     std::map<Address, MessageBuffer*> m_block_map;
     typedef std::vector<MessageBuffer*> MsgVecType;
@@ -148,6 +147,9 @@ class AbstractController : public ClockedObject, public Consumer
     unsigned int m_in_ports;
     unsigned int m_cur_in_port;
     int m_number_of_TBEs;
+    int m_transitions_per_cycle;
+    int m_buffer_size;
+    Cycles m_recycle_latency;
 
     //! Map from physical network number to the Message Buffer.
     std::map<uint32_t, MessageBuffer*> peerQueueMap;
diff --git a/src/mem/ruby/slicc_interface/Controller.py b/src/mem/ruby/slicc_interface/Controller.py
index f8242322e..638d50b61 100644
--- a/src/mem/ruby/slicc_interface/Controller.py
+++ b/src/mem/ruby/slicc_interface/Controller.py
@@ -36,7 +36,8 @@ class RubyController(ClockedObject):
     cxx_header = "mem/ruby/slicc_interface/AbstractController.hh"
     abstract = True
     version = Param.Int("")
-    cntrl_id = Param.Int("")
+    cluster_id = Param.UInt32(0, "Id of this controller's cluster")
+
     transitions_per_cycle = \
         Param.Int(32, "no. of  SLICC state machine transitions per cycle")
     buffer_size = Param.Int(0, "max buffer size 0 means infinite")
diff --git a/src/mem/ruby/slicc_interface/RubySlicc_ComponentMapping.hh b/src/mem/ruby/slicc_interface/RubySlicc_ComponentMapping.hh
index cb9830446..4df57c712 100644
--- a/src/mem/ruby/slicc_interface/RubySlicc_ComponentMapping.hh
+++ b/src/mem/ruby/slicc_interface/RubySlicc_ComponentMapping.hh
@@ -58,7 +58,7 @@ inline NetDest
 broadcast(MachineType type)
 {
     NetDest dest;
-    for (int i = 0; i < MachineType_base_count(type); i++) {
+    for (NodeID i = 0; i < MachineType_base_count(type); i++) {
         MachineID mach = {type, i};
         dest.add(mach);
     }
@@ -67,12 +67,14 @@ broadcast(MachineType type)
 
 inline MachineID
 mapAddressToRange(const Address & addr, MachineType type, int low_bit,
-                  int num_bits)
+                  int num_bits, int cluster_id = 0)
 {
     MachineID mach = {type, 0};
     if (num_bits == 0)
-        return mach;
-    mach.num = addr.bitSelect(low_bit, low_bit + num_bits - 1);
+        mach.num = cluster_id;
+    else
+        mach.num = addr.bitSelect(low_bit, low_bit + num_bits - 1)
+            + (1 << num_bits) * cluster_id;
     return mach;
 }
 
diff --git a/src/mem/slicc/ast/ObjDeclAST.py b/src/mem/slicc/ast/ObjDeclAST.py
index 6469bc25a..a87ede724 100644
--- a/src/mem/slicc/ast/ObjDeclAST.py
+++ b/src/mem/slicc/ast/ObjDeclAST.py
@@ -70,6 +70,8 @@ class ObjDeclAST(DeclAST):
             c_code = "m_version"
         elif self.ident == "machineID":
             c_code = "m_machineID"
+        elif self.ident == "clusterID":
+            c_code = "m_clusterID"
         elif machine:
             c_code = "(*m_%s_%s_ptr)" % (machine.ident, self.ident)
         else: