mem-ruby: Change MOESI_CMP_Dir L2 addressing

L1 controller selects the L2 to message based on the assigned address
ranges instead of explicitly interleaving bits in the L1 controller. This
simplifies the L1 controller implementation a bit and allows for more
flexibility when changing the address->controller mapping.

Change-Id: Ie67999bb977566939432a5045f65dbd2da81816a
Signed-off-by: Tiago Muck <tiago.muck@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/18410
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>

2 files changed, 62 insertions, 38 deletions

diff --git a/configs/ruby/MOESI_CMP_directory.py b/configs/ruby/MOESI_CMP_directory.py
index 5ce31dfa1..e3385fa77 100644
--- a/configs/ruby/MOESI_CMP_directory.py
+++ b/configs/ruby/MOESI_CMP_directory.py
@@ -1,3 +1,15 @@
+# Copyright (c) 2019 ARM Limited
+# All rights reserved.
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder.  You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
 # Copyright (c) 2006-2007 The Regents of The University of Michigan
 # Copyright (c) 2009 Advanced Micro Devices, Inc.
 # All rights reserved.
@@ -64,7 +76,6 @@ def create_system(options, full_system, system, dma_ports, bootmem,
     # Must create the individual controllers before the network to ensure the
     # controller constructors are called before the network constructor
     #
-    l2_bits = int(math.log(options.num_l2caches, 2))
     block_size_bits = int(math.log(options.cacheline_size, 2))
 
     for i in range(options.num_cpus):
@@ -94,7 +105,6 @@ def create_system(options, full_system, system, dma_ports, bootmem,
 
         l1_cntrl = L1Cache_Controller(version=i, L1Icache=l1i_cache,
                                       L1Dcache=l1d_cache,
-                                      l2_select_num_bits=l2_bits,
                                       send_evictions=send_evicts(options),
                                       transitions_per_cycle=options.ports,
                                       clk_domain=clk_domain,
@@ -124,7 +134,21 @@ def create_system(options, full_system, system, dma_ports, bootmem,
         l1_cntrl.triggerQueue = MessageBuffer(ordered = True)
 
 
-    l2_index_start = block_size_bits + l2_bits
+    # Create the L2s interleaved addr ranges
+    l2_addr_ranges = []
+    l2_bits = int(math.log(options.num_l2caches, 2))
+    numa_bit = block_size_bits + l2_bits - 1
+    sysranges = [] + system.mem_ranges
+    if bootmem: sysranges.append(bootmem.range)
+    for i in range(options.num_l2caches):
+        ranges = []
+        for r in sysranges:
+            addr_range = AddrRange(r.start, size = r.size(),
+                                    intlvHighBit = numa_bit,
+                                    intlvBits = l2_bits,
+                                    intlvMatch = i)
+            ranges.append(addr_range)
+        l2_addr_ranges.append(ranges)
 
     for i in range(options.num_l2caches):
         #
@@ -132,12 +156,13 @@ def create_system(options, full_system, system, dma_ports, bootmem,
         #
         l2_cache = L2Cache(size = options.l2_size,
                            assoc = options.l2_assoc,
-                           start_index_bit = l2_index_start)
+                           start_index_bit = block_size_bits + l2_bits)
 
         l2_cntrl = L2Cache_Controller(version = i,
                                       L2cache = l2_cache,
                                       transitions_per_cycle = options.ports,
-                                      ruby_system = ruby_system)
+                                      ruby_system = ruby_system,
+                                      addr_ranges = l2_addr_ranges[i])
 
         exec("ruby_system.l2_cntrl%d = l2_cntrl" % i)
         l2_cntrl_nodes.append(l2_cntrl)
diff --git a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
index f1512672c..11c821b97 100644
--- a/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
+++ b/src/mem/protocol/MOESI_CMP_directory-L1cache.sm
@@ -38,11 +38,10 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-machine(MachineType:L1Cache, "Directory protocol")
+machine(MachineType:L1Cache, "L1 cache protocol")
  : Sequencer * sequencer;
    CacheMemory * L1Icache;
    CacheMemory * L1Dcache;
-   int l2_select_num_bits;
    Cycles request_latency := 2;
    Cycles use_timeout_latency := 50;
    bool send_evictions;
@@ -151,10 +150,10 @@ machine(MachineType:L1Cache, "Directory protocol")
   void unset_cache_entry();
   void set_tbe(TBE b);
   void unset_tbe();
+  MachineID mapAddressToMachine(Addr addr, MachineType mtype);
 
   TBETable TBEs, template="<L1Cache_TBE>", constructor="m_number_of_TBEs";
   TimerTable useTimerTable;
-  int l2_select_low_bit, default="RubySystem::getBlockSizeBits()";
 
   Entry getCacheEntry(Addr addr), return_by_pointer="yes" {
     Entry L1Dcache_entry := static_cast(Entry, "pointer", L1Dcache.lookup(addr));
@@ -446,8 +445,8 @@ machine(MachineType:L1Cache, "Directory protocol")
         out_msg.Type := CoherenceRequestType:GETS;
         out_msg.Requestor := machineID;
         out_msg.RequestorMachine := MachineType:L1Cache;
-        out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+        out_msg.Destination.add(mapAddressToMachine(address,
+                                                    MachineType:L2Cache));
         out_msg.MessageSize := MessageSizeType:Request_Control;
         out_msg.AccessMode := in_msg.AccessMode;
         out_msg.Prefetch := in_msg.Prefetch;
@@ -462,8 +461,8 @@ machine(MachineType:L1Cache, "Directory protocol")
         out_msg.Type := CoherenceRequestType:GETX;
         out_msg.Requestor := machineID;
         out_msg.RequestorMachine := MachineType:L1Cache;
-        out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+        out_msg.Destination.add(mapAddressToMachine(address,
+                                                    MachineType:L2Cache));
         out_msg.MessageSize := MessageSizeType:Request_Control;
         out_msg.AccessMode := in_msg.AccessMode;
         out_msg.Prefetch := in_msg.Prefetch;
@@ -477,8 +476,8 @@ machine(MachineType:L1Cache, "Directory protocol")
       out_msg.Type := CoherenceRequestType:PUTX;
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+      out_msg.Destination.add(mapAddressToMachine(address,
+                                                  MachineType:L2Cache));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -489,8 +488,8 @@ machine(MachineType:L1Cache, "Directory protocol")
       out_msg.Type := CoherenceRequestType:PUTO;
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+      out_msg.Destination.add(mapAddressToMachine(address,
+                                                  MachineType:L2Cache));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -501,8 +500,8 @@ machine(MachineType:L1Cache, "Directory protocol")
       out_msg.Type := CoherenceRequestType:PUTS;
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+      out_msg.Destination.add(mapAddressToMachine(address,
+                                                  MachineType:L2Cache));
       out_msg.MessageSize := MessageSizeType:Writeback_Control;
     }
   }
@@ -516,8 +515,8 @@ machine(MachineType:L1Cache, "Directory protocol")
           out_msg.Type := CoherenceResponseType:DATA;
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
-          out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+          out_msg.Destination.add(mapAddressToMachine(address,
+                                                      MachineType:L2Cache));
           out_msg.DataBlk := cache_entry.DataBlk;
           // out_msg.Dirty := cache_entry.Dirty;
           out_msg.Dirty := false;
@@ -551,8 +550,8 @@ machine(MachineType:L1Cache, "Directory protocol")
       out_msg.Type := CoherenceResponseType:DATA;
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+      out_msg.Destination.add(mapAddressToMachine(address,
+                                                  MachineType:L2Cache));
       out_msg.DataBlk := cache_entry.DataBlk;
       out_msg.Dirty := cache_entry.Dirty;
       out_msg.Acks := 0; // irrelevant
@@ -569,8 +568,8 @@ machine(MachineType:L1Cache, "Directory protocol")
           out_msg.Type := CoherenceResponseType:DATA_EXCLUSIVE;
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
-          out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+          out_msg.Destination.add(mapAddressToMachine(address,
+                                                      MachineType:L2Cache));
           out_msg.DataBlk := cache_entry.DataBlk;
           out_msg.Dirty := cache_entry.Dirty;
           out_msg.Acks := in_msg.Acks;
@@ -614,8 +613,8 @@ machine(MachineType:L1Cache, "Directory protocol")
           out_msg.Type := CoherenceResponseType:ACK;
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
-          out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+          out_msg.Destination.add(mapAddressToMachine(address,
+                                                      MachineType:L2Cache));
           out_msg.Acks := 0 - 1; // -1
           out_msg.MessageSize := MessageSizeType:Response_Control;
         }
@@ -629,8 +628,8 @@ machine(MachineType:L1Cache, "Directory protocol")
       out_msg.Type := CoherenceResponseType:UNBLOCK;
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+      out_msg.Destination.add(mapAddressToMachine(address,
+                                                  MachineType:L2Cache));
       out_msg.MessageSize := MessageSizeType:Unblock_Control;
     }
   }
@@ -641,8 +640,8 @@ machine(MachineType:L1Cache, "Directory protocol")
       out_msg.Type := CoherenceResponseType:UNBLOCK_EXCLUSIVE;
       out_msg.Sender := machineID;
       out_msg.SenderMachine := MachineType:L1Cache;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+      out_msg.Destination.add(mapAddressToMachine(address,
+                                                  MachineType:L2Cache));
       out_msg.MessageSize := MessageSizeType:Unblock_Control;
     }
   }
@@ -752,8 +751,8 @@ machine(MachineType:L1Cache, "Directory protocol")
         out_msg.Type := CoherenceResponseType:DMA_ACK;
         out_msg.Sender := machineID;
         out_msg.SenderMachine := MachineType:L1Cache;
-        out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+        out_msg.Destination.add(mapAddressToMachine(address,
+                                                    MachineType:L2Cache));
         out_msg.Dirty := false;
         out_msg.Acks := 1;
         out_msg.MessageSize := MessageSizeType:Response_Control;
@@ -785,8 +784,8 @@ machine(MachineType:L1Cache, "Directory protocol")
           out_msg.Type := CoherenceResponseType:DATA;
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
-          out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+          out_msg.Destination.add(mapAddressToMachine(address,
+                                                      MachineType:L2Cache));
           out_msg.DataBlk := tbe.DataBlk;
           // out_msg.Dirty := tbe.Dirty;
           out_msg.Dirty := false;
@@ -819,8 +818,8 @@ machine(MachineType:L1Cache, "Directory protocol")
           out_msg.Type := CoherenceResponseType:DATA_EXCLUSIVE;
           out_msg.Sender := machineID;
           out_msg.SenderMachine := MachineType:L1Cache;
-          out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+          out_msg.Destination.add(mapAddressToMachine(address,
+                                                      MachineType:L2Cache));
           out_msg.DataBlk := tbe.DataBlk;
           out_msg.Dirty := tbe.Dirty;
           out_msg.Acks := in_msg.Acks;
@@ -837,8 +836,8 @@ machine(MachineType:L1Cache, "Directory protocol")
       out_msg.addr := address;
       out_msg.Requestor := machineID;
       out_msg.RequestorMachine := MachineType:L1Cache;
-      out_msg.Destination.add(mapAddressToRange(address, MachineType:L2Cache,
-              l2_select_low_bit, l2_select_num_bits, intToID(0)));
+      out_msg.Destination.add(mapAddressToMachine(address,
+                                                  MachineType:L2Cache));
       if (tbe.Dirty) {
         out_msg.Type := CoherenceRequestType:WRITEBACK_DIRTY_DATA;
       } else {