ruby: interface with classic memory controller

This patch is the final in the series.  The whole series and this patch in
particular were written with the aim of interfacing ruby's directory controller
with the memory controller in the classic memory system.  This is being done
since ruby's memory controller has not being kept up to date with the changes
going on in DRAMs.  Classic's memory controller is more up to date and
supports multiple different types of DRAM.  This also brings classic and
ruby ever more close.  The patch also changes ruby's memory controller to
expose the same interface.

13 files changed, 75 insertions, 151 deletions

diff --git a/configs/common/MemConfig.py b/configs/common/MemConfig.py
index d203118aa..57066426f 100644
--- a/configs/common/MemConfig.py
+++ b/configs/common/MemConfig.py
@@ -54,7 +54,8 @@ _mem_aliases_all = [
     ("lpddr2_s4_1066_x32", "LPDDR2_S4_1066_x32"),
     ("lpddr3_1600_x32", "LPDDR3_1600_x32"),
     ("wio_200_x128", "WideIO_200_x128"),
-    ("dramsim2", "DRAMSim2")
+    ("dramsim2", "DRAMSim2"),
+    ("ruby_memory", "RubyMemoryControl")
     ]
 
 # Filtered list of aliases. Only aliases for existing memory
diff --git a/configs/example/fs.py b/configs/example/fs.py
index abf8fe966..727f69339 100644
--- a/configs/example/fs.py
+++ b/configs/example/fs.py
@@ -137,8 +137,6 @@ def build_test_system(np):
 
         Ruby.create_system(options, True, test_sys, test_sys.iobus,
                            test_sys._dma_ports)
-        test_sys.physmem = [SimpleMemory(range = r, null = True)
-                          for r in test_sys.mem_ranges]
 
         # Create a seperate clock domain for Ruby
         test_sys.ruby.clk_domain = SrcClockDomain(clock = options.ruby_clock,
diff --git a/configs/example/ruby_direct_test.py b/configs/example/ruby_direct_test.py
index 6773aea6d..857909ba9 100644
--- a/configs/example/ruby_direct_test.py
+++ b/configs/example/ruby_direct_test.py
@@ -48,7 +48,7 @@ m5_root = os.path.dirname(config_root)
 parser = optparse.OptionParser()
 Options.addCommonOptions(parser)
 
-parser.add_option("-l", "--requests", metavar="N", default=100,
+parser.add_option("--requests", metavar="N", default=100,
                   help="Stop after N requests")
 parser.add_option("-f", "--wakeup_freq", metavar="N", default=10,
                   help="Wakeup every N cycles")
@@ -87,13 +87,8 @@ else:
     print "Error: unknown direct test generator"
     sys.exit(1)
 
-#
-# Create the M5 system.  Note that the Memory Object isn't
-# actually used by the rubytester, but is included to support the
-# M5 memory size == Ruby memory size checks
-#
-system = System(physmem = SimpleMemory(),
-                mem_ranges = [AddrRange(options.mem_size)])
+# Create the M5 system.
+system = System(mem_ranges = [AddrRange(options.mem_size)])
 
 
 # Create a top-level voltage domain and clock domain
@@ -102,12 +97,9 @@ system.voltage_domain = VoltageDomain(voltage = options.sys_voltage)
 system.clk_domain = SrcClockDomain(clock = options.sys_clock,
                                    voltage_domain = system.voltage_domain)
 
-#
 # Create the ruby random tester
-#
-system.cpu = RubyDirectedTester(requests_to_complete = \
-                                   options.requests,
-                                   generator = generator)
+system.cpu = RubyDirectedTester(requests_to_complete = options.requests,
+                                generator = generator)
 
 Ruby.create_system(options, False, system)
 
@@ -121,7 +113,7 @@ for ruby_port in system.ruby._cpu_ports:
     #
     # Tie the ruby tester ports to the ruby cpu ports
     #
-    system.tester.cpuPort = ruby_port.slave
+    system.cpu.cpuPort = ruby_port.slave
 
 # -----------------------
 # run simulation
diff --git a/configs/example/ruby_mem_test.py b/configs/example/ruby_mem_test.py
index 15684d153..f5e6d2a82 100644
--- a/configs/example/ruby_mem_test.py
+++ b/configs/example/ruby_mem_test.py
@@ -107,7 +107,6 @@ cpus = [ MemTest(atomic = False,
 system = System(cpu = cpus,
                 funcmem = SimpleMemory(in_addr_map = False),
                 funcbus = NoncoherentXBar(),
-                physmem = SimpleMemory(),
                 clk_domain = SrcClockDomain(clock = options.sys_clock),
                 mem_ranges = [AddrRange(options.mem_size)])
 
diff --git a/configs/example/ruby_random_test.py b/configs/example/ruby_random_test.py
index 7cde5b86c..225b3d23b 100644
--- a/configs/example/ruby_random_test.py
+++ b/configs/example/ruby_random_test.py
@@ -97,8 +97,7 @@ tester = RubyTester(check_flush = check_flush,
 # actually used by the rubytester, but is included to support the
 # M5 memory size == Ruby memory size checks
 #
-system = System(cpu = tester, physmem = SimpleMemory(),
-                mem_ranges = [AddrRange(options.mem_size)])
+system = System(cpu = tester, mem_ranges = [AddrRange(options.mem_size)])
 
 # Create a top-level voltage domain and clock domain
 system.voltage_domain = VoltageDomain(voltage = options.sys_voltage)
diff --git a/configs/example/se.py b/configs/example/se.py
index 461ebf11c..e0535c726 100644
--- a/configs/example/se.py
+++ b/configs/example/se.py
@@ -225,11 +225,6 @@ if options.ruby:
         print >> sys.stderr, "Ruby requires TimingSimpleCPU or O3CPU!!"
         sys.exit(1)
 
-    # Use SimpleMemory with the null option since this memory is only used
-    # for determining which addresses are within the range of the memory.
-    # No space allocation is required.
-    system.physmem = SimpleMemory(range=AddrRange(options.mem_size),
-                              null = True)
     options.use_map = True
     Ruby.create_system(options, False, system)
     assert(options.num_cpus == len(system.ruby._cpu_ports))
diff --git a/configs/ruby/MESI_Three_Level.py b/configs/ruby/MESI_Three_Level.py
index fe2e6aef5..f9ded25f1 100644
--- a/configs/ruby/MESI_Three_Level.py
+++ b/configs/ruby/MESI_Three_Level.py
@@ -182,22 +182,12 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
         #
         # Create the Ruby objects associated with the directory controller
         #
-
-        mem_cntrl = RubyMemoryControl(
-                              clk_domain = ruby_system.memctrl_clk_domain,
-                              version = i,
-                              ruby_system = ruby_system)
-
         dir_size = MemorySize('0B')
         dir_size.value = mem_module_size
 
         dir_cntrl = Directory_Controller(version = i,
-                                         directory = \
-                                         RubyDirectoryMemory(version = i,
-                                                             size = dir_size,
-                                                             use_map =
-                                                           options.use_map),
-                                         memBuffer = mem_cntrl,
+                                         directory = RubyDirectoryMemory(
+                                             version = i, size = dir_size),
                                          transitions_per_cycle = options.ports,
                                          ruby_system = ruby_system)
 
diff --git a/configs/ruby/MESI_Two_Level.py b/configs/ruby/MESI_Two_Level.py
index 6cc4efcb9..b7bdd1447 100644
--- a/configs/ruby/MESI_Two_Level.py
+++ b/configs/ruby/MESI_Two_Level.py
@@ -162,25 +162,12 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_divider=3)
 
     for i in xrange(options.num_dirs):
-        #
-        # Create the Ruby objects associated with the directory controller
-        #
-
-        mem_cntrl = RubyMemoryControl(
-                              clk_domain = ruby_system.memctrl_clk_domain,
-                              version = i,
-                              ruby_system = ruby_system)
-
         dir_size = MemorySize('0B')
         dir_size.value = mem_module_size
 
         dir_cntrl = Directory_Controller(version = i,
-                                         directory = \
-                                         RubyDirectoryMemory(version = i,
-                                                             size = dir_size,
-                                                             use_map =
-                                                           options.use_map),
-                                         memBuffer = mem_cntrl,
+                                         directory = RubyDirectoryMemory(
+                                             version = i, size = dir_size),
                                          transitions_per_cycle = options.ports,
                                          ruby_system = ruby_system)
 
diff --git a/configs/ruby/MI_example.py b/configs/ruby/MI_example.py
index de1f3e924..2dd064b55 100644
--- a/configs/ruby/MI_example.py
+++ b/configs/ruby/MI_example.py
@@ -117,27 +117,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_divider=3)
 
     for i in xrange(options.num_dirs):
-        #
-        # Create the Ruby objects associated with the directory controller
-        #
-
-        mem_cntrl = RubyMemoryControl(
-                              clk_domain = ruby_system.memctrl_clk_domain,
-                              version = i,
-                              ruby_system = ruby_system)
-
         dir_size = MemorySize('0B')
         dir_size.value = mem_module_size
-
         dir_cntrl = Directory_Controller(version = i,
-                                         directory = \
-                                         RubyDirectoryMemory( \
-                                                    version = i,
-                                                    size = dir_size,
-                                                    use_map = options.use_map,
-                                                    map_levels = \
-                                                      options.map_levels),
-                                         memBuffer = mem_cntrl,
+                                         directory = RubyDirectoryMemory(
+                                             version = i, size = dir_size),
                                          transitions_per_cycle = options.ports,
                                          ruby_system = ruby_system)
 
diff --git a/configs/ruby/MOESI_CMP_directory.py b/configs/ruby/MOESI_CMP_directory.py
index 3fb9c55a7..9c4bab434 100644
--- a/configs/ruby/MOESI_CMP_directory.py
+++ b/configs/ruby/MOESI_CMP_directory.py
@@ -156,24 +156,12 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_divider=3)
 
     for i in xrange(options.num_dirs):
-        #
-        # Create the Ruby objects associated with the directory controller
-        #
-
-        mem_cntrl = RubyMemoryControl(
-                              clk_domain = ruby_system.memctrl_clk_domain,
-                              version = i,
-                              ruby_system = ruby_system)
-
         dir_size = MemorySize('0B')
         dir_size.value = mem_module_size
 
         dir_cntrl = Directory_Controller(version = i,
-                                         directory = \
-                                         RubyDirectoryMemory(version = i,
-                                             size = dir_size,
-                                             use_map = options.use_map),
-                                         memBuffer = mem_cntrl,
+                                         directory = RubyDirectoryMemory(
+                                             version = i, size = dir_size),
                                          transitions_per_cycle = options.ports,
                                          ruby_system = ruby_system)
 
diff --git a/configs/ruby/MOESI_CMP_token.py b/configs/ruby/MOESI_CMP_token.py
index bedc444bf..26cd625b5 100644
--- a/configs/ruby/MOESI_CMP_token.py
+++ b/configs/ruby/MOESI_CMP_token.py
@@ -180,24 +180,12 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_divider=3)
 
     for i in xrange(options.num_dirs):
-        #
-        # Create the Ruby objects associated with the directory controller
-        #
-
-        mem_cntrl = RubyMemoryControl(
-                              clk_domain = ruby_system.memctrl_clk_domain,
-                              version = i,
-                              ruby_system = ruby_system)
-
         dir_size = MemorySize('0B')
         dir_size.value = mem_module_size
 
         dir_cntrl = Directory_Controller(version = i,
-                                         directory = \
-                                         RubyDirectoryMemory(version = i,
-                                             use_map = options.use_map,
-                                             size = dir_size),
-                                         memBuffer = mem_cntrl,
+                                         directory = RubyDirectoryMemory(
+                                             version = i, size = dir_size),
                                          l2_select_num_bits = l2_bits,
                                          transitions_per_cycle = options.ports,
                                          ruby_system = ruby_system)
diff --git a/configs/ruby/MOESI_hammer.py b/configs/ruby/MOESI_hammer.py
index 2f9aaebc7..740c6783e 100644
--- a/configs/ruby/MOESI_hammer.py
+++ b/configs/ruby/MOESI_hammer.py
@@ -170,15 +170,6 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_divider=3)
 
     for i in xrange(options.num_dirs):
-        #
-        # Create the Ruby objects associated with the directory controller
-        #
-
-        mem_cntrl = RubyMemoryControl(
-                              clk_domain = ruby_system.memctrl_clk_domain,
-                              version = i,
-                              ruby_system = ruby_system)
-
         dir_size = MemorySize('0B')
         dir_size.value = mem_module_size
 
@@ -186,17 +177,9 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                          start_index_bit = pf_start_bit)
 
         dir_cntrl = Directory_Controller(version = i,
-                                         directory = \
-                                         RubyDirectoryMemory( \
-                                                    version = i,
-                                                    size = dir_size,
-                                                    use_map = options.use_map,
-                                                    map_levels = \
-                                                    options.map_levels,
-                                                    numa_high_bit = \
-                                                      options.numa_high_bit),
+                                         directory = RubyDirectoryMemory(
+                                            version = i, size = dir_size),
                                          probeFilter = pf,
-                                         memBuffer = mem_cntrl,
                                          probe_filter_enabled = options.pf_on,
                                          full_bit_dir_enabled = options.dir_on,
                                          transitions_per_cycle = options.ports,
diff --git a/configs/ruby/Ruby.py b/configs/ruby/Ruby.py
index b7fd5078b..35ff66a0c 100644
--- a/configs/ruby/Ruby.py
+++ b/configs/ruby/Ruby.py
@@ -45,6 +45,7 @@ from m5.objects import *
 from m5.defines import buildEnv
 from m5.util import addToPath, fatal
 
+import MemConfig
 addToPath('../topologies')
 
 def define_options(parser):
@@ -75,22 +76,67 @@ def define_options(parser):
                       help="high order address bit to use for numa mapping. " \
                            "0 = highest bit, not specified = lowest bit")
 
-    # ruby sparse memory options
-    parser.add_option("--use-map", action="store_true", default=False)
-    parser.add_option("--map-levels", type="int", default=4)
-
     parser.add_option("--recycle-latency", type="int", default=10,
                       help="Recycle latency for ruby controller input buffers")
 
     parser.add_option("--random_seed", type="int", default=1234,
                       help="Used for seeding the random number generator")
 
-    parser.add_option("--ruby_stats", type="string", default="ruby.stats")
-
     protocol = buildEnv['PROTOCOL']
     exec "import %s" % protocol
     eval("%s.define_options(parser)" % protocol)
 
+def setup_memory_controllers(system, ruby, dir_cntrls, options):
+    ruby.block_size_bytes = options.cacheline_size
+    ruby.memory_size_bits = 48
+    block_size_bits = int(math.log(options.cacheline_size, 2))
+
+    if options.numa_high_bit:
+        numa_bit = options.numa_high_bit
+    else:
+        # if the numa_bit is not specified, set the directory bits as the
+        # lowest bits above the block offset bits, and the numa_bit as the
+        # highest of those directory bits
+        dir_bits = int(math.log(options.num_dirs, 2))
+        numa_bit = block_size_bits + dir_bits - 1
+
+    index = 0
+    mem_ctrls = []
+    crossbars = []
+
+    # Sets bits to be used for interleaving.  Creates memory controllers
+    # attached to a directory controller.  A separate controller is created
+    # for each address range as the abstract memory can handle only one
+    # contiguous address range as of now.
+    for dir_cntrl in dir_cntrls:
+        dir_cntrl.directory.numa_high_bit = numa_bit
+
+        crossbar = None
+        if len(system.mem_ranges) > 1:
+            crossbar = NoncoherentXBar()
+            crossbars.append(crossbar)
+            dir_cntrl.memory = crossbar.slave
+
+        for r in system.mem_ranges:
+            mem_ctrl = MemConfig.create_mem_ctrl(
+                MemConfig.get(options.mem_type), r, index, options.num_dirs,
+                int(math.log(options.num_dirs, 2)), options.cacheline_size)
+
+            mem_ctrls.append(mem_ctrl)
+
+            if crossbar != None:
+                mem_ctrl.port = crossbar.master
+            else:
+                mem_ctrl.port = dir_cntrl.memory
+
+        index += 1
+
+    system.mem_ctrls = mem_ctrls
+
+    if len(crossbars) > 0:
+        ruby.crossbars = crossbars
+
+
 def create_topology(controllers, options):
     """ Called from create_system in configs/ruby/<protocol>.py
         Must return an object which is a subclass of BaseTopology
@@ -103,7 +149,7 @@ def create_topology(controllers, options):
 
 def create_system(options, full_system, system, piobus = None, dma_ports = []):
 
-    system.ruby = RubySystem(no_mem_vec = options.use_map)
+    system.ruby = RubySystem()
     ruby = system.ruby
 
     # Set the network classes based on the command line options
@@ -169,33 +215,7 @@ def create_system(options, full_system, system, piobus = None, dma_ports = []):
         network.enable_fault_model = True
         network.fault_model = FaultModel()
 
-    # Loop through the directory controlers.
-    # Determine the total memory size of the ruby system and verify it is equal
-    # to physmem.  However, if Ruby memory is using sparse memory in SE
-    # mode, then the system should not back-up the memory state with
-    # the Memory Vector and thus the memory size bytes should stay at 0.
-    # Also set the numa bits to the appropriate values.
-    total_mem_size = MemorySize('0B')
-
-    ruby.block_size_bytes = options.cacheline_size
-    block_size_bits = int(math.log(options.cacheline_size, 2))
-
-    if options.numa_high_bit:
-        numa_bit = options.numa_high_bit
-    else:
-        # if the numa_bit is not specified, set the directory bits as the
-        # lowest bits above the block offset bits, and the numa_bit as the
-        # highest of those directory bits
-        dir_bits = int(math.log(options.num_dirs, 2))
-        numa_bit = block_size_bits + dir_bits - 1
-
-    for dir_cntrl in dir_cntrls:
-        total_mem_size.value += dir_cntrl.directory.size.value
-        dir_cntrl.directory.numa_high_bit = numa_bit
-
-    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
-    assert(total_mem_size.value == phys_mem_size)
-    ruby.mem_size = total_mem_size
+    setup_memory_controllers(system, ruby, dir_cntrls, options)
 
     # Connect the cpu sequencers and the piobus
     if piobus != None: