config: Move the memory instantiation outside FSConfig

This patch moves the instantiation of the memory controller outside
FSConfig and instead relies on the mem_ranges to pass the information
to the caller (e.g. fs.py or one of the regression scripts). The main
motivation for this change is to expose the structural composition of
the memory system and allow more tuning and configuration without
adding a large number of options to the makeSystem functions.

The patch updates the relevant example scripts to maintain the current
functionality. As the order that ports are connected to the memory bus
changes (in certain regresisons), some bus stats are shuffled
around. For example, what used to be layer 0 is now layer 1.

Going forward, options will be added to support the addition of
multi-channel memory controllers.

10 files changed, 61 insertions, 80 deletions

diff --git a/configs/common/FSConfig.py b/configs/common/FSConfig.py
index 422de971e..d560e8d96 100644
--- a/configs/common/FSConfig.py
+++ b/configs/common/FSConfig.py
@@ -55,7 +55,7 @@ class MemBus(CoherentBus):
     default = Self.badaddr_responder.pio
 
 
-def makeLinuxAlphaSystem(mem_mode, MemClass, mdesc = None):
+def makeLinuxAlphaSystem(mem_mode, mdesc = None):
     IO_address_space_base = 0x80000000000
     class BaseTsunami(Tsunami):
         ethernet = NSGigE(pci_bus=0, pci_dev=1, pci_func=0)
@@ -73,11 +73,9 @@ def makeLinuxAlphaSystem(mem_mode, MemClass, mdesc = None):
     # base address (including the PCI config space)
     self.bridge = Bridge(delay='50ns',
                          ranges = [AddrRange(IO_address_space_base, Addr.max)])
-    self.physmem = MemClass(range = AddrRange(mdesc.mem()))
-    self.mem_ranges = [self.physmem.range]
+    self.mem_ranges = [AddrRange(mdesc.mem())]
     self.bridge.master = self.iobus.slave
     self.bridge.slave = self.membus.master
-    self.physmem.port = self.membus.master
     self.disk0 = CowIdeDisk(driveID='master')
     self.disk2 = CowIdeDisk(driveID='master')
     self.disk0.childImage(mdesc.disk())
@@ -104,15 +102,13 @@ def makeLinuxAlphaSystem(mem_mode, MemClass, mdesc = None):
 
     return self
 
-def makeLinuxAlphaRubySystem(mem_mode, MemClass, mdesc = None):
+def makeLinuxAlphaRubySystem(mem_mode, mdesc = None):
     class BaseTsunami(Tsunami):
         ethernet = NSGigE(pci_bus=0, pci_dev=1, pci_func=0)
         ide = IdeController(disks=[Parent.disk0, Parent.disk2],
                             pci_func=0, pci_dev=0, pci_bus=0)
-        
-    physmem = MemClass(range = AddrRange(mdesc.mem()))
-    self = LinuxAlphaSystem(physmem = physmem)
-    self.mem_ranges = [self.physmem.range]
+    self = LinuxAlphaSystem()
+    self.mem_ranges = [AddrRange(mdesc.mem())]
     if not mdesc:
         # generic system
         mdesc = SysConfig()
@@ -121,13 +117,6 @@ def makeLinuxAlphaRubySystem(mem_mode, MemClass, mdesc = None):
     # Create pio bus to connect all device pio ports to rubymem's pio port
     self.piobus = NoncoherentBus()
 
-    #
-    # Pio functional accesses from devices need direct access to memory
-    # RubyPort currently does support functional accesses.  Therefore provide
-    # the piobus a direct connection to physical memory
-    #
-    self.piobus.master = physmem.port
-
     self.disk0 = CowIdeDisk(driveID='master')
     self.disk2 = CowIdeDisk(driveID='master')
     self.disk0.childImage(mdesc.disk())
@@ -157,7 +146,7 @@ def makeLinuxAlphaRubySystem(mem_mode, MemClass, mdesc = None):
 
     return self
 
-def makeSparcSystem(mem_mode, MemClass, mdesc = None):
+def makeSparcSystem(mem_mode, mdesc = None):
     # Constants from iob.cc and uart8250.cc
     iob_man_addr = 0x9800000000
     uart_pio_size = 8
@@ -180,13 +169,10 @@ def makeSparcSystem(mem_mode, MemClass, mdesc = None):
     self.t1000 = T1000()
     self.t1000.attachOnChipIO(self.membus)
     self.t1000.attachIO(self.iobus)
-    self.physmem = MemClass(range = AddrRange(Addr('1MB'), size = '64MB'))
-    self.physmem2 = MemClass(range = AddrRange(Addr('2GB'), size ='256MB'))
-    self.mem_ranges = [self.physmem.range, self.physmem2.range]
+    self.mem_ranges = [AddrRange(Addr('1MB'), size = '64MB'),
+                       AddrRange(Addr('2GB'), size ='256MB')]
     self.bridge.master = self.iobus.slave
     self.bridge.slave = self.membus.master
-    self.physmem.port = self.membus.master
-    self.physmem2.port = self.membus.master
     self.rom.port = self.membus.master
     self.nvram.port = self.membus.master
     self.hypervisor_desc.port = self.membus.master
@@ -225,7 +211,7 @@ def makeSparcSystem(mem_mode, MemClass, mdesc = None):
 
     return self
 
-def makeArmSystem(mem_mode, machine_type, MemClass, mdesc = None,
+def makeArmSystem(mem_mode, machine_type, mdesc = None,
                   dtb_filename = None, bare_metal=False):
     assert machine_type
 
@@ -273,8 +259,7 @@ def makeArmSystem(mem_mode, machine_type, MemClass, mdesc = None,
     if bare_metal:
         # EOT character on UART will end the simulation
         self.realview.uart.end_on_eot = True
-        self.physmem = MemClass(range = AddrRange(Addr(mdesc.mem())))
-        self.mem_ranges = [self.physmem.range]
+        self.mem_ranges = [AddrRange(mdesc.mem())]
     else:
         self.kernel = binary('vmlinux.arm.smp.fb.2.6.38.8')
         if dtb_filename is not None:
@@ -288,11 +273,8 @@ def makeArmSystem(mem_mode, machine_type, MemClass, mdesc = None,
 
         boot_flags = 'earlyprintk console=ttyAMA0 lpj=19988480 norandmaps ' + \
                      'rw loglevel=8 mem=%s root=/dev/sda1' % mdesc.mem()
-
-        self.physmem = MemClass(range = AddrRange(self.realview.mem_start_addr,
-                                                  size = mdesc.mem()),
-                                conf_table_reported = True)
-        self.mem_ranges = [self.physmem.range]
+        self.mem_ranges = [AddrRange(self.realview.mem_start_addr,
+                                     size = mdesc.mem())]
         self.realview.setupBootLoader(self.membus, self, binary)
         self.gic_cpu_addr = self.realview.gic.cpu_addr
         self.flags_addr = self.realview.realview_io.pio_addr + 0x30
@@ -300,8 +282,6 @@ def makeArmSystem(mem_mode, machine_type, MemClass, mdesc = None,
         if mdesc.disk().lower().count('android'):
             boot_flags += " init=/init "
         self.boot_osflags = boot_flags
-
-    self.physmem.port = self.membus.master
     self.realview.attachOnChipIO(self.membus, self.bridge)
     self.realview.attachIO(self.iobus)
     self.intrctrl = IntrControl()
@@ -313,7 +293,7 @@ def makeArmSystem(mem_mode, machine_type, MemClass, mdesc = None,
     return self
 
 
-def makeLinuxMipsSystem(mem_mode, MemClass, mdesc = None):
+def makeLinuxMipsSystem(mem_mode, mdesc = None):
     class BaseMalta(Malta):
         ethernet = NSGigE(pci_bus=0, pci_dev=1, pci_func=0)
         ide = IdeController(disks=[Parent.disk0, Parent.disk2],
@@ -327,11 +307,9 @@ def makeLinuxMipsSystem(mem_mode, MemClass, mdesc = None):
     self.iobus = NoncoherentBus()
     self.membus = MemBus()
     self.bridge = Bridge(delay='50ns')
-    self.physmem = MemClass(range = AddrRange('1GB'))
-    self.mem_ranges = [self.physmem.range]
+    self.mem_ranges = [AddrRange('1GB')]
     self.bridge.master = self.iobus.slave
     self.bridge.slave = self.membus.master
-    self.physmem.port = self.membus.master
     self.disk0 = CowIdeDisk(driveID='master')
     self.disk2 = CowIdeDisk(driveID='master')
     self.disk0.childImage(mdesc.disk())
@@ -369,7 +347,6 @@ def connectX86ClassicSystem(x86_sys, numCPUs):
     APIC_range_size = 1 << 12;
 
     x86_sys.membus = MemBus()
-    x86_sys.physmem.port = x86_sys.membus.master
 
     # North Bridge
     x86_sys.iobus = NoncoherentBus()
@@ -409,19 +386,13 @@ def connectX86RubySystem(x86_sys):
     # North Bridge
     x86_sys.piobus = NoncoherentBus()
 
-    #
-    # Pio functional accesses from devices need direct access to memory
-    # RubyPort currently does support functional accesses.  Therefore provide
-    # the piobus a direct connection to physical memory
-    #
-    x86_sys.piobus.master = x86_sys.physmem.port
     # add the ide to the list of dma devices that later need to attach to
     # dma controllers
     x86_sys._dma_ports = [x86_sys.pc.south_bridge.ide.dma]
     x86_sys.pc.attachIO(x86_sys.piobus, x86_sys._dma_ports)
 
 
-def makeX86System(mem_mode, MemClass, numCPUs = 1, mdesc = None, self = None,
+def makeX86System(mem_mode, numCPUs = 1, mdesc = None, self = None,
                   Ruby = False):
     if self == None:
         self = X86System()
@@ -434,8 +405,7 @@ def makeX86System(mem_mode, MemClass, numCPUs = 1, mdesc = None, self = None,
     self.mem_mode = mem_mode
 
     # Physical memory
-    self.physmem = MemClass(range = AddrRange(mdesc.mem()))
-    self.mem_ranges = [self.physmem.range]
+    self.mem_ranges = [AddrRange(mdesc.mem())]
 
     # Platform
     self.pc = Pc()
@@ -518,17 +488,16 @@ def makeX86System(mem_mode, MemClass, numCPUs = 1, mdesc = None, self = None,
     self.intel_mp_table.base_entries = base_entries
     self.intel_mp_table.ext_entries = ext_entries
 
-def makeLinuxX86System(mem_mode, MemClass, numCPUs = 1, mdesc = None,
+def makeLinuxX86System(mem_mode, numCPUs = 1, mdesc = None,
                        Ruby = False):
     self = LinuxX86System()
 
     # Build up the x86 system and then specialize it for Linux
-    makeX86System(mem_mode, MemClass, numCPUs, mdesc, self, Ruby)
+    makeX86System(mem_mode, numCPUs, mdesc, self, Ruby)
 
     # We assume below that there's at least 1MB of memory. We'll require 2
     # just to avoid corner cases.
-    phys_mem_size = sum(map(lambda mem: mem.range.size(),
-                            self.memories.unproxy(self)))
+    phys_mem_size = sum(map(lambda r: r.size(), self.mem_ranges))
     assert(phys_mem_size >= 0x200000)
 
     self.e820_table.entries = \
diff --git a/configs/example/fs.py b/configs/example/fs.py
index 028148404..037a54b75 100644
--- a/configs/example/fs.py
+++ b/configs/example/fs.py
@@ -102,17 +102,16 @@ else:
 np = options.num_cpus
 
 if buildEnv['TARGET_ISA'] == "alpha":
-    test_sys = makeLinuxAlphaSystem(test_mem_mode, TestMemClass, bm[0])
+    test_sys = makeLinuxAlphaSystem(test_mem_mode, bm[0])
 elif buildEnv['TARGET_ISA'] == "mips":
-    test_sys = makeLinuxMipsSystem(test_mem_mode, TestMemClass, bm[0])
+    test_sys = makeLinuxMipsSystem(test_mem_mode, bm[0])
 elif buildEnv['TARGET_ISA'] == "sparc":
-    test_sys = makeSparcSystem(test_mem_mode, TestMemClass, bm[0])
+    test_sys = makeSparcSystem(test_mem_mode, bm[0])
 elif buildEnv['TARGET_ISA'] == "x86":
-    test_sys = makeLinuxX86System(test_mem_mode, TestMemClass,
-                                  options.num_cpus, bm[0])
+    test_sys = makeLinuxX86System(test_mem_mode, options.num_cpus, bm[0])
 elif buildEnv['TARGET_ISA'] == "arm":
-    test_sys = makeArmSystem(test_mem_mode, options.machine_type,
-                             TestMemClass, bm[0], options.dtb_filename,
+    test_sys = makeArmSystem(test_mem_mode, options.machine_type, bm[0],
+                             options.dtb_filename,
                              bare_metal=options.bare_metal)
 else:
     fatal("Incapable of building %s full system!", buildEnv['TARGET_ISA'])
@@ -164,18 +163,24 @@ for i in xrange(np):
 
 CacheConfig.config_cache(options, test_sys)
 
+# Create the appropriate memory controllers and connect them to the
+# memory bus
+test_sys.mem_ctrls = [TestMemClass(range = r, conf_table_reported = True)
+                      for r in test_sys.mem_ranges]
+for i in xrange(len(test_sys.mem_ctrls)):
+    test_sys.mem_ctrls[i].port = test_sys.membus.master
+
 if len(bm) == 2:
     if buildEnv['TARGET_ISA'] == 'alpha':
-        drive_sys = makeLinuxAlphaSystem(drive_mem_mode, DriveMemClass, bm[1])
+        drive_sys = makeLinuxAlphaSystem(drive_mem_mode, bm[1])
     elif buildEnv['TARGET_ISA'] == 'mips':
-        drive_sys = makeLinuxMipsSystem(drive_mem_mode, DriveMemClass, bm[1])
+        drive_sys = makeLinuxMipsSystem(drive_mem_mode, bm[1])
     elif buildEnv['TARGET_ISA'] == 'sparc':
-        drive_sys = makeSparcSystem(drive_mem_mode, DriveMemClass, bm[1])
+        drive_sys = makeSparcSystem(drive_mem_mode, bm[1])
     elif buildEnv['TARGET_ISA'] == 'x86':
-        drive_sys = makeX86System(drive_mem_mode, DriveMemClass, np, bm[1])
+        drive_sys = makeX86System(drive_mem_mode, np, bm[1])
     elif buildEnv['TARGET_ISA'] == 'arm':
-        drive_sys = makeArmSystem(drive_mem_mode, options.machine_type,
-                                  DriveMemClass, bm[1])
+        drive_sys = makeArmSystem(drive_mem_mode, options.machine_type, bm[1])
 
     # Create a source clock for the system and set the clock period
     drive_sys.clk_domain = SrcClockDomain(clock =  options.sys_clock)
@@ -201,6 +206,13 @@ if len(bm) == 2:
     drive_sys.iobridge.slave = drive_sys.iobus.master
     drive_sys.iobridge.master = drive_sys.membus.slave
 
+    # Create the appropriate memory controllers and connect them to the
+    # memory bus
+    drive_sys.mem_ctrls = [DriveMemClass(range = r, conf_table_reported = True)
+                           for r in drive_sys.mem_ranges]
+    for i in xrange(len(drive_sys.mem_ctrls)):
+        drive_sys.mem_ctrls[i].port = drive_sys.membus.master
+
     drive_sys.init_param = options.init_param
     root = makeDualRoot(True, test_sys, drive_sys, options.etherdump)
 elif len(bm) == 1:
diff --git a/configs/example/ruby_fs.py b/configs/example/ruby_fs.py
index a1293a08c..a254841d2 100644
--- a/configs/example/ruby_fs.py
+++ b/configs/example/ruby_fs.py
@@ -83,10 +83,9 @@ if not (options.cpu_type == "detailed" or options.cpu_type == "timing"):
 TestMemClass = Simulation.setMemClass(options)
 
 if buildEnv['TARGET_ISA'] == "alpha":
-    system = makeLinuxAlphaRubySystem(test_mem_mode, TestMemClass, bm[0])
+    system = makeLinuxAlphaRubySystem(test_mem_mode, bm[0])
 elif buildEnv['TARGET_ISA'] == "x86":
-    system = makeLinuxX86System(test_mem_mode, TestMemClass,
-                                options.num_cpus, bm[0], True)
+    system = makeLinuxX86System(test_mem_mode, options.num_cpus, bm[0], True)
     Simulation.setWorkCountOptions(system, options)
 else:
     fatal("incapable of building non-alpha or non-x86 full system!")
@@ -127,5 +126,13 @@ for (i, cpu) in enumerate(system.cpu):
 
     system.ruby._cpu_ruby_ports[i].access_phys_mem = True
 
+# Create the appropriate memory controllers and connect them to the
+# PIO bus
+system.mem_ctrls = [TestMemClass(range = r,
+                                 conf_table_reported = True)
+                    for r in system.mem_ranges]
+for i in xrange(len(system.physmem)):
+    system.mem_ctrls[i].port = system.piobus.master
+
 root = Root(full_system = True, system = system)
 Simulation.run(options, root, system, FutureClass)
diff --git a/configs/ruby/MESI_CMP_directory.py b/configs/ruby/MESI_CMP_directory.py
index e263b4e5a..8cd74cde6 100644
--- a/configs/ruby/MESI_CMP_directory.py
+++ b/configs/ruby/MESI_CMP_directory.py
@@ -140,8 +140,7 @@ def create_system(options, system, piobus, dma_ports, ruby_system):
         
         cntrl_count += 1
 
-    phys_mem_size = sum(map(lambda mem: mem.range.size(),
-                            system.memories.unproxy(system)))
+    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
     assert(phys_mem_size % options.num_dirs == 0)
     mem_module_size = phys_mem_size / options.num_dirs
 
diff --git a/configs/ruby/MI_example.py b/configs/ruby/MI_example.py
index 896a01a61..44687e493 100644
--- a/configs/ruby/MI_example.py
+++ b/configs/ruby/MI_example.py
@@ -105,8 +105,7 @@ def create_system(options, system, piobus, dma_ports, ruby_system):
 
         cntrl_count += 1
 
-    phys_mem_size = sum(map(lambda mem: mem.range.size(),
-                            system.memories.unproxy(system)))
+    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
     assert(phys_mem_size % options.num_dirs == 0)
     mem_module_size = phys_mem_size / options.num_dirs
 
diff --git a/configs/ruby/MOESI_CMP_directory.py b/configs/ruby/MOESI_CMP_directory.py
index bc1f7d641..33c62f7a3 100644
--- a/configs/ruby/MOESI_CMP_directory.py
+++ b/configs/ruby/MOESI_CMP_directory.py
@@ -135,8 +135,7 @@ def create_system(options, system, piobus, dma_ports, ruby_system):
 
         cntrl_count += 1
 
-    phys_mem_size = sum(map(lambda mem: mem.range.size(),
-                            system.memories.unproxy(system)))
+    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
     assert(phys_mem_size % options.num_dirs == 0)
     mem_module_size = phys_mem_size / options.num_dirs
 
diff --git a/configs/ruby/MOESI_CMP_token.py b/configs/ruby/MOESI_CMP_token.py
index 4ede788a8..43b9f97e8 100644
--- a/configs/ruby/MOESI_CMP_token.py
+++ b/configs/ruby/MOESI_CMP_token.py
@@ -156,8 +156,7 @@ def create_system(options, system, piobus, dma_ports, ruby_system):
 
         cntrl_count += 1
 
-    phys_mem_size = sum(map(lambda mem: mem.range.size(),
-                            system.memories.unproxy(system)))
+    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
     assert(phys_mem_size % options.num_dirs == 0)
     mem_module_size = phys_mem_size / options.num_dirs
 
diff --git a/configs/ruby/MOESI_hammer.py b/configs/ruby/MOESI_hammer.py
index da8b004ee..124845ab8 100644
--- a/configs/ruby/MOESI_hammer.py
+++ b/configs/ruby/MOESI_hammer.py
@@ -131,8 +131,7 @@ def create_system(options, system, piobus, dma_ports, ruby_system):
 
         cntrl_count += 1
 
-    phys_mem_size = sum(map(lambda mem: mem.range.size(),
-                            system.memories.unproxy(system)))
+    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
     assert(phys_mem_size % options.num_dirs == 0)
     mem_module_size = phys_mem_size / options.num_dirs
 
diff --git a/configs/ruby/Network_test.py b/configs/ruby/Network_test.py
index d7b04cd31..6dfcc0b59 100644
--- a/configs/ruby/Network_test.py
+++ b/configs/ruby/Network_test.py
@@ -103,8 +103,7 @@ def create_system(options, system, piobus, dma_ports, ruby_system):
 
         cntrl_count += 1
 
-    phys_mem_size = sum(map(lambda mem: mem.range.size(),
-                            system.memories.unproxy(system)))
+    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
     assert(phys_mem_size % options.num_dirs == 0)
     mem_module_size = phys_mem_size / options.num_dirs
 
diff --git a/configs/ruby/Ruby.py b/configs/ruby/Ruby.py
index a35ef4f09..b6cc7a5e8 100644
--- a/configs/ruby/Ruby.py
+++ b/configs/ruby/Ruby.py
@@ -183,8 +183,7 @@ def create_system(options, system, piobus = None, dma_ports = []):
         total_mem_size.value += dir_cntrl.directory.size.value
         dir_cntrl.directory.numa_high_bit = numa_bit
 
-    phys_mem_size = sum(map(lambda mem: mem.range.size(),
-                            system.memories.unproxy(system)))
+    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
     assert(total_mem_size.value == phys_mem_size)
 
     ruby_profiler = RubyProfiler(ruby_system = ruby,