arch-arm, configs: Treat the bootloader rom as cacheable memory

Prior to this changeset the bootloader rom (instantiated as a
SimpleMemory) in ruby Arm systems was treated as an IO device and it
was fronted by a DMA controller. This changeset moves the bootloader
rom and adds it to the system as another memory with a dedicated
directory controller.

Change-Id: I094fed031cdef7f77a939d94f948d967b349b7e0
Reviewed-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/8741
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>

15 files changed, 89 insertions, 44 deletions

diff --git a/configs/common/FSConfig.py b/configs/common/FSConfig.py
index 42cfafed4..17498c42b 100644
--- a/configs/common/FSConfig.py
+++ b/configs/common/FSConfig.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2012, 2015-2017 ARM Limited
+# Copyright (c) 2010-2012, 2015-2018 ARM Limited
 # All rights reserved.
 #
 # The license below extends only to copyright in the software and shall
@@ -326,8 +326,10 @@ def makeArmSystem(mem_mode, machine_type, num_cpus=1, mdesc=None,
         # iobus, as gem5's membus is only used for initialization and
         # SST doesn't use it.  Attaching nvmem to iobus solves this issue.
         # During initialization, system_port -> membus -> iobus -> nvmem.
-        if external_memory or ruby:
+        if external_memory:
             self.realview.setupBootLoader(self.iobus,  self, binary)
+        elif ruby:
+            self.realview.setupBootLoader(None, self, binary)
         else:
             self.realview.setupBootLoader(self.membus, self, binary)
         self.gic_cpu_addr = self.realview.gic.cpu_addr
@@ -386,8 +388,6 @@ def makeArmSystem(mem_mode, machine_type, num_cpus=1, mdesc=None,
     elif ruby:
         self._dma_ports = [ ]
         self.realview.attachOnChipIO(self.iobus, dma_ports=self._dma_ports)
-        # Force Ruby to treat the boot ROM as an IO device.
-        self.realview.nvmem.in_addr_map = False
         self.realview.attachIO(self.iobus, dma_ports=self._dma_ports)
     else:
         self.realview.attachOnChipIO(self.membus, self.bridge)
diff --git a/configs/example/fs.py b/configs/example/fs.py
index 0f87e2b3a..4031fd05e 100644
--- a/configs/example/fs.py
+++ b/configs/example/fs.py
@@ -152,8 +152,9 @@ def build_test_system(np):
         test_sys.kvm_vm = KvmVM()
 
     if options.ruby:
+        bootmem = getattr(test_sys, 'bootmem', None)
         Ruby.create_system(options, True, test_sys, test_sys.iobus,
-                           test_sys._dma_ports)
+                           test_sys._dma_ports, bootmem)
 
         # Create a seperate clock domain for Ruby
         test_sys.ruby.clk_domain = SrcClockDomain(clock = options.ruby_clock,
diff --git a/configs/ruby/GPU_RfO.py b/configs/ruby/GPU_RfO.py
index 832ea4422..3331ab2c5 100644
--- a/configs/ruby/GPU_RfO.py
+++ b/configs/ruby/GPU_RfO.py
@@ -427,7 +427,8 @@ def define_options(parser):
     parser.add_option("--tcc-dir-factor", type='int', default=4,
                       help="TCCdir size = factor *(TCPs + TCC)")
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'GPU_RfO':
         panic("This script requires the GPU_RfO protocol to be built.")
 
diff --git a/configs/ruby/GPU_VIPER.py b/configs/ruby/GPU_VIPER.py
index e4ba18089..37136af3c 100644
--- a/configs/ruby/GPU_VIPER.py
+++ b/configs/ruby/GPU_VIPER.py
@@ -390,7 +390,8 @@ def define_options(parser):
     parser.add_option("--noL1", action = "store_true", default = False,
                       help = "bypassL1")
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'GPU_VIPER':
         panic("This script requires the GPU_VIPER protocol to be built.")
 
diff --git a/configs/ruby/GPU_VIPER_Baseline.py b/configs/ruby/GPU_VIPER_Baseline.py
index 978d4cc39..ec569252e 100644
--- a/configs/ruby/GPU_VIPER_Baseline.py
+++ b/configs/ruby/GPU_VIPER_Baseline.py
@@ -373,7 +373,8 @@ def define_options(parser):
     parser.add_option("--noL2", action = "store_true", default = False,
                       help = "bypassL2")
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'GPU_VIPER_Baseline':
         panic("This script requires the" \
         "GPU_VIPER_Baseline protocol to be built.")
diff --git a/configs/ruby/GPU_VIPER_Region.py b/configs/ruby/GPU_VIPER_Region.py
index 8b59c047d..5f8293bbe 100644
--- a/configs/ruby/GPU_VIPER_Region.py
+++ b/configs/ruby/GPU_VIPER_Region.py
@@ -442,7 +442,8 @@ def define_options(parser):
         action="store_true", default=False)
     parser.add_option("--use-L3-on-WT", action="store_true", default=False)
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'GPU_VIPER_Region':
         panic("This script requires the GPU_VIPER_Region protocol to be built.")
 
diff --git a/configs/ruby/Garnet_standalone.py b/configs/ruby/Garnet_standalone.py
index 5c173ce17..168f84dd4 100644
--- a/configs/ruby/Garnet_standalone.py
+++ b/configs/ruby/Garnet_standalone.py
@@ -42,7 +42,8 @@ class L1Cache(RubyCache): pass
 def define_options(parser):
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'Garnet_standalone':
         panic("This script requires Garnet_standalone protocol to be built.")
 
@@ -99,9 +100,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
         l1_cntrl.responseFromCache = MessageBuffer()
         l1_cntrl.forwardFromCache = MessageBuffer()
 
-
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
-                                         ruby_system)
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer()
@@ -112,4 +115,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
     all_cntrls = l1_cntrl_nodes + dir_cntrl_nodes
     ruby_system.network.number_of_virtual_networks = 3
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)
diff --git a/configs/ruby/MESI_Three_Level.py b/configs/ruby/MESI_Three_Level.py
index 5d9d5b2f8..7c80e48ad 100644
--- a/configs/ruby/MESI_Three_Level.py
+++ b/configs/ruby/MESI_Three_Level.py
@@ -49,7 +49,8 @@ def define_options(parser):
             caches private to clusters")
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MESI_Three_Level':
         fatal("This script requires the MESI_Three_Level protocol to be\
@@ -199,8 +200,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
     ruby_system.memctrl_clk_domain = DerivedClockDomain(
             clk_domain = ruby_system.clk_domain, clk_divider = 3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
-                                         ruby_system)
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer()
@@ -259,4 +263,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 3
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)
diff --git a/configs/ruby/MESI_Two_Level.py b/configs/ruby/MESI_Two_Level.py
index 844c62af4..b488b9d51 100644
--- a/configs/ruby/MESI_Two_Level.py
+++ b/configs/ruby/MESI_Two_Level.py
@@ -43,7 +43,8 @@ class L2Cache(RubyCache): pass
 def define_options(parser):
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MESI_Two_Level':
         fatal("This script requires the MESI_Two_Level protocol to be built.")
@@ -173,8 +174,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_domain = ruby_system.clk_domain,
                                           clk_divider = 3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
-                                         ruby_system)
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer()
@@ -231,4 +235,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 3
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)
diff --git a/configs/ruby/MI_example.py b/configs/ruby/MI_example.py
index eb881e55c..c92bb2036 100644
--- a/configs/ruby/MI_example.py
+++ b/configs/ruby/MI_example.py
@@ -42,7 +42,8 @@ class L1Cache(RubyCache): pass
 def define_options(parser):
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MI_example':
         panic("This script requires the MI_example protocol to be built.")
@@ -125,8 +126,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_domain=ruby_system.clk_domain,
                                           clk_divider=3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
-                                         ruby_system)
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer(ordered = True)
@@ -188,4 +192,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 5
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)
diff --git a/configs/ruby/MOESI_AMD_Base.py b/configs/ruby/MOESI_AMD_Base.py
index cdbb6f600..a5193878e 100644
--- a/configs/ruby/MOESI_AMD_Base.py
+++ b/configs/ruby/MOESI_AMD_Base.py
@@ -209,7 +209,8 @@ def define_options(parser):
     parser.add_option("--num-tbes", type="int", default=256)
     parser.add_option("--l2-latency", type="int", default=50) # load to use
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'MOESI_AMD_Base':
         panic("This script requires the MOESI_AMD_Base protocol.")
 
diff --git a/configs/ruby/MOESI_CMP_directory.py b/configs/ruby/MOESI_CMP_directory.py
index cbb061d32..80d0bc106 100644
--- a/configs/ruby/MOESI_CMP_directory.py
+++ b/configs/ruby/MOESI_CMP_directory.py
@@ -43,7 +43,8 @@ class L2Cache(RubyCache): pass
 def define_options(parser):
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MOESI_CMP_directory':
         panic("This script requires the MOESI_CMP_directory protocol to be built.")
@@ -165,8 +166,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_divider=3)
 
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
-                                         ruby_system)
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer()
@@ -236,4 +240,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 3
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)
diff --git a/configs/ruby/MOESI_CMP_token.py b/configs/ruby/MOESI_CMP_token.py
index 7c9871970..25b18a0b8 100644
--- a/configs/ruby/MOESI_CMP_token.py
+++ b/configs/ruby/MOESI_CMP_token.py
@@ -50,7 +50,8 @@ def define_options(parser):
     parser.add_option("--allow-atomic-migration", action="store_true",
           help="allow migratory sharing for atomic only accessed blocks")
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MOESI_CMP_token':
         panic("This script requires the MOESI_CMP_token protocol to be built.")
@@ -190,8 +191,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_domain=ruby_system.clk_domain,
                                           clk_divider=3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
-                                         ruby_system)
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         dir_cntrl.l2_select_num_bits = l2_bits
         # Connect the directory controllers and the network
@@ -264,4 +268,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 6
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)
diff --git a/configs/ruby/MOESI_hammer.py b/configs/ruby/MOESI_hammer.py
index 9f615f931..a2e902df3 100644
--- a/configs/ruby/MOESI_hammer.py
+++ b/configs/ruby/MOESI_hammer.py
@@ -52,7 +52,8 @@ def define_options(parser):
     parser.add_option("--dir-on", action="store_true",
           help="Hammer: enable Full-bit Directory")
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MOESI_hammer':
         panic("This script requires the MOESI_hammer protocol to be built.")
@@ -172,8 +173,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_domain=ruby_system.clk_domain,
                                           clk_divider=3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
-                                         ruby_system)
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         pf = ProbeFilter(size = pf_size, assoc = 4,
                          start_index_bit = pf_start_bit)
@@ -254,4 +258,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 6
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)
diff --git a/configs/ruby/Ruby.py b/configs/ruby/Ruby.py
index 9e110f84d..f1415b062 100644
--- a/configs/ruby/Ruby.py
+++ b/configs/ruby/Ruby.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2012, 2017 ARM Limited
+# Copyright (c) 2012, 2017-2018 ARM Limited
 # All rights reserved.
 #
 # The license below extends only to copyright in the software and shall
@@ -135,7 +135,8 @@ def create_topology(controllers, options):
     topology = eval("Topo.%s(controllers)" % options.topology)
     return topology
 
-def create_system(options, full_system, system, piobus = None, dma_ports = []):
+def create_system(options, full_system, system, piobus = None, dma_ports = [],
+                  bootmem=None):
 
     system.ruby = RubySystem()
     ruby = system.ruby
@@ -150,7 +151,7 @@ def create_system(options, full_system, system, piobus = None, dma_ports = []):
     try:
         (cpu_sequencers, dir_cntrls, topology) = \
              eval("%s.create_system(options, full_system, system, dma_ports,\
-                                    ruby)"
+                                    bootmem, ruby)"
                   % protocol)
     except:
         print("Error: could not create sytem for ruby protocol %s" % protocol)
@@ -198,7 +199,8 @@ def create_system(options, full_system, system, piobus = None, dma_ports = []):
         ruby.phys_mem = SimpleMemory(range=system.mem_ranges[0],
                                      in_addr_map=False)
 
-def create_directories(options, mem_ranges, ruby_system):
+def create_directories(options, mem_ranges, bootmem, ruby_system,
+                       system):
     dir_cntrl_nodes = []
     if options.numa_high_bit:
         numa_bit = options.numa_high_bit
@@ -227,7 +229,17 @@ def create_directories(options, mem_ranges, ruby_system):
 
         exec("ruby_system.dir_cntrl%d = dir_cntrl" % i)
         dir_cntrl_nodes.append(dir_cntrl)
-    return dir_cntrl_nodes
+
+    if bootmem is not None:
+        rom_dir_cntrl = Directory_Controller()
+        rom_dir_cntrl.directory = RubyDirectoryMemory()
+        rom_dir_cntrl.ruby_system = ruby_system
+        rom_dir_cntrl.version = i + 1
+        rom_dir_cntrl.memory = bootmem.port
+        rom_dir_cntrl.addr_ranges = bootmem.range
+        return (dir_cntrl_nodes, rom_dir_cntrl)
+
+    return (dir_cntrl_nodes, None)
 
 def send_evicts(options):
     # currently, 2 scenarios warrant forwarding evictions to the CPU: