ruby: garnet2.0

Revamped version of garnet with more optimized single-cycle routers,
more configurability, and cleaner code.

6 files changed, 134 insertions, 38 deletions

diff --git a/configs/network/Network.py b/configs/network/Network.py
index 871682ff1..3c15a4f79 100644
--- a/configs/network/Network.py
+++ b/configs/network/Network.py
@@ -38,25 +38,44 @@ def define_options(parser):
 
     parser.add_option("--topology", type="string", default="Crossbar",
                       help="check configs/topologies for complete set")
-    parser.add_option("--mesh-rows", type="int", default=1,
+    parser.add_option("--mesh-rows", type="int", default=0,
                       help="the number of rows in the mesh topology")
-    parser.add_option("--garnet-network", type="choice",
-                      choices=['fixed', 'flexible'], help="'fixed'|'flexible'")
-    parser.add_option("--network-fault-model", action="store_true", default=False,
-                      help="enable network fault model: see src/mem/ruby/network/fault_model/")
+    parser.add_option("--network", type="choice", default="simple",
+                      choices=['simple', 'garnet2.0'],
+                      help="'simple'|'garnet2.0'")
+    parser.add_option("--router-latency", action="store", type="int",
+                      default=1,
+                      help="""number of pipeline stages in the garnet router.
+                            Has to be >= 1.
+                            Can be over-ridden on a per router basis
+                            in the topology file.""")
+    parser.add_option("--link-latency", action="store", type="int", default=1,
+                      help="""latency of each link the simple/garnet networks.
+                            Has to be >= 1.
+                            Can be over-ridden on a per link basis
+                            in the topology file.""")
+    parser.add_option("--link-width-bits", action="store", type="int",
+                      default=128,
+                      help="width in bits for all links inside garnet.")
+    parser.add_option("--vcs-per-vnet", action="store", type="int", default=4,
+                      help="""number of virtual channels per virtual network
+                            inside garnet network.""")
+    parser.add_option("--routing-algorithm", action="store", type="int",
+                      default=0,
+                      help="""routing algorithm in network.
+                            0: weight-based table
+                            1: XY (for Mesh. see garnet2.0/RoutingUnit.cc)
+                            2: Custom (see garnet2.0/RoutingUnit.cc""")
+    parser.add_option("--network-fault-model", action="store_true",
+                      default=False,
+                      help="""enable network fault model:
+                            see src/mem/ruby/network/fault_model/""")
 
 
 def create_network(options, ruby):
 
     # Set the network classes based on the command line options
-    if options.garnet_network == "fixed":
-        NetworkClass = GarnetNetwork_d
-        IntLinkClass = GarnetIntLink_d
-        ExtLinkClass = GarnetExtLink_d
-        RouterClass = GarnetRouter_d
-        InterfaceClass = GarnetNetworkInterface_d
-
-    elif options.garnet_network == "flexible":
+    if options.network == "garnet2.0":
         NetworkClass = GarnetNetwork
         IntLinkClass = GarnetIntLink
         ExtLinkClass = GarnetExtLink
@@ -79,7 +98,13 @@ def create_network(options, ruby):
 
 def init_network(options, network, InterfaceClass):
 
-    if options.garnet_network is None:
+    if options.network == "garnet2.0":
+        network.num_rows = options.mesh_rows
+        network.vcs_per_vnet = options.vcs_per_vnet
+        network.ni_flit_size = options.link_width_bits / 8
+        network.routing_algorithm = options.routing_algorithm
+
+    if options.network == "simple":
         network.setup_buffers()
 
     if InterfaceClass != None:
@@ -88,6 +113,6 @@ def init_network(options, network, InterfaceClass):
         network.netifs = netifs
 
     if options.network_fault_model:
-        assert(options.garnet_network == "fixed")
+        assert(options.network == "garnet2.0")
         network.enable_fault_model = True
         network.fault_model = FaultModel()
diff --git a/configs/topologies/Crossbar.py b/configs/topologies/Crossbar.py
index 35d20de34..447b1c597 100644
--- a/configs/topologies/Crossbar.py
+++ b/configs/topologies/Crossbar.py
@@ -35,6 +35,12 @@ class Crossbar(SimpleTopology):
     description='Crossbar'
 
     def makeTopology(self, options, network, IntLink, ExtLink, Router):
+
+        # default values for link latency and router latency.
+        # Can be over-ridden on a per link/router basis
+        link_latency = options.link_latency # used by simple and garnet
+        router_latency = options.router_latency # only used by garnet
+
         # Create an individual router for each controller plus one more for
         # the centralized crossbar.  The large numbers of routers are needed
         # because external links do not model outgoing bandwidth in the
@@ -45,7 +51,8 @@ class Crossbar(SimpleTopology):
         xbar = routers[len(self.nodes)] # the crossbar router is the last router created
         network.routers = routers
 
-        ext_links = [ExtLink(link_id=i, ext_node=n, int_node=routers[i])
+        ext_links = [ExtLink(link_id=i, ext_node=n, int_node=routers[i],
+                             latency = link_latency)
                         for (i, n) in enumerate(self.nodes)]
         network.ext_links = ext_links
 
@@ -55,13 +62,15 @@ class Crossbar(SimpleTopology):
         for i in range(len(self.nodes)):
             int_links.append(IntLink(link_id=(link_count+i),
                                      src_node=routers[i],
-                                     dst_node=xbar))
+                                     dst_node=xbar,
+                                     latency = link_latency))
 
         link_count += len(self.nodes)
 
         for i in range(len(self.nodes)):
             int_links.append(IntLink(link_id=(link_count+i),
                                      src_node=xbar,
-                                     dst_node=routers[i]))
+                                     dst_node=routers[i],
+                                     latency = link_latency))
 
         network.int_links = int_links
diff --git a/configs/topologies/MeshDirCorners_XY.py b/configs/topologies/MeshDirCorners_XY.py
index 47eb48086..46f3c6fdd 100644
--- a/configs/topologies/MeshDirCorners_XY.py
+++ b/configs/topologies/MeshDirCorners_XY.py
@@ -47,6 +47,12 @@ class MeshDirCorners_XY(SimpleTopology):
         num_routers = options.num_cpus
         num_rows = options.mesh_rows
 
+        # default values for link latency and router latency.
+        # Can be over-ridden on a per link/router basis
+        link_latency = options.link_latency # used by simple and garnet
+        router_latency = options.router_latency # only used by garnet
+
+
         # First determine which nodes are cache cntrls vs. dirs vs. dma
         cache_nodes = []
         dir_nodes = []
@@ -64,7 +70,7 @@ class MeshDirCorners_XY(SimpleTopology):
         # and evenly divisible.  Also the number of caches must be a
         # multiple of the number of routers and the number of directories
         # must be four.
-        assert(num_rows <= num_routers)
+        assert(num_rows > 0 and num_rows <= num_routers)
         num_columns = int(num_routers / num_rows)
         assert(num_columns * num_rows == num_routers)
         caches_per_router, remainder = divmod(len(cache_nodes), num_routers)
@@ -72,7 +78,8 @@ class MeshDirCorners_XY(SimpleTopology):
         assert(len(dir_nodes) == 4)
 
         # Create the routers in the mesh
-        routers = [Router(router_id=i) for i in range(num_routers)]
+        routers = [Router(router_id=i, latency = router_latency) \
+            for i in range(num_routers)]
         network.routers = routers
 
         # link counter to set unique link ids
@@ -84,28 +91,34 @@ class MeshDirCorners_XY(SimpleTopology):
             cntrl_level, router_id = divmod(i, num_routers)
             assert(cntrl_level < caches_per_router)
             ext_links.append(ExtLink(link_id=link_count, ext_node=n,
-                                    int_node=routers[router_id]))
+                                    int_node=routers[router_id],
+                                    latency = link_latency))
             link_count += 1
 
         # Connect the dir nodes to the corners.
         ext_links.append(ExtLink(link_id=link_count, ext_node=dir_nodes[0],
-                                int_node=routers[0]))
+                                int_node=routers[0],
+                                latency = link_latency))
         link_count += 1
         ext_links.append(ExtLink(link_id=link_count, ext_node=dir_nodes[1],
-                                int_node=routers[num_columns - 1]))
+                                int_node=routers[num_columns - 1],
+                                latency = link_latency))
         link_count += 1
         ext_links.append(ExtLink(link_id=link_count, ext_node=dir_nodes[2],
-                                int_node=routers[num_routers - num_columns]))
+                                int_node=routers[num_routers - num_columns],
+                                latency = link_latency))
         link_count += 1
         ext_links.append(ExtLink(link_id=link_count, ext_node=dir_nodes[3],
-                                int_node=routers[num_routers - 1]))
+                                int_node=routers[num_routers - 1],
+                                latency = link_latency))
         link_count += 1
 
         # Connect the dma nodes to router 0.  These should only be DMA nodes.
         for (i, node) in enumerate(dma_nodes):
             assert(node.type == 'DMA_Controller')
             ext_links.append(ExtLink(link_id=link_count, ext_node=node,
-                                     int_node=routers[0]))
+                                     int_node=routers[0],
+                                     latency = link_latency))
 
         network.ext_links = ext_links
 
@@ -121,6 +134,9 @@ class MeshDirCorners_XY(SimpleTopology):
                     int_links.append(IntLink(link_id=link_count,
                                              src_node=routers[east_out],
                                              dst_node=routers[west_in],
+                                             src_outport="East",
+                                             dst_inport="West",
+                                             latency = link_latency,
                                              weight=1))
                     link_count += 1
 
@@ -133,6 +149,9 @@ class MeshDirCorners_XY(SimpleTopology):
                     int_links.append(IntLink(link_id=link_count,
                                              src_node=routers[west_out],
                                              dst_node=routers[east_in],
+                                             src_outport="West",
+                                             dst_inport="East",
+                                             latency = link_latency,
                                              weight=1))
                     link_count += 1
 
@@ -145,6 +164,9 @@ class MeshDirCorners_XY(SimpleTopology):
                     int_links.append(IntLink(link_id=link_count,
                                              src_node=routers[north_out],
                                              dst_node=routers[south_in],
+                                             src_outport="North",
+                                             dst_inport="South",
+                                             latency = link_latency,
                                              weight=2))
                     link_count += 1
 
@@ -157,6 +179,9 @@ class MeshDirCorners_XY(SimpleTopology):
                     int_links.append(IntLink(link_id=link_count,
                                              src_node=routers[south_out],
                                              dst_node=routers[north_in],
+                                             src_outport="South",
+                                             dst_inport="North",
+                                             latency = link_latency,
                                              weight=2))
                     link_count += 1
 
diff --git a/configs/topologies/Mesh_XY.py b/configs/topologies/Mesh_XY.py
index 134a5c07d..652ac16d8 100644
--- a/configs/topologies/Mesh_XY.py
+++ b/configs/topologies/Mesh_XY.py
@@ -53,15 +53,22 @@ class Mesh_XY(SimpleTopology):
         num_routers = options.num_cpus
         num_rows = options.mesh_rows
 
+        # default values for link latency and router latency.
+        # Can be over-ridden on a per link/router basis
+        link_latency = options.link_latency # used by simple and garnet
+        router_latency = options.router_latency # only used by garnet
+
+
         # There must be an evenly divisible number of cntrls to routers
         # Also, obviously the number or rows must be <= the number of routers
         cntrls_per_router, remainder = divmod(len(nodes), num_routers)
-        assert(num_rows <= num_routers)
+        assert(num_rows > 0 and num_rows <= num_routers)
         num_columns = int(num_routers / num_rows)
         assert(num_columns * num_rows == num_routers)
 
         # Create the routers in the mesh
-        routers = [Router(router_id=i) for i in range(num_routers)]
+        routers = [Router(router_id=i, latency = router_latency) \
+            for i in range(num_routers)]
         network.routers = routers
 
         # link counter to set unique link ids
@@ -83,7 +90,8 @@ class Mesh_XY(SimpleTopology):
             cntrl_level, router_id = divmod(i, num_routers)
             assert(cntrl_level < cntrls_per_router)
             ext_links.append(ExtLink(link_id=link_count, ext_node=n,
-                                    int_node=routers[router_id]))
+                                    int_node=routers[router_id],
+                                    latency = link_latency))
             link_count += 1
 
         # Connect the remainding nodes to router 0.  These should only be
@@ -92,7 +100,8 @@ class Mesh_XY(SimpleTopology):
             assert(node.type == 'DMA_Controller')
             assert(i < remainder)
             ext_links.append(ExtLink(link_id=link_count, ext_node=node,
-                                    int_node=routers[0]))
+                                    int_node=routers[0],
+                                    latency = link_latency))
             link_count += 1
 
         network.ext_links = ext_links
@@ -111,6 +120,7 @@ class Mesh_XY(SimpleTopology):
                                              dst_node=routers[west_in],
                                              src_outport="East",
                                              dst_inport="West",
+                                             latency = link_latency,
                                              weight=1))
                     link_count += 1
 
@@ -125,6 +135,7 @@ class Mesh_XY(SimpleTopology):
                                              dst_node=routers[east_in],
                                              src_outport="West",
                                              dst_inport="East",
+                                             latency = link_latency,
                                              weight=1))
                     link_count += 1
 
@@ -139,6 +150,7 @@ class Mesh_XY(SimpleTopology):
                                              dst_node=routers[south_in],
                                              src_outport="North",
                                              dst_inport="South",
+                                             latency = link_latency,
                                              weight=2))
                     link_count += 1
 
@@ -153,6 +165,7 @@ class Mesh_XY(SimpleTopology):
                                              dst_node=routers[north_in],
                                              src_outport="South",
                                              dst_inport="North",
+                                             latency = link_latency,
                                              weight=2))
                     link_count += 1
 
diff --git a/configs/topologies/Mesh_westfirst.py b/configs/topologies/Mesh_westfirst.py
index 3af894f93..6139f6742 100644
--- a/configs/topologies/Mesh_westfirst.py
+++ b/configs/topologies/Mesh_westfirst.py
@@ -58,15 +58,21 @@ class Mesh_westfirst(SimpleTopology):
         num_routers = options.num_cpus
         num_rows = options.mesh_rows
 
+        # default values for link latency and router latency.
+        # Can be over-ridden on a per link/router basis
+        link_latency = options.link_latency # used by simple and garnet
+        router_latency = options.router_latency # only used by garnet
+
         # There must be an evenly divisible number of cntrls to routers
         # Also, obviously the number or rows must be <= the number of routers
         cntrls_per_router, remainder = divmod(len(nodes), num_routers)
-        assert(num_rows <= num_routers)
+        assert(num_rows > 0 and num_rows <= num_routers)
         num_columns = int(num_routers / num_rows)
         assert(num_columns * num_rows == num_routers)
 
         # Create the routers in the mesh
-        routers = [Router(router_id=i) for i in range(num_routers)]
+        routers = [Router(router_id=i, latency=router_latency) \
+            for i in range(num_routers)]
         network.routers = routers
 
         # link counter to set unique link ids
@@ -88,7 +94,8 @@ class Mesh_westfirst(SimpleTopology):
             cntrl_level, router_id = divmod(i, num_routers)
             assert(cntrl_level < cntrls_per_router)
             ext_links.append(ExtLink(link_id=link_count, ext_node=n,
-                                    int_node=routers[router_id]))
+                                    int_node=routers[router_id],
+                                    latency = link_latency))
             link_count += 1
 
         # Connect the remainding nodes to router 0.  These should only be
@@ -97,7 +104,8 @@ class Mesh_westfirst(SimpleTopology):
             assert(node.type == 'DMA_Controller')
             assert(i < remainder)
             ext_links.append(ExtLink(link_id=link_count, ext_node=node,
-                                    int_node=routers[0]))
+                                    int_node=routers[0],
+                                    latency = link_latency))
             link_count += 1
 
         network.ext_links = ext_links
@@ -114,6 +122,7 @@ class Mesh_westfirst(SimpleTopology):
                     int_links.append(IntLink(link_id=link_count,
                                              src_node=routers[east_out],
                                              dst_node=routers[west_in],
+                                             latency = link_latency,
                                              weight=2))
                     link_count += 1
 
@@ -126,6 +135,7 @@ class Mesh_westfirst(SimpleTopology):
                     int_links.append(IntLink(link_id=link_count,
                                              src_node=routers[west_out],
                                              dst_node=routers[east_in],
+                                             latency = link_latency,
                                              weight=1))
                     link_count += 1
 
@@ -139,6 +149,7 @@ class Mesh_westfirst(SimpleTopology):
                     int_links.append(IntLink(link_id=link_count,
                                              src_node=routers[north_out],
                                              dst_node=routers[south_in],
+                                             latency = link_latency,
                                              weight=2))
                     link_count += 1
 
@@ -151,6 +162,7 @@ class Mesh_westfirst(SimpleTopology):
                     int_links.append(IntLink(link_id=link_count,
                                              src_node=routers[south_out],
                                              dst_node=routers[north_in],
+                                             latency = link_latency,
                                              weight=2))
                     link_count += 1
 
diff --git a/configs/topologies/Pt2Pt.py b/configs/topologies/Pt2Pt.py
index 006d00c2f..6cbf5ad85 100644
--- a/configs/topologies/Pt2Pt.py
+++ b/configs/topologies/Pt2Pt.py
@@ -42,11 +42,22 @@ class Pt2Pt(SimpleTopology):
     def makeTopology(self, options, network, IntLink, ExtLink, Router):
         nodes = self.nodes
 
-        # Create an individual router for each controller, and connect all to all.
-        routers = [Router(router_id=i) for i in range(len(nodes))]
+        # default values for link latency and router latency.
+        # Can be over-ridden on a per link/router basis
+        link_latency = options.link_latency # used by simple and garnet
+        router_latency = options.router_latency # only used by garnet
+
+        # Create an individual router for each controller,
+        # and connect all to all.
+        # Since this is a high-radix router, router_latency should
+        # accordingly be set to a higher value than the default
+        # (which is 1 for mesh routers)
+        routers = [Router(router_id=i, latency = router_latency) \
+            for i in range(len(nodes))]
         network.routers = routers
 
-        ext_links = [ExtLink(link_id=i, ext_node=n, int_node=routers[i])
+        ext_links = [ExtLink(link_id=i, ext_node=n, int_node=routers[i],
+                     latency = link_latency)
                     for (i, n) in enumerate(nodes)]
         network.ext_links = ext_links
 
@@ -58,6 +69,7 @@ class Pt2Pt(SimpleTopology):
                     link_count += 1
                     int_links.append(IntLink(link_id=link_count,
                                              src_node=routers[i],
-                                             dst_node=routers[j]))
+                                             dst_node=routers[j],
+                                             latency = link_latency))
 
         network.int_links = int_links