ruby: fixed DirectoryMemory's numa_high_bit configuration

This fix includes the off-by-one bit selection bug for numa mapping.

3 files changed, 33 insertions, 8 deletions

diff --git a/configs/ruby/Ruby.py b/configs/ruby/Ruby.py
index 07a6632dd..71add4b61 100644
--- a/configs/ruby/Ruby.py
+++ b/configs/ruby/Ruby.py
@@ -27,6 +27,7 @@
 #
 # Authors: Brad Beckmann
 
+import math
 import m5
 from m5.objects import *
 from m5.defines import buildEnv
@@ -42,7 +43,8 @@ def define_options(parser):
 
     # ruby mapping options
     parser.add_option("--numa-high-bit", type="int", default=none,
-                      help="high order address bit to use for numa mapping")
+                      help="high order address bit to use for numa mapping. " \
+                           "0 = highest bit, not specified = lowest bit")
 
     # ruby sparse memory options
     parser.add_option("--use-map", action="store_true", default=False)
@@ -88,14 +90,32 @@ def create_system(options, system, piobus = None, dma_devices = []):
         network = SimpleNetwork(topology = net_topology)
 
     #
+    # Loop through the directory controlers.
     # Determine the total memory size of the ruby system and verify it is equal
     # to physmem.  However, if Ruby memory is using sparse memory in SE 
     # mode, then the system should not back-up the memory state with
     # the Memory Vector and thus the memory size bytes should stay at 0.
+    # Also set the numa bits to the appropriate values.
     #
     total_mem_size = MemorySize('0B')
+
+    dir_bits = int(math.log(options.num_dirs, 2))
+
+    if options.numa_high_bit:
+        numa_bit = options.numa_high_bit
+    else:
+        # if not specified, use the lowest bits above the block offest
+        if options.numa_high_bit == 0:
+            if dir_bits > 0:
+                # add 5 because bits 0-5 are the block offset
+                numa_bit = dir_bits + 5
+            else:
+                numa_bit = 6
+        
     for dir_cntrl in dir_cntrls:
         total_mem_size.value += dir_cntrl.directory.size.value
+        dir_cntrl.directory.numa_high_bit = numa_bit
+        
     physmem_size = long(system.physmem.range.second) - \
                      long(system.physmem.range.first) + 1
     assert(total_mem_size.value == physmem_size)
diff --git a/src/mem/ruby/common/Address.hh b/src/mem/ruby/common/Address.hh
index dc58d012f..23b683d69 100644
--- a/src/mem/ruby/common/Address.hh
+++ b/src/mem/ruby/common/Address.hh
@@ -187,7 +187,7 @@ Address::bitRemove(int small, int big) const
         physical_address_t higher_bits = m_address & mask;
 
         // Shift the valid high bits over the removed section
-        higher_bits = higher_bits >> (big - small);
+        higher_bits = higher_bits >> (big - small + 1);
         return (higher_bits | lower_bits);
     }
 }
diff --git a/src/mem/ruby/system/DirectoryMemory.cc b/src/mem/ruby/system/DirectoryMemory.cc
index 65b61c935..fbb48d7f5 100644
--- a/src/mem/ruby/system/DirectoryMemory.cc
+++ b/src/mem/ruby/system/DirectoryMemory.cc
@@ -71,7 +71,7 @@ DirectoryMemory::init()
     m_total_size_bytes += m_size_bytes;
 
     if (m_numa_high_bit == 0) {
-        m_numa_high_bit = RubySystem::getMemorySizeBits();
+        m_numa_high_bit = RubySystem::getMemorySizeBits() - 1;
     }
     assert(m_numa_high_bit != 0);
 }
@@ -125,7 +125,7 @@ DirectoryMemory::mapAddressToDirectoryVersion(PhysAddress address)
     if (m_num_directories_bits == 0)
         return 0;
 
-    uint64 ret = address.bitSelect(m_numa_high_bit - m_num_directories_bits,
+    uint64 ret = address.bitSelect(m_numa_high_bit - m_num_directories_bits + 1,
                                    m_numa_high_bit);
     return ret;
 }
@@ -140,10 +140,15 @@ DirectoryMemory::isPresent(PhysAddress address)
 uint64
 DirectoryMemory::mapAddressToLocalIdx(PhysAddress address)
 {
-    uint64 ret = address.bitRemove(m_numa_high_bit - m_num_directories_bits,
-                                   m_numa_high_bit)
-        >> (RubySystem::getBlockSizeBits());
-    return ret;
+    uint64 ret;
+    if (m_num_directories_bits > 0) {
+        ret = address.bitRemove(m_numa_high_bit - m_num_directories_bits + 1,
+                                m_numa_high_bit);
+    } else {
+        ret = address.getAddress();
+    }
+
+    return ret >> (RubySystem::getBlockSizeBits());
 }
 
 Directory_Entry&