ruby: record fully busy cycle with in the controller

This patch does several things. First, the counter for fully busy cycles for a
controller is now kept with in the controller, instead of being part of the profiler.
Second, the topology class no longer keeps an array of controllers which was only
used for printing stats. Instead, ruby system will now ask each controller to print
the stats. Thirdly, the statistical variable for recording how many different types
were created is being moved in to the controller from the profiler. Note that for
printing, the profiler will collate results from different controllers.

16 files changed, 155 insertions, 106 deletions

diff --git a/src/mem/protocol/MOESI_CMP_directory.slicc b/src/mem/protocol/MOESI_CMP_directory.slicc
index 0bba349b5..215a8016f 100644
--- a/src/mem/protocol/MOESI_CMP_directory.slicc
+++ b/src/mem/protocol/MOESI_CMP_directory.slicc
@@ -1,7 +1,7 @@
 protocol "MOESI_CMP_directory";
 include "RubySlicc_interfaces.slicc";
 include "MOESI_CMP_directory-msg.sm";
-include "MOESI_CMP_directory-L2cache.sm";
 include "MOESI_CMP_directory-L1cache.sm";
+include "MOESI_CMP_directory-L2cache.sm";
 include "MOESI_CMP_directory-dma.sm";
 include "MOESI_CMP_directory-dir.sm";
diff --git a/src/mem/ruby/common/Global.cc b/src/mem/ruby/common/Global.cc
index a2ed4e6ae..407e37307 100644
--- a/src/mem/ruby/common/Global.cc
+++ b/src/mem/ruby/common/Global.cc
@@ -28,4 +28,7 @@
 
 #include "mem/ruby/common/Global.hh"
 
+using namespace std;
+
 RubySystem* g_system_ptr = 0;
+vector<map<uint32_t, AbstractController *> > g_abs_controls;
diff --git a/src/mem/ruby/common/Global.hh b/src/mem/ruby/common/Global.hh
index d6597be89..671f423f0 100644
--- a/src/mem/ruby/common/Global.hh
+++ b/src/mem/ruby/common/Global.hh
@@ -29,10 +29,16 @@
 #ifndef __MEM_RUBY_COMMON_GLOBAL_HH__
 #define __MEM_RUBY_COMMON_GLOBAL_HH__
 
+#include <map>
+#include <vector>
+
 #include "base/str.hh"
 
 class RubySystem;
 extern RubySystem* g_system_ptr;
 
+class AbstractController;
+extern std::vector<std::map<uint32_t, AbstractController *> > g_abs_controls;
+
 #endif // __MEM_RUBY_COMMON_GLOBAL_HH__
 
diff --git a/src/mem/ruby/network/Topology.cc b/src/mem/ruby/network/Topology.cc
index 7ccc9e3e2..bfda3c49b 100644
--- a/src/mem/ruby/network/Topology.cc
+++ b/src/mem/ruby/network/Topology.cc
@@ -89,8 +89,7 @@ Topology::Topology(const Params *p)
         AbstractController *abs_cntrl = ext_link->params()->ext_node;
         BasicRouter *router = ext_link->params()->int_node;
 
-        // Store the controller and ExtLink pointers for later
-        m_controller_vector.push_back(abs_cntrl);
+        // Store the ExtLink pointers for later
         m_ext_link_vector.push_back(ext_link);
 
         int ext_idx1 = abs_cntrl->params()->cntrl_id;
@@ -265,22 +264,6 @@ Topology::makeLink(Network *net, SwitchID src, SwitchID dest,
     }
 }
 
-void
-Topology::printStats(std::ostream& out) const
-{
-    for (int cntrl = 0; cntrl < m_controller_vector.size(); cntrl++) {
-        m_controller_vector[cntrl]->printStats(out);
-    }
-}
-
-void
-Topology::clearStats()
-{
-    for (int cntrl = 0; cntrl < m_controller_vector.size(); cntrl++) {
-        m_controller_vector[cntrl]->clearStats();
-    }
-}
-
 // The following all-pairs shortest path algorithm is based on the
 // discussion from Cormen et al., Chapter 26.1.
 void
diff --git a/src/mem/ruby/network/Topology.hh b/src/mem/ruby/network/Topology.hh
index 578340ab9..d37341ea1 100644
--- a/src/mem/ruby/network/Topology.hh
+++ b/src/mem/ruby/network/Topology.hh
@@ -78,8 +78,6 @@ class Topology : public SimObject
     void initNetworkPtr(Network* net_ptr);
 
     const std::string getName() { return m_name; }
-    void printStats(std::ostream& out) const;
-    void clearStats();
     void print(std::ostream& out) const { out << "[Topology]"; }
 
   protected:
@@ -99,7 +97,6 @@ class Topology : public SimObject
     NodeID m_nodes;
     int m_number_of_switches;
 
-    std::vector<AbstractController*> m_controller_vector;
     std::vector<BasicExtLink*> m_ext_link_vector;
     std::vector<BasicIntLink*> m_int_link_vector;
 
diff --git a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
index e3196a0e7..b66a96989 100644
--- a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
+++ b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc
@@ -146,7 +146,6 @@ BaseGarnetNetwork::printStats(ostream& out) const
     printPerformanceStats(out);
     printLinkStats(out);
     printPowerStats(out);
-    m_topology_ptr->printStats(out);
 }
 
 void
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc
index 9df9ed3a5..9a8201a05 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.cc
+++ b/src/mem/ruby/network/simple/SimpleNetwork.cc
@@ -304,7 +304,6 @@ SimpleNetwork::printStats(ostream& out) const
     for (int i = 0; i < m_switch_ptr_vector.size(); i++) {
         m_switch_ptr_vector[i]->printStats(out);
     }
-    m_topology_ptr->printStats(out);
 }
 
 void
@@ -313,7 +312,6 @@ SimpleNetwork::clearStats()
     for (int i = 0; i < m_switch_ptr_vector.size(); i++) {
         m_switch_ptr_vector[i]->clearStats();
     }
-    m_topology_ptr->clearStats();
 }
 
 void
diff --git a/src/mem/ruby/profiler/Profiler.cc b/src/mem/ruby/profiler/Profiler.cc
index 9a01ce45a..546934d52 100644
--- a/src/mem/ruby/profiler/Profiler.cc
+++ b/src/mem/ruby/profiler/Profiler.cc
@@ -171,6 +171,59 @@ Profiler::print(ostream& out) const
 }
 
 void
+Profiler::printRequestProfile(ostream &out)
+{
+    out << "Request vs. RubySystem State Profile" << endl;
+    out << "--------------------------------" << endl;
+    out << endl;
+
+    map<string, uint64_t> m_requestProfileMap;
+    uint64_t m_requests = 0;
+
+    for (uint32_t i = 0; i < MachineType_NUM; i++) {
+        for (map<uint32_t, AbstractController*>::iterator it =
+                  g_abs_controls[i].begin();
+             it != g_abs_controls[i].end(); ++it) {
+
+            AbstractController *ctr = (*it).second;
+            map<string, uint64_t> mp = ctr->getRequestProfileMap();
+
+            for (map<string, uint64_t>::iterator jt = mp.begin();
+                 jt != mp.end(); ++jt) {
+
+                map<string, uint64_t>::iterator kt =
+                    m_requestProfileMap.find((*jt).first);
+                if (kt != m_requestProfileMap.end()) {
+                    (*kt).second += (*jt).second;
+                } else {
+                    m_requestProfileMap[(*jt).first] = (*jt).second;
+                }
+            }
+
+            m_requests += ctr->getRequestCount();
+        }
+    }
+
+    map<string, uint64_t>::const_iterator i = m_requestProfileMap.begin();
+    map<string, uint64_t>::const_iterator end = m_requestProfileMap.end();
+    for (; i != end; ++i) {
+        const string &key = i->first;
+        uint64_t count = i->second;
+
+        double percent = (100.0 * double(count)) / double(m_requests);
+        vector<string> items;
+        tokenize(items, key, ':');
+        vector<string>::iterator j = items.begin();
+        vector<string>::iterator end = items.end();
+        for (; j != end; ++i)
+            out << setw(10) << *j;
+        out << setw(11) << count;
+        out << setw(14) << percent << endl;
+    }
+    out << endl;
+}
+
+void
 Profiler::printStats(ostream& out, bool short_stats)
 {
     out << endl;
@@ -237,13 +290,17 @@ Profiler::printStats(ostream& out, bool short_stats)
 
     if (!short_stats) {
         out << "Busy Controller Counts:" << endl;
-        for (int i = 0; i < MachineType_NUM; i++) {
-            int size = MachineType_base_count((MachineType)i);
-            for (int j = 0; j < size; j++) {
+        for (uint32_t i = 0; i < MachineType_NUM; i++) {
+            uint32_t size = MachineType_base_count((MachineType)i);
+
+            for (uint32_t j = 0; j < size; j++) {
                 MachineID machID;
                 machID.type = (MachineType)i;
                 machID.num = j;
-                out << machID << ":" << m_busyControllerCount[i][j] << "  ";
+
+                AbstractController *ctr =
+                    (*(g_abs_controls[i].find(j))).second;
+                out << machID << ":" << ctr->getFullyBusyCycles() << "  ";
                 if ((j + 1) % 8 == 0) {
                     out << endl;
                 }
@@ -365,27 +422,7 @@ Profiler::printStats(ostream& out, bool short_stats)
     }
 
     if (!short_stats) {
-        out << "Request vs. RubySystem State Profile" << endl;
-        out << "--------------------------------" << endl;
-        out << endl;
-
-        map<string, int>::const_iterator i = m_requestProfileMap.begin();
-        map<string, int>::const_iterator end = m_requestProfileMap.end();
-        for (; i != end; ++i) {
-            const string &key = i->first;
-            int count = i->second;
-
-            double percent = (100.0 * double(count)) / double(m_requests);
-            vector<string> items;
-            tokenize(items, key, ':');
-            vector<string>::iterator j = items.begin();
-            vector<string>::iterator end = items.end();
-            for (; j != end; ++i)
-                out << setw(10) << *j;
-            out << setw(11) << count;
-            out << setw(14) << percent << endl;
-        }
-        out << endl;
+        printRequestProfile(out);
 
         out << "filter_action: " << m_filter_action_histogram << endl;
 
@@ -449,14 +486,6 @@ Profiler::clearStats()
         }
     }
 
-    m_busyControllerCount.resize(MachineType_NUM); // all machines
-    for (int i = 0; i < MachineType_NUM; i++) {
-        int size = MachineType_base_count((MachineType)i);
-        m_busyControllerCount[i].resize(size);
-        for (int j = 0; j < size; j++) {
-            m_busyControllerCount[i][j] = 0;
-        }
-    }
     m_busyBankCount = 0;
 
     m_delayedCyclesHistogram.clear();
@@ -511,12 +540,6 @@ Profiler::clearStats()
     m_cache_to_cache = 0;
     m_memory_to_cache = 0;
 
-    // clear HashMaps
-    m_requestProfileMap.clear();
-
-    // count requests profiled
-    m_requests = 0;
-
     m_outstanding_requests.clear();
     m_outstanding_persistent_requests.clear();
 
@@ -581,23 +604,6 @@ Profiler::profileMsgDelay(uint32_t virtualNetwork, Time delayCycles)
     }
 }
 
-// profiles original cache requests including PUTs
-void
-Profiler::profileRequest(const string& requestStr)
-{
-    m_requests++;
-
-    // if it doesn't exist, conveniently, it will be created with the
-    // default value which is 0
-    m_requestProfileMap[requestStr]++;
-}
-
-void
-Profiler::controllerBusy(MachineID machID)
-{
-    m_busyControllerCount[(int)machID.type][(int)machID.num]++;
-}
-
 void
 Profiler::profilePFWait(Time waitTime)
 {
diff --git a/src/mem/ruby/profiler/Profiler.hh b/src/mem/ruby/profiler/Profiler.hh
index 5f78f279b..5b370de54 100644
--- a/src/mem/ruby/profiler/Profiler.hh
+++ b/src/mem/ruby/profiler/Profiler.hh
@@ -171,6 +171,9 @@ class Profiler : public SimObject
     bool getAllInstructions() { return m_all_instructions; }
 
   private:
+    void printRequestProfile(std::ostream &out);
+
+  private:
     // Private copy constructor and assignment operator
     Profiler(const Profiler& obj);
     Profiler& operator=(const Profiler& obj);
@@ -187,7 +190,6 @@ class Profiler : public SimObject
     Time m_ruby_start;
     time_t m_real_time_start_time;
 
-    std::vector<std::vector<int64_t> > m_busyControllerCount;
     int64_t m_busyBankCount;
     Histogram m_multicast_retry_histogram;
 
@@ -234,9 +236,6 @@ class Profiler : public SimObject
     Histogram m_average_latency_estimate;
 
     m5::hash_set<Address> m_watch_address_set;
-    // counts all initiated cache request including PUTs
-    int m_requests;
-    std::map<std::string, int> m_requestProfileMap;
 
     //added by SS
     bool m_hot_lines;
diff --git a/src/mem/ruby/slicc_interface/AbstractController.cc b/src/mem/ruby/slicc_interface/AbstractController.cc
index 359512afc..adf411f82 100644
--- a/src/mem/ruby/slicc_interface/AbstractController.cc
+++ b/src/mem/ruby/slicc_interface/AbstractController.cc
@@ -30,7 +30,8 @@
 #include "mem/ruby/system/System.hh"
 
 AbstractController::AbstractController(const Params *p)
-    : ClockedObject(p), Consumer(this)
+    : ClockedObject(p), Consumer(this), m_fully_busy_cycles(0),
+    m_request_count(0)
 {
     m_version = p->version;
     m_transitions_per_cycle = p->transitions_per_cycle;
@@ -38,5 +39,27 @@ AbstractController::AbstractController(const Params *p)
     m_recycle_latency = p->recycle_latency;
     m_number_of_TBEs = p->number_of_TBEs;
     m_is_blocking = false;
-    p->ruby_system->registerAbstractController(this);
+}
+
+void
+AbstractController::init()
+{
+    params()->ruby_system->registerAbstractController(this);
+}
+
+void
+AbstractController::clearStats()
+{
+    m_requestProfileMap.clear();
+    m_request_count = 0;
+}
+
+void
+AbstractController::profileRequest(const std::string &request)
+{
+    m_request_count++;
+
+    // if it doesn't exist, conveniently, it will be created with the
+    // default value which is 0
+    m_requestProfileMap[request]++;
 }
diff --git a/src/mem/ruby/slicc_interface/AbstractController.hh b/src/mem/ruby/slicc_interface/AbstractController.hh
index 40356cac5..0e3af44a1 100644
--- a/src/mem/ruby/slicc_interface/AbstractController.hh
+++ b/src/mem/ruby/slicc_interface/AbstractController.hh
@@ -51,6 +51,7 @@ class AbstractController : public ClockedObject, public Consumer
   public:
     typedef RubyControllerParams Params;
     AbstractController(const Params *p);
+    void init();
     const Params *params() const { return (const Params *)_params; }
     virtual MessageBuffer* getMandatoryQueue() const = 0;
     virtual const int & getVersion() const = 0;
@@ -84,12 +85,22 @@ class AbstractController : public ClockedObject, public Consumer
     virtual void enqueuePrefetch(const Address&, const RubyRequestType&)
     { fatal("Prefetches not implemented!");}
 
+  public:
+    MachineID getMachineID() const { return m_machineID; }
+    uint64_t getFullyBusyCycles() const { return m_fully_busy_cycles; }
+    uint64_t getRequestCount() const { return m_request_count; }
+    const std::map<std::string, uint64_t>& getRequestProfileMap() const
+    { return m_requestProfileMap; }
+
+  protected:
+    //! Profiles original cache requests including PUTs
+    void profileRequest(const std::string &request);
+
   protected:
     int m_transitions_per_cycle;
     int m_buffer_size;
     int m_recycle_latency;
     std::string m_name;
-    std::map<std::string, std::string> m_cfg;
     NodeID m_version;
     Network* m_net_ptr;
     MachineID m_machineID;
@@ -101,6 +112,15 @@ class AbstractController : public ClockedObject, public Consumer
     int m_max_in_port_rank;
     int m_cur_in_port_rank;
     int m_number_of_TBEs;
+
+    //! Counter for the number of cycles when the transitions carried out
+    //! were equal to the maximum allowed
+    uint64_t m_fully_busy_cycles;
+
+    //! Map for couting requests of different types. The controller should
+    //! call requisite function for updating the count.
+    std::map<std::string, uint64_t> m_requestProfileMap;
+    uint64_t m_request_count;
 };
 
 #endif // __MEM_RUBY_SLICC_INTERFACE_ABSTRACTCONTROLLER_HH__
diff --git a/src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.cc b/src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.cc
index 2dd8671a1..b8503c2cb 100644
--- a/src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.cc
+++ b/src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.cc
@@ -36,16 +36,6 @@
 using namespace std;
 
 void
-profile_request(const string& L1CacheState, const string& L2CacheState,
-                const string& directoryState, const string& requestType)
-{
-    string requestStr = L1CacheState + ":" + L2CacheState + ":" +
-        directoryState + ":" + requestType;
-
-    g_system_ptr->getProfiler()->profileRequest(requestStr);
-}
-
-void
 profile_outstanding_request(int outstanding)
 {
     g_system_ptr->getProfiler()->profileOutstandingRequest(outstanding);
diff --git a/src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.hh b/src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.hh
index 799671435..1796d9442 100644
--- a/src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.hh
+++ b/src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.hh
@@ -46,10 +46,6 @@ void profile_outstanding_persistent_request(int outstanding);
 void profile_outstanding_request(int outstanding);
 void profile_sharing(const Address& addr, AccessType type, NodeID requestor,
                      const Set& sharers, const Set& owner);
-void profile_request(const std::string& L1CacheStateStr,
-                     const std::string& L2CacheStateStr,
-                     const std::string& directoryStateStr,
-                     const std::string& requestTypeStr);
 void profile_miss(const RubyRequest& msg, NodeID id);
 void profile_token_retry(const Address& addr, AccessType type, int count);
 void profile_filter_action(int action);
diff --git a/src/mem/ruby/system/MachineID.hh b/src/mem/ruby/system/MachineID.hh
index 18beac5d8..0bcd10efc 100644
--- a/src/mem/ruby/system/MachineID.hh
+++ b/src/mem/ruby/system/MachineID.hh
@@ -38,7 +38,11 @@
 struct MachineID
 {
     MachineType type;
-    int num;  // range: 0 ... number of this machine's components in system - 1
+    //! range: 0 ... number of this machine's components in system - 1
+    uint32_t num;
+
+    MachineType getType() const { return type; }
+    uint32_t getNum() const { return num; }
 };
 
 inline std::string
diff --git a/src/mem/ruby/system/System.cc b/src/mem/ruby/system/System.cc
index 01e50ef0b..d2c7d357b 100644
--- a/src/mem/ruby/system/System.cc
+++ b/src/mem/ruby/system/System.cc
@@ -72,7 +72,6 @@ RubySystem::RubySystem(const Params *p)
         m_memory_size_bits = ceilLog2(m_memory_size_bytes);
     }
 
-    g_system_ptr = this;
     if (p->no_mem_vec) {
         m_mem_vec_ptr = NULL;
     } else {
@@ -86,6 +85,12 @@ RubySystem::RubySystem(const Params *p)
 
     m_warmup_enabled = false;
     m_cooldown_enabled = false;
+
+    // Setup the global variables used in Ruby
+    g_system_ptr = this;
+
+    // Resize to the size of different machine types
+    g_abs_controls.resize(MachineType_NUM);
 }
 
 void
@@ -111,6 +116,9 @@ void
 RubySystem::registerAbstractController(AbstractController* cntrl)
 {
   m_abs_cntrl_vec.push_back(cntrl);
+
+  MachineID id = cntrl->getMachineID();
+  g_abs_controls[id.getType()][id.getNum()] = cntrl;
 }
 
 void
@@ -144,6 +152,15 @@ RubySystem::printStats(ostream& out)
 
     m_profiler_ptr->printStats(out);
     m_network_ptr->printStats(out);
+
+    for (uint32_t i = 0;i < g_abs_controls.size(); ++i) {
+        for (map<uint32_t, AbstractController *>::iterator it =
+                g_abs_controls[i].begin();
+             it != g_abs_controls[i].end(); ++it) {
+
+            ((*it).second)->printStats(out);
+        }
+    }
 }
 
 void
@@ -397,6 +414,9 @@ RubySystem::resetStats()
 {
     m_profiler_ptr->clearStats();
     m_network_ptr->clearStats();
+    for (uint32_t cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
+        m_abs_cntrl_vec[cntrl]->clearStats();
+    }
 }
 
 bool
diff --git a/src/mem/slicc/symbols/StateMachine.py b/src/mem/slicc/symbols/StateMachine.py
index c676574cc..d1e7dc147 100644
--- a/src/mem/slicc/symbols/StateMachine.py
+++ b/src/mem/slicc/symbols/StateMachine.py
@@ -663,7 +663,11 @@ $vid->setDescription("[Version " + to_string(m_version) + ", ${ident}, name=${{v
                 code('m_profiler.possibleTransition($state, $event);')
 
         code.dedent()
-        code('}')
+        code('''
+    AbstractController::init();
+    clearStats();
+}
+''')
 
         has_mandatory_q = False
         for port in self.in_ports:
@@ -839,6 +843,7 @@ void $c_ident::clearStats() {
 
         code('''
     m_profiler.clearStats();
+    AbstractController::clearStats();
 }
 ''')
 
@@ -1047,7 +1052,7 @@ ${ident}_Controller::wakeup()
         assert(counter <= m_transitions_per_cycle);
         if (counter == m_transitions_per_cycle) {
             // Count how often we are fully utilized
-            g_system_ptr->getProfiler()->controllerBusy(m_machineID);
+            m_fully_busy_cycles++;
 
             // Wakeup in another cycle and try again
             scheduleEvent(1);