From 6f38428abbe106d63318dd86bfa82a03b6b368ee Mon Sep 17 00:00:00 2001 From: Andreas Sandberg Date: Wed, 26 Jun 2019 18:58:24 +0100 Subject: stats: Add support for hierarchical stats This change makes the stat system aware of the hierarchical nature of stats. The aim is to achieve the following goals: * Make the SimObject hierarchy explicit in the stat system (i.e., get rid of name() + ".foo"). This makes stat naming less fragile and makes it possible to implement hierarchical formats like XML/HDF5/JSON in a clean way. * Make it more convenient to split stats into a separate struct/class that can be bound to a SimObject. This makes the namespace cleaner and makes stat accesses a bit more obvious. * Make it possible to build groups of stats in C++ that can be used in subcomponents in a SimObject (similar to what we do for checkpoint sections). This makes it easier to structure large components. * Enable partial stat dumps. Some of our internal users have been asking for this since a full stat dump can be large. * Enable better stat access from Python. This changeset implements solves the first three points by introducing a class (Stats::Group) that owns statistics belonging to the same object. SimObjects inherit from Stats::Group since they typically have statistics. New-style statistics need to be associated with a parent group at instantiation time. Instantiation typically sets the name and the description, other parameters need to be set by overriding Group::regStats() just like with legacy stats. Simple objects with scalar stats can typically avoid implementing regStats() altogether since the stat name and description are both specified in the constructor. For convenience reasons, statistics groups can be merged into other groups. This means that a SimObject can create a stat struct that inherits from Stats::Group and merge it into the parent group (SimObject). This can make the code cleaner since statistics tracking gets grouped into a single object. Stat visitors have a new API to expose the group structure. The Output::beginGroup(name) method is called at the beginning of a group and the Output::endGroup() method is called when all stats, and sub-groups, have been visited. Flat formats (e.g., the text format) typically need to maintain a stack to track the full path to a stat. Legacy, flat, statistics are still supported after applying this change. These stats don't belong to any group and stat visitors will not see a Output::beginGroup(name) call before their corresponding Output::visit() methods are called. Change-Id: I9025d61dfadeabcc8ecf30813ab2060def455648 Signed-off-by: Andreas Sandberg Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/19368 Tested-by: kokoro Reviewed-by: Jason Lowe-Power Reviewed-by: Daniel Carvalho --- src/python/m5/SimObject.py | 4 +-- src/python/m5/simulate.py | 3 +- src/python/m5/stats/__init__.py | 73 ++++++++++++++++++++++++++++++++++++----- 3 files changed, 67 insertions(+), 13 deletions(-) (limited to 'src/python/m5') diff --git a/src/python/m5/SimObject.py b/src/python/m5/SimObject.py index 317ae6c33..3d2f123b9 100644 --- a/src/python/m5/SimObject.py +++ b/src/python/m5/SimObject.py @@ -1086,7 +1086,7 @@ class SimObject(object): abstract = True cxx_header = "sim/sim_object.hh" - cxx_extra_bases = [ "Drainable", "Serializable" ] + cxx_extra_bases = [ "Drainable", "Serializable", "Stats::Group" ] eventq_index = Param.UInt32(Parent.eventq_index, "Event Queue Index") cxx_exports = [ @@ -1094,8 +1094,6 @@ class SimObject(object): PyBindMethod("initState"), PyBindMethod("memInvalidate"), PyBindMethod("memWriteback"), - PyBindMethod("regStats"), - PyBindMethod("resetStats"), PyBindMethod("regProbePoints"), PyBindMethod("regProbeListeners"), PyBindMethod("startup"), diff --git a/src/python/m5/simulate.py b/src/python/m5/simulate.py index f8da1fb32..1369fb5ed 100644 --- a/src/python/m5/simulate.py +++ b/src/python/m5/simulate.py @@ -124,7 +124,8 @@ def instantiate(ckpt_dir=None): for obj in root.descendants(): obj.init() # Do a third pass to initialize statistics - for obj in root.descendants(): obj.regStats() + stats._bindStatHierarchy(root) + root.regStats() # Do a fourth pass to initialize probe points for obj in root.descendants(): obj.regProbePoints() diff --git a/src/python/m5/stats/__init__.py b/src/python/m5/stats/__init__.py index 019c7ebb8..d9174d387 100644 --- a/src/python/m5/stats/__init__.py +++ b/src/python/m5/stats/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 ARM Limited +# Copyright (c) 2017, 2019 Arm Limited # All rights reserved. # # The license below extends only to copyright in the software and shall @@ -158,6 +158,29 @@ def initSimStats(): _m5.stats.initSimStats() _m5.stats.registerPythonStatsHandlers() +def _visit_groups(root, visitor): + for group in root.getStatGroups().values(): + visitor(group) + _visit_groups(group, visitor) + +def _visit_stats(root, visitor): + def for_each_stat(g): + for stat in g.getStats(): + visitor(g, stat) + _visit_groups(root, for_each_stat) + +def _bindStatHierarchy(root): + def _bind_obj(name, obj): + if m5.SimObject.isSimObjectVector(obj): + for idx, obj in enumerate(obj): + _bind_obj("{}{}".format(name, idx), obj) + else: + root.addStatGroup(name, obj.getCCObject()) + _bindStatHierarchy(obj) + + for name, obj in root._children.items(): + _bind_obj(name, obj) + names = [] stats_dict = {} stats_list = [] @@ -166,10 +189,7 @@ def enable(): enabled, all statistics must be created and initialized and once the package is enabled, no more statistics can be created.''' - global stats_list - stats_list = list(_m5.stats.statsList()) - - for stat in stats_list: + def check_stat(group, stat): if not stat.check() or not stat.baseCheck(): fatal("statistic '%s' (%d) was not properly initialized " \ "by a regStats() function\n", stat.name, stat.id) @@ -177,21 +197,57 @@ def enable(): if not (stat.flags & flags.display): stat.name = "__Stat%06d" % stat.id + + # Legacy stat + global stats_list + stats_list = list(_m5.stats.statsList()) + + for stat in stats_list: + check_stat(None, stat) + stats_list.sort(key=lambda s: s.name.split('.')) for stat in stats_list: stats_dict[stat.name] = stat stat.enable() + + # New stats + _visit_stats(Root.getInstance(), check_stat) + _visit_stats(Root.getInstance(), lambda g, s: s.enable()) + _m5.stats.enable(); def prepare(): '''Prepare all stats for data access. This must be done before dumping and serialization.''' + # Legacy stats for stat in stats_list: stat.prepare() + # New stats + _visit_stats(Root.getInstance(), lambda g, s: s.prepare()) + lastDump = 0 + +def _dump_to_visitor(visitor): + # Legacy stats + for stat in stats_list: + stat.visit(visitor) + + # New stats + def dump_group(group): + for stat in group.getStats(): + stat.visit(visitor) + + for n, g in group.getStatGroups().items(): + visitor.beginGroup(n) + dump_group(g) + visitor.endGroup() + + dump_group(Root.getInstance()) + + def dump(): '''Dump all statistics data to the registered outputs''' @@ -210,8 +266,7 @@ def dump(): for output in outputList: if output.valid(): output.begin() - for stat in stats_list: - stat.visit(output) + _dump_to_visitor(output) output.end() def reset(): @@ -220,9 +275,9 @@ def reset(): # call reset stats on all SimObjects root = Root.getInstance() if root: - for obj in root.descendants(): obj.resetStats() + root.resetStats() - # call any other registered stats reset callbacks + # call any other registered legacy stats reset callbacks for stat in stats_list: stat.reset() -- cgit v1.2.3