path: root/util/config/m5config.py

from __future__ import generators

import os
import re
import sys

#####################################################################
#
# M5 Python Configuration Utility
#
# The basic idea is to write simple Python programs that build Python
# objects corresponding to M5 SimObjects for the deisred simulation
# configuration.  For now, the Python emits a .ini file that can be
# parsed by M5.  In the future, some tighter integration between M5
# and the Python interpreter may allow bypassing the .ini file.
#
# Each SimObject class in M5 is represented by a Python class with the
# same name.  The Python inheritance tree mirrors the M5 C++ tree
# (e.g., SimpleCPU derives from BaseCPU in both cases, and all
# SimObjects inherit from a single SimObject base class).  To specify
# an instance of an M5 SimObject in a configuration, the user simply
# instantiates the corresponding Python object.  The parameters for
# that SimObject are given by assigning to attributes of the Python
# object, either using keyword assignment in the constructor or in
# separate assignment statements.  For example:
#
# cache = BaseCache('my_cache', root, size=64*K)
# cache.hit_latency = 3
# cache.assoc = 8
#
# (The first two constructor arguments specify the name of the created
# cache and its parent node in the hierarchy.)
#
# The magic lies in the mapping of the Python attributes for SimObject
# classes to the actual SimObject parameter specifications.  This
# allows parameter validity checking in the Python code.  Continuing
# the example above, the statements "cache.blurfl=3" or
# "cache.assoc='hello'" would both result in runtime errors in Python,
# since the BaseCache object has no 'blurfl' parameter and the 'assoc'
# parameter requires an integer, respectively.  This magic is done
# primarily by overriding the special __setattr__ method that controls
# assignment to object attributes.
#
# The Python module provides another class, ConfigNode, which is a
# superclass of SimObject.  ConfigNode implements the parent/child
# relationship for building the configuration hierarchy tree.
# Concrete instances of ConfigNode can be used to group objects in the
# hierarchy, but do not correspond to SimObjects themselves (like a
# .ini section with "children=" but no "type=".
#
# Once a set of Python objects have been instantiated in a hierarchy,
# calling 'instantiate(obj)' (where obj is the root of the hierarchy)
# will generate a .ini file.  See simple-4cpu.py for an example
# (corresponding to m5-test/simple-4cpu.ini).
#
#####################################################################

#####################################################################
#
# ConfigNode/SimObject classes
#
# The Python class hierarchy rooted by ConfigNode (which is the base
# class of SimObject, which in turn is the base class of all other M5
# SimObject classes) has special attribute behavior.  In general, an
# object in this hierarchy has three categories of attribute-like
# things:
#
# 1. Regular Python methods and variables.  These must start with an
# underscore to be treated normally.
#
# 2. SimObject parameters.  These values are stored as normal Python
# attributes, but all assignments to these attributes are checked
# against the pre-defined set of parameters stored in the class's
# _param_dict dictionary.  Assignments to attributes that do not
# correspond to predefined parameters, or that are not of the correct
# type, incur runtime errors.
#
# 3. Hierarchy children.  The child nodes of a ConfigNode are stored
# in the node's _children dictionary, but can be accessed using the
# Python attribute dot-notation (just as they are printed out by the
# simulator).  Children cannot be created using attribute assigment;
# they must be added by specifying the parent node in the child's
# constructor or using the '+=' operator.

# The SimObject parameters are the most complex, for a few reasons.
# First, both parameter descriptions and parameter values are
# inherited.  Thus parameter description lookup must go up the
# inheritance chain like normal attribute lookup, but this behavior
# must be explicitly coded since the lookup occurs in each class's
# _param_dict attribute.  Second, because parameter values can be set
# on SimObject classes (to implement default values), the parameter
# checking behavior must be enforced on class attribute assignments as
# well as instance attribute assignments.  Finally, because we allow
# class specialization via inheritance (e.g., see the L1Cache class in
# the simple-4cpu.py example), we must do parameter checking even on
# class instantiation.  To provide all these features, we use a
# metaclass to define most of the SimObject parameter behavior for
# this class hierarchy.
#
#####################################################################

# The metaclass for ConfigNode (and thus for everything that dervies
# from ConfigNode, including SimObject).  This class controls how new
# classes that derive from ConfigNode are instantiated, and provides
# inherited class behavior (just like a class controls how instances
# of that class are instantiated, and provides inherited instance
# behavior).
class MetaConfigNode(type):

    # __new__ is called before __init__, and is where the statements
    # in the body of the class definition get loaded into the class's
    # __dict__.  We intercept this to filter out parameter assignments
    # and only allow "private" attributes to be passed to the base
    # __new__ (starting with underscore).
    def __new__(cls, name, bases, dict):
        priv_keys = [k for k in dict.iterkeys() if k.startswith('_')]
        priv_dict = {}
        for k in priv_keys: priv_dict[k] = dict[k]; del dict[k]
        # entries left in dict will get passed to __init__, where we'll
        # deal with them as params.
        return super(MetaConfigNode, cls).__new__(cls, name, bases, priv_dict)

    # initialization: start out with an empty param dict (makes life
    # simpler if we can assume _param_dict is always valid).  Also
    # build inheritance list to simplify searching for inherited
    # params.  Finally set parameters specified in class definition
    # (if any).
    def __init__(cls, name, bases, dict):
        super(MetaConfigNode, cls).__init__(cls, name, bases, {})
        # initialize _param_dict to empty
        cls._param_dict = {}
        # __mro__ is the ordered list of classes Python uses for
        # method resolution.  We want to pick out the ones that have a
        # _param_dict attribute for doing parameter lookups.
        cls._param_bases = \
                         [c for c in cls.__mro__ if hasattr(c, '_param_dict')]
        # initialize attributes with values from class definition
        for (pname, value) in dict.items():
            try:
                setattr(cls, pname, value)
            except Exception, exc:
                print "Error setting '%s' to '%s' on class '%s'\n" \
                      % (pname, value, cls.__name__), exc

    # set the class's parameter dictionary (called when loading
    # class descriptions)
    def set_param_dict(cls, param_dict):
        # should only be called once (current one should be empty one
        # from __init__)
        assert not cls._param_dict
        cls._param_dict = param_dict
        # initialize attributes with default values
        for (pname, param) in param_dict.items():
            try:
                setattr(cls, pname, param.default)
            except Exception, exc:
                print "Error setting '%s' default on class '%s'\n" \
                      % (pname, cls.__name__), exc

    # Lookup a parameter description by name in the given class.  Use
    # the _param_bases list defined in __init__ to go up the
    # inheritance hierarchy if necessary.
    def lookup_param(cls, param_name):
        for c in cls._param_bases:
            param = c._param_dict.get(param_name)
            if param: return param
        return None

    # Set attribute (called on foo.attr_name = value when foo is an
    # instance of class cls).
    def __setattr__(cls, attr_name, value):
        # normal processing for private attributes
        if attr_name.startswith('_'):
            object.__setattr__(cls, attr_name, value)
            return
        # no '_': must be SimObject param
        param = cls.lookup_param(attr_name)
        if not param:
            raise AttributeError, \
                  "Class %s has no parameter %s" % (cls.__name__, attr_name)
        # It's ok: set attribute by delegating to 'object' class.
        # Note the use of param.make_value() to verify/canonicalize
        # the assigned value
        object.__setattr__(cls, attr_name, param.make_value(value))

    # generator that iterates across all parameters for this class and
    # all classes it inherits from
    def all_param_names(cls):
        for c in cls._param_bases:
            for p in c._param_dict.iterkeys():
                yield p

# The ConfigNode class is the root of the special hierarchy.  Most of
# the code in this class deals with the configuration hierarchy itself
# (parent/child node relationships).
class ConfigNode(object):
    # Specify metaclass.  Any class inheriting from ConfigNode will
    # get this metaclass.
    __metaclass__ = MetaConfigNode

    # Constructor.  Since bare ConfigNodes don't have parameters, just
    # worry about the name and the parent/child stuff.
    def __init__(self, _name, _parent=None):
        # Type-check _name
        if type(_name) != str:
            if isinstance(_name, ConfigNode):
                # special case message for common error of trying to
                # coerce a SimObject to the wrong type
                raise TypeError, \
                      "Attempt to coerce %s to %s" \
                      % (_name.__class__.__name__, self.__class__.__name__)
            else:
                raise TypeError, \
                      "%s name must be string (was %s, %s)" \
                      % (self.__class__.__name__, _name, type(_name))
        # if specified, parent must be a subclass of ConfigNode
        if _parent != None and not isinstance(_parent, ConfigNode):
            raise TypeError, \
                  "%s parent must be ConfigNode subclass (was %s, %s)" \
                  % (self.__class__.__name__, _name, type(_name))
        self._name = _name
        self._parent = _parent
        self._children = {}
        if (_parent):
            _parent.__addChild(self)
        # Set up absolute path from root.
        if (_parent and _parent._path != 'Universe'):
            self._path = _parent._path + '.' + self._name
        else:
            self._path = self._name

    # When printing (e.g. to .ini file), just give the name.
    def __str__(self):
        return self._name

    # Catch attribute accesses that could be requesting children, and
    # satisfy them.  Note that __getattr__ is called only if the
    # regular attribute lookup fails, so private and parameter lookups
    # will already be satisfied before we ever get here.
    def __getattr__(self, name):
        try:
            return self._children[name]
        except KeyError:
            raise AttributeError, \
                  "Node '%s' has no attribute or child '%s'" \
                  % (self._name, name)

    # Set attribute.  All attribute assignments go through here.  Must
    # be private attribute (starts with '_') or valid parameter entry.
    # Basically identical to MetaConfigClass.__setattr__(), except
    # this handles instances rather than class attributes.
    def __setattr__(self, attr_name, value):
        if attr_name.startswith('_'):
            object.__setattr__(self, attr_name, value)
            return
        # not private; look up as param
        param = self.__class__.lookup_param(attr_name)
        if not param:
            raise AttributeError, \
                  "Class %s has no parameter %s" \
                  % (self.__class__.__name__, attr_name)
        # It's ok: set attribute by delegating to 'object' class.
        # Note the use of param.make_value() to verify/canonicalize
        # the assigned value
        object.__setattr__(self, attr_name, param.make_value(value))

    # Add a child to this node.
    def __addChild(self, new_child):
        # set child's parent before calling this function
        assert new_child._parent == self
        if not isinstance(new_child, ConfigNode):
            raise TypeError, \
                  "ConfigNode child must also be of class ConfigNode"
        if new_child._name in self._children:
            raise AttributeError, \
                  "Node '%s' already has a child '%s'" \
                  % (self._name, new_child._name)
        self._children[new_child._name] = new_child

    # operator overload for '+='.  You can say "node += child" to add
    # # a child that was created with parent=None.  An early attempt
    # at # playing with syntax; turns out not to be that useful.
    def __iadd__(self, new_child):
        if new_child._parent != None:
            raise AttributeError, \
                  "Node '%s' already has a parent" % new_child._name
        new_child._parent = self
        self.__addChild(new_child)
        return self

    # Print instance info to .ini file.
    def _instantiate(self):
        print '[' + self._path + ']'	# .ini section header
        if self._children:
            # instantiate children in sorted order for backward
            # compatibility (else we can end up with cpu1 before cpu0).
            child_names = self._children.keys()
            child_names.sort()
            print 'children =',
            for child_name in child_names:
                print child_name,
            print
        self._instantiateParams()
        print
        # recursively dump out children
        if self._children:
            for child_name in child_names:
                self._children[child_name]._instantiate()

    # ConfigNodes have no parameters.  Overridden by SimObject.
    def _instantiateParams(self):
        pass

# SimObject is a minimal extension of ConfigNode, implementing a
# hierarchy node that corresponds to an M5 SimObject.  It prints out a
# "type=" line to indicate its SimObject class, prints out the
# assigned parameters corresponding to its class, and allows
# parameters to be set by keyword in the constructor.  Note that most
# of the heavy lifting for the SimObject param handling is done in the
# MetaConfigNode metaclass.

class SimObject(ConfigNode):
    # initialization: like ConfigNode, but handle keyword-based
    # parameter initializers.
    def __init__(self, _name, _parent=None, **params):
        ConfigNode.__init__(self, _name, _parent)
        for param, value in params.items():
            setattr(self, param, value)

    # print type and parameter values to .ini file
    def _instantiateParams(self):
        print "type =", self.__class__._name
        for pname in self.__class__.all_param_names():
            value = getattr(self, pname)
            if value != None:
                print pname, '=', value

#####################################################################
#
# Parameter description classes
#
# The _param_dict dictionary in each class maps parameter names to
# either a Param or a VectorParam object.  These objects contain the
# parameter description string, the parameter type, and the default
# value (loaded from the PARAM section of the .odesc files).  The
# make_value() method on these objects is used to force whatever value
# is assigned to the parameter to the appropriate type.
#
# Note that the default values are loaded into the class's attribute
# space when the parameter dictionary is initialized (in
# MetaConfigNode.set_param_dict()); after that point they aren't
# used.
#
#####################################################################

# Force parameter value (rhs of '=') to ptype (or None, which means
# not set).
def make_param_value(ptype, value):
    # nothing to do if None or already correct type
    if value == None or isinstance(value, ptype):
        return value
    # this type conversion will raise an exception if it's illegal
    return ptype(value)

# Regular parameter.
class Param(object):
    # Constructor.  E.g., Param(Int, "number of widgets", 5)
    def __init__(self, ptype, desc, default=None):
        self.ptype = ptype
        self.desc = desc
        self.default = default

    # Convert assigned value to appropriate type.
    def make_value(self, value):
        return make_param_value(self.ptype, value)

# The _VectorParamValue class is a wrapper for vector-valued
# parameters.  The leading underscore indicates that users shouldn't
# see this class; it's magically generated by VectorParam.  The
# parameter values are stored in the 'value' field as a Python list of
# whatever type the parameter is supposed to be.  The only purpose of
# storing these instead of a raw Python list is that we can override
# the __str__() method to not print out '[' and ']' in the .ini file.
class _VectorParamValue(object):
    def __init__(self, list):
        self.value = list

    def __str__(self):
        return ' '.join(map(str, self.value))

# Vector-valued parameter description.  Just like Param, except that
# the value is a vector (list) of the specified type instead of a
# single value.
class VectorParam(object):
    # Constructor.  The resulting parameter will be a list of ptype.
    def __init__(self, ptype, desc, default=None):
        self.ptype = ptype
        self.desc = desc
        self.default = default

    # Convert assigned value to appropriate type.  If the RHS is not a
    # list or tuple, it generates a single-element list.
    def make_value(self, value):
        if value == None: return value
        if isinstance(value, list) or isinstance(value, tuple):
            # list: coerce each element into new list
            val_list = [make_param_value(self.ptype, v) for v in
                        iter(value)]
        else:
            # singleton: coerce & wrap in a list
            val_list = [ make_param_value(self.ptype, value) ]
        # wrap list in _VectorParamValue (see above)
        return _VectorParamValue(val_list)

#####################################################################
#
# Parameter Types
#
# Though native Python types could be used to specify parameter types
# (the 'ptype' field of the Param and VectorParam classes), it's more
# flexible to define our own set of types.  This gives us more control
# over how Python expressions are converted to values (via the
# __init__() constructor) and how these values are printed out (via
# the __str__() conversion method).  Eventually we'll need these types
# to correspond to distinct C++ types as well.
#
#####################################################################

# Integer parameter type.
class Int(object):
    # Constructor.  Value must be Python int or long (long integer).
    def __init__(self, value):
        t = type(value)
        if t == int or t == long:
            self.value = value
        else:
            raise TypeError, "Int param got value %s %s" % (repr(value), t)

    # Use Python string conversion.  Note that this puts an 'L' on the
    # end of long integers; we can strip that off here if it gives us
    # trouble.
    def __str__(self):
        return str(self.value)

# Counter, Addr, and Tick are just aliases for Int for now.
class Counter(Int):
    pass

class Addr(Int):
    pass

class Tick(Int):
    pass

# Boolean parameter type.
class Bool(object):

    # Constructor.  Typically the value will be one of the Python bool
    # constants True or False (or the aliases true and false below).
    # Also need to take integer 0 or 1 values since bool was not a
    # distinct type in Python 2.2.  Parse a bunch of boolean-sounding
    # strings too just for kicks.
    def __init__(self, value):
        t = type(value)
        if t == bool:
            self.value = value
        elif t == int or t == long:
            if value == 1:
                self.value = True
            elif value == 0:
                self.value = False
        elif t == str:
            v = value.lower()
            if v == "true" or v == "t" or v == "yes" or v == "y":
                self.value = True
            elif v == "false" or v == "f" or v == "no" or v == "n":
                self.value = False
        # if we didn't set it yet, it must not be something we understand
        if not hasattr(self, 'value'):
            raise TypeError, "Bool param got value %s %s" % (repr(value), t)

    # Generate printable string version.
    def __str__(self):
        if self.value: return "true"
        else: return "false"

# String-valued parameter.
class String(object):
    # Constructor.  Value must be Python string.
    def __init__(self, value):
        t = type(value)
        if t == str:
            self.value = value
        else:
            raise TypeError, "String param got value %s %s" % (repr(value), t)

    # Generate printable string version.  Not too tricky.
    def __str__(self):
        return self.value


# Enumerated types are a little more complex.  The user specifies the
# type as Enum(foo) where foo is either a list or dictionary of
# alternatives (typically strings, but not necessarily so).  (In the
# long run, the integer value of the parameter will be the list index
# or the corresponding dictionary value.  For now, since we only check
# that the alternative is valid and then spit it into a .ini file,
# there's not much point in using the dictionary.)

# What Enum() must do is generate a new type encapsulating the
# provided list/dictionary so that specific values of the parameter
# can be instances of that type.  We define two hidden internal
# classes (_ListEnum and _DictEnum) to serve as base classes, then
# derive the new type from the appropriate base class on the fly.


# Base class for list-based Enum types.
class _ListEnum(object):
    # Constructor.  Value must be a member of the type's map list.
    def __init__(self, value):
        if value in self.map:
            self.value = value
            self.index = self.map.index(value)
        else:
            raise TypeError, "Enum param got bad value '%s' (not in %s)" \
                  % (value, self.map)

    # Generate printable string version of value.
    def __str__(self):
        return str(self.value)

class _DictEnum(object):
    # Constructor.  Value must be a key in the type's map dictionary.
    def __init__(self, value):
        if value in self.map:
            self.value = value
            self.index = self.map[value]
        else:
            raise TypeError, "Enum param got bad value '%s' (not in %s)" \
                  % (value, self.map.keys())

    # Generate printable string version of value.
    def __str__(self):
        return str(self.value)

# Enum metaclass... calling Enum(foo) generates a new type (class)
# that derives from _ListEnum or _DictEnum as appropriate.
class Enum(type):
    # counter to generate unique names for generated classes
    counter = 1

    def __new__(cls, map):
        if isinstance(map, dict):
            base = _DictEnum
            keys = map.keys()
        elif isinstance(map, list):
            base = _ListEnum
            keys = map
        else:
            raise TypeError, "Enum map must be list or dict (got %s)" % map
        classname = "Enum%04d" % Enum.counter
        Enum.counter += 1
        # New class derives from selected base, and gets a 'map'
        # attribute containing the specified list or dict.
        return type.__new__(cls, classname, (base,), { 'map': map })


#
# "Constants"... handy aliases for various values.
#

# For compatibility with C++ bool constants.
false = False
true = True

# Some memory range specifications use this as a default upper bound.
MAX_ADDR = 2 ** 63

# For power-of-two sizing, e.g. 64*K gives an integer value 65536.
K = 1024
M = K*K
G = K*M

#####################################################################
#
# Object description loading.
#
# The final step is to define the classes corresponding to M5 objects
# and their parameters.  These classes are described in .odesc files
# in the source tree.  This code walks the tree to find those files
# and loads up the descriptions (by evaluating them in pieces as
# Python code).
#
#
# Because SimObject classes inherit from other SimObject classes, and
# can use arbitrary other SimObject classes as parameter types, we
# have to do this in three steps:
#
# 1. Walk the tree to find all the .odesc files.  Note that the base
# of the filename *must* match the class name.  This step builds a
# mapping from class names to file paths.
#
# 2. Start generating empty class definitions (via def_class()) using
# the OBJECT field of the .odesc files to determine inheritance.
# def_class() recurses on demand to define needed base classes before
# derived classes.
#
# 3. Now that all of the classes are defined, go through the .odesc
# files one more time loading the parameter descriptions.
#
#####################################################################

# dictionary: maps object names to file paths
odesc_file = {}

# dictionary: maps object names to boolean flag indicating whether
# class definition was loaded yet.  Since SimObject is defined in
# m5.config.py, count it as loaded.
odesc_loaded = { 'SimObject': True }

# Find odesc files in namelist and initialize odesc_file and
# odesc_loaded dictionaries.  Called via os.path.walk() (see below).
def find_odescs(process, dirpath, namelist):
    # Prune out SCCS directories so we don't process s.*.odesc files.
    i = 0
    while i < len(namelist):
        if namelist[i] == "SCCS":
            del namelist[i]
        else:
            i = i + 1
    # Find .odesc files and record them.
    for name in namelist:
        if name.endswith('.odesc'):
            objname = name[:name.rindex('.odesc')]
            path = os.path.join(dirpath, name)
            if odesc_file.has_key(objname):
                print "Warning: duplicate object names:", \
                      odesc_file[objname], path
            odesc_file[objname] = path
            odesc_loaded[objname] = False


# Regular expression string for parsing .odesc files.
file_re_string = r'''
^OBJECT: \s* (\w+) \s* \( \s* (\w+) \s* \)
\s*
^PARAMS: \s*\n ( (\s+.*\n)* )
'''

# Compiled regular expression object.
file_re = re.compile(file_re_string, re.MULTILINE | re.VERBOSE)

# .odesc file parsing function.  Takes a filename and returns tuple of
# object name, object base, and parameter description section.
def parse_file(path):
    f = open(path, 'r').read()
    m = file_re.search(f)
    if not m:
        print "Can't parse", path
        sys.exit(1)
    return (m.group(1), m.group(2), m.group(3))

# Define SimObject class based on description in specified filename.
# Class itself is empty except for _name attribute; parameter
# descriptions will be loaded later.  Will recurse to define base
# classes as needed before defining specified class.
def def_class(path):
    # load & parse file
    (obj, parent, params) = parse_file(path)
    # check to see if base class is defined yet; define it if not
    if not odesc_loaded.has_key(parent):
        print "No .odesc file found for", parent
        sys.exit(1)
    if not odesc_loaded[parent]:
        def_class(odesc_file[parent])
    # define the class.
    s = "class %s(%s): _name = '%s'" % (obj, parent, obj)
    try:
        # execute in global namespace, so new class will be globally
        # visible
        exec s in globals()
    except Exception, exc:
        print "Object error in %s:" % path, exc
    # mark this file as loaded
    odesc_loaded[obj] = True

# Munge an arbitrary Python code string to get it to execute (mostly
# dealing with indentation).  Stolen from isa_parser.py... see
# comments there for a more detailed description.
def fixPythonIndentation(s):
    # get rid of blank lines first
    s = re.sub(r'(?m)^\s*\n', '', s);
    if (s != '' and re.match(r'[ \t]', s[0])):
        s = 'if 1:\n' + s
    return s

# Load parameter descriptions from .odesc file.  Object class must
# already be defined.
def def_params(path):
    # load & parse file
    (obj_name, parent_name, param_code) = parse_file(path)
    # initialize param dict
    param_dict = {}
    # execute parameter descriptions.
    try:
        # "in globals(), param_dict" makes exec use the current
        # globals as the global namespace (so all of the Param
        # etc. objects are visible) and param_dict as the local
        # namespace (so the newly defined parameter variables will be
        # entered into param_dict).
        exec fixPythonIndentation(param_code) in globals(), param_dict
    except Exception, exc:
        print "Param error in %s:" % path, exc
        return
    # Convert object name string to Python class object
    obj = eval(obj_name)
    # Set the object's parameter description dictionary (see MetaConfigNode).
    obj.set_param_dict(param_dict)


# Walk directory tree to find .odesc files.
# Someday we'll have to make the root path an argument instead of
# hard-coding it.  For now the assumption is you're running this in
# util/config.
root = '../..'
os.path.walk(root, find_odescs, None)

# Iterate through file dictionary and define classes.
for objname, path in odesc_file.iteritems():
    if not odesc_loaded[objname]:
        def_class(path)

# Iterate through files again and load parameters.
for path in odesc_file.itervalues():
    def_params(path)

#####################################################################

# The final hook to generate .ini files.  Called from configuration
# script once config is built.
def instantiate(*objs):
    for obj in objs:
        obj._instantiate()