From f82d4e2364e9df683b6891c04efaa861093a50ec Mon Sep 17 00:00:00 2001 From: Nathan Binkert Date: Sat, 14 Jun 2008 20:19:49 -0700 Subject: python: Move various utility classes into a new m5.util package so they're all in the same place. This also involves having just one jobfile.py and moving it into the utils directory to avoid duplication. Lots of improvements to the utility as well. --HG-- rename : src/python/m5/attrdict.py => src/python/m5/util/attrdict.py rename : util/pbs/jobfile.py => src/python/m5/util/jobfile.py rename : src/python/m5/util.py => src/python/m5/util/misc.py rename : src/python/m5/multidict.py => src/python/m5/util/multidict.py rename : util/stats/orderdict.py => src/python/m5/util/orderdict.py --- src/python/SConscript | 8 +- src/python/m5/SimObject.py | 1 - src/python/m5/attrdict.py | 61 ----- src/python/m5/main.py | 2 +- src/python/m5/multidict.py | 182 -------------- src/python/m5/util.py | 59 ----- src/python/m5/util/__init__.py | 32 +++ src/python/m5/util/attrdict.py | 70 ++++++ src/python/m5/util/jobfile.py | 450 +++++++++++++++++++++++++++++++++ src/python/m5/util/misc.py | 87 +++++++ src/python/m5/util/multidict.py | 182 ++++++++++++++ src/python/m5/util/orderdict.py | 80 ++++++ util/batch/jobfile.py | 539 ---------------------------------------- util/pbs/jobfile.py | 539 ---------------------------------------- util/stats/orderdict.py | 80 ------ 15 files changed, 907 insertions(+), 1465 deletions(-) delete mode 100644 src/python/m5/attrdict.py delete mode 100644 src/python/m5/multidict.py delete mode 100644 src/python/m5/util.py create mode 100644 src/python/m5/util/__init__.py create mode 100644 src/python/m5/util/attrdict.py create mode 100644 src/python/m5/util/jobfile.py create mode 100644 src/python/m5/util/misc.py create mode 100644 src/python/m5/util/multidict.py create mode 100644 src/python/m5/util/orderdict.py delete mode 100644 util/batch/jobfile.py delete mode 100644 util/pbs/jobfile.py delete mode 100644 util/stats/orderdict.py diff --git a/src/python/SConscript b/src/python/SConscript index b39c9ea9c..0785b4307 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -36,18 +36,20 @@ Source('swig/pyobject.cc') PySource('m5', 'm5/__init__.py') PySource('m5', 'm5/SimObject.py') -PySource('m5', 'm5/attrdict.py') PySource('m5', 'm5/convert.py') PySource('m5', 'm5/event.py') PySource('m5', 'm5/main.py') -PySource('m5', 'm5/multidict.py') PySource('m5', 'm5/params.py') PySource('m5', 'm5/proxy.py') PySource('m5', 'm5/simulate.py') PySource('m5', 'm5/smartdict.py') PySource('m5', 'm5/stats.py') PySource('m5', 'm5/ticks.py') -PySource('m5', 'm5/util.py') +PySource('m5.util', 'm5/util/__init__.py') +PySource('m5.util', 'm5/util/attrdict.py') +PySource('m5.util', 'm5/util/jobfile.py') +PySource('m5.util', 'm5/util/misc.py') +PySource('m5.util', 'm5/util/multidict.py') SwigSource('m5.internal', 'swig/core.i') SwigSource('m5.internal', 'swig/debug.i') diff --git a/src/python/m5/SimObject.py b/src/python/m5/SimObject.py index d1aec44b3..41ed3df9e 100644 --- a/src/python/m5/SimObject.py +++ b/src/python/m5/SimObject.py @@ -32,7 +32,6 @@ import sys, types import proxy import m5 from util import * -from multidict import multidict # These utility functions have to come first because they're # referenced in params.py... otherwise they won't be defined when we diff --git a/src/python/m5/attrdict.py b/src/python/m5/attrdict.py deleted file mode 100644 index 4ee7f1b8c..000000000 --- a/src/python/m5/attrdict.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2006 The Regents of The University of Michigan -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Authors: Nathan Binkert - -__all__ = [ 'attrdict' ] - -class attrdict(dict): - def __getattr__(self, attr): - if attr in self: - return self.__getitem__(attr) - return super(attrdict, self).__getattribute__(attr) - - def __setattr__(self, attr, value): - if attr in dir(self): - return super(attrdict, self).__setattr__(attr, value) - return self.__setitem__(attr, value) - - def __delattr__(self, attr): - if attr in self: - return self.__delitem__(attr) - return super(attrdict, self).__delattr__(attr, value) - -if __name__ == '__main__': - x = attrdict() - x.y = 1 - x['z'] = 2 - print x['y'], x.y - print x['z'], x.z - print dir(x) - print x - - print - - del x['y'] - del x.z - print dir(x) - print(x) diff --git a/src/python/m5/main.py b/src/python/m5/main.py index cbdd65492..5c3324224 100644 --- a/src/python/m5/main.py +++ b/src/python/m5/main.py @@ -33,7 +33,7 @@ import os import socket import sys -from attrdict import attrdict +from util import attrdict import defines import traceflags diff --git a/src/python/m5/multidict.py b/src/python/m5/multidict.py deleted file mode 100644 index b5cd700ef..000000000 --- a/src/python/m5/multidict.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright (c) 2005 The Regents of The University of Michigan -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Authors: Nathan Binkert - -__all__ = [ 'multidict' ] - -class multidict(object): - def __init__(self, parent = {}, **kwargs): - self.local = dict(**kwargs) - self.parent = parent - self.deleted = {} - - def __str__(self): - return str(dict(self.items())) - - def __repr__(self): - return `dict(self.items())` - - def __contains__(self, key): - return self.local.has_key(key) or self.parent.has_key(key) - - def __delitem__(self, key): - try: - del self.local[key] - except KeyError, e: - if key in self.parent: - self.deleted[key] = True - else: - raise KeyError, e - - def __setitem__(self, key, value): - self.deleted.pop(key, False) - self.local[key] = value - - def __getitem__(self, key): - try: - return self.local[key] - except KeyError, e: - if not self.deleted.get(key, False) and key in self.parent: - return self.parent[key] - else: - raise KeyError, e - - def __len__(self): - return len(self.local) + len(self.parent) - - def next(self): - for key,value in self.local.items(): - yield key,value - - if self.parent: - for key,value in self.parent.next(): - if key not in self.local and key not in self.deleted: - yield key,value - - def has_key(self, key): - return key in self - - def iteritems(self): - for item in self.next(): - yield item - - def items(self): - return [ item for item in self.next() ] - - def iterkeys(self): - for key,value in self.next(): - yield key - - def keys(self): - return [ key for key,value in self.next() ] - - def itervalues(self): - for key,value in self.next(): - yield value - - def values(self): - return [ value for key,value in self.next() ] - - def get(self, key, default=None): - try: - return self[key] - except KeyError, e: - return default - - def setdefault(self, key, default): - try: - return self[key] - except KeyError: - self.deleted.pop(key, False) - self.local[key] = default - return default - - def _dump(self): - print 'multidict dump' - node = self - while isinstance(node, multidict): - print ' ', node.local - node = node.parent - - def _dumpkey(self, key): - values = [] - node = self - while isinstance(node, multidict): - if key in node.local: - values.append(node.local[key]) - node = node.parent - print key, values - -if __name__ == '__main__': - test1 = multidict() - test2 = multidict(test1) - test3 = multidict(test2) - test4 = multidict(test3) - - test1['a'] = 'test1_a' - test1['b'] = 'test1_b' - test1['c'] = 'test1_c' - test1['d'] = 'test1_d' - test1['e'] = 'test1_e' - - test2['a'] = 'test2_a' - del test2['b'] - test2['c'] = 'test2_c' - del test1['a'] - - test2.setdefault('f', multidict) - - print 'test1>', test1.items() - print 'test2>', test2.items() - #print test1['a'] - print test1['b'] - print test1['c'] - print test1['d'] - print test1['e'] - - print test2['a'] - #print test2['b'] - print test2['c'] - print test2['d'] - print test2['e'] - - for key in test2.iterkeys(): - print key - - test2.get('g', 'foo') - #test2.get('b') - test2.get('b', 'bar') - test2.setdefault('b', 'blah') - print test1 - print test2 - print `test2` - - print len(test2) - - test3['a'] = [ 0, 1, 2, 3 ] - - print test4 diff --git a/src/python/m5/util.py b/src/python/m5/util.py deleted file mode 100644 index 28b8b1b94..000000000 --- a/src/python/m5/util.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2004-2006 The Regents of The University of Michigan -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Authors: Steve Reinhardt -# Nathan Binkert - -############################# -# -# Utility classes & methods -# -############################# - -class Singleton(type): - def __call__(cls, *args, **kwargs): - if hasattr(cls, '_instance'): - return cls._instance - - cls._instance = super(Singleton, cls).__call__(*args, **kwargs) - return cls._instance - -# Apply method to object. -# applyMethod(obj, 'meth', ) is equivalent to obj.meth() -def applyMethod(obj, meth, *args, **kwargs): - return getattr(obj, meth)(*args, **kwargs) - -# If the first argument is an (non-sequence) object, apply the named -# method with the given arguments. If the first argument is a -# sequence, apply the method to each element of the sequence (a la -# 'map'). -def applyOrMap(objOrSeq, meth, *args, **kwargs): - if not isinstance(objOrSeq, (list, tuple)): - return applyMethod(objOrSeq, meth, *args, **kwargs) - else: - return [applyMethod(o, meth, *args, **kwargs) for o in objOrSeq] - - diff --git a/src/python/m5/util/__init__.py b/src/python/m5/util/__init__.py new file mode 100644 index 000000000..f82de696a --- /dev/null +++ b/src/python/m5/util/__init__.py @@ -0,0 +1,32 @@ +# Copyright (c) 2008 The Hewlett-Packard Development Company +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + +from attrdict import attrdict, optiondict +from misc import * +from multidict import multidict +import jobfile diff --git a/src/python/m5/util/attrdict.py b/src/python/m5/util/attrdict.py new file mode 100644 index 000000000..44479c456 --- /dev/null +++ b/src/python/m5/util/attrdict.py @@ -0,0 +1,70 @@ +# Copyright (c) 2006 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + +__all__ = [ 'attrdict', 'optiondict' ] + +class attrdict(dict): + def __getattr__(self, attr): + if attr in self: + return self.__getitem__(attr) + return super(attrdict, self).__getattribute__(attr) + + def __setattr__(self, attr, value): + if attr in dir(self): + return super(attrdict, self).__setattr__(attr, value) + return self.__setitem__(attr, value) + + def __delattr__(self, attr): + if attr in self: + return self.__delitem__(attr) + return super(attrdict, self).__delattr__(attr, value) + +class optiondict(attrdict): + def __getattr__(self, attr): + try: + return super(optiondict, self).__getattr__(attr) + except AttributeError: + #d = optionsdict() + #setattr(self, attr, d) + return None + +if __name__ == '__main__': + x = attrdict() + x.y = 1 + x['z'] = 2 + print x['y'], x.y + print x['z'], x.z + print dir(x) + print x + + print + + del x['y'] + del x.z + print dir(x) + print(x) diff --git a/src/python/m5/util/jobfile.py b/src/python/m5/util/jobfile.py new file mode 100644 index 000000000..5e015c4ad --- /dev/null +++ b/src/python/m5/util/jobfile.py @@ -0,0 +1,450 @@ +# Copyright (c) 2005-2006 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + +import sys + +from attrdict import attrdict, optiondict +from misc import crossproduct, flatten + +class Data(object): + def __init__(self, name, desc, **kwargs): + self.name = name + self.desc = desc + self.__dict__.update(kwargs) + + def update(self, obj): + if not isinstance(obj, Data): + raise AttributeError, "can only update from Data object" + + for k,v in obj.__dict__.iteritems(): + if not k.startswith('_'): + self.__dict__[k] = v + if hasattr(self, 'system') and hasattr(obj, 'system'): + if self.system != obj.system: + raise AttributeError, \ + "conflicting values for system: '%s'/'%s'" % \ + (self.system, obj.system) + + def printinfo(self): + if self.name: + print 'name: %s' % self.name + if self.desc: + print 'desc: %s' % self.desc + try: + if self.system: + print 'system: %s' % self.system + except AttributeError: + pass + + def printverbose(self): + for key in self: + val = self[key] + if isinstance(val, dict): + import pprint + val = pprint.pformat(val) + print '%-20s = %s' % (key, val) + print + + def __contains__(self, attr): + if attr.startswith('_'): + return False + return attr in self.__dict__ + + def __getitem__(self, key): + if key.startswith('_'): + raise KeyError, "Key '%s' not found" % attr + return self.__dict__[key] + + def __iter__(self): + keys = self.__dict__.keys() + keys.sort() + for key in keys: + if not key.startswith('_'): + yield key + + def optiondict(self): + result = optiondict() + for key in self: + result[key] = self[key] + return result + + def __str__(self): + return self.name + +class Job(Data): + def __init__(self, options): + super(Job, self).__init__('', '') + + config = options[0]._config + for opt in options: + if opt._config != config: + raise AttributeError, \ + "All options are not from the same Configuration" + + self._config = config + self._groups = [ opt._group for opt in options ] + self._options = options + + self.update(self._config) + for group in self._groups: + self.update(group) + + self._is_checkpoint = True + + for option in self._options: + self.update(option) + if not option._group._checkpoint: + self._is_checkpoint = False + + if option._suboption: + self.update(option._suboption) + self._is_checkpoint = False + + names = [ ] + for opt in self._options: + if opt.name: + names.append(opt.name) + self.name = ':'.join(names) + + descs = [ ] + for opt in self._options: + if opt.desc: + descs.append(opt.desc) + self.desc = ', '.join(descs) + + self._checkpoint = None + if not self._is_checkpoint: + opts = [] + for opt in options: + cpt = opt._group._checkpoint + if not cpt: + continue + if isinstance(cpt, Option): + opt = cpt.clone(suboptions=False) + else: + opt = opt.clone(suboptions=False) + + opts.append(opt) + + if opts: + self._checkpoint = Job(opts) + + def clone(self): + return Job(self._options) + + def printinfo(self): + super(Job, self).printinfo() + if self._checkpoint: + print 'checkpoint: %s' % self._checkpoint.name + print 'config: %s' % self._config.name + print 'groups: %s' % [ g.name for g in self._groups ] + print 'options: %s' % [ o.name for o in self._options ] + super(Job, self).printverbose() + +class SubOption(Data): + def __init__(self, name, desc, **kwargs): + super(SubOption, self).__init__(name, desc, **kwargs) + self._number = None + +class Option(Data): + def __init__(self, name, desc, **kwargs): + super(Option, self).__init__(name, desc, **kwargs) + self._suboptions = [] + self._suboption = None + self._number = None + + def __getattribute__(self, attr): + if attr == 'name': + name = self.__dict__[attr] + if self._suboption is not None: + name = '%s:%s' % (name, self._suboption.name) + return name + + if attr == 'desc': + desc = [ self.__dict__[attr] ] + if self._suboption is not None and self._suboption.desc: + desc.append(self._suboption.desc) + return ', '.join(desc) + + return super(Option, self).__getattribute__(attr) + + def suboption(self, name, desc, **kwargs): + subo = SubOption(name, desc, **kwargs) + subo._config = self._config + subo._group = self._group + subo._option = self + subo._number = len(self._suboptions) + self._suboptions.append(subo) + return subo + + def clone(self, suboptions=True): + option = Option(self.__dict__['name'], self.__dict__['desc']) + option.update(self) + option._group = self._group + option._config = self._config + option._number = self._number + if suboptions: + option._suboptions.extend(self._suboptions) + option._suboption = self._suboption + return option + + def subopts(self): + if not self._suboptions: + return [ self ] + + subopts = [] + for subo in self._suboptions: + option = self.clone() + option._suboption = subo + subopts.append(option) + + return subopts + + def printinfo(self): + super(Option, self).printinfo() + print 'config: %s' % self._config.name + super(Option, self).printverbose() + +class Group(Data): + def __init__(self, name, desc, **kwargs): + super(Group, self).__init__(name, desc, **kwargs) + self._options = [] + self._number = None + self._checkpoint = False + + def option(self, name, desc, **kwargs): + opt = Option(name, desc, **kwargs) + opt._config = self._config + opt._group = self + opt._number = len(self._options) + self._options.append(opt) + return opt + + def options(self): + return self._options + + def subopts(self): + subopts = [] + for opt in self._options: + for subo in opt.subopts(): + subopts.append(subo) + return subopts + + def printinfo(self): + super(Group, self).printinfo() + print 'config: %s' % self._config.name + print 'options: %s' % [ o.name for o in self._options ] + super(Group, self).printverbose() + +class Configuration(Data): + def __init__(self, name, desc, **kwargs): + super(Configuration, self).__init__(name, desc, **kwargs) + self._groups = [] + self._posfilters = [] + self._negfilters = [] + + def group(self, name, desc, **kwargs): + grp = Group(name, desc, **kwargs) + grp._config = self + grp._number = len(self._groups) + self._groups.append(grp) + return grp + + def groups(self): + return self._groups + + def checkchildren(self, kids): + for kid in kids: + if kid._config != self: + raise AttributeError, "child from the wrong configuration" + + def sortgroups(self, groups): + groups = [ (grp._number, grp) for grp in groups ] + groups.sort() + return [ grp[1] for grp in groups ] + + def options(self, groups=None, checkpoint=False): + if groups is None: + groups = self._groups + self.checkchildren(groups) + groups = self.sortgroups(groups) + if checkpoint: + groups = [ grp for grp in groups if grp._checkpoint ] + optgroups = [ g.options() for g in groups ] + else: + optgroups = [ g.subopts() for g in groups ] + if not optgroups: + return + for options in crossproduct(optgroups): + for opt in options: + cpt = opt._group._checkpoint + if not isinstance(cpt, bool) and cpt != opt: + if checkpoint: + break + else: + yield options + else: + if checkpoint: + yield options + + def addfilter(self, filt, pos=True): + import re + filt = re.compile(filt) + if pos: + self._posfilters.append(filt) + else: + self._negfilters.append(filt) + + def jobfilter(self, job): + for filt in self._negfilters: + if filt.match(job.name): + return False + + if not self._posfilters: + return True + + for filt in self._posfilters: + if filt.match(job.name): + return True + + return False + + def checkpoints(self, groups=None): + for options in self.options(groups, True): + job = Job(options) + if self.jobfilter(job): + yield job + + def jobs(self, groups=None): + for options in self.options(groups, False): + job = Job(options) + if self.jobfilter(job): + yield job + + def alljobs(self, groups=None): + for options in self.options(groups, True): + yield Job(options) + for options in self.options(groups, False): + yield Job(options) + + def find(self, jobname): + for job in self.alljobs(): + if job.name == jobname: + return job + else: + raise AttributeError, "job '%s' not found" % jobname + + def job(self, options): + self.checkchildren(options) + options = [ (opt._group._number, opt) for opt in options ] + options.sort() + options = [ opt[1] for opt in options ] + job = Job(options) + return job + + def printinfo(self): + super(Configuration, self).printinfo() + print 'groups: %s' % [ g.name for g in self._groups ] + super(Configuration, self).printverbose() + +def JobFile(jobfile): + from os.path import expanduser, isfile, join as joinpath + filename = expanduser(jobfile) + + # Can't find filename in the current path, search sys.path + if not isfile(filename): + for path in sys.path: + testname = joinpath(path, filename) + if isfile(testname): + filename = testname + break + else: + raise AttributeError, \ + "Could not find file '%s'" % jobfile + + data = {} + execfile(filename, data) + if 'conf' not in data: + raise ImportError, 'cannot import name conf from %s' % jobfile + conf = data['conf'] + import jobfile + if not isinstance(conf, Configuration): + raise AttributeError, \ + 'conf in jobfile: %s (%s) is not type %s' % \ + (jobfile, type(conf), Configuration) + return conf + +def main(conf=None): + import sys + + usage = 'Usage: %s [-b] [-c] [-v] ' % sys.argv[0] + + try: + import getopt + opts, args = getopt.getopt(sys.argv[1:], '-bcv') + except getopt.GetoptError: + sys.exit(usage) + + both = False + checkpoint = False + verbose = False + for opt,arg in opts: + if opt == '-b': + both = True + checkpoint = True + if opt == '-c': + checkpoint = True + if opt == '-v': + verbose = True + + if conf is None: + if len(args) != 1: + raise AttributeError, usage + conf = JobFile(args[0]) + else: + if len(args) != 0: + raise AttributeError, usage + + if both: + jobs = conf.alljobs() + elif checkpoint: + jobs = conf.checkpoints() + else: + jobs = conf.jobs() + + for job in jobs: + if verbose: + job.printinfo() + else: + cpt = '' + if job._checkpoint: + cpt = job._checkpoint.name + print job.name, cpt + +if __name__ == '__main__': + main() diff --git a/src/python/m5/util/misc.py b/src/python/m5/util/misc.py new file mode 100644 index 000000000..094e3ed9a --- /dev/null +++ b/src/python/m5/util/misc.py @@ -0,0 +1,87 @@ +# Copyright (c) 2004-2006 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Steve Reinhardt +# Nathan Binkert + +############################# +# +# Utility classes & methods +# +############################# + +class Singleton(type): + def __call__(cls, *args, **kwargs): + if hasattr(cls, '_instance'): + return cls._instance + + cls._instance = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instance + +# Apply method to object. +# applyMethod(obj, 'meth', ) is equivalent to obj.meth() +def applyMethod(obj, meth, *args, **kwargs): + return getattr(obj, meth)(*args, **kwargs) + +# If the first argument is an (non-sequence) object, apply the named +# method with the given arguments. If the first argument is a +# sequence, apply the method to each element of the sequence (a la +# 'map'). +def applyOrMap(objOrSeq, meth, *args, **kwargs): + if not isinstance(objOrSeq, (list, tuple)): + return applyMethod(objOrSeq, meth, *args, **kwargs) + else: + return [applyMethod(o, meth, *args, **kwargs) for o in objOrSeq] + +def crossproduct(items): + if not isinstance(items, (list, tuple)): + raise AttributeError, 'crossproduct works only on sequences' + + if not items: + yield None + return + + current = items[0] + remainder = items[1:] + + if not hasattr(current, '__iter__'): + current = [ current ] + + for item in current: + for rem in crossproduct(remainder): + data = [ item ] + if rem: + data += rem + yield data + +def flatten(items): + if not isinstance(items, (list, tuple)): + yield items + return + + for item in items: + for flat in flatten(item): + yield flat diff --git a/src/python/m5/util/multidict.py b/src/python/m5/util/multidict.py new file mode 100644 index 000000000..b5cd700ef --- /dev/null +++ b/src/python/m5/util/multidict.py @@ -0,0 +1,182 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + +__all__ = [ 'multidict' ] + +class multidict(object): + def __init__(self, parent = {}, **kwargs): + self.local = dict(**kwargs) + self.parent = parent + self.deleted = {} + + def __str__(self): + return str(dict(self.items())) + + def __repr__(self): + return `dict(self.items())` + + def __contains__(self, key): + return self.local.has_key(key) or self.parent.has_key(key) + + def __delitem__(self, key): + try: + del self.local[key] + except KeyError, e: + if key in self.parent: + self.deleted[key] = True + else: + raise KeyError, e + + def __setitem__(self, key, value): + self.deleted.pop(key, False) + self.local[key] = value + + def __getitem__(self, key): + try: + return self.local[key] + except KeyError, e: + if not self.deleted.get(key, False) and key in self.parent: + return self.parent[key] + else: + raise KeyError, e + + def __len__(self): + return len(self.local) + len(self.parent) + + def next(self): + for key,value in self.local.items(): + yield key,value + + if self.parent: + for key,value in self.parent.next(): + if key not in self.local and key not in self.deleted: + yield key,value + + def has_key(self, key): + return key in self + + def iteritems(self): + for item in self.next(): + yield item + + def items(self): + return [ item for item in self.next() ] + + def iterkeys(self): + for key,value in self.next(): + yield key + + def keys(self): + return [ key for key,value in self.next() ] + + def itervalues(self): + for key,value in self.next(): + yield value + + def values(self): + return [ value for key,value in self.next() ] + + def get(self, key, default=None): + try: + return self[key] + except KeyError, e: + return default + + def setdefault(self, key, default): + try: + return self[key] + except KeyError: + self.deleted.pop(key, False) + self.local[key] = default + return default + + def _dump(self): + print 'multidict dump' + node = self + while isinstance(node, multidict): + print ' ', node.local + node = node.parent + + def _dumpkey(self, key): + values = [] + node = self + while isinstance(node, multidict): + if key in node.local: + values.append(node.local[key]) + node = node.parent + print key, values + +if __name__ == '__main__': + test1 = multidict() + test2 = multidict(test1) + test3 = multidict(test2) + test4 = multidict(test3) + + test1['a'] = 'test1_a' + test1['b'] = 'test1_b' + test1['c'] = 'test1_c' + test1['d'] = 'test1_d' + test1['e'] = 'test1_e' + + test2['a'] = 'test2_a' + del test2['b'] + test2['c'] = 'test2_c' + del test1['a'] + + test2.setdefault('f', multidict) + + print 'test1>', test1.items() + print 'test2>', test2.items() + #print test1['a'] + print test1['b'] + print test1['c'] + print test1['d'] + print test1['e'] + + print test2['a'] + #print test2['b'] + print test2['c'] + print test2['d'] + print test2['e'] + + for key in test2.iterkeys(): + print key + + test2.get('g', 'foo') + #test2.get('b') + test2.get('b', 'bar') + test2.setdefault('b', 'blah') + print test1 + print test2 + print `test2` + + print len(test2) + + test3['a'] = [ 0, 1, 2, 3 ] + + print test4 diff --git a/src/python/m5/util/orderdict.py b/src/python/m5/util/orderdict.py new file mode 100644 index 000000000..3f755d299 --- /dev/null +++ b/src/python/m5/util/orderdict.py @@ -0,0 +1,80 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + +__all__ = [ 'orderdict' ] + +class orderdict(dict): + def __init__(self, d = {}): + self._keys = d.keys() + super(orderdict, self).__init__(d) + + def __setitem__(self, key, item): + super(orderdict, self).__setitem__(key, item) + if not hasattr(self, '_keys'): + self._keys = [key,] + if key not in self._keys: + self._keys.append(key) + + def __delitem__(self, key): + super(orderdict, self).__delitem__(key) + self._keys.remove(key) + + def clear(self): + super(orderdict, self).clear() + self._keys = [] + + def items(self): + for i in self._keys: + yield i, self[i] + + def keys(self): + return self._keys + + def popitem(self): + if len(self._keys) == 0: + raise KeyError('dictionary is empty') + else: + key = self._keys[-1] + val = self[key] + del self[key] + return key, val + + def setdefault(self, key, failobj = None): + super(orderdict, self).setdefault(key, failobj) + if key not in self._keys: + self._keys.append(key) + + def update(self, d): + for key in d.keys(): + if not self.has_key(key): + self._keys.append(key) + super(orderdict, self).update(d) + + def values(self): + for i in self._keys: + yield self[i] diff --git a/util/batch/jobfile.py b/util/batch/jobfile.py deleted file mode 100644 index b78d7f3e1..000000000 --- a/util/batch/jobfile.py +++ /dev/null @@ -1,539 +0,0 @@ -# Copyright (c) 2006 The Regents of The University of Michigan -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Authors: Kevin Lim - -import sys - -class ternary(object): - def __new__(cls, *args): - if len(args) > 1: - raise TypeError, \ - '%s() takes at most 1 argument (%d given)' % \ - (cls.__name__, len(args)) - - if args: - if not isinstance(args[0], (bool, ternary)): - raise TypeError, \ - '%s() argument must be True, False, or Any' % \ - cls.__name__ - return args[0] - return super(ternary, cls).__new__(cls) - - def __bool__(self): - return True - - def __neg__(self): - return self - - def __eq__(self, other): - return True - - def __ne__(self, other): - return False - - def __str__(self): - return 'Any' - - def __repr__(self): - return 'Any' - -Any = ternary() - -class Flags(dict): - def __init__(self, *args, **kwargs): - super(Flags, self).__init__() - self.update(*args, **kwargs) - - def __getattr__(self, attr): - return self[attr] - - def __setattr__(self, attr, value): - self[attr] = value - - def __setitem__(self, item, value): - return super(Flags, self).__setitem__(item, ternary(value)) - - def __getitem__(self, item): - if item not in self: - return False - return super(Flags, self).__getitem__(item) - - def update(self, *args, **kwargs): - for arg in args: - if isinstance(arg, Flags): - super(Flags, self).update(arg) - elif isinstance(arg, dict): - for key,val in kwargs.iteritems(): - self[key] = val - else: - raise AttributeError, \ - 'flags not of type %s or %s, but %s' % \ - (Flags, dict, type(arg)) - - for key,val in kwargs.iteritems(): - self[key] = val - - def match(self, *args, **kwargs): - match = Flags(*args, **kwargs) - - for key,value in match.iteritems(): - if self[key] != value: - return False - - return True - -def crossproduct(items): - if not isinstance(items, (list, tuple)): - raise AttributeError, 'crossproduct works only on sequences' - - if not items: - yield None - return - - current = items[0] - remainder = items[1:] - - if not hasattr(current, '__iter__'): - current = [ current ] - - for item in current: - for rem in crossproduct(remainder): - data = [ item ] - if rem: - data += rem - yield data - -def flatten(items): - if not isinstance(items, (list, tuple)): - yield items - return - - for item in items: - for flat in flatten(item): - yield flat - -class Data(object): - def __init__(self, name, desc, **kwargs): - self.name = name - self.desc = desc - self.system = None - self.flags = Flags() - self.env = {} - for k,v in kwargs.iteritems(): - setattr(self, k, v) - - def update(self, obj): - if not isinstance(obj, Data): - raise AttributeError, "can only update from Data object" - - self.env.update(obj.env) - self.flags.update(obj.flags) - if obj.system: - if self.system and self.system != obj.system: - raise AttributeError, \ - "conflicting values for system: '%s'/'%s'" % \ - (self.system, obj.system) - self.system = obj.system - - def printinfo(self): - if self.name: - print 'name: %s' % self.name - if self.desc: - print 'desc: %s' % self.desc - if self.system: - print 'system: %s' % self.system - - def printverbose(self): - print 'flags:' - keys = self.flags.keys() - keys.sort() - for key in keys: - print ' %s = %s' % (key, self.flags[key]) - print 'env:' - keys = self.env.keys() - keys.sort() - for key in keys: - print ' %s = %s' % (key, self.env[key]) - print - - def __str__(self): - return self.name - -class Job(Data): - def __init__(self, options): - super(Job, self).__init__('', '') - self.setoptions(options) - - self.checkpoint = False - opts = [] - for opt in options: - cpt = opt.group.checkpoint - if not cpt: - self.checkpoint = True - continue - if isinstance(cpt, Option): - opt = cpt.clone(suboptions=False) - else: - opt = opt.clone(suboptions=False) - - opts.append(opt) - - if not opts: - self.checkpoint = False - - if self.checkpoint: - self.checkpoint = Job(opts) - - def clone(self): - return Job(self.options) - - def __getattribute__(self, attr): - if attr == 'name': - names = [ ] - for opt in self.options: - if opt.name: - names.append(opt.name) - return ':'.join(names) - - if attr == 'desc': - descs = [ ] - for opt in self.options: - if opt.desc: - descs.append(opt.desc) - return ', '.join(descs) - - return super(Job, self).__getattribute__(attr) - - def setoptions(self, options): - config = options[0].config - for opt in options: - if opt.config != config: - raise AttributeError, \ - "All options are not from the same Configuration" - - self.config = config - self.groups = [ opt.group for opt in options ] - self.options = options - - self.update(self.config) - for group in self.groups: - self.update(group) - - for option in self.options: - self.update(option) - if option._suboption: - self.update(option._suboption) - - def printinfo(self): - super(Job, self).printinfo() - if self.checkpoint: - print 'checkpoint: %s' % self.checkpoint.name - print 'config: %s' % self.config.name - print 'groups: %s' % [ g.name for g in self.groups ] - print 'options: %s' % [ o.name for o in self.options ] - super(Job, self).printverbose() - -class SubOption(Data): - def __init__(self, name, desc, **kwargs): - super(SubOption, self).__init__(name, desc, **kwargs) - self.number = None - -class Option(Data): - def __init__(self, name, desc, **kwargs): - super(Option, self).__init__(name, desc, **kwargs) - self._suboptions = [] - self._suboption = None - self.number = None - - def __getattribute__(self, attr): - if attr == 'name': - name = self.__dict__[attr] - if self._suboption is not None: - name = '%s:%s' % (name, self._suboption.name) - return name - - if attr == 'desc': - desc = [ self.__dict__[attr] ] - if self._suboption is not None and self._suboption.desc: - desc.append(self._suboption.desc) - return ', '.join(desc) - - - return super(Option, self).__getattribute__(attr) - - def suboption(self, name, desc, **kwargs): - subo = SubOption(name, desc, **kwargs) - subo.config = self.config - subo.group = self.group - subo.option = self - subo.number = len(self._suboptions) - self._suboptions.append(subo) - return subo - - def clone(self, suboptions=True): - option = Option(self.__dict__['name'], self.__dict__['desc']) - option.update(self) - option.group = self.group - option.config = self.config - option.number = self.number - if suboptions: - option._suboptions.extend(self._suboptions) - option._suboption = self._suboption - return option - - def subopts(self): - if not self._suboptions: - return [ self ] - - subopts = [] - for subo in self._suboptions: - option = self.clone() - option._suboption = subo - subopts.append(option) - - return subopts - - def printinfo(self): - super(Option, self).printinfo() - print 'config: %s' % self.config.name - super(Option, self).printverbose() - -class Group(Data): - def __init__(self, name, desc, **kwargs): - super(Group, self).__init__(name, desc, **kwargs) - self._options = [] - self.checkpoint = False - self.number = None - - def option(self, name, desc, **kwargs): - opt = Option(name, desc, **kwargs) - opt.config = self.config - opt.group = self - opt.number = len(self._options) - self._options.append(opt) - return opt - - def options(self): - return self._options - - def subopts(self): - subopts = [] - for opt in self._options: - for subo in opt.subopts(): - subopts.append(subo) - return subopts - - def printinfo(self): - super(Group, self).printinfo() - print 'config: %s' % self.config.name - print 'options: %s' % [ o.name for o in self._options ] - super(Group, self).printverbose() - -class Configuration(Data): - def __init__(self, name, desc, **kwargs): - super(Configuration, self).__init__(name, desc, **kwargs) - self._groups = [] - self._posfilters = [] - self._negfilters = [] - - def group(self, name, desc, **kwargs): - grp = Group(name, desc, **kwargs) - grp.config = self - grp.number = len(self._groups) - self._groups.append(grp) - return grp - - def groups(self, flags=Flags(), sign=True): - if not flags: - return self._groups - - return [ grp for grp in self._groups if sign ^ grp.flags.match(flags) ] - - def checkchildren(self, kids): - for kid in kids: - if kid.config != self: - raise AttributeError, "child from the wrong configuration" - - def sortgroups(self, groups): - groups = [ (grp.number, grp) for grp in groups ] - groups.sort() - return [ grp[1] for grp in groups ] - - def options(self, groups = None, checkpoint = False): - if groups is None: - groups = self._groups - self.checkchildren(groups) - groups = self.sortgroups(groups) - if checkpoint: - groups = [ grp for grp in groups if grp.checkpoint ] - optgroups = [ g.options() for g in groups ] - else: - optgroups = [ g.subopts() for g in groups ] - for options in crossproduct(optgroups): - for opt in options: - cpt = opt.group.checkpoint - if not isinstance(cpt, bool) and cpt != opt: - if checkpoint: - break - else: - yield options - else: - if checkpoint: - yield options - - def addfilter(self, filt, pos=True): - import re - filt = re.compile(filt) - if pos: - self._posfilters.append(filt) - else: - self._negfilters.append(filt) - - def jobfilter(self, job): - for filt in self._negfilters: - if filt.match(job.name): - return False - - if not self._posfilters: - return True - - for filt in self._posfilters: - if filt.match(job.name): - return True - - return False - - def checkpoints(self, groups = None): - for options in self.options(groups, True): - job = Job(options) - if self.jobfilter(job): - yield job - - def jobs(self, groups = None): - for options in self.options(groups, False): - job = Job(options) - if self.jobfilter(job): - yield job - - def alljobs(self, groups = None): - for options in self.options(groups, True): - yield Job(options) - for options in self.options(groups, False): - yield Job(options) - - def find(self, jobname): - for job in self.alljobs(): - if job.name == jobname: - return job - else: - raise AttributeError, "job '%s' not found" % jobname - - def job(self, options): - self.checkchildren(options) - options = [ (opt.group.number, opt) for opt in options ] - options.sort() - options = [ opt[1] for opt in options ] - job = Job(options) - return job - - def printinfo(self): - super(Configuration, self).printinfo() - print 'groups: %s' % [ g.name for g in self._grouips ] - super(Configuration, self).printverbose() - -def JobFile(jobfile): - from os.path import expanduser, isfile, join as joinpath - filename = expanduser(jobfile) - - # Can't find filename in the current path, search sys.path - if not isfile(filename): - for path in sys.path: - testname = joinpath(path, filename) - if isfile(testname): - filename = testname - break - else: - raise AttributeError, \ - "Could not find file '%s'" % jobfile - - data = {} - execfile(filename, data) - if 'conf' not in data: - raise ImportError, 'cannot import name conf from %s' % jobfile - conf = data['conf'] - import jobfile - if not isinstance(conf, Configuration): - raise AttributeError, \ - 'conf in jobfile: %s (%s) is not type %s' % \ - (jobfile, type(conf), Configuration) - return conf - -if __name__ == '__main__': - from jobfile import * - import sys - - usage = 'Usage: %s [-b] [-c] [-v] ' % sys.argv[0] - - try: - import getopt - opts, args = getopt.getopt(sys.argv[1:], '-bcv') - except getopt.GetoptError: - sys.exit(usage) - - if len(args) != 1: - raise AttributeError, usage - - both = False - checkpoint = False - verbose = False - for opt,arg in opts: - if opt == '-b': - both = True - checkpoint = True - if opt == '-c': - checkpoint = True - if opt == '-v': - verbose = True - - jobfile = args[0] - conf = JobFile(jobfile) - - if both: - gen = conf.alljobs() - elif checkpoint: - gen = conf.checkpoints() - else: - gen = conf.jobs() - - for job in gen: - if not verbose: - cpt = '' - if job.checkpoint: - cpt = job.checkpoint.name - print job.name, cpt - else: - job.printinfo() diff --git a/util/pbs/jobfile.py b/util/pbs/jobfile.py deleted file mode 100644 index fd19b3bf5..000000000 --- a/util/pbs/jobfile.py +++ /dev/null @@ -1,539 +0,0 @@ -# Copyright (c) 2005-2006 The Regents of The University of Michigan -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Authors: Nathan Binkert - -import sys - -class ternary(object): - def __new__(cls, *args): - if len(args) > 1: - raise TypeError, \ - '%s() takes at most 1 argument (%d given)' % \ - (cls.__name__, len(args)) - - if args: - if not isinstance(args[0], (bool, ternary)): - raise TypeError, \ - '%s() argument must be True, False, or Any' % \ - cls.__name__ - return args[0] - return super(ternary, cls).__new__(cls) - - def __bool__(self): - return True - - def __neg__(self): - return self - - def __eq__(self, other): - return True - - def __ne__(self, other): - return False - - def __str__(self): - return 'Any' - - def __repr__(self): - return 'Any' - -Any = ternary() - -class Flags(dict): - def __init__(self, *args, **kwargs): - super(Flags, self).__init__() - self.update(*args, **kwargs) - - def __getattr__(self, attr): - return self[attr] - - def __setattr__(self, attr, value): - self[attr] = value - - def __setitem__(self, item, value): - return super(Flags, self).__setitem__(item, ternary(value)) - - def __getitem__(self, item): - if item not in self: - return False - return super(Flags, self).__getitem__(item) - - def update(self, *args, **kwargs): - for arg in args: - if isinstance(arg, Flags): - super(Flags, self).update(arg) - elif isinstance(arg, dict): - for key,val in kwargs.iteritems(): - self[key] = val - else: - raise AttributeError, \ - 'flags not of type %s or %s, but %s' % \ - (Flags, dict, type(arg)) - - for key,val in kwargs.iteritems(): - self[key] = val - - def match(self, *args, **kwargs): - match = Flags(*args, **kwargs) - - for key,value in match.iteritems(): - if self[key] != value: - return False - - return True - -def crossproduct(items): - if not isinstance(items, (list, tuple)): - raise AttributeError, 'crossproduct works only on sequences' - - if not items: - yield None - return - - current = items[0] - remainder = items[1:] - - if not hasattr(current, '__iter__'): - current = [ current ] - - for item in current: - for rem in crossproduct(remainder): - data = [ item ] - if rem: - data += rem - yield data - -def flatten(items): - if not isinstance(items, (list, tuple)): - yield items - return - - for item in items: - for flat in flatten(item): - yield flat - -class Data(object): - def __init__(self, name, desc, **kwargs): - self.name = name - self.desc = desc - self.system = None - self.flags = Flags() - self.env = {} - for k,v in kwargs.iteritems(): - setattr(self, k, v) - - def update(self, obj): - if not isinstance(obj, Data): - raise AttributeError, "can only update from Data object" - - self.env.update(obj.env) - self.flags.update(obj.flags) - if obj.system: - if self.system and self.system != obj.system: - raise AttributeError, \ - "conflicting values for system: '%s'/'%s'" % \ - (self.system, obj.system) - self.system = obj.system - - def printinfo(self): - if self.name: - print 'name: %s' % self.name - if self.desc: - print 'desc: %s' % self.desc - if self.system: - print 'system: %s' % self.system - - def printverbose(self): - print 'flags:' - keys = self.flags.keys() - keys.sort() - for key in keys: - print ' %s = %s' % (key, self.flags[key]) - print 'env:' - keys = self.env.keys() - keys.sort() - for key in keys: - print ' %s = %s' % (key, self.env[key]) - print - - def __str__(self): - return self.name - -class Job(Data): - def __init__(self, options): - super(Job, self).__init__('', '') - self.setoptions(options) - - self.checkpoint = False - opts = [] - for opt in options: - cpt = opt.group.checkpoint - if not cpt: - self.checkpoint = True - continue - if isinstance(cpt, Option): - opt = cpt.clone(suboptions=False) - else: - opt = opt.clone(suboptions=False) - - opts.append(opt) - - if not opts: - self.checkpoint = False - - if self.checkpoint: - self.checkpoint = Job(opts) - - def clone(self): - return Job(self.options) - - def __getattribute__(self, attr): - if attr == 'name': - names = [ ] - for opt in self.options: - if opt.name: - names.append(opt.name) - return ':'.join(names) - - if attr == 'desc': - descs = [ ] - for opt in self.options: - if opt.desc: - descs.append(opt.desc) - return ', '.join(descs) - - return super(Job, self).__getattribute__(attr) - - def setoptions(self, options): - config = options[0].config - for opt in options: - if opt.config != config: - raise AttributeError, \ - "All options are not from the same Configuration" - - self.config = config - self.groups = [ opt.group for opt in options ] - self.options = options - - self.update(self.config) - for group in self.groups: - self.update(group) - - for option in self.options: - self.update(option) - if option._suboption: - self.update(option._suboption) - - def printinfo(self): - super(Job, self).printinfo() - if self.checkpoint: - print 'checkpoint: %s' % self.checkpoint.name - print 'config: %s' % self.config.name - print 'groups: %s' % [ g.name for g in self.groups ] - print 'options: %s' % [ o.name for o in self.options ] - super(Job, self).printverbose() - -class SubOption(Data): - def __init__(self, name, desc, **kwargs): - super(SubOption, self).__init__(name, desc, **kwargs) - self.number = None - -class Option(Data): - def __init__(self, name, desc, **kwargs): - super(Option, self).__init__(name, desc, **kwargs) - self._suboptions = [] - self._suboption = None - self.number = None - - def __getattribute__(self, attr): - if attr == 'name': - name = self.__dict__[attr] - if self._suboption is not None: - name = '%s:%s' % (name, self._suboption.name) - return name - - if attr == 'desc': - desc = [ self.__dict__[attr] ] - if self._suboption is not None and self._suboption.desc: - desc.append(self._suboption.desc) - return ', '.join(desc) - - - return super(Option, self).__getattribute__(attr) - - def suboption(self, name, desc, **kwargs): - subo = SubOption(name, desc, **kwargs) - subo.config = self.config - subo.group = self.group - subo.option = self - subo.number = len(self._suboptions) - self._suboptions.append(subo) - return subo - - def clone(self, suboptions=True): - option = Option(self.__dict__['name'], self.__dict__['desc']) - option.update(self) - option.group = self.group - option.config = self.config - option.number = self.number - if suboptions: - option._suboptions.extend(self._suboptions) - option._suboption = self._suboption - return option - - def subopts(self): - if not self._suboptions: - return [ self ] - - subopts = [] - for subo in self._suboptions: - option = self.clone() - option._suboption = subo - subopts.append(option) - - return subopts - - def printinfo(self): - super(Option, self).printinfo() - print 'config: %s' % self.config.name - super(Option, self).printverbose() - -class Group(Data): - def __init__(self, name, desc, **kwargs): - super(Group, self).__init__(name, desc, **kwargs) - self._options = [] - self.checkpoint = False - self.number = None - - def option(self, name, desc, **kwargs): - opt = Option(name, desc, **kwargs) - opt.config = self.config - opt.group = self - opt.number = len(self._options) - self._options.append(opt) - return opt - - def options(self): - return self._options - - def subopts(self): - subopts = [] - for opt in self._options: - for subo in opt.subopts(): - subopts.append(subo) - return subopts - - def printinfo(self): - super(Group, self).printinfo() - print 'config: %s' % self.config.name - print 'options: %s' % [ o.name for o in self._options ] - super(Group, self).printverbose() - -class Configuration(Data): - def __init__(self, name, desc, **kwargs): - super(Configuration, self).__init__(name, desc, **kwargs) - self._groups = [] - self._posfilters = [] - self._negfilters = [] - - def group(self, name, desc, **kwargs): - grp = Group(name, desc, **kwargs) - grp.config = self - grp.number = len(self._groups) - self._groups.append(grp) - return grp - - def groups(self, flags=Flags(), sign=True): - if not flags: - return self._groups - - return [ grp for grp in self._groups if sign ^ grp.flags.match(flags) ] - - def checkchildren(self, kids): - for kid in kids: - if kid.config != self: - raise AttributeError, "child from the wrong configuration" - - def sortgroups(self, groups): - groups = [ (grp.number, grp) for grp in groups ] - groups.sort() - return [ grp[1] for grp in groups ] - - def options(self, groups = None, checkpoint = False): - if groups is None: - groups = self._groups - self.checkchildren(groups) - groups = self.sortgroups(groups) - if checkpoint: - groups = [ grp for grp in groups if grp.checkpoint ] - optgroups = [ g.options() for g in groups ] - else: - optgroups = [ g.subopts() for g in groups ] - for options in crossproduct(optgroups): - for opt in options: - cpt = opt.group.checkpoint - if not isinstance(cpt, bool) and cpt != opt: - if checkpoint: - break - else: - yield options - else: - if checkpoint: - yield options - - def addfilter(self, filt, pos=True): - import re - filt = re.compile(filt) - if pos: - self._posfilters.append(filt) - else: - self._negfilters.append(filt) - - def jobfilter(self, job): - for filt in self._negfilters: - if filt.match(job.name): - return False - - if not self._posfilters: - return True - - for filt in self._posfilters: - if filt.match(job.name): - return True - - return False - - def checkpoints(self, groups = None): - for options in self.options(groups, True): - job = Job(options) - if self.jobfilter(job): - yield job - - def jobs(self, groups = None): - for options in self.options(groups, False): - job = Job(options) - if self.jobfilter(job): - yield job - - def alljobs(self, groups = None): - for options in self.options(groups, True): - yield Job(options) - for options in self.options(groups, False): - yield Job(options) - - def find(self, jobname): - for job in self.alljobs(): - if job.name == jobname: - return job - else: - raise AttributeError, "job '%s' not found" % jobname - - def job(self, options): - self.checkchildren(options) - options = [ (opt.group.number, opt) for opt in options ] - options.sort() - options = [ opt[1] for opt in options ] - job = Job(options) - return job - - def printinfo(self): - super(Configuration, self).printinfo() - print 'groups: %s' % [ g.name for g in self._grouips ] - super(Configuration, self).printverbose() - -def JobFile(jobfile): - from os.path import expanduser, isfile, join as joinpath - filename = expanduser(jobfile) - - # Can't find filename in the current path, search sys.path - if not isfile(filename): - for path in sys.path: - testname = joinpath(path, filename) - if isfile(testname): - filename = testname - break - else: - raise AttributeError, \ - "Could not find file '%s'" % jobfile - - data = {} - execfile(filename, data) - if 'conf' not in data: - raise ImportError, 'cannot import name conf from %s' % jobfile - conf = data['conf'] - import jobfile - if not isinstance(conf, Configuration): - raise AttributeError, \ - 'conf in jobfile: %s (%s) is not type %s' % \ - (jobfile, type(conf), Configuration) - return conf - -if __name__ == '__main__': - from jobfile import * - import sys - - usage = 'Usage: %s [-b] [-c] [-v] ' % sys.argv[0] - - try: - import getopt - opts, args = getopt.getopt(sys.argv[1:], '-bcv') - except getopt.GetoptError: - sys.exit(usage) - - if len(args) != 1: - raise AttributeError, usage - - both = False - checkpoint = False - verbose = False - for opt,arg in opts: - if opt == '-b': - both = True - checkpoint = True - if opt == '-c': - checkpoint = True - if opt == '-v': - verbose = True - - jobfile = args[0] - conf = JobFile(jobfile) - - if both: - gen = conf.alljobs() - elif checkpoint: - gen = conf.checkpoints() - else: - gen = conf.jobs() - - for job in gen: - if not verbose: - cpt = '' - if job.checkpoint: - cpt = job.checkpoint.name - print job.name, cpt - else: - job.printinfo() diff --git a/util/stats/orderdict.py b/util/stats/orderdict.py deleted file mode 100644 index 3f755d299..000000000 --- a/util/stats/orderdict.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2005 The Regents of The University of Michigan -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Authors: Nathan Binkert - -__all__ = [ 'orderdict' ] - -class orderdict(dict): - def __init__(self, d = {}): - self._keys = d.keys() - super(orderdict, self).__init__(d) - - def __setitem__(self, key, item): - super(orderdict, self).__setitem__(key, item) - if not hasattr(self, '_keys'): - self._keys = [key,] - if key not in self._keys: - self._keys.append(key) - - def __delitem__(self, key): - super(orderdict, self).__delitem__(key) - self._keys.remove(key) - - def clear(self): - super(orderdict, self).clear() - self._keys = [] - - def items(self): - for i in self._keys: - yield i, self[i] - - def keys(self): - return self._keys - - def popitem(self): - if len(self._keys) == 0: - raise KeyError('dictionary is empty') - else: - key = self._keys[-1] - val = self[key] - del self[key] - return key, val - - def setdefault(self, key, failobj = None): - super(orderdict, self).setdefault(key, failobj) - if key not in self._keys: - self._keys.append(key) - - def update(self, d): - for key in d.keys(): - if not self.has_key(key): - self._keys.append(key) - super(orderdict, self).update(d) - - def values(self): - for i in self._keys: - yield self[i] -- cgit v1.2.3