summaryrefslogtreecommitdiff
path: root/util
diff options
context:
space:
mode:
authorAli Saidi <saidi@eecs.umich.edu>2005-11-02 14:56:18 -0500
committerAli Saidi <saidi@eecs.umich.edu>2005-11-02 14:56:18 -0500
commit3b66cb49ecf29e762f4659ed174ca76b8f553a1e (patch)
treeb8f29795c7abf7c93882881252aff716fb33ee02 /util
parent0523736b96b2779f8a33c2315c94be55d0a4d9c7 (diff)
parenta0829a7780b110a912ffc250d424b6dfe3586e62 (diff)
downloadgem5-3b66cb49ecf29e762f4659ed174ca76b8f553a1e.tar.xz
Merge zizzer:/bk/m5
into zeep.eecs.umich.edu:/z/saidi/work/m5 --HG-- extra : convert_revision : 3cc23080d19cc464a8ba7c1c93b6e5d45af7d463
Diffstat (limited to 'util')
-rwxr-xr-xutil/pbs/pbs.py4
-rwxr-xr-xutil/pbs/send.py7
-rwxr-xr-xutil/qdo16
-rw-r--r--util/stats/db.py72
-rw-r--r--util/stats/display.py97
-rw-r--r--util/stats/info.py669
-rw-r--r--util/stats/output.py52
-rw-r--r--util/stats/profile.py13
-rwxr-xr-xutil/stats/stats.py96
9 files changed, 507 insertions, 519 deletions
diff --git a/util/pbs/pbs.py b/util/pbs/pbs.py
index 292ddf1f9..21c2cb89d 100755
--- a/util/pbs/pbs.py
+++ b/util/pbs/pbs.py
@@ -147,6 +147,8 @@ class qsub:
flags.append('e')
if len(flags):
self.cmd.append('-m ' + flags)
+ else:
+ self.cmd.append('-mn')
if self.name:
self.cmd.append("-N%s" % self.name)
@@ -158,7 +160,7 @@ class qsub:
self.cmd.append('-q' + self.queue)
if self.afterok:
- self.cmd.append('-Wdepend=afterok:%s' % self.after)
+ self.cmd.append('-Wdepend=afterok:%s' % self.afterok)
self.cmd.extend(args)
self.script = script
diff --git a/util/pbs/send.py b/util/pbs/send.py
index 1f48b2b0e..54bba931d 100755
--- a/util/pbs/send.py
+++ b/util/pbs/send.py
@@ -98,7 +98,7 @@ Usage:
try:
import getopt
- opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lq:Rt:v')
+ opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v')
except getopt.GetoptError:
sys.exit(usage)
@@ -115,6 +115,7 @@ docpts = False
doruns = True
runflag = False
node_type = 'FAST'
+update = True
for opt,arg in opts:
if opt == '-C':
@@ -134,6 +135,8 @@ for opt,arg in opts:
jfile = arg
if opt == '-l':
listonly = True
+ if opt == '-n':
+ update = False
if opt == '-q':
queue = arg
if opt == '-R':
@@ -154,7 +157,7 @@ from job import JobDir, date
conf = jobfile.JobFile(jfile)
-if not listonly and not onlyecho and isdir(conf.linkdir):
+if update and not listonly and not onlyecho and isdir(conf.linkdir):
if verbose:
print 'Checking for outdated files in Link directory'
if not isdir(conf.basedir):
diff --git a/util/qdo b/util/qdo
index 3a475b420..339d944a7 100755
--- a/util/qdo
+++ b/util/qdo
@@ -45,7 +45,11 @@ optparser.add_option('-o', dest='stdout_file',
help='command stdout output file')
optparser.add_option('-l', dest='save_log', action='store_true',
help='save qsub output log file')
-optparser.add_option('-q', dest='qsub_timeout', type='int',
+optparser.add_option('-N', dest='job_name',
+ help='qsub job name')
+optparser.add_option('-q', dest='dest_queue',
+ help='qsub destination queue')
+optparser.add_option('--qwait', dest='qsub_timeout', type='int',
help='qsub queue wait timeout', default=30*60)
optparser.add_option('-t', dest='cmd_timeout', type='int',
help='command execution timeout', default=600*60)
@@ -56,6 +60,12 @@ if cmd == []:
print >>sys.stderr, "%s: missing command" % progname
sys.exit(1)
+# If we want to do this, need to add check here to make sure cmd[0] is
+# a valid PBS job name, else qsub will die on us.
+#
+#if not options.job_name:
+# options.job_name = cmd[0]
+
cwd = os.getcwd()
# Deal with systems where /n is a symlink to /.automount
@@ -137,6 +147,10 @@ if False and len(cmd) > 50:
print "%s: running %s on poolfs" % (progname, cmd[0])
else:
shell_cmd = 'qsub -I -S /bin/sh'
+ if options.job_name:
+ shell_cmd += ' -N "%s"' % options.job_name
+ if options.dest_queue:
+ shell_cmd += ' -q ' + options.dest_queue
shell = Shell(shell_cmd)
diff --git a/util/stats/db.py b/util/stats/db.py
index ab005e67b..1ece6df88 100644
--- a/util/stats/db.py
+++ b/util/stats/db.py
@@ -103,6 +103,20 @@ class Node(object):
def __str__(self):
return self.name
+class Result(object):
+ def __init__(self, x, y):
+ self.data = {}
+ self.x = x
+ self.y = y
+
+ def __contains__(self, run):
+ return run in self.data
+
+ def __getitem__(self, run):
+ if run not in self.data:
+ self.data[run] = [ [ 0.0 ] * self.y for i in xrange(self.x) ]
+ return self.data[run]
+
class Database(object):
def __init__(self):
self.host = 'zizzer.pool'
@@ -135,7 +149,23 @@ class Database(object):
self.runs = None
self.bins = None
self.ticks = None
- self.__dict__['get'] = type(self).sum
+ self.method = 'sum'
+ self._method = type(self).sum
+
+ def get(self, job, stat):
+ run = self.allRunNames.get(job.name, None)
+ if run is None:
+ print 'run "%s" not found' % job
+ return None
+
+ from info import scalar, vector, value, values, total, len
+ stat.system = self[job.system]
+ if scalar(stat):
+ return value(stat, run.run)
+ if vector(stat):
+ return values(stat, run.run)
+
+ return None
def query(self, sql):
self.cursor.execute(sql)
@@ -203,7 +233,7 @@ class Database(object):
self.query('select * from stats')
import info
for result in self.cursor.fetchall():
- stat = info.NewStat(StatData(result))
+ stat = info.NewStat(self, StatData(result))
self.append(stat)
self.allStats.append(stat)
self.allStatIds[stat.stat] = stat
@@ -421,30 +451,17 @@ class Database(object):
def stdev(self, stat, bins, ticks):
return self.outer('stddev', 'sum', stat, bins, ticks)
- def __getattribute__(self, attr):
- if attr != 'get':
- return super(Database, self).__getattribute__(attr)
-
- if self.__dict__['get'] == type(self).sum:
- return 'sum'
- elif self.__dict__['get'] == type(self).avg:
- return 'avg'
- elif self.__dict__['get'] == type(self).stdev:
- return 'stdev'
- else:
- return ''
-
def __setattr__(self, attr, value):
- if attr != 'get':
- super(Database, self).__setattr__(attr, value)
+ super(Database, self).__setattr__(attr, value)
+ if attr != 'method':
return
if value == 'sum':
- self.__dict__['get'] = type(self).sum
+ self._method = self.sum
elif value == 'avg':
- self.__dict__['get'] = type(self).avg
+ self._method = self.avg
elif value == 'stdev':
- self.__dict__['get'] = type(self).stdev
+ self._method = self.stdev
else:
raise AttributeError, "can only set get to: sum | avg | stdev"
@@ -453,10 +470,12 @@ class Database(object):
bins = self.bins
if ticks is None:
ticks = self.ticks
- sql = self.__dict__['get'](self, stat, bins, ticks)
+ sql = self._method(self, stat, bins, ticks)
self.query(sql)
runs = {}
+ xmax = 0
+ ymax = 0
for x in self.cursor.fetchall():
data = Data(x)
if not runs.has_key(data.run):
@@ -464,8 +483,17 @@ class Database(object):
if not runs[data.run].has_key(data.x):
runs[data.run][data.x] = {}
+ xmax = max(xmax, data.x)
+ ymax = max(ymax, data.y)
runs[data.run][data.x][data.y] = data.data
- return runs
+
+ results = Result(xmax + 1, ymax + 1)
+ for run,data in runs.iteritems():
+ result = results[run]
+ for x,ydata in data.iteritems():
+ for y,data in ydata.iteritems():
+ result[x][y] = data
+ return results
def __getitem__(self, key):
return self.stattop[key]
diff --git a/util/stats/display.py b/util/stats/display.py
index 629684ca4..fbcff5c70 100644
--- a/util/stats/display.py
+++ b/util/stats/display.py
@@ -26,7 +26,7 @@
class Value:
def __init__(self, value, precision, percent = False):
- self.value = value
+ self.value = float(value)
self.precision = precision
self.percent = percent
def __str__(self):
@@ -90,61 +90,60 @@ class Print:
class VectorDisplay:
def display(self):
+ if not self.value:
+ return
+
p = Print()
p.flags = self.flags
p.precision = self.precision
- if isinstance(self.value, (list, tuple)):
- if not len(self.value):
- return
-
- mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
- mycdf = 0.0
+ if not isinstance(self.value, (list, tuple)):
+ p.name = self.name
+ p.desc = self.desc
+ p.value = self.value
+ p.display()
+ return
- value = self.value
+ mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
+ mycdf = 0.0
- if display_all:
- subnames = [ '[%d]' % i for i in range(len(value)) ]
- else:
- subnames = [''] * len(value)
-
- if self.__dict__.has_key('subnames'):
- for i,each in enumerate(self.subnames):
- if len(each) > 0:
- subnames[i] = '.%s' % each
-
- subdescs = [self.desc]*len(value)
- if self.__dict__.has_key('subdescs'):
- for i in xrange(min(len(value), len(self.subdescs))):
- subdescs[i] = self.subdescs[i]
-
- for val,sname,sdesc in map(None, value, subnames, subdescs):
- if mytotal > 0.0:
- mypdf = float(val) / float(mytotal)
- mycdf += mypdf
- if (self.flags & flags_pdf):
- p.pdf = mypdf
- p.cdf = mycdf
-
- if len(sname) == 0:
- continue
-
- p.name = self.name + sname
- p.desc = sdesc
- p.value = val
- p.display()
-
- if (self.flags & flags_total):
- if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
- if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
- p.name = self.name + '.total'
- p.desc = self.desc
- p.value = mytotal
- p.display()
+ value = self.value
+ if display_all:
+ subnames = [ '[%d]' % i for i in range(len(value)) ]
else:
- p.name = self.name
- p.desc = self.desc
- p.value = self.value
+ subnames = [''] * len(value)
+
+ if self.__dict__.has_key('subnames'):
+ for i,each in enumerate(self.subnames):
+ if len(each) > 0:
+ subnames[i] = '.%s' % each
+
+ subdescs = [self.desc]*len(value)
+ if self.__dict__.has_key('subdescs'):
+ for i in xrange(min(len(value), len(self.subdescs))):
+ subdescs[i] = self.subdescs[i]
+
+ for val,sname,sdesc in map(None, value, subnames, subdescs):
+ if mytotal > 0.0:
+ mypdf = float(val) / float(mytotal)
+ mycdf += mypdf
+ if (self.flags & flags_pdf):
+ p.pdf = mypdf
+ p.cdf = mycdf
+
+ if len(sname) == 0:
+ continue
+
+ p.name = self.name + sname
+ p.desc = sdesc
+ p.value = val
p.display()
+ if (self.flags & flags_total):
+ if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
+ if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
+ p.name = self.name + '.total'
+ p.desc = self.desc
+ p.value = mytotal
+ p.display()
diff --git a/util/stats/info.py b/util/stats/info.py
index ae5d3211f..889af6d53 100644
--- a/util/stats/info.py
+++ b/util/stats/info.py
@@ -27,391 +27,347 @@
from __future__ import division
import operator, re, types
-source = None
-display_run = 0
-global globalTicks
-globalTicks = None
-
-def total(f):
- if isinstance(f, FormulaStat):
- v = f.value
- else:
- v = f
-
- f = FormulaStat()
- if isinstance(v, (list, tuple)):
- f.value = reduce(operator.add, v)
- else:
- f.value = v
-
- return f
-
-def unaryop(op, f):
- if isinstance(f, FormulaStat):
- v = f.value
- else:
- v = f
-
- if isinstance(v, (list, tuple)):
- return map(op, v)
- else:
- return op(v)
-
-def zerodiv(lv, rv):
- if rv == 0.0:
- return 0.0
- else:
- return operator.truediv(lv, rv)
-
-def wrapop(op, lv, rv):
- if isinstance(lv, str):
- return lv
-
- if isinstance(rv, str):
- return rv
-
- return op(lv, rv)
-
-def same(lrun, rrun):
- for lx,rx in zip(lrun.keys(),rrun.keys()):
- if lx != rx:
- print 'lx != rx'
- print lx, rx
- print lrun.keys()
- print rrun.keys()
- return False
- for ly,ry in zip(lrun[lx].keys(),rrun[rx].keys()):
- if ly != ry:
- print 'ly != ry'
- print ly, ry
- print lrun[lx].keys()
- print rrun[rx].keys()
- return False
- return True
-
-
-def binaryop(op, lf, rf):
- result = {}
-
- if isinstance(lf, FormulaStat) and isinstance(rf, FormulaStat):
- lv = lf.value
- rv = rf.value
-
- theruns = []
- for r in lv.keys():
- if rv.has_key(r):
- if same(lv[r], rv[r]):
- theruns.append(r)
- else:
- raise AttributeError
-
- for run in theruns:
- result[run] = {}
- for x in lv[run].keys():
- result[run][x] = {}
- for y in lv[run][x].keys():
- result[run][x][y] = wrapop(op, lv[run][x][y],
- rv[run][x][y])
- elif isinstance(lf, FormulaStat):
- lv = lf.value
- for run in lv.keys():
- result[run] = {}
- for x in lv[run].keys():
- result[run][x] = {}
- for y in lv[run][x].keys():
- result[run][x][y] = wrapop(op, lv[run][x][y], rf)
- elif isinstance(rf, FormulaStat):
- rv = rf.value
- for run in rv.keys():
- result[run] = {}
- for x in rv[run].keys():
- result[run][x] = {}
- for y in rv[run][x].keys():
- result[run][x][y] = wrapop(op, lf, rv[run][x][y])
-
+def unproxy(proxy):
+ if hasattr(proxy, '__unproxy__'):
+ return proxy.__unproxy__()
+
+ return proxy
+
+def scalar(stat):
+ stat = unproxy(stat)
+ assert(stat.__scalar__() != stat.__vector__())
+ return stat.__scalar__()
+
+def vector(stat):
+ stat = unproxy(stat)
+ assert(stat.__scalar__() != stat.__vector__())
+ return stat.__vector__()
+
+def value(stat, *args):
+ stat = unproxy(stat)
+ return stat.__value__(*args)
+
+def values(stat, run):
+ stat = unproxy(stat)
+ result = []
+ for i in xrange(len(stat)):
+ val = value(stat, run, i)
+ if val is None:
+ return None
+ result.append(val)
return result
-def sums(x, y):
- if isinstance(x, (list, tuple)):
- return map(lambda x, y: x + y, x, y)
- else:
- return x + y
-
-def alltrue(seq):
- return reduce(lambda x, y: x and y, seq)
-
-def allfalse(seq):
- return not reduce(lambda x, y: x or y, seq)
-
-def enumerate(seq):
- return map(None, range(len(seq)), seq)
-
-def cmp(a, b):
- if a < b:
- return -1
- elif a == b:
- return 0
- else:
- return 1
-
-class Statistic(object):
-
- def __init__(self, data):
- self.__dict__.update(data.__dict__)
- if not self.__dict__.has_key('value'):
- self.__dict__['value'] = None
- if not self.__dict__.has_key('bins'):
- self.__dict__['bins'] = None
- if not self.__dict__.has_key('ticks'):
- self.__dict__['ticks'] = None
- if 'vc' not in self.__dict__:
- self.vc = {}
-
- def __getattribute__(self, attr):
- if attr == 'ticks':
- if self.__dict__['ticks'] != globalTicks:
- self.__dict__['value'] = None
- self.__dict__['ticks'] = globalTicks
- return self.__dict__['ticks']
- if attr == 'value':
- if self.__dict__['ticks'] != globalTicks:
- if self.__dict__['ticks'] != None and \
- len(self.__dict__['ticks']) == 1:
- self.vc[self.__dict__['ticks'][0]] = self.__dict__['value']
- self.__dict__['ticks'] = globalTicks
- if len(globalTicks) == 1 and self.vc.has_key(globalTicks[0]):
- self.__dict__['value'] = self.vc[globalTicks[0]]
- else:
- self.__dict__['value'] = None
- if self.__dict__['value'] == None:
- self.__dict__['value'] = self.getValue()
- return self.__dict__['value']
- else:
- return super(Statistic, self).__getattribute__(attr)
-
- def __setattr__(self, attr, value):
- if attr == 'bins' or attr == 'ticks':
- if attr == 'bins':
- if value is not None:
- value = source.getBin(value)
- #elif attr == 'ticks' and type(value) is str:
- # value = [ int(x) for x in value.split() ]
-
- self.__dict__[attr] = value
- self.__dict__['value'] = None
- self.vc = {}
- else:
- super(Statistic, self).__setattr__(attr, value)
+def total(stat, run):
+ return sum(values(stat, run))
- def getValue(self):
- raise AttributeError, 'getValue() must be defined'
-
- def zero(self):
- return False
+def len(stat):
+ stat = unproxy(stat)
+ return stat.__len__()
- def __ne__(self, other):
- return not (self == other)
+class Value(object):
+ def __scalar__(self):
+ raise AttributeError, "must define __scalar__ for %s" % (type (self))
+ def __vector__(self):
+ raise AttributeError, "must define __vector__ for %s" % (type (self))
- def __str__(self):
- return '%f' % (float(self))
-
-class FormulaStat(object):
def __add__(self, other):
- f = FormulaStat()
- f.value = binaryop(operator.add, self, other)
- return f
+ return BinaryProxy(operator.__add__, self, other)
def __sub__(self, other):
- f = FormulaStat()
- f.value = binaryop(operator.sub, self, other)
- return f
+ return BinaryProxy(operator.__sub__, self, other)
def __mul__(self, other):
- f = FormulaStat()
- f.value = binaryop(operator.mul, self, other)
- return f
+ return BinaryProxy(operator.__mul__, self, other)
+ def __div__(self, other):
+ return BinaryProxy(operator.__div__, self, other)
def __truediv__(self, other):
- f = FormulaStat()
- f.value = binaryop(zerodiv, self, other)
- return f
- def __mod__(self, other):
- f = FormulaStat()
- f.value = binaryop(operator.mod, self, other)
- return f
+ return BinaryProxy(operator.__truediv__, self, other)
+ def __floordiv__(self, other):
+ return BinaryProxy(operator.__floordiv__, self, other)
+
def __radd__(self, other):
- f = FormulaStat()
- f.value = binaryop(operator.add, other, self)
- return f
+ return BinaryProxy(operator.__add__, other, self)
def __rsub__(self, other):
- f = FormulaStat()
- f.value = binaryop(operator.sub, other, self)
- return f
+ return BinaryProxy(operator.__sub__, other, self)
def __rmul__(self, other):
- f = FormulaStat()
- f.value = binaryop(operator.mul, other, self)
- return f
+ return BinaryProxy(operator.__mul__, other, self)
+ def __rdiv__(self, other):
+ return BinaryProxy(operator.__div__, other, self)
def __rtruediv__(self, other):
- f = FormulaStat()
- f.value = binaryop(zerodiv, other, self)
- return f
- def __rmod__(self, other):
- f = FormulaStat()
- f.value = binaryop(operator.mod, other, self)
- return f
+ return BinaryProxy(operator.__truediv__, other, self)
+ def __rfloordiv__(self, other):
+ return BinaryProxy(operator.__floordiv__, other, self)
+
def __neg__(self):
- f = FormulaStat()
- f.value = unaryop(operator.neg, self)
- return f
- def __getitem__(self, idx):
- f = FormulaStat()
- f.value = {}
- for key in self.value.keys():
- f.value[key] = {}
- f.value[key][0] = {}
- f.value[key][0][0] = self.value[key][idx][0]
- return f
-
- def __float__(self):
- if isinstance(self.value, FormulaStat):
- return float(self.value)
- if not self.value.has_key(display_run):
- return (1e300*1e300)
- if len(self.value[display_run]) == 1:
- return self.value[display_run][0][0]
- else:
- #print self.value[display_run]
- return self.value[display_run][4][0]
- #raise ValueError
+ return UnaryProxy(operator.__neg__, self)
+ def __pos__(self):
+ return UnaryProxy(operator.__pos__, self)
+ def __abs__(self):
+ return UnaryProxy(operator.__abs__, self)
+
+class Scalar(Value):
+ def __scalar__(self):
+ return True
- def display(self):
- import display
- d = display.VectorDisplay()
- d.flags = 0
- d.precision = 1
- d.name = 'formula'
- d.desc = 'formula'
- val = self.value[display_run]
- d.value = [ val[x][0] for x in val.keys() ]
- d.display()
+ def __vector__(self):
+ return False
+ def __value__(self, run):
+ raise AttributeError, '__value__ must be defined'
-class Scalar(Statistic,FormulaStat):
- def getValue(self):
- return source.data(self, self.bins, self.ticks)
+class VectorItemProxy(Value):
+ def __init__(self, proxy, index):
+ self.proxy = proxy
+ self.index = index
- def display(self):
- import display
- p = display.Print()
- p.name = self.name
- p.desc = self.desc
- p.value = float(self)
- p.flags = self.flags
- p.precision = self.precision
- if display.all or (self.flags & flags.printable):
- p.display()
+ def __scalar__(self):
+ return True
- def comparable(self, other):
- return self.name == other.name
+ def __vector__(self):
+ return False
- def __eq__(self, other):
- return self.value == other.value
+ def __value__(self, run):
+ return value(self.proxy, run, self.index)
- def __isub__(self, other):
- self.value -= other.value
- return self
+class Vector(Value):
+ def __scalar__(self):
+ return False
- def __iadd__(self, other):
- self.value += other.value
- return self
+ def __vector__(self):
+ return True
- def __itruediv__(self, other):
- if not other:
- return self
- self.value /= other
- return self
+ def __value__(self, run, index):
+ raise AttributeError, '__value__ must be defined'
-class Vector(Statistic,FormulaStat):
- def getValue(self):
- return source.data(self, self.bins, self.ticks);
+ def __getitem__(self, index):
+ return VectorItemProxy(self, index)
- def display(self):
- import display
- if not display.all and not (self.flags & flags.printable):
- return
+class ScalarConstant(Scalar):
+ def __init__(self, constant):
+ self.constant = constant
+ def __value__(self, run):
+ return self.constant
- d = display.VectorDisplay()
- d.__dict__.update(self.__dict__)
- d.display()
+class VectorConstant(Vector):
+ def __init__(self, constant):
+ self.constant = constant
+ def __value__(self, run, index):
+ return self.constant[index]
+ def __len__(self):
+ return len(self.constant)
- def comparable(self, other):
- return self.name == other.name and \
- len(self.value) == len(other.value)
+def WrapValue(value):
+ if isinstance(value, (int, long, float)):
+ return ScalarConstant(value)
+ if isinstance(value, (list, tuple)):
+ return VectorConstant(value)
+ if isinstance(value, Value):
+ return value
- def __eq__(self, other):
- if isinstance(self.value, (list, tuple)) != \
- isinstance(other.value, (list, tuple)):
- return False
+ raise AttributeError, 'Only values can be wrapped'
- if isinstance(self.value, (list, tuple)):
- if len(self.value) != len(other.value):
- return False
- else:
- for v1,v2 in zip(self.value, other.value):
- if v1 != v2:
- return False
- return True
- else:
- return self.value == other.value
+class Statistic(object):
+ def __getattr__(self, attr):
+ if attr in ('data', 'x', 'y'):
+ result = self.source.data(self, self.bins, self.ticks)
+ self.data = result.data
+ self.x = result.x
+ self.y = result.y
+ return super(Statistic, self).__getattribute__(attr)
- def __isub__(self, other):
- self.value = binaryop(operator.sub, self.value, other.value)
- return self
+ def __setattr__(self, attr, value):
+ if attr == 'stat':
+ raise AttributeError, '%s is read only' % stat
+ if attr in ('source', 'bins', 'ticks'):
+ if getattr(self, attr) != value:
+ if hasattr(self, 'data'):
+ delattr(self, 'data')
+
+ super(Statistic, self).__setattr__(attr, value)
+
+class ValueProxy(Value):
+ def __getattr__(self, attr):
+ if attr == '__value__':
+ if scalar(self):
+ return self.__scalarvalue__
+ if vector(self):
+ return self.__vectorvalue__
+ if attr == '__len__':
+ if vector(self):
+ return self.__vectorlen__
+ return super(ValueProxy, self).__getattribute__(attr)
+
+class UnaryProxy(ValueProxy):
+ def __init__(self, op, arg):
+ self.op = op
+ self.arg = WrapValue(arg)
+
+ def __scalar__(self):
+ return scalar(self.arg)
+
+ def __vector__(self):
+ return vector(self.arg)
+
+ def __scalarvalue__(self, run):
+ val = value(self.arg, run)
+ if val is None:
+ return None
+ return self.op(val)
+
+ def __vectorvalue__(self, run, index):
+ val = value(self.arg, run, index)
+ if val is None:
+ return None
+ return self.op(val)
+
+ def __vectorlen__(self):
+ return len(unproxy(self.arg))
+
+class BinaryProxy(ValueProxy):
+ def __init__(self, op, arg0, arg1):
+ super(BinaryProxy, self).__init__()
+ self.op = op
+ self.arg0 = WrapValue(arg0)
+ self.arg1 = WrapValue(arg1)
+
+ def __scalar__(self):
+ return scalar(self.arg0) and scalar(self.arg1)
+
+ def __vector__(self):
+ return vector(self.arg0) or vector(self.arg1)
+
+ def __scalarvalue__(self, run):
+ val0 = value(self.arg0, run)
+ val1 = value(self.arg1, run)
+ if val0 is None or val1 is None:
+ return None
+ return self.op(val0, val1)
+
+ def __vectorvalue__(self, run, index):
+ if scalar(self.arg0):
+ val0 = value(self.arg0, run)
+ if vector(self.arg0):
+ val0 = value(self.arg0, run, index)
+ if scalar(self.arg1):
+ val1 = value(self.arg1, run)
+ if vector(self.arg1):
+ val1 = value(self.arg1, run, index)
+
+ if val0 is None or val1 is None:
+ return None
+
+ return self.op(val0, val1)
+
+ def __vectorlen__(self):
+ if vector(self.arg0) and scalar(self.arg1):
+ return len(self.arg0)
+ if scalar(self.arg0) and vector(self.arg1):
+ return len(self.arg1)
+
+ len0 = len(self.arg0)
+ len1 = len(self.arg1)
+
+ if len0 != len1:
+ raise AttributeError, \
+ "vectors of different lengths %d != %d" % (len0, len1)
+
+ return len0
+
+class Proxy(Value):
+ def __init__(self, name, dict):
+ self.name = name
+ self.dict = dict
+
+ def __unproxy__(self):
+ return unproxy(self.dict[self.name])
+
+ def __getitem__(self, index):
+ return ItemProxy(self, index)
+
+ def __getattr__(self, attr):
+ return AttrProxy(self, attr)
+
+class ItemProxy(Proxy):
+ def __init__(self, proxy, index):
+ self.proxy = proxy
+ self.index = index
+
+ def __unproxy__(self):
+ return unproxy(unproxy(self.proxy)[self.index])
- def __iadd__(self, other):
- self.value = binaryop(operator.add, self.value, other.value)
- return self
+class AttrProxy(Proxy):
+ def __init__(self, proxy, attr):
+ self.proxy = proxy
+ self.attr = attr
- def __itruediv__(self, other):
- if not other:
- return self
- if isinstance(self.value, (list, tuple)):
- for i in xrange(len(self.value)):
- self.value[i] /= other
- else:
- self.value /= other
- return self
+ def __unproxy__(self):
+ return unproxy(getattr(unproxy(self.proxy), self.attr))
-class Formula(Vector):
- def getValue(self):
- formula = re.sub(':', '__', self.formula)
- x = eval(formula, source.stattop)
- return x.value
+class ProxyGroup(object):
+ def __init__(self, dict=None, **kwargs):
+ self.__dict__['dict'] = {}
- def comparable(self, other):
- return self.name == other.name and \
- compare(self.dist, other.dist)
+ if dict is not None:
+ self.dict.update(dict)
- def __eq__(self, other):
- return self.value == other.value
+ if kwargs:
+ self.dict.update(kwargs)
- def __isub__(self, other):
- return self
+ def __getattr__(self, name):
+ return Proxy(name, self.dict)
- def __iadd__(self, other):
- return self
+ def __setattr__(self, attr, value):
+ self.dict[attr] = value
- def __itruediv__(self, other):
- if not other:
- return self
- return self
+class ScalarStat(Statistic,Scalar):
+ def __value__(self, run):
+ if run not in self.data:
+ return None
+ return self.data[run][0][0]
+
+ def display(self, run=None):
+ import display
+ p = display.Print()
+ p.name = self.name
+ p.desc = self.desc
+ p.value = value(self, run)
+ p.flags = self.flags
+ p.precision = self.precision
+ if display.all or (self.flags & flags.printable):
+ p.display()
+
+class VectorStat(Statistic,Vector):
+ def __value__(self, run, item):
+ if run not in self.data:
+ return None
+ return self.data[run][item][0]
+
+ def __len__(self):
+ return self.x
-class SimpleDist(object):
+ def display(self, run=None):
+ import display
+ d = display.VectorDisplay()
+ d.name = self.name
+ d.desc = self.desc
+ d.value = [ value(self, run, i) for i in xrange(len(self)) ]
+ d.flags = self.flags
+ d.precision = self.precision
+ d.display()
+
+class Formula(Value):
+ def __getattribute__(self, attr):
+ if attr not in ( '__scalar__', '__vector__', '__value__', '__len__' ):
+ return super(Formula, self).__getattribute__(attr)
+
+ formula = re.sub(':', '__', self.formula)
+ value = eval(formula, self.source.stattop)
+ return getattr(value, attr)
+
+class SimpleDist(Statistic):
def __init__(self, sums, squares, samples):
self.sums = sums
self.squares = squares
self.samples = samples
- def getValue(self):
- return 0.0
-
def display(self, name, desc, flags, precision):
import display
p = display.Print()
@@ -482,9 +438,6 @@ class FullDist(SimpleDist):
self.bsize = bsize
self.size = size
- def getValue(self):
- return 0.0
-
def display(self, name, desc, flags, precision):
import display
p = display.Print()
@@ -574,9 +527,6 @@ class FullDist(SimpleDist):
return self
class Dist(Statistic):
- def getValue(self):
- return 0.0
-
def display(self):
import display
if not display.all and not (self.flags & flags.printable):
@@ -606,9 +556,6 @@ class Dist(Statistic):
return self
class VectorDist(Statistic):
- def getValue(self):
- return 0.0
-
def display(self):
import display
if not display.all and not (self.flags & flags.printable):
@@ -694,9 +641,6 @@ class VectorDist(Statistic):
return self
class Vector2d(Statistic):
- def getValue(self):
- return 0.0
-
def display(self):
import display
if not display.all and not (self.flags & flags.printable):
@@ -748,20 +692,25 @@ class Vector2d(Statistic):
return self
return self
-def NewStat(data):
+def NewStat(source, data):
stat = None
if data.type == 'SCALAR':
- stat = Scalar(data)
+ stat = ScalarStat()
elif data.type == 'VECTOR':
- stat = Vector(data)
+ stat = VectorStat()
elif data.type == 'DIST':
- stat = Dist(data)
+ stat = Dist()
elif data.type == 'VECTORDIST':
- stat = VectorDist(data)
+ stat = VectorDist()
elif data.type == 'VECTOR2D':
- stat = Vector2d(data)
+ stat = Vector2d()
elif data.type == 'FORMULA':
- stat = Formula(data)
+ stat = Formula()
+
+ stat.__dict__['source'] = source
+ stat.__dict__['bins'] = None
+ stat.__dict__['ticks'] = None
+ stat.__dict__.update(data.__dict__)
return stat
diff --git a/util/stats/output.py b/util/stats/output.py
index f793749f7..cf76f291e 100644
--- a/util/stats/output.py
+++ b/util/stats/output.py
@@ -26,24 +26,8 @@
#
# Authors: Nathan Binkert
-class dbinfo(object):
- def get(self, job, stat):
- import info
-
- run = info.source.allRunNames.get(job.name, None)
- if run is None:
- print 'run "%s" not found' % job
- return None
-
- stat.system = info.source[job.system]
- info.display_run = run.run;
- val = float(stat)
- if val == 1e300*1e300:
- return None
- return val
-
class StatOutput(object):
- def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None):
+ def __init__(self, name, jobfile, info, stat=None, binstats=None):
self.name = name
self.jobfile = jobfile
self.stat = stat
@@ -141,7 +125,7 @@ class StatOutput(object):
data = zeros((len(groupopts), len(baropts)), Float)
data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ]
enabled = False
- stacked = None
+ stacked = 0
for g,gopt in enumerate(groupopts):
for b,bopt in enumerate(baropts):
job = self.jobfile.job(options + [ gopt, bopt ])
@@ -149,17 +133,28 @@ class StatOutput(object):
continue
val = self.info.get(job, self.stat)
- if val is None:
- val = 0.0
- curstacked = isinstance(val, (list, tuple))
- if stacked is None:
- stacked = curstacked
- else:
- if stacked != curstacked:
- raise ValueError, "some stats stacked, some not"
+ if isinstance(val, (list, tuple)):
+ if len(val) == 1:
+ val = val[0]
+ else:
+ stacked = len(val)
data[g][b] = val
+ if stacked == 0:
+ for i in xrange(len(groupopts)):
+ for j in xrange(len(baropts)):
+ if data[i][j] is None:
+ data[i][j] = 0.0
+ else:
+ for i in xrange(len(groupopts)):
+ for j in xrange(len(baropts)):
+ val = data[i][j]
+ if val is None:
+ data[i][j] = [] * stacked
+ elif len(val) != stacked:
+ raise ValueError, "some stats stacked, some not"
+
data = array(data)
if data.sum() == 0:
continue
@@ -167,7 +162,10 @@ class StatOutput(object):
bar_descs = [ opt.desc for opt in baropts ]
group_descs = [ opt.desc for opt in groupopts ]
if stacked:
- legend = self.info.rcategories
+ try:
+ legend = self.info.rcategories
+ except:
+ legend = [ str(i) for i in xrange(stacked) ]
else:
legend = bar_descs
diff --git a/util/stats/profile.py b/util/stats/profile.py
index cf946452b..57b854a30 100644
--- a/util/stats/profile.py
+++ b/util/stats/profile.py
@@ -316,7 +316,7 @@ class Profile(object):
symbols.tree.dot(dot, threshold=threshold)
dot.write(symbols.filename[:-3] + 'dot')
- def write_txt(self, jobfile=None, jobs=None):
+ def write_txt(self, jobfile=None, jobs=None, limit=None):
if jobs is None:
jobs = [ job for job in jobfile.jobs() ]
@@ -327,7 +327,7 @@ class Profile(object):
continue
output = file(symbols.filename[:-3] + 'txt', 'w')
- symbols.display(output)
+ symbols.display(output, limit)
def display(self, jobfile=None, jobs=None, limit=None):
if jobs is None:
@@ -453,6 +453,8 @@ if __name__ == '__main__':
else:
profile = PCProfile()
+ if not categorize:
+ profile.categorize = None
profile.inputdir(jobfile.rootdir)
if graph:
@@ -470,11 +472,6 @@ if __name__ == '__main__':
profile.cpu = cpu
profile.write_dot(jobfile=jobfile, threshold=threshold)
- if not categorize:
- for cpu in cpus:
- profile.cpu = cpu
- profile.categorize = None
-
if textout:
for cpu in cpus:
profile.cpu = cpu
@@ -482,5 +479,7 @@ if __name__ == '__main__':
if not graph and not textout and not dodot:
for cpu in cpus:
+ if not categorize:
+ profile.categorize = None
profile.cpu = cpu
profile.display(jobfile=jobfile, limit=numsyms)
diff --git a/util/stats/stats.py b/util/stats/stats.py
index 3f6d162bc..61f54dede 100755
--- a/util/stats/stats.py
+++ b/util/stats/stats.py
@@ -98,29 +98,27 @@ def commands(options, command, args):
raise CommandException
- import db, info
- info.source = db.Database()
- info.source.host = options.host
- info.source.db = options.db
- info.source.passwd = options.passwd
- info.source.user = options.user
- info.source.connect()
- #info.source.update_dict(globals())
-
- if type(options.get) is str:
- info.source.get = options.get
+ import db
+ source = db.Database()
+ source.host = options.host
+ source.db = options.db
+ source.passwd = options.passwd
+ source.user = options.user
+ source.connect()
+ #source.update_dict(globals())
+
+ if type(options.method) is str:
+ source.method = options.method
if options.runs is None:
- runs = info.source.allRuns
+ runs = source.allRuns
else:
rx = re.compile(options.runs)
runs = []
- for run in info.source.allRuns:
+ for run in source.allRuns:
if rx.match(run.name):
runs.append(run)
- info.display_run = runs[0].run
-
if command == 'runs':
user = None
opts, args = getopts(args, '-u')
@@ -129,14 +127,14 @@ def commands(options, command, args):
for o,a in opts:
if o == '-u':
user = a
- info.source.listRuns(user)
+ source.listRuns(user)
return
if command == 'stats':
if len(args) == 0:
- info.source.listStats()
+ source.listStats()
elif len(args) == 1:
- info.source.listStats(args[0])
+ source.listStats(args[0])
else:
raise CommandException
@@ -144,9 +142,9 @@ def commands(options, command, args):
if command == 'bins':
if len(args) == 0:
- info.source.listBins()
+ source.listBins()
elif len(args) == 1:
- info.source.listBins(args[0])
+ source.listBins(args[0])
else:
raise CommandException
@@ -154,9 +152,9 @@ def commands(options, command, args):
if command == 'formulas':
if len(args) == 0:
- info.source.listFormulas()
+ source.listFormulas()
elif len(args) == 1:
- info.source.listFormulas(args[0])
+ source.listFormulas(args[0])
else:
raise CommandException
@@ -166,7 +164,7 @@ def commands(options, command, args):
if len(args):
raise CommandException
- info.source.listTicks(runs)
+ source.listTicks(runs)
return
if command == 'stability':
@@ -177,8 +175,8 @@ def commands(options, command, args):
merge = int(args[0])
except ValueError:
usage()
- stats = info.source.getStat(args[1])
- info.source.get = "sum"
+ stats = source.getStat(args[1])
+ source.method = 'sum'
def disp(*args):
print "%-35s %12s %12s %4s %5s %5s %5s %10s" % args
@@ -195,18 +193,17 @@ def commands(options, command, args):
#loop through all the selected runs
for run in runs:
- info.display_run = run.run;
- runTicks = info.source.retTicks([ run ])
+ runTicks = source.retTicks([ run ])
#throw away the first one, it's 0
runTicks.pop(0)
- info.globalTicks = runTicks
+ source.ticks = runTicks
avg = 0
stdev = 0
numoutsideavg = 0
numoutside1std = 0
numoutside2std = 0
pairRunTicks = []
- if float(stat) == 1e300*1e300:
+ if value(stat, run.run) == 1e300*1e300:
continue
for t in range(0, len(runTicks)-(merge-1), merge):
tempPair = []
@@ -215,17 +212,17 @@ def commands(options, command, args):
pairRunTicks.append(tempPair)
#loop through all the various ticks for each run
for tick in pairRunTicks:
- info.globalTicks = tick
- avg += float(stat)
+ source.ticks = tick
+ avg += value(stat, run.run)
avg /= len(pairRunTicks)
for tick in pairRunTicks:
- info.globalTicks = tick
- val = float(stat)
+ source.ticks = tick
+ val = value(stat, run.run)
stdev += pow((val-avg),2)
stdev = math.sqrt(stdev / len(pairRunTicks))
for tick in pairRunTicks:
- info.globalTicks = tick
- val = float(stat)
+ source.ticks = tick
+ val = value(stat, run.run)
if (val < (avg * .9)) or (val > (avg * 1.1)):
numoutsideavg += 1
if (val < (avg - stdev)) or (val > (avg + stdev)):
@@ -264,9 +261,9 @@ def commands(options, command, args):
if options.ticks:
if not options.graph:
print 'only displaying sample %s' % options.ticks
- info.globalTicks = [ int(x) for x in options.ticks.split() ]
+ source.ticks = [ int(x) for x in options.ticks.split() ]
- from output import StatOutput
+ import output
def display():
if options.graph:
@@ -280,12 +277,12 @@ def commands(options, command, args):
raise CommandException
if command == 'stat':
- stats = info.source.getStat(args[0])
+ stats = source.getStat(args[0])
if command == 'formula':
stats = eval(args[0])
for stat in stats:
- output = StatOutput(stat.name, options.jobfile)
+ output = output.StatOutput(stat.name, options.jobfile, source)
output.stat = stat
output.label = stat.name
display()
@@ -295,12 +292,11 @@ def commands(options, command, args):
if len(args):
raise CommandException
- system = info.source.__dict__[options.system]
-
- from proxy import ProxyGroup
- sim_ticks = info.source['sim_ticks']
- sim_seconds = info.source['sim_seconds']
- proxy = ProxyGroup(system = info.source[options.system])
+ system = source.__dict__[options.system]
+ from info import ProxyGroup
+ sim_ticks = source['sim_ticks']
+ sim_seconds = source['sim_seconds']
+ proxy = ProxyGroup(system = source[options.system])
system = proxy.system
etherdev = system.tsunami.etherdev0
@@ -309,7 +305,7 @@ def commands(options, command, args):
packets = etherdev.rxPackets + etherdev.txPackets
bps = etherdev.rxBandwidth + etherdev.txBandwidth
- output = StatOutput(command, options.jobfile)
+ output = output.StatOutput(command, options.jobfile, source)
if command == 'usertime':
import copy
@@ -460,7 +456,7 @@ if __name__ == '__main__':
options.user = getpass.getuser()
options.runs = None
options.system = 'client'
- options.get = None
+ options.method = None
options.binned = False
options.graph = False
options.ticks = False
@@ -469,7 +465,7 @@ if __name__ == '__main__':
options.jobfile = None
options.all = False
- opts, args = getopts(sys.argv[1:], '-BEFG:Jad:g:h:j:pr:s:u:T:')
+ opts, args = getopts(sys.argv[1:], '-BEFJad:g:h:j:m:pr:s:u:T:')
for o,a in opts:
if o == '-B':
options.binned = True
@@ -477,8 +473,6 @@ if __name__ == '__main__':
options.printmode = 'E'
if o == '-F':
options.printmode = 'F'
- if o == '-G':
- options.get = a
if o == '-a':
options.all = True
if o == '-d':
@@ -492,6 +486,8 @@ if __name__ == '__main__':
jobfilename = None
if o == '-j':
jobfilename = a
+ if o == '-m':
+ options.method = a
if o == '-p':
options.passwd = getpass.getpass()
if o == '-r':