diff options
| -rw-r--r-- | util/stats/db.py | 72 | ||||
| -rw-r--r-- | util/stats/display.py | 97 | ||||
| -rw-r--r-- | util/stats/info.py | 669 | ||||
| -rw-r--r-- | util/stats/output.py | 52 | ||||
| -rwxr-xr-x | util/stats/stats.py | 96 |
5 files changed, 478 insertions, 508 deletions
diff --git a/util/stats/db.py b/util/stats/db.py index ab005e67b..0d321a360 100644 --- a/util/stats/db.py +++ b/util/stats/db.py @@ -103,6 +103,20 @@ class Node(object): def __str__(self): return self.name +class Result(object): + def __init__(self, x, y): + self.data = {} + self.x = x + self.y = y + + def __contains__(self, run): + return run in self.data + + def __getitem__(self, run): + if run not in self.data: + self.data[run] = [ [ 0.0 ] * self.y for i in xrange(self.x) ] + return self.data[run] + class Database(object): def __init__(self): self.host = 'zizzer.pool' @@ -135,7 +149,23 @@ class Database(object): self.runs = None self.bins = None self.ticks = None - self.__dict__['get'] = type(self).sum + self.method = 'sum' + self._method = type(self).sum + + def get(self, job, stat): + run = self.allRunNames.get(job.name, None) + if run is None: + print 'run "%s" not found' % job + return None + + from info import scalar, vector, value, total, len + stat.system = self[job.system] + if scalar(stat): + return value(stat, run.run) + if vector(stat): + return values(stat, run.run) + + return None def query(self, sql): self.cursor.execute(sql) @@ -203,7 +233,7 @@ class Database(object): self.query('select * from stats') import info for result in self.cursor.fetchall(): - stat = info.NewStat(StatData(result)) + stat = info.NewStat(self, StatData(result)) self.append(stat) self.allStats.append(stat) self.allStatIds[stat.stat] = stat @@ -421,30 +451,17 @@ class Database(object): def stdev(self, stat, bins, ticks): return self.outer('stddev', 'sum', stat, bins, ticks) - def __getattribute__(self, attr): - if attr != 'get': - return super(Database, self).__getattribute__(attr) - - if self.__dict__['get'] == type(self).sum: - return 'sum' - elif self.__dict__['get'] == type(self).avg: - return 'avg' - elif self.__dict__['get'] == type(self).stdev: - return 'stdev' - else: - return '' - def __setattr__(self, attr, value): - if attr != 'get': - super(Database, self).__setattr__(attr, value) + super(Database, self).__setattr__(attr, value) + if attr != 'method': return if value == 'sum': - self.__dict__['get'] = type(self).sum + self._method = self.sum elif value == 'avg': - self.__dict__['get'] = type(self).avg + self._method = self.avg elif value == 'stdev': - self.__dict__['get'] = type(self).stdev + self._method = self.stdev else: raise AttributeError, "can only set get to: sum | avg | stdev" @@ -453,10 +470,12 @@ class Database(object): bins = self.bins if ticks is None: ticks = self.ticks - sql = self.__dict__['get'](self, stat, bins, ticks) + sql = self._method(self, stat, bins, ticks) self.query(sql) runs = {} + xmax = 0 + ymax = 0 for x in self.cursor.fetchall(): data = Data(x) if not runs.has_key(data.run): @@ -464,8 +483,17 @@ class Database(object): if not runs[data.run].has_key(data.x): runs[data.run][data.x] = {} + xmax = max(xmax, data.x) + ymax = max(ymax, data.y) runs[data.run][data.x][data.y] = data.data - return runs + + results = Result(xmax + 1, ymax + 1) + for run,data in runs.iteritems(): + result = results[run] + for x,ydata in data.iteritems(): + for y,data in ydata.iteritems(): + result[x][y] = data + return results def __getitem__(self, key): return self.stattop[key] diff --git a/util/stats/display.py b/util/stats/display.py index 629684ca4..fbcff5c70 100644 --- a/util/stats/display.py +++ b/util/stats/display.py @@ -26,7 +26,7 @@ class Value: def __init__(self, value, precision, percent = False): - self.value = value + self.value = float(value) self.precision = precision self.percent = percent def __str__(self): @@ -90,61 +90,60 @@ class Print: class VectorDisplay: def display(self): + if not self.value: + return + p = Print() p.flags = self.flags p.precision = self.precision - if isinstance(self.value, (list, tuple)): - if not len(self.value): - return - - mytotal = reduce(lambda x,y: float(x) + float(y), self.value) - mycdf = 0.0 + if not isinstance(self.value, (list, tuple)): + p.name = self.name + p.desc = self.desc + p.value = self.value + p.display() + return - value = self.value + mytotal = reduce(lambda x,y: float(x) + float(y), self.value) + mycdf = 0.0 - if display_all: - subnames = [ '[%d]' % i for i in range(len(value)) ] - else: - subnames = [''] * len(value) - - if self.__dict__.has_key('subnames'): - for i,each in enumerate(self.subnames): - if len(each) > 0: - subnames[i] = '.%s' % each - - subdescs = [self.desc]*len(value) - if self.__dict__.has_key('subdescs'): - for i in xrange(min(len(value), len(self.subdescs))): - subdescs[i] = self.subdescs[i] - - for val,sname,sdesc in map(None, value, subnames, subdescs): - if mytotal > 0.0: - mypdf = float(val) / float(mytotal) - mycdf += mypdf - if (self.flags & flags_pdf): - p.pdf = mypdf - p.cdf = mycdf - - if len(sname) == 0: - continue - - p.name = self.name + sname - p.desc = sdesc - p.value = val - p.display() - - if (self.flags & flags_total): - if (p.__dict__.has_key('pdf')): del p.__dict__['pdf'] - if (p.__dict__.has_key('cdf')): del p.__dict__['cdf'] - p.name = self.name + '.total' - p.desc = self.desc - p.value = mytotal - p.display() + value = self.value + if display_all: + subnames = [ '[%d]' % i for i in range(len(value)) ] else: - p.name = self.name - p.desc = self.desc - p.value = self.value + subnames = [''] * len(value) + + if self.__dict__.has_key('subnames'): + for i,each in enumerate(self.subnames): + if len(each) > 0: + subnames[i] = '.%s' % each + + subdescs = [self.desc]*len(value) + if self.__dict__.has_key('subdescs'): + for i in xrange(min(len(value), len(self.subdescs))): + subdescs[i] = self.subdescs[i] + + for val,sname,sdesc in map(None, value, subnames, subdescs): + if mytotal > 0.0: + mypdf = float(val) / float(mytotal) + mycdf += mypdf + if (self.flags & flags_pdf): + p.pdf = mypdf + p.cdf = mycdf + + if len(sname) == 0: + continue + + p.name = self.name + sname + p.desc = sdesc + p.value = val p.display() + if (self.flags & flags_total): + if (p.__dict__.has_key('pdf')): del p.__dict__['pdf'] + if (p.__dict__.has_key('cdf')): del p.__dict__['cdf'] + p.name = self.name + '.total' + p.desc = self.desc + p.value = mytotal + p.display() diff --git a/util/stats/info.py b/util/stats/info.py index ae5d3211f..9932d7922 100644 --- a/util/stats/info.py +++ b/util/stats/info.py @@ -27,391 +27,347 @@ from __future__ import division import operator, re, types -source = None -display_run = 0 -global globalTicks -globalTicks = None - -def total(f): - if isinstance(f, FormulaStat): - v = f.value - else: - v = f - - f = FormulaStat() - if isinstance(v, (list, tuple)): - f.value = reduce(operator.add, v) - else: - f.value = v - - return f - -def unaryop(op, f): - if isinstance(f, FormulaStat): - v = f.value - else: - v = f - - if isinstance(v, (list, tuple)): - return map(op, v) - else: - return op(v) - -def zerodiv(lv, rv): - if rv == 0.0: - return 0.0 - else: - return operator.truediv(lv, rv) - -def wrapop(op, lv, rv): - if isinstance(lv, str): - return lv - - if isinstance(rv, str): - return rv - - return op(lv, rv) - -def same(lrun, rrun): - for lx,rx in zip(lrun.keys(),rrun.keys()): - if lx != rx: - print 'lx != rx' - print lx, rx - print lrun.keys() - print rrun.keys() - return False - for ly,ry in zip(lrun[lx].keys(),rrun[rx].keys()): - if ly != ry: - print 'ly != ry' - print ly, ry - print lrun[lx].keys() - print rrun[rx].keys() - return False - return True - - -def binaryop(op, lf, rf): - result = {} - - if isinstance(lf, FormulaStat) and isinstance(rf, FormulaStat): - lv = lf.value - rv = rf.value - - theruns = [] - for r in lv.keys(): - if rv.has_key(r): - if same(lv[r], rv[r]): - theruns.append(r) - else: - raise AttributeError - - for run in theruns: - result[run] = {} - for x in lv[run].keys(): - result[run][x] = {} - for y in lv[run][x].keys(): - result[run][x][y] = wrapop(op, lv[run][x][y], - rv[run][x][y]) - elif isinstance(lf, FormulaStat): - lv = lf.value - for run in lv.keys(): - result[run] = {} - for x in lv[run].keys(): - result[run][x] = {} - for y in lv[run][x].keys(): - result[run][x][y] = wrapop(op, lv[run][x][y], rf) - elif isinstance(rf, FormulaStat): - rv = rf.value - for run in rv.keys(): - result[run] = {} - for x in rv[run].keys(): - result[run][x] = {} - for y in rv[run][x].keys(): - result[run][x][y] = wrapop(op, lf, rv[run][x][y]) - +def unproxy(proxy): + if hasattr(proxy, '__unproxy__'): + return proxy.__unproxy__() + + return proxy + +def scalar(stat): + stat = unproxy(stat) + assert(stat.__scalar__() != stat.__vector__()) + return stat.__scalar__() + +def vector(stat): + stat = unproxy(stat) + assert(stat.__scalar__() != stat.__vector__()) + return stat.__vector__() + +def value(stat, *args): + stat = unproxy(stat) + return stat.__value__(*args) + +def values(stat, run): + stat = unproxy(stat) + result = [] + for i in xrange(len(stat)): + val = value(stat, run.run, i) + if val is None: + return None + result.append(val) return result -def sums(x, y): - if isinstance(x, (list, tuple)): - return map(lambda x, y: x + y, x, y) - else: - return x + y - -def alltrue(seq): - return reduce(lambda x, y: x and y, seq) - -def allfalse(seq): - return not reduce(lambda x, y: x or y, seq) - -def enumerate(seq): - return map(None, range(len(seq)), seq) - -def cmp(a, b): - if a < b: - return -1 - elif a == b: - return 0 - else: - return 1 - -class Statistic(object): - - def __init__(self, data): - self.__dict__.update(data.__dict__) - if not self.__dict__.has_key('value'): - self.__dict__['value'] = None - if not self.__dict__.has_key('bins'): - self.__dict__['bins'] = None - if not self.__dict__.has_key('ticks'): - self.__dict__['ticks'] = None - if 'vc' not in self.__dict__: - self.vc = {} - - def __getattribute__(self, attr): - if attr == 'ticks': - if self.__dict__['ticks'] != globalTicks: - self.__dict__['value'] = None - self.__dict__['ticks'] = globalTicks - return self.__dict__['ticks'] - if attr == 'value': - if self.__dict__['ticks'] != globalTicks: - if self.__dict__['ticks'] != None and \ - len(self.__dict__['ticks']) == 1: - self.vc[self.__dict__['ticks'][0]] = self.__dict__['value'] - self.__dict__['ticks'] = globalTicks - if len(globalTicks) == 1 and self.vc.has_key(globalTicks[0]): - self.__dict__['value'] = self.vc[globalTicks[0]] - else: - self.__dict__['value'] = None - if self.__dict__['value'] == None: - self.__dict__['value'] = self.getValue() - return self.__dict__['value'] - else: - return super(Statistic, self).__getattribute__(attr) - - def __setattr__(self, attr, value): - if attr == 'bins' or attr == 'ticks': - if attr == 'bins': - if value is not None: - value = source.getBin(value) - #elif attr == 'ticks' and type(value) is str: - # value = [ int(x) for x in value.split() ] - - self.__dict__[attr] = value - self.__dict__['value'] = None - self.vc = {} - else: - super(Statistic, self).__setattr__(attr, value) +def total(stat, run): + return sum(values(stat, run)) - def getValue(self): - raise AttributeError, 'getValue() must be defined' - - def zero(self): - return False +def len(stat): + stat = unproxy(stat) + return stat.__len__() - def __ne__(self, other): - return not (self == other) +class Value(object): + def __scalar__(self): + raise AttributeError, "must define __scalar__ for %s" % (type (self)) + def __vector__(self): + raise AttributeError, "must define __vector__ for %s" % (type (self)) - def __str__(self): - return '%f' % (float(self)) - -class FormulaStat(object): def __add__(self, other): - f = FormulaStat() - f.value = binaryop(operator.add, self, other) - return f + return BinaryProxy(operator.__add__, self, other) def __sub__(self, other): - f = FormulaStat() - f.value = binaryop(operator.sub, self, other) - return f + return BinaryProxy(operator.__sub__, self, other) def __mul__(self, other): - f = FormulaStat() - f.value = binaryop(operator.mul, self, other) - return f + return BinaryProxy(operator.__mul__, self, other) + def __div__(self, other): + return BinaryProxy(operator.__div__, self, other) def __truediv__(self, other): - f = FormulaStat() - f.value = binaryop(zerodiv, self, other) - return f - def __mod__(self, other): - f = FormulaStat() - f.value = binaryop(operator.mod, self, other) - return f + return BinaryProxy(operator.__truediv__, self, other) + def __floordiv__(self, other): + return BinaryProxy(operator.__floordiv__, self, other) + def __radd__(self, other): - f = FormulaStat() - f.value = binaryop(operator.add, other, self) - return f + return BinaryProxy(operator.__add__, other, self) def __rsub__(self, other): - f = FormulaStat() - f.value = binaryop(operator.sub, other, self) - return f + return BinaryProxy(operator.__sub__, other, self) def __rmul__(self, other): - f = FormulaStat() - f.value = binaryop(operator.mul, other, self) - return f + return BinaryProxy(operator.__mul__, other, self) + def __rdiv__(self, other): + return BinaryProxy(operator.__div__, other, self) def __rtruediv__(self, other): - f = FormulaStat() - f.value = binaryop(zerodiv, other, self) - return f - def __rmod__(self, other): - f = FormulaStat() - f.value = binaryop(operator.mod, other, self) - return f + return BinaryProxy(operator.__truediv__, other, self) + def __rfloordiv__(self, other): + return BinaryProxy(operator.__floordiv__, other, self) + def __neg__(self): - f = FormulaStat() - f.value = unaryop(operator.neg, self) - return f - def __getitem__(self, idx): - f = FormulaStat() - f.value = {} - for key in self.value.keys(): - f.value[key] = {} - f.value[key][0] = {} - f.value[key][0][0] = self.value[key][idx][0] - return f - - def __float__(self): - if isinstance(self.value, FormulaStat): - return float(self.value) - if not self.value.has_key(display_run): - return (1e300*1e300) - if len(self.value[display_run]) == 1: - return self.value[display_run][0][0] - else: - #print self.value[display_run] - return self.value[display_run][4][0] - #raise ValueError + return UnaryProxy(operator.__neg__, self) + def __pos__(self): + return UnaryProxy(operator.__pos__, self) + def __abs__(self): + return UnaryProxy(operator.__abs__, self) + +class Scalar(Value): + def __scalar__(self): + return True - def display(self): - import display - d = display.VectorDisplay() - d.flags = 0 - d.precision = 1 - d.name = 'formula' - d.desc = 'formula' - val = self.value[display_run] - d.value = [ val[x][0] for x in val.keys() ] - d.display() + def __vector__(self): + return False + def __value__(self, run): + raise AttributeError, '__value__ must be defined' -class Scalar(Statistic,FormulaStat): - def getValue(self): - return source.data(self, self.bins, self.ticks) +class VectorItemProxy(Value): + def __init__(self, proxy, index): + self.proxy = proxy + self.index = index - def display(self): - import display - p = display.Print() - p.name = self.name - p.desc = self.desc - p.value = float(self) - p.flags = self.flags - p.precision = self.precision - if display.all or (self.flags & flags.printable): - p.display() + def __scalar__(self): + return True - def comparable(self, other): - return self.name == other.name + def __vector__(self): + return False - def __eq__(self, other): - return self.value == other.value + def __value__(self, run): + return value(self.proxy, run, self.index) - def __isub__(self, other): - self.value -= other.value - return self +class Vector(Value): + def __scalar__(self): + return False - def __iadd__(self, other): - self.value += other.value - return self + def __vector__(self): + return True - def __itruediv__(self, other): - if not other: - return self - self.value /= other - return self + def __value__(self, run, index): + raise AttributeError, '__value__ must be defined' -class Vector(Statistic,FormulaStat): - def getValue(self): - return source.data(self, self.bins, self.ticks); + def __getitem__(self, index): + return VectorItemProxy(self, index) - def display(self): - import display - if not display.all and not (self.flags & flags.printable): - return +class ScalarConstant(Scalar): + def __init__(self, constant): + self.constant = constant + def __value__(self, run): + return self.constant - d = display.VectorDisplay() - d.__dict__.update(self.__dict__) - d.display() +class VectorConstant(Vector): + def __init__(self, constant): + self.constant = constant + def __value__(self, run, index): + return self.constant[index] + def __len__(self): + return len(self.constant) - def comparable(self, other): - return self.name == other.name and \ - len(self.value) == len(other.value) +def WrapValue(value): + if isinstance(value, (int, long, float)): + return ScalarConstant(value) + if isinstance(value, (list, tuple)): + return VectorConstant(value) + if isinstance(value, Value): + return value - def __eq__(self, other): - if isinstance(self.value, (list, tuple)) != \ - isinstance(other.value, (list, tuple)): - return False + raise AttributeError, 'Only values can be wrapped' - if isinstance(self.value, (list, tuple)): - if len(self.value) != len(other.value): - return False - else: - for v1,v2 in zip(self.value, other.value): - if v1 != v2: - return False - return True - else: - return self.value == other.value +class Statistic(object): + def __getattr__(self, attr): + if attr in ('data', 'x', 'y'): + result = self.source.data(self, self.bins, self.ticks) + self.data = result.data + self.x = result.x + self.y = result.y + return super(Statistic, self).__getattribute__(attr) - def __isub__(self, other): - self.value = binaryop(operator.sub, self.value, other.value) - return self + def __setattr__(self, attr, value): + if attr == 'stat': + raise AttributeError, '%s is read only' % stat + if attr in ('source', 'bins', 'ticks'): + if getattr(self, attr) != value: + if hasattr(self, 'data'): + delattr(self, 'data') + + super(Statistic, self).__setattr__(attr, value) + +class ValueProxy(Value): + def __getattr__(self, attr): + if attr == '__value__': + if scalar(self): + return self.__scalarvalue__ + if vector(self): + return self.__vectorvalue__ + if attr == '__len__': + if vector(self): + return self.__vectorlen__ + return super(ValueProxy, self).__getattribute__(attr) + +class UnaryProxy(ValueProxy): + def __init__(self, op, arg): + self.op = op + self.arg = WrapValue(arg) + + def __scalar__(self): + return scalar(self.arg) + + def __vector__(self): + return vector(self.arg) + + def __scalarvalue__(self, run): + val = value(self.arg, run) + if val is None: + return None + return self.op(val) + + def __vectorvalue__(self, run, index): + val = value(self.arg, run, index) + if val is None: + return None + return self.op(val) + + def __vectorlen__(self): + return len(unproxy(self.arg)) + +class BinaryProxy(ValueProxy): + def __init__(self, op, arg0, arg1): + super(BinaryProxy, self).__init__() + self.op = op + self.arg0 = WrapValue(arg0) + self.arg1 = WrapValue(arg1) + + def __scalar__(self): + return scalar(self.arg0) and scalar(self.arg1) + + def __vector__(self): + return vector(self.arg0) or vector(self.arg1) + + def __scalarvalue__(self, run): + val0 = value(self.arg0, run) + val1 = value(self.arg1, run) + if val0 is None or val1 is None: + return None + return self.op(val0, val1) + + def __vectorvalue__(self, run, index): + if scalar(self.arg0): + val0 = value(self.arg0, run) + if vector(self.arg0): + val0 = value(self.arg0, run, index) + if scalar(self.arg1): + val1 = value(self.arg1, run) + if vector(self.arg1): + val1 = value(self.arg1, run, index) + + if val0 is None or val1 is None: + return None + + return self.op(val0, val1) + + def __vectorlen__(self): + if vector(self.arg0) and scalar(self.arg1): + return len(self.arg0) + if scalar(self.arg0) and vector(self.arg1): + return len(self.arg1) + + len0 = len(self.arg0) + len1 = len(self.arg1) + + if len0 != len1: + raise AttributeError, \ + "vectors of different lengths %d != %d" % (len0, len1) + + return len0 + +class Proxy(Value): + def __init__(self, name, dict): + self.name = name + self.dict = dict + + def __unproxy__(self): + return unproxy(self.dict[self.name]) + + def __getitem__(self, index): + return ItemProxy(self, index) + + def __getattr__(self, attr): + return AttrProxy(self, attr) + +class ItemProxy(Proxy): + def __init__(self, proxy, index): + self.proxy = proxy + self.index = index + + def __unproxy__(self): + return unproxy(unproxy(self.proxy)[self.index]) - def __iadd__(self, other): - self.value = binaryop(operator.add, self.value, other.value) - return self +class AttrProxy(Proxy): + def __init__(self, proxy, attr): + self.proxy = proxy + self.attr = attr - def __itruediv__(self, other): - if not other: - return self - if isinstance(self.value, (list, tuple)): - for i in xrange(len(self.value)): - self.value[i] /= other - else: - self.value /= other - return self + def __unproxy__(self): + return unproxy(getattr(unproxy(self.proxy), self.attr)) -class Formula(Vector): - def getValue(self): - formula = re.sub(':', '__', self.formula) - x = eval(formula, source.stattop) - return x.value +class ProxyGroup(object): + def __init__(self, dict=None, **kwargs): + self.__dict__['dict'] = {} - def comparable(self, other): - return self.name == other.name and \ - compare(self.dist, other.dist) + if dict is not None: + self.dict.update(dict) - def __eq__(self, other): - return self.value == other.value + if kwargs: + self.dict.update(kwargs) - def __isub__(self, other): - return self + def __getattr__(self, name): + return Proxy(name, self.dict) - def __iadd__(self, other): - return self + def __setattr__(self, attr, value): + self.dict[attr] = value - def __itruediv__(self, other): - if not other: - return self - return self +class ScalarStat(Statistic,Scalar): + def __value__(self, run): + if run not in self.data: + return None + return self.data[run][0][0] + + def display(self, run=None): + import display + p = display.Print() + p.name = self.name + p.desc = self.desc + p.value = value(self, run) + p.flags = self.flags + p.precision = self.precision + if display.all or (self.flags & flags.printable): + p.display() + +class VectorStat(Statistic,Vector): + def __value__(self, run, item): + if run not in self.data: + return None + return self.data[run][item][0] + + def __len__(self): + return self.x -class SimpleDist(object): + def display(self, run=None): + import display + d = display.VectorDisplay() + d.name = self.name + d.desc = self.desc + d.value = [ value(self, run, i) for i in xrange(len(self)) ] + d.flags = self.flags + d.precision = self.precision + d.display() + +class Formula(Value): + def __getattribute__(self, attr): + if attr not in ( '__scalar__', '__vector__', '__value__', '__len__' ): + return super(Formula, self).__getattribute__(attr) + + formula = re.sub(':', '__', self.formula) + value = eval(formula, self.source.stattop) + return getattr(value, attr) + +class SimpleDist(Statistic): def __init__(self, sums, squares, samples): self.sums = sums self.squares = squares self.samples = samples - def getValue(self): - return 0.0 - def display(self, name, desc, flags, precision): import display p = display.Print() @@ -482,9 +438,6 @@ class FullDist(SimpleDist): self.bsize = bsize self.size = size - def getValue(self): - return 0.0 - def display(self, name, desc, flags, precision): import display p = display.Print() @@ -574,9 +527,6 @@ class FullDist(SimpleDist): return self class Dist(Statistic): - def getValue(self): - return 0.0 - def display(self): import display if not display.all and not (self.flags & flags.printable): @@ -606,9 +556,6 @@ class Dist(Statistic): return self class VectorDist(Statistic): - def getValue(self): - return 0.0 - def display(self): import display if not display.all and not (self.flags & flags.printable): @@ -694,9 +641,6 @@ class VectorDist(Statistic): return self class Vector2d(Statistic): - def getValue(self): - return 0.0 - def display(self): import display if not display.all and not (self.flags & flags.printable): @@ -748,20 +692,25 @@ class Vector2d(Statistic): return self return self -def NewStat(data): +def NewStat(source, data): stat = None if data.type == 'SCALAR': - stat = Scalar(data) + stat = ScalarStat() elif data.type == 'VECTOR': - stat = Vector(data) + stat = VectorStat() elif data.type == 'DIST': - stat = Dist(data) + stat = Dist() elif data.type == 'VECTORDIST': - stat = VectorDist(data) + stat = VectorDist() elif data.type == 'VECTOR2D': - stat = Vector2d(data) + stat = Vector2d() elif data.type == 'FORMULA': - stat = Formula(data) + stat = Formula() + + stat.__dict__['source'] = source + stat.__dict__['bins'] = None + stat.__dict__['ticks'] = None + stat.__dict__.update(data.__dict__) return stat diff --git a/util/stats/output.py b/util/stats/output.py index f793749f7..cf76f291e 100644 --- a/util/stats/output.py +++ b/util/stats/output.py @@ -26,24 +26,8 @@ # # Authors: Nathan Binkert -class dbinfo(object): - def get(self, job, stat): - import info - - run = info.source.allRunNames.get(job.name, None) - if run is None: - print 'run "%s" not found' % job - return None - - stat.system = info.source[job.system] - info.display_run = run.run; - val = float(stat) - if val == 1e300*1e300: - return None - return val - class StatOutput(object): - def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None): + def __init__(self, name, jobfile, info, stat=None, binstats=None): self.name = name self.jobfile = jobfile self.stat = stat @@ -141,7 +125,7 @@ class StatOutput(object): data = zeros((len(groupopts), len(baropts)), Float) data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ] enabled = False - stacked = None + stacked = 0 for g,gopt in enumerate(groupopts): for b,bopt in enumerate(baropts): job = self.jobfile.job(options + [ gopt, bopt ]) @@ -149,17 +133,28 @@ class StatOutput(object): continue val = self.info.get(job, self.stat) - if val is None: - val = 0.0 - curstacked = isinstance(val, (list, tuple)) - if stacked is None: - stacked = curstacked - else: - if stacked != curstacked: - raise ValueError, "some stats stacked, some not" + if isinstance(val, (list, tuple)): + if len(val) == 1: + val = val[0] + else: + stacked = len(val) data[g][b] = val + if stacked == 0: + for i in xrange(len(groupopts)): + for j in xrange(len(baropts)): + if data[i][j] is None: + data[i][j] = 0.0 + else: + for i in xrange(len(groupopts)): + for j in xrange(len(baropts)): + val = data[i][j] + if val is None: + data[i][j] = [] * stacked + elif len(val) != stacked: + raise ValueError, "some stats stacked, some not" + data = array(data) if data.sum() == 0: continue @@ -167,7 +162,10 @@ class StatOutput(object): bar_descs = [ opt.desc for opt in baropts ] group_descs = [ opt.desc for opt in groupopts ] if stacked: - legend = self.info.rcategories + try: + legend = self.info.rcategories + except: + legend = [ str(i) for i in xrange(stacked) ] else: legend = bar_descs diff --git a/util/stats/stats.py b/util/stats/stats.py index af9ae5575..e481a794c 100755 --- a/util/stats/stats.py +++ b/util/stats/stats.py @@ -98,29 +98,27 @@ def commands(options, command, args): raise CommandException - import db, info - info.source = db.Database() - info.source.host = options.host - info.source.db = options.db - info.source.passwd = options.passwd - info.source.user = options.user - info.source.connect() - #info.source.update_dict(globals()) - - if type(options.get) is str: - info.source.get = options.get + import db + source = db.Database() + source.host = options.host + source.db = options.db + source.passwd = options.passwd + source.user = options.user + source.connect() + #source.update_dict(globals()) + + if type(options.method) is str: + source.method = options.method if options.runs is None: - runs = info.source.allRuns + runs = source.allRuns else: rx = re.compile(options.runs) runs = [] - for run in info.source.allRuns: + for run in source.allRuns: if rx.match(run.name): runs.append(run) - info.display_run = runs[0].run - if command == 'runs': user = None opts, args = getopts(args, '-u') @@ -129,14 +127,14 @@ def commands(options, command, args): for o,a in opts: if o == '-u': user = a - info.source.listRuns(user) + source.listRuns(user) return if command == 'stats': if len(args) == 0: - info.source.listStats() + source.listStats() elif len(args) == 1: - info.source.listStats(args[0]) + source.listStats(args[0]) else: raise CommandException @@ -144,9 +142,9 @@ def commands(options, command, args): if command == 'bins': if len(args) == 0: - info.source.listBins() + source.listBins() elif len(args) == 1: - info.source.listBins(args[0]) + source.listBins(args[0]) else: raise CommandException @@ -154,9 +152,9 @@ def commands(options, command, args): if command == 'formulas': if len(args) == 0: - info.source.listFormulas() + source.listFormulas() elif len(args) == 1: - info.source.listFormulas(args[0]) + source.listFormulas(args[0]) else: raise CommandException @@ -166,7 +164,7 @@ def commands(options, command, args): if len(args): raise CommandException - info.source.listTicks(runs) + source.listTicks(runs) return if command == 'stability': @@ -177,8 +175,8 @@ def commands(options, command, args): merge = int(args[0]) except ValueError: usage() - stats = info.source.getStat(args[1]) - info.source.get = "sum" + stats = source.getStat(args[1]) + source.method = 'sum' def disp(*args): print "%-20s %12s %12s %4s %5s %5s %5s %10s" % args @@ -195,18 +193,17 @@ def commands(options, command, args): #loop through all the selected runs for run in runs: - info.display_run = run.run; - runTicks = info.source.retTicks([ run ]) + runTicks = source.retTicks([ run ]) #throw away the first one, it's 0 runTicks.pop(0) - info.globalTicks = runTicks + source.ticks = runTicks avg = 0 stdev = 0 numoutsideavg = 0 numoutside1std = 0 numoutside2std = 0 pairRunTicks = [] - if float(stat) == 1e300*1e300: + if value(stat, run.run) == 1e300*1e300: continue for t in range(0, len(runTicks)-(merge-1), merge): tempPair = [] @@ -215,17 +212,17 @@ def commands(options, command, args): pairRunTicks.append(tempPair) #loop through all the various ticks for each run for tick in pairRunTicks: - info.globalTicks = tick - avg += float(stat) + source.ticks = tick + avg += value(stat, run.run) avg /= len(pairRunTicks) for tick in pairRunTicks: - info.globalTicks = tick - val = float(stat) + source.ticks = tick + val = value(stat, run.run) stdev += pow((val-avg),2) stdev = math.sqrt(stdev / len(pairRunTicks)) for tick in pairRunTicks: - info.globalTicks = tick - val = float(stat) + source.ticks = tick + val = value(stat, run.run) if (val < (avg * .9)) or (val > (avg * 1.1)): numoutsideavg += 1 if (val < (avg - stdev)) or (val > (avg + stdev)): @@ -264,9 +261,9 @@ def commands(options, command, args): if options.ticks: if not options.graph: print 'only displaying sample %s' % options.ticks - info.globalTicks = [ int(x) for x in options.ticks.split() ] + source.ticks = [ int(x) for x in options.ticks.split() ] - from output import StatOutput + import output def display(): if options.graph: @@ -280,12 +277,12 @@ def commands(options, command, args): raise CommandException if command == 'stat': - stats = info.source.getStat(args[0]) + stats = source.getStat(args[0]) if command == 'formula': stats = eval(args[0]) for stat in stats: - output = StatOutput(stat.name, options.jobfile) + output = output.StatOutput(stat.name, options.jobfile, source) output.stat = stat output.label = stat.name display() @@ -295,12 +292,11 @@ def commands(options, command, args): if len(args): raise CommandException - system = info.source.__dict__[options.system] - - from proxy import ProxyGroup - sim_ticks = info.source['sim_ticks'] - sim_seconds = info.source['sim_seconds'] - proxy = ProxyGroup(system = info.source[options.system]) + system = source.__dict__[options.system] + from info import ProxyGroup + sim_ticks = source['sim_ticks'] + sim_seconds = source['sim_seconds'] + proxy = ProxyGroup(system = source[options.system]) system = proxy.system etherdev = system.tsunami.etherdev0 @@ -309,7 +305,7 @@ def commands(options, command, args): packets = etherdev.rxPackets + etherdev.txPackets bps = etherdev.rxBandwidth + etherdev.txBandwidth - output = StatOutput(command, options.jobfile) + output = output.StatOutput(command, options.jobfile, source) if command == 'usertime': import copy @@ -460,7 +456,7 @@ if __name__ == '__main__': options.user = getpass.getuser() options.runs = None options.system = 'client' - options.get = None + options.method = None options.binned = False options.graph = False options.ticks = False @@ -469,7 +465,7 @@ if __name__ == '__main__': options.jobfile = None options.all = False - opts, args = getopts(sys.argv[1:], '-BEFG:Jad:g:h:j:pr:s:u:T:') + opts, args = getopts(sys.argv[1:], '-BEFJad:g:h:j:m:pr:s:u:T:') for o,a in opts: if o == '-B': options.binned = True @@ -477,8 +473,6 @@ if __name__ == '__main__': options.printmode = 'E' if o == '-F': options.printmode = 'F' - if o == '-G': - options.get = a if o == '-a': options.all = True if o == '-d': @@ -492,6 +486,8 @@ if __name__ == '__main__': jobfilename = None if o == '-j': jobfilename = a + if o == '-m': + options.method = a if o == '-p': options.passwd = getpass.getpass() if o == '-r': |