From e00237e49e3cb171a1235f5de43587e8eb31ec2c Mon Sep 17 00:00:00 2001
From: Nathan Binkert <binkertn@umich.edu>
Date: Fri, 21 Oct 2005 16:29:13 -0400
Subject: Major cleanup of the statistics handling code

util/stats/db.py:
    Build a result object as the result of a query operation so it is
    easier to populate and contains a bit more information than just
    a big dict.  Also change the next level data into a matrix instead
    of a dict of dicts.
    Move the "get" function into the Database object.  (The get function
    is used by the output parsing function as the interface for accessing
    backend storage, same interface for profile stuff.)
    Change the old get variable to the method variable, it describes how
    the get works, (whether using sum, stdev, etc.)
util/stats/display.py:
    Clean up the display functions, mostly formatting.
    Handle values the way they should be now.
util/stats/info.py:
    Totally re-work how values are accessed from their data store.
    Access individual values on demand instead of calculating everything
    and passing up a huge result from the bottom.
    This impacts the way that proxying works, and in general, everything
    is now esentially a proxy for the lower level database.  Provide new
    operators: unproxy, scalar, vector, value, values, total, and len which
    retrieve the proper result from the object they are called on.
    Move the ProxyGroup stuff (proxies of proxies!) here from the now gone
    proxy.py file and integrate the shared parts of the code.  The ProxyGroup
    stuff allows you to write formulas without specifying the statistics
    until evaluation time.

    Get rid of global variables!
util/stats/output.py:
    Move the dbinfo stuff into the Database itself.  Each source should
    have it's own get() function for accessing it's data.
    This get() function behaves a bit differently than before in that it
    can return vectors as well, deal with these vectors and with no result
    conditions better.
util/stats/stats.py:
    the info module no longer has the source global variable, just
    create the database source and pass it around as necessary

--HG--
extra : convert_revision : 8e5aa228e5d3ae8068ef9c40f65b3a2f9e7c0cff
---
 util/stats/db.py      |  72 ++++--
 util/stats/display.py |  97 ++++----
 util/stats/info.py    | 669 +++++++++++++++++++++++---------------------------
 util/stats/output.py  |  52 ++--
 util/stats/stats.py   |  96 ++++----
 5 files changed, 478 insertions(+), 508 deletions(-)

(limited to 'util/stats')

diff --git a/util/stats/db.py b/util/stats/db.py
index ab005e67b..0d321a360 100644
--- a/util/stats/db.py
+++ b/util/stats/db.py
@@ -103,6 +103,20 @@ class Node(object):
     def __str__(self):
         return self.name
 
+class Result(object):
+    def __init__(self, x, y):
+        self.data = {}
+        self.x = x
+        self.y = y
+
+    def __contains__(self, run):
+        return run in self.data
+
+    def __getitem__(self, run):
+        if run not in self.data:
+            self.data[run] = [ [ 0.0 ] * self.y for i in xrange(self.x) ]
+        return self.data[run]
+
 class Database(object):
     def __init__(self):
         self.host = 'zizzer.pool'
@@ -135,7 +149,23 @@ class Database(object):
         self.runs = None
         self.bins = None
         self.ticks = None
-        self.__dict__['get'] = type(self).sum
+        self.method = 'sum'
+        self._method = type(self).sum
+
+    def get(self, job, stat):
+        run = self.allRunNames.get(job.name, None)
+        if run is None:
+            print 'run "%s" not found' % job
+            return None
+
+        from info import scalar, vector, value, total, len
+        stat.system = self[job.system]
+        if scalar(stat):
+            return value(stat, run.run)
+        if vector(stat):
+            return values(stat, run.run)
+
+        return None
 
     def query(self, sql):
         self.cursor.execute(sql)
@@ -203,7 +233,7 @@ class Database(object):
         self.query('select * from stats')
         import info
         for result in self.cursor.fetchall():
-            stat = info.NewStat(StatData(result))
+            stat = info.NewStat(self, StatData(result))
             self.append(stat)
             self.allStats.append(stat)
             self.allStatIds[stat.stat] = stat
@@ -421,30 +451,17 @@ class Database(object):
     def stdev(self, stat, bins, ticks):
         return self.outer('stddev', 'sum', stat, bins, ticks)
 
-    def __getattribute__(self, attr):
-        if attr != 'get':
-            return super(Database, self).__getattribute__(attr)
-
-        if self.__dict__['get'] == type(self).sum:
-            return 'sum'
-        elif self.__dict__['get'] == type(self).avg:
-            return 'avg'
-        elif self.__dict__['get'] == type(self).stdev:
-            return 'stdev'
-        else:
-            return ''
-
     def __setattr__(self, attr, value):
-        if attr != 'get':
-            super(Database, self).__setattr__(attr, value)
+        super(Database, self).__setattr__(attr, value)
+        if attr != 'method':
             return
 
         if value == 'sum':
-            self.__dict__['get'] = type(self).sum
+            self._method = self.sum
         elif value == 'avg':
-            self.__dict__['get'] = type(self).avg
+            self._method = self.avg
         elif value == 'stdev':
-            self.__dict__['get'] = type(self).stdev
+            self._method = self.stdev
         else:
             raise AttributeError, "can only set get to: sum | avg | stdev"
 
@@ -453,10 +470,12 @@ class Database(object):
             bins = self.bins
         if ticks is None:
             ticks = self.ticks
-        sql = self.__dict__['get'](self, stat, bins, ticks)
+        sql = self._method(self, stat, bins, ticks)
         self.query(sql)
 
         runs = {}
+        xmax = 0
+        ymax = 0
         for x in self.cursor.fetchall():
             data = Data(x)
             if not runs.has_key(data.run):
@@ -464,8 +483,17 @@ class Database(object):
             if not runs[data.run].has_key(data.x):
                 runs[data.run][data.x] = {}
 
+            xmax = max(xmax, data.x)
+            ymax = max(ymax, data.y)
             runs[data.run][data.x][data.y] = data.data
-        return runs
+
+        results = Result(xmax + 1, ymax + 1)
+        for run,data in runs.iteritems():
+            result = results[run]
+            for x,ydata in data.iteritems():
+                for y,data in ydata.iteritems():
+                    result[x][y] = data
+        return results
 
     def __getitem__(self, key):
         return self.stattop[key]
diff --git a/util/stats/display.py b/util/stats/display.py
index 629684ca4..fbcff5c70 100644
--- a/util/stats/display.py
+++ b/util/stats/display.py
@@ -26,7 +26,7 @@
 
 class Value:
     def __init__(self, value, precision, percent = False):
-        self.value = value
+        self.value = float(value)
         self.precision = precision
         self.percent = percent
     def __str__(self):
@@ -90,61 +90,60 @@ class Print:
 
 class VectorDisplay:
     def display(self):
+        if not self.value:
+            return
+
         p = Print()
         p.flags = self.flags
         p.precision = self.precision
 
-        if isinstance(self.value, (list, tuple)):
-            if not len(self.value):
-                return
-
-            mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
-            mycdf = 0.0
+        if not isinstance(self.value, (list, tuple)):
+            p.name = self.name
+            p.desc = self.desc
+            p.value = self.value
+            p.display()
+            return
 
-            value = self.value
+        mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
+        mycdf = 0.0
 
-            if display_all:
-                subnames = [ '[%d]' % i for i in range(len(value)) ]
-            else:
-                subnames = [''] * len(value)
-
-            if self.__dict__.has_key('subnames'):
-                for i,each in enumerate(self.subnames):
-                    if len(each) > 0:
-                        subnames[i] = '.%s' % each
-
-            subdescs = [self.desc]*len(value)
-            if self.__dict__.has_key('subdescs'):
-                for i in xrange(min(len(value), len(self.subdescs))):
-                    subdescs[i] = self.subdescs[i]
-
-            for val,sname,sdesc in map(None, value, subnames, subdescs):
-                if mytotal > 0.0:
-                    mypdf = float(val) / float(mytotal)
-                    mycdf += mypdf
-                    if (self.flags & flags_pdf):
-                        p.pdf = mypdf
-                        p.cdf = mycdf
-
-                if len(sname) == 0:
-                    continue
-
-                p.name = self.name + sname
-                p.desc = sdesc
-                p.value = val
-                p.display()
-
-            if (self.flags & flags_total):
-                if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
-                if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
-                p.name = self.name + '.total'
-                p.desc = self.desc
-                p.value = mytotal
-                p.display()
+        value = self.value
 
+        if display_all:
+            subnames = [ '[%d]' % i for i in range(len(value)) ]
         else:
-            p.name = self.name
-            p.desc = self.desc
-            p.value = self.value
+            subnames = [''] * len(value)
+
+        if self.__dict__.has_key('subnames'):
+            for i,each in enumerate(self.subnames):
+                if len(each) > 0:
+                    subnames[i] = '.%s' % each
+
+        subdescs = [self.desc]*len(value)
+        if self.__dict__.has_key('subdescs'):
+            for i in xrange(min(len(value), len(self.subdescs))):
+                subdescs[i] = self.subdescs[i]
+
+        for val,sname,sdesc in map(None, value, subnames, subdescs):
+            if mytotal > 0.0:
+                mypdf = float(val) / float(mytotal)
+                mycdf += mypdf
+                if (self.flags & flags_pdf):
+                    p.pdf = mypdf
+                    p.cdf = mycdf
+
+            if len(sname) == 0:
+                continue
+
+            p.name = self.name + sname
+            p.desc = sdesc
+            p.value = val
             p.display()
 
+        if (self.flags & flags_total):
+            if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
+            if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
+            p.name = self.name + '.total'
+            p.desc = self.desc
+            p.value = mytotal
+            p.display()
diff --git a/util/stats/info.py b/util/stats/info.py
index ae5d3211f..9932d7922 100644
--- a/util/stats/info.py
+++ b/util/stats/info.py
@@ -27,391 +27,347 @@
 from __future__ import division
 import operator, re, types
 
-source = None
-display_run = 0
-global globalTicks
-globalTicks = None
-
-def total(f):
-    if isinstance(f, FormulaStat):
-        v = f.value
-    else:
-        v = f
-
-    f = FormulaStat()
-    if isinstance(v, (list, tuple)):
-        f.value = reduce(operator.add, v)
-    else:
-        f.value = v
-
-    return f
-
-def unaryop(op, f):
-    if isinstance(f, FormulaStat):
-        v = f.value
-    else:
-        v = f
-
-    if isinstance(v, (list, tuple)):
-        return map(op, v)
-    else:
-        return op(v)
-
-def zerodiv(lv, rv):
-    if rv == 0.0:
-        return 0.0
-    else:
-        return operator.truediv(lv, rv)
-
-def wrapop(op, lv, rv):
-    if isinstance(lv, str):
-        return lv
-
-    if isinstance(rv, str):
-        return rv
-
-    return op(lv, rv)
-
-def same(lrun, rrun):
-    for lx,rx in zip(lrun.keys(),rrun.keys()):
-        if lx != rx:
-            print 'lx != rx'
-            print lx, rx
-            print lrun.keys()
-            print rrun.keys()
-            return False
-        for ly,ry in zip(lrun[lx].keys(),rrun[rx].keys()):
-            if ly != ry:
-                print 'ly != ry'
-                print ly, ry
-                print lrun[lx].keys()
-                print rrun[rx].keys()
-                return False
-    return True
-
-
-def binaryop(op, lf, rf):
-    result = {}
-
-    if isinstance(lf, FormulaStat) and isinstance(rf, FormulaStat):
-        lv = lf.value
-        rv = rf.value
-
-        theruns = []
-        for r in lv.keys():
-            if rv.has_key(r):
-                if same(lv[r], rv[r]):
-                    theruns.append(r)
-                else:
-                    raise AttributeError
-
-        for run in theruns:
-            result[run] = {}
-            for x in lv[run].keys():
-                result[run][x] = {}
-                for y in lv[run][x].keys():
-                    result[run][x][y] = wrapop(op, lv[run][x][y],
-                                               rv[run][x][y])
-    elif isinstance(lf, FormulaStat):
-        lv = lf.value
-        for run in lv.keys():
-            result[run] = {}
-            for x in lv[run].keys():
-                result[run][x] = {}
-                for y in lv[run][x].keys():
-                    result[run][x][y] = wrapop(op, lv[run][x][y], rf)
-    elif isinstance(rf, FormulaStat):
-        rv = rf.value
-        for run in rv.keys():
-            result[run] = {}
-            for x in rv[run].keys():
-                result[run][x] = {}
-                for y in rv[run][x].keys():
-                    result[run][x][y] = wrapop(op, lf, rv[run][x][y])
-
+def unproxy(proxy):
+    if hasattr(proxy, '__unproxy__'):
+        return proxy.__unproxy__()
+
+    return proxy
+
+def scalar(stat):
+    stat = unproxy(stat)
+    assert(stat.__scalar__() != stat.__vector__())
+    return stat.__scalar__()
+
+def vector(stat):
+    stat = unproxy(stat)
+    assert(stat.__scalar__() != stat.__vector__())
+    return stat.__vector__()
+
+def value(stat, *args):
+    stat = unproxy(stat)
+    return stat.__value__(*args)
+
+def values(stat, run):
+    stat = unproxy(stat)
+    result = []
+    for i in xrange(len(stat)):
+        val = value(stat, run.run, i)
+        if val is None:
+            return None
+        result.append(val)
     return result
 
-def sums(x, y):
-    if isinstance(x, (list, tuple)):
-        return map(lambda x, y: x + y, x, y)
-    else:
-        return x + y
-
-def alltrue(seq):
-    return reduce(lambda x, y: x and y, seq)
-
-def allfalse(seq):
-    return not reduce(lambda x, y: x or y, seq)
-
-def enumerate(seq):
-    return map(None, range(len(seq)), seq)
-
-def cmp(a, b):
-    if a < b:
-        return -1
-    elif a == b:
-        return 0
-    else:
-        return 1
-
-class Statistic(object):
-
-    def __init__(self, data):
-        self.__dict__.update(data.__dict__)
-        if not self.__dict__.has_key('value'):
-            self.__dict__['value'] = None
-        if not self.__dict__.has_key('bins'):
-            self.__dict__['bins'] = None
-        if not self.__dict__.has_key('ticks'):
-            self.__dict__['ticks'] = None
-        if 'vc' not in self.__dict__:
-            self.vc = {}
-
-    def __getattribute__(self, attr):
-        if attr == 'ticks':
-            if self.__dict__['ticks'] != globalTicks:
-                self.__dict__['value'] = None
-                self.__dict__['ticks'] = globalTicks
-            return self.__dict__['ticks']
-        if attr == 'value':
-            if self.__dict__['ticks'] != globalTicks:
-                if self.__dict__['ticks'] != None and \
-                                    len(self.__dict__['ticks']) == 1:
-                    self.vc[self.__dict__['ticks'][0]] = self.__dict__['value']
-                self.__dict__['ticks'] = globalTicks
-                if len(globalTicks) == 1 and self.vc.has_key(globalTicks[0]):
-                    self.__dict__['value'] = self.vc[globalTicks[0]]
-                else:
-                    self.__dict__['value'] = None
-            if self.__dict__['value'] == None:
-                self.__dict__['value'] = self.getValue()
-            return self.__dict__['value']
-        else:
-            return super(Statistic, self).__getattribute__(attr)
-
-    def __setattr__(self, attr, value):
-        if attr == 'bins' or attr == 'ticks':
-            if attr == 'bins':
-                if value is not None:
-                    value = source.getBin(value)
-            #elif attr == 'ticks' and type(value) is str:
-            #    value = [ int(x) for x in value.split() ]
-
-            self.__dict__[attr] = value
-            self.__dict__['value'] = None
-            self.vc = {}
-        else:
-            super(Statistic, self).__setattr__(attr, value)
+def total(stat, run):
+    return sum(values(stat, run))
 
-    def getValue(self):
-        raise AttributeError, 'getValue() must be defined'
-
-    def zero(self):
-        return False
+def len(stat):
+    stat = unproxy(stat)
+    return stat.__len__()
 
-    def __ne__(self, other):
-        return not (self == other)
+class Value(object):
+    def __scalar__(self):
+        raise AttributeError, "must define __scalar__ for %s" % (type (self))
+    def __vector__(self):
+        raise AttributeError, "must define __vector__ for %s" % (type (self))
 
-    def __str__(self):
-        return '%f' % (float(self))
-
-class FormulaStat(object):
     def __add__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.add, self, other)
-        return f
+        return BinaryProxy(operator.__add__, self, other)
     def __sub__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.sub, self, other)
-        return f
+        return BinaryProxy(operator.__sub__, self, other)
     def __mul__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.mul, self, other)
-        return f
+        return BinaryProxy(operator.__mul__, self, other)
+    def __div__(self, other):
+        return BinaryProxy(operator.__div__, self, other)
     def __truediv__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(zerodiv, self, other)
-        return f
-    def __mod__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.mod, self, other)
-        return f
+        return BinaryProxy(operator.__truediv__, self, other)
+    def __floordiv__(self, other):
+        return BinaryProxy(operator.__floordiv__, self, other)
+
     def __radd__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.add, other, self)
-        return f
+        return BinaryProxy(operator.__add__, other, self)
     def __rsub__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.sub, other, self)
-        return f
+        return BinaryProxy(operator.__sub__, other, self)
     def __rmul__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.mul, other, self)
-        return f
+        return BinaryProxy(operator.__mul__, other, self)
+    def __rdiv__(self, other):
+        return BinaryProxy(operator.__div__, other, self)
     def __rtruediv__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(zerodiv, other, self)
-        return f
-    def __rmod__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.mod, other, self)
-        return f
+        return BinaryProxy(operator.__truediv__, other, self)
+    def __rfloordiv__(self, other):
+        return BinaryProxy(operator.__floordiv__, other, self)
+
     def __neg__(self):
-        f = FormulaStat()
-        f.value = unaryop(operator.neg, self)
-        return f
-    def __getitem__(self, idx):
-        f = FormulaStat()
-        f.value = {}
-        for key in self.value.keys():
-            f.value[key] = {}
-            f.value[key][0] = {}
-            f.value[key][0][0] = self.value[key][idx][0]
-        return f
-
-    def __float__(self):
-        if isinstance(self.value, FormulaStat):
-            return float(self.value)
-        if not self.value.has_key(display_run):
-            return (1e300*1e300)
-        if len(self.value[display_run]) == 1:
-            return self.value[display_run][0][0]
-        else:
-            #print self.value[display_run]
-            return self.value[display_run][4][0]
-            #raise ValueError
+        return UnaryProxy(operator.__neg__, self)
+    def __pos__(self):
+        return UnaryProxy(operator.__pos__, self)
+    def __abs__(self):
+        return UnaryProxy(operator.__abs__, self)
+
+class Scalar(Value):
+    def __scalar__(self):
+        return True
 
-    def display(self):
-        import display
-        d = display.VectorDisplay()
-        d.flags = 0
-        d.precision = 1
-        d.name = 'formula'
-        d.desc = 'formula'
-        val = self.value[display_run]
-        d.value = [ val[x][0] for x in val.keys() ]
-        d.display()
+    def __vector__(self):
+        return False
 
+    def __value__(self, run):
+        raise AttributeError, '__value__ must be defined'
 
-class Scalar(Statistic,FormulaStat):
-    def getValue(self):
-        return source.data(self, self.bins, self.ticks)
+class VectorItemProxy(Value):
+    def __init__(self, proxy, index):
+        self.proxy = proxy
+        self.index = index
 
-    def display(self):
-        import display
-        p = display.Print()
-        p.name = self.name
-        p.desc = self.desc
-        p.value = float(self)
-        p.flags = self.flags
-        p.precision = self.precision
-        if display.all or (self.flags & flags.printable):
-            p.display()
+    def __scalar__(self):
+        return True
 
-    def comparable(self, other):
-        return self.name == other.name
+    def __vector__(self):
+        return False
 
-    def __eq__(self, other):
-        return self.value == other.value
+    def __value__(self, run):
+        return value(self.proxy, run, self.index)
 
-    def __isub__(self, other):
-        self.value -= other.value
-        return self
+class Vector(Value):
+    def __scalar__(self):
+        return False
 
-    def __iadd__(self, other):
-        self.value += other.value
-        return self
+    def __vector__(self):
+        return True
 
-    def __itruediv__(self, other):
-        if not other:
-            return self
-        self.value /= other
-        return self
+    def __value__(self, run, index):
+        raise AttributeError, '__value__ must be defined'
 
-class Vector(Statistic,FormulaStat):
-    def getValue(self):
-        return source.data(self, self.bins, self.ticks);
+    def __getitem__(self, index):
+        return VectorItemProxy(self, index)
 
-    def display(self):
-        import display
-        if not display.all and not (self.flags & flags.printable):
-            return
+class ScalarConstant(Scalar):
+    def __init__(self, constant):
+        self.constant = constant
+    def __value__(self, run):
+        return self.constant
 
-        d = display.VectorDisplay()
-        d.__dict__.update(self.__dict__)
-        d.display()
+class VectorConstant(Vector):
+    def __init__(self, constant):
+        self.constant = constant
+    def __value__(self, run, index):
+        return self.constant[index]
+    def __len__(self):
+        return len(self.constant)
 
-    def comparable(self, other):
-        return self.name == other.name and \
-               len(self.value) == len(other.value)
+def WrapValue(value):
+    if isinstance(value, (int, long, float)):
+        return ScalarConstant(value)
+    if isinstance(value, (list, tuple)):
+        return VectorConstant(value)
+    if isinstance(value, Value):
+        return value
 
-    def __eq__(self, other):
-        if isinstance(self.value, (list, tuple)) != \
-               isinstance(other.value, (list, tuple)):
-            return False
+    raise AttributeError, 'Only values can be wrapped'
 
-        if isinstance(self.value, (list, tuple)):
-            if len(self.value) != len(other.value):
-                return False
-            else:
-                for v1,v2 in zip(self.value, other.value):
-                    if v1 != v2:
-                        return False
-                return True
-        else:
-            return self.value == other.value
+class Statistic(object):
+    def __getattr__(self, attr):
+        if attr in ('data', 'x', 'y'):
+            result = self.source.data(self, self.bins, self.ticks)
+            self.data = result.data
+            self.x = result.x
+            self.y = result.y
+        return super(Statistic, self).__getattribute__(attr)
 
-    def __isub__(self, other):
-        self.value = binaryop(operator.sub, self.value, other.value)
-        return self
+    def __setattr__(self, attr, value):
+        if attr == 'stat':
+            raise AttributeError, '%s is read only' % stat
+        if attr in ('source', 'bins', 'ticks'):
+            if getattr(self, attr) != value:
+                if hasattr(self, 'data'):
+                    delattr(self, 'data')
+
+        super(Statistic, self).__setattr__(attr, value)
+
+class ValueProxy(Value):
+    def __getattr__(self, attr):
+        if attr == '__value__':
+            if scalar(self):
+                return self.__scalarvalue__
+            if vector(self):
+                return self.__vectorvalue__
+        if attr == '__len__':
+            if vector(self):
+                return self.__vectorlen__
+        return super(ValueProxy, self).__getattribute__(attr)
+
+class UnaryProxy(ValueProxy):
+    def __init__(self, op, arg):
+        self.op = op
+        self.arg = WrapValue(arg)
+
+    def __scalar__(self):
+        return scalar(self.arg)
+
+    def __vector__(self):
+        return vector(self.arg)
+
+    def __scalarvalue__(self, run):
+        val = value(self.arg, run)
+        if val is None:
+            return None
+        return self.op(val)
+
+    def __vectorvalue__(self, run, index):
+        val = value(self.arg, run, index)
+        if val is None:
+            return None
+        return self.op(val)
+
+    def __vectorlen__(self):
+        return len(unproxy(self.arg))
+
+class BinaryProxy(ValueProxy):
+    def __init__(self, op, arg0, arg1):
+        super(BinaryProxy, self).__init__()
+        self.op = op
+        self.arg0 = WrapValue(arg0)
+        self.arg1 = WrapValue(arg1)
+
+    def __scalar__(self):
+        return scalar(self.arg0) and scalar(self.arg1)
+
+    def __vector__(self):
+        return vector(self.arg0) or vector(self.arg1)
+
+    def __scalarvalue__(self, run):
+        val0 = value(self.arg0, run)
+        val1 = value(self.arg1, run)
+        if val0 is None or val1 is None:
+            return None
+        return self.op(val0, val1)
+
+    def __vectorvalue__(self, run, index):
+        if scalar(self.arg0):
+            val0 = value(self.arg0, run)
+        if vector(self.arg0):
+            val0 = value(self.arg0, run, index)
+        if scalar(self.arg1):
+            val1 = value(self.arg1, run)
+        if vector(self.arg1):
+            val1 = value(self.arg1, run, index)
+
+        if val0 is None or val1 is None:
+            return None
+
+        return self.op(val0, val1)
+
+    def __vectorlen__(self):
+        if vector(self.arg0) and scalar(self.arg1):
+            return len(self.arg0)
+        if scalar(self.arg0) and vector(self.arg1):
+            return len(self.arg1)
+
+        len0 = len(self.arg0)
+        len1 = len(self.arg1)
+
+        if len0 != len1:
+            raise AttributeError, \
+                  "vectors of different lengths %d != %d" % (len0, len1)
+
+        return len0
+
+class Proxy(Value):
+    def __init__(self, name, dict):
+        self.name = name
+        self.dict = dict
+
+    def __unproxy__(self):
+        return unproxy(self.dict[self.name])
+
+    def __getitem__(self, index):
+        return ItemProxy(self, index)
+
+    def __getattr__(self, attr):
+        return AttrProxy(self, attr)
+
+class ItemProxy(Proxy):
+    def __init__(self, proxy, index):
+        self.proxy = proxy
+        self.index = index
+
+    def __unproxy__(self):
+        return unproxy(unproxy(self.proxy)[self.index])
 
-    def __iadd__(self, other):
-        self.value = binaryop(operator.add, self.value, other.value)
-        return self
+class AttrProxy(Proxy):
+    def __init__(self, proxy, attr):
+        self.proxy = proxy
+        self.attr = attr
 
-    def __itruediv__(self, other):
-        if not other:
-            return self
-        if isinstance(self.value, (list, tuple)):
-            for i in xrange(len(self.value)):
-                self.value[i] /= other
-        else:
-            self.value /= other
-        return self
+    def __unproxy__(self):
+        return unproxy(getattr(unproxy(self.proxy), self.attr))
 
-class Formula(Vector):
-    def getValue(self):
-        formula = re.sub(':', '__', self.formula)
-        x = eval(formula, source.stattop)
-        return x.value
+class ProxyGroup(object):
+    def __init__(self, dict=None, **kwargs):
+        self.__dict__['dict'] = {}
 
-    def comparable(self, other):
-        return self.name == other.name and \
-               compare(self.dist, other.dist)
+        if dict is not None:
+            self.dict.update(dict)
 
-    def __eq__(self, other):
-        return self.value == other.value
+        if kwargs:
+            self.dict.update(kwargs)
 
-    def __isub__(self, other):
-        return self
+    def __getattr__(self, name):
+        return Proxy(name, self.dict)
 
-    def __iadd__(self, other):
-        return self
+    def __setattr__(self, attr, value):
+        self.dict[attr] = value
 
-    def __itruediv__(self, other):
-        if not other:
-            return self
-        return self
+class ScalarStat(Statistic,Scalar):
+    def __value__(self, run):
+        if run not in self.data:
+            return None
+        return self.data[run][0][0]
+
+    def display(self, run=None):
+        import display
+        p = display.Print()
+        p.name = self.name
+        p.desc = self.desc
+        p.value = value(self, run)
+        p.flags = self.flags
+        p.precision = self.precision
+        if display.all or (self.flags & flags.printable):
+            p.display()
+
+class VectorStat(Statistic,Vector):
+    def __value__(self, run, item):
+        if run not in self.data:
+            return None
+        return self.data[run][item][0]
+
+    def __len__(self):
+        return self.x
 
-class SimpleDist(object):
+    def display(self, run=None):
+        import display
+        d = display.VectorDisplay()
+        d.name = self.name
+        d.desc = self.desc
+        d.value = [ value(self, run, i) for i in xrange(len(self)) ]
+        d.flags = self.flags
+        d.precision = self.precision
+        d.display()
+
+class Formula(Value):
+    def __getattribute__(self, attr):
+        if attr not in ( '__scalar__', '__vector__', '__value__', '__len__' ):
+            return super(Formula, self).__getattribute__(attr)
+
+        formula = re.sub(':', '__', self.formula)
+        value = eval(formula, self.source.stattop)
+        return getattr(value, attr)
+
+class SimpleDist(Statistic):
     def __init__(self, sums, squares, samples):
         self.sums = sums
         self.squares = squares
         self.samples = samples
 
-    def getValue(self):
-        return 0.0
-
     def display(self, name, desc, flags, precision):
         import display
         p = display.Print()
@@ -482,9 +438,6 @@ class FullDist(SimpleDist):
         self.bsize = bsize
         self.size = size
 
-    def getValue(self):
-        return 0.0
-
     def display(self, name, desc, flags, precision):
         import display
         p = display.Print()
@@ -574,9 +527,6 @@ class FullDist(SimpleDist):
         return self
 
 class Dist(Statistic):
-    def getValue(self):
-        return 0.0
-
     def display(self):
         import display
         if not display.all and not (self.flags & flags.printable):
@@ -606,9 +556,6 @@ class Dist(Statistic):
         return self
 
 class VectorDist(Statistic):
-    def getValue(self):
-        return 0.0
-
     def display(self):
         import display
         if not display.all and not (self.flags & flags.printable):
@@ -694,9 +641,6 @@ class VectorDist(Statistic):
         return self
 
 class Vector2d(Statistic):
-    def getValue(self):
-        return 0.0
-
     def display(self):
         import display
         if not display.all and not (self.flags & flags.printable):
@@ -748,20 +692,25 @@ class Vector2d(Statistic):
             return self
         return self
 
-def NewStat(data):
+def NewStat(source, data):
     stat = None
     if data.type == 'SCALAR':
-        stat = Scalar(data)
+        stat = ScalarStat()
     elif data.type == 'VECTOR':
-        stat = Vector(data)
+        stat = VectorStat()
     elif data.type == 'DIST':
-        stat = Dist(data)
+        stat = Dist()
     elif data.type == 'VECTORDIST':
-        stat = VectorDist(data)
+        stat = VectorDist()
     elif data.type == 'VECTOR2D':
-        stat = Vector2d(data)
+        stat = Vector2d()
     elif data.type == 'FORMULA':
-        stat = Formula(data)
+        stat = Formula()
+
+    stat.__dict__['source'] = source
+    stat.__dict__['bins'] = None
+    stat.__dict__['ticks'] = None
+    stat.__dict__.update(data.__dict__)
 
     return stat
 
diff --git a/util/stats/output.py b/util/stats/output.py
index f793749f7..cf76f291e 100644
--- a/util/stats/output.py
+++ b/util/stats/output.py
@@ -26,24 +26,8 @@
 #
 # Authors: Nathan Binkert
 
-class dbinfo(object):
-    def get(self, job, stat):
-        import info
-
-        run = info.source.allRunNames.get(job.name, None)
-        if run is None:
-            print 'run "%s" not found' % job
-            return None
-
-        stat.system = info.source[job.system]
-        info.display_run = run.run;
-        val = float(stat)
-        if val == 1e300*1e300:
-            return None
-        return val
-
 class StatOutput(object):
-    def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None):
+    def __init__(self, name, jobfile, info, stat=None, binstats=None):
         self.name = name
         self.jobfile = jobfile
         self.stat = stat
@@ -141,7 +125,7 @@ class StatOutput(object):
             data = zeros((len(groupopts), len(baropts)), Float)
             data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ]
             enabled = False
-            stacked = None
+            stacked = 0
             for g,gopt in enumerate(groupopts):
                 for b,bopt in enumerate(baropts):
                     job = self.jobfile.job(options + [ gopt, bopt ])
@@ -149,17 +133,28 @@ class StatOutput(object):
                         continue
 
                     val = self.info.get(job, self.stat)
-                    if val is None:
-                        val = 0.0
-                    curstacked = isinstance(val, (list, tuple))
-                    if stacked is None:
-                        stacked = curstacked
-                    else:
-                        if stacked != curstacked:
-                            raise ValueError, "some stats stacked, some not"
+                    if isinstance(val, (list, tuple)):
+                        if len(val) == 1:
+                            val = val[0]
+                        else:
+                            stacked = len(val)
 
                     data[g][b] = val
 
+            if stacked == 0:
+                for i in xrange(len(groupopts)):
+                    for j in xrange(len(baropts)):
+                        if data[i][j] is None:
+                            data[i][j] = 0.0
+            else:
+                for i in xrange(len(groupopts)):
+                    for j in xrange(len(baropts)):
+                        val = data[i][j]
+                        if val is None:
+                            data[i][j] = [] * stacked
+                        elif len(val) != stacked:
+                            raise ValueError, "some stats stacked, some not"
+
             data = array(data)
             if data.sum() == 0:
                 continue
@@ -167,7 +162,10 @@ class StatOutput(object):
             bar_descs = [ opt.desc for opt in baropts ]
             group_descs = [ opt.desc for opt in groupopts ]
             if stacked:
-                legend = self.info.rcategories
+                try:
+                    legend = self.info.rcategories
+                except:
+                    legend = [ str(i) for i in xrange(stacked) ]
             else:
                 legend = bar_descs
 
diff --git a/util/stats/stats.py b/util/stats/stats.py
index af9ae5575..e481a794c 100755
--- a/util/stats/stats.py
+++ b/util/stats/stats.py
@@ -98,29 +98,27 @@ def commands(options, command, args):
 
         raise CommandException
 
-    import db, info
-    info.source = db.Database()
-    info.source.host = options.host
-    info.source.db = options.db
-    info.source.passwd = options.passwd
-    info.source.user = options.user
-    info.source.connect()
-    #info.source.update_dict(globals())
-
-    if type(options.get) is str:
-        info.source.get = options.get
+    import db
+    source = db.Database()
+    source.host = options.host
+    source.db = options.db
+    source.passwd = options.passwd
+    source.user = options.user
+    source.connect()
+    #source.update_dict(globals())
+
+    if type(options.method) is str:
+        source.method = options.method
 
     if options.runs is None:
-        runs = info.source.allRuns
+        runs = source.allRuns
     else:
         rx = re.compile(options.runs)
         runs = []
-        for run in info.source.allRuns:
+        for run in source.allRuns:
             if rx.match(run.name):
                 runs.append(run)
 
-    info.display_run = runs[0].run
-
     if command == 'runs':
         user = None
         opts, args = getopts(args, '-u')
@@ -129,14 +127,14 @@ def commands(options, command, args):
         for o,a in opts:
             if o == '-u':
                 user = a
-        info.source.listRuns(user)
+        source.listRuns(user)
         return
 
     if command == 'stats':
         if len(args) == 0:
-            info.source.listStats()
+            source.listStats()
         elif len(args) == 1:
-            info.source.listStats(args[0])
+            source.listStats(args[0])
         else:
             raise CommandException
 
@@ -144,9 +142,9 @@ def commands(options, command, args):
 
     if command == 'bins':
         if len(args) == 0:
-            info.source.listBins()
+            source.listBins()
         elif len(args) == 1:
-            info.source.listBins(args[0])
+            source.listBins(args[0])
         else:
             raise CommandException
 
@@ -154,9 +152,9 @@ def commands(options, command, args):
 
     if command == 'formulas':
         if len(args) == 0:
-            info.source.listFormulas()
+            source.listFormulas()
         elif len(args) == 1:
-            info.source.listFormulas(args[0])
+            source.listFormulas(args[0])
         else:
             raise CommandException
 
@@ -166,7 +164,7 @@ def commands(options, command, args):
         if len(args):
             raise CommandException
 
-        info.source.listTicks(runs)
+        source.listTicks(runs)
         return
 
     if command == 'stability':
@@ -177,8 +175,8 @@ def commands(options, command, args):
             merge = int(args[0])
         except ValueError:
             usage()
-        stats = info.source.getStat(args[1])
-        info.source.get = "sum"
+        stats = source.getStat(args[1])
+        source.method = 'sum'
 
         def disp(*args):
             print "%-20s %12s %12s %4s %5s %5s %5s %10s" % args
@@ -195,18 +193,17 @@ def commands(options, command, args):
 
             #loop through all the selected runs
             for run in runs:
-                info.display_run = run.run;
-                runTicks = info.source.retTicks([ run ])
+                runTicks = source.retTicks([ run ])
                 #throw away the first one, it's 0
                 runTicks.pop(0)
-                info.globalTicks = runTicks
+                source.ticks = runTicks
                 avg = 0
                 stdev = 0
                 numoutsideavg  = 0
                 numoutside1std = 0
                 numoutside2std = 0
                 pairRunTicks = []
-                if float(stat) == 1e300*1e300:
+                if value(stat, run.run) == 1e300*1e300:
                     continue
                 for t in range(0, len(runTicks)-(merge-1), merge):
                     tempPair = []
@@ -215,17 +212,17 @@ def commands(options, command, args):
                     pairRunTicks.append(tempPair)
                 #loop through all the various ticks for each run
                 for tick in pairRunTicks:
-                    info.globalTicks = tick
-                    avg += float(stat)
+                    source.ticks = tick
+                    avg += value(stat, run.run)
                 avg /= len(pairRunTicks)
                 for tick in pairRunTicks:
-                    info.globalTicks = tick
-                    val = float(stat)
+                    source.ticks = tick
+                    val = value(stat, run.run)
                     stdev += pow((val-avg),2)
                 stdev = math.sqrt(stdev / len(pairRunTicks))
                 for tick in pairRunTicks:
-                    info.globalTicks = tick
-                    val = float(stat)
+                    source.ticks = tick
+                    val = value(stat, run.run)
                     if (val < (avg * .9)) or (val > (avg * 1.1)):
                         numoutsideavg += 1
                     if (val < (avg - stdev)) or (val > (avg + stdev)):
@@ -264,9 +261,9 @@ def commands(options, command, args):
     if options.ticks:
         if not options.graph:
             print 'only displaying sample %s' % options.ticks
-        info.globalTicks = [ int(x) for x in options.ticks.split() ]
+        source.ticks = [ int(x) for x in options.ticks.split() ]
 
-    from output import StatOutput
+    import output
 
     def display():
         if options.graph:
@@ -280,12 +277,12 @@ def commands(options, command, args):
             raise CommandException
 
         if command == 'stat':
-            stats = info.source.getStat(args[0])
+            stats = source.getStat(args[0])
         if command == 'formula':
             stats = eval(args[0])
 
         for stat in stats:
-            output = StatOutput(stat.name, options.jobfile)
+            output = output.StatOutput(stat.name, options.jobfile, source)
             output.stat = stat
             output.label = stat.name
             display()
@@ -295,12 +292,11 @@ def commands(options, command, args):
     if len(args):
         raise CommandException
 
-    system = info.source.__dict__[options.system]
-
-    from proxy import ProxyGroup
-    sim_ticks = info.source['sim_ticks']
-    sim_seconds = info.source['sim_seconds']
-    proxy = ProxyGroup(system = info.source[options.system])
+    system = source.__dict__[options.system]
+    from info import ProxyGroup
+    sim_ticks = source['sim_ticks']
+    sim_seconds = source['sim_seconds']
+    proxy = ProxyGroup(system = source[options.system])
     system = proxy.system
 
     etherdev = system.tsunami.etherdev0
@@ -309,7 +305,7 @@ def commands(options, command, args):
     packets = etherdev.rxPackets + etherdev.txPackets
     bps = etherdev.rxBandwidth + etherdev.txBandwidth
 
-    output = StatOutput(command, options.jobfile)
+    output = output.StatOutput(command, options.jobfile, source)
 
     if command == 'usertime':
         import copy
@@ -460,7 +456,7 @@ if __name__ == '__main__':
     options.user = getpass.getuser()
     options.runs = None
     options.system = 'client'
-    options.get = None
+    options.method = None
     options.binned = False
     options.graph = False
     options.ticks = False
@@ -469,7 +465,7 @@ if __name__ == '__main__':
     options.jobfile = None
     options.all = False
 
-    opts, args = getopts(sys.argv[1:], '-BEFG:Jad:g:h:j:pr:s:u:T:')
+    opts, args = getopts(sys.argv[1:], '-BEFJad:g:h:j:m:pr:s:u:T:')
     for o,a in opts:
         if o == '-B':
             options.binned = True
@@ -477,8 +473,6 @@ if __name__ == '__main__':
             options.printmode = 'E'
         if o == '-F':
             options.printmode = 'F'
-        if o == '-G':
-            options.get = a
         if o == '-a':
             options.all = True
         if o == '-d':
@@ -492,6 +486,8 @@ if __name__ == '__main__':
             jobfilename = None
         if o == '-j':
             jobfilename = a
+        if o == '-m':
+            options.method = a
         if o == '-p':
             options.passwd = getpass.getpass()
         if o == '-r':
-- 
cgit v1.2.3


From 12822443a595eb1260f63ba756b110d5f91f0da6 Mon Sep 17 00:00:00 2001
From: Nathan Binkert <binkertn@umich.edu>
Date: Fri, 21 Oct 2005 16:32:48 -0400
Subject: Minor updates to the profile code.

util/stats/profile.py:
    Pass around the number of symbols limit
    deal with categorization a bit better.

--HG--
extra : convert_revision : 908410e296efd4514f2dfc0eb9e6e42834585560
---
 util/stats/profile.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

(limited to 'util/stats')

diff --git a/util/stats/profile.py b/util/stats/profile.py
index cf946452b..57b854a30 100644
--- a/util/stats/profile.py
+++ b/util/stats/profile.py
@@ -316,7 +316,7 @@ class Profile(object):
             symbols.tree.dot(dot, threshold=threshold)
             dot.write(symbols.filename[:-3] + 'dot')
 
-    def write_txt(self, jobfile=None, jobs=None):
+    def write_txt(self, jobfile=None, jobs=None, limit=None):
         if jobs is None:
             jobs = [ job for job in jobfile.jobs() ]
 
@@ -327,7 +327,7 @@ class Profile(object):
                 continue
 
             output = file(symbols.filename[:-3] + 'txt', 'w')
-            symbols.display(output)
+            symbols.display(output, limit)
 
     def display(self, jobfile=None, jobs=None, limit=None):
         if jobs is None:
@@ -453,6 +453,8 @@ if __name__ == '__main__':
         else:
             profile = PCProfile()
 
+        if not categorize:
+            profile.categorize = None
         profile.inputdir(jobfile.rootdir)
 
         if graph:
@@ -470,11 +472,6 @@ if __name__ == '__main__':
                 profile.cpu = cpu
                 profile.write_dot(jobfile=jobfile, threshold=threshold)
 
-        if not categorize:
-            for cpu in cpus:
-                profile.cpu = cpu
-                profile.categorize = None
-
         if textout:
             for cpu in cpus:
                 profile.cpu = cpu
@@ -482,5 +479,7 @@ if __name__ == '__main__':
 
         if not graph and not textout and not dodot:
             for cpu in cpus:
+                if not categorize:
+                    profile.categorize = None
                 profile.cpu = cpu
                 profile.display(jobfile=jobfile, limit=numsyms)
-- 
cgit v1.2.3


From 13bf7e27ac7a20c80fb079fbaff0d6091a1e1c18 Mon Sep 17 00:00:00 2001
From: Nathan Binkert <binkertn@umich.edu>
Date: Fri, 21 Oct 2005 17:28:47 -0400
Subject: Fix a couple of bug in the values() vector accessor

util/stats/db.py:
    need to import the values function
util/stats/info.py:
    it's just run

--HG--
extra : convert_revision : 3cb67d8112a1a5fdf761b73732859a71f585bd1f
---
 util/stats/db.py   | 2 +-
 util/stats/info.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'util/stats')

diff --git a/util/stats/db.py b/util/stats/db.py
index 0d321a360..1ece6df88 100644
--- a/util/stats/db.py
+++ b/util/stats/db.py
@@ -158,7 +158,7 @@ class Database(object):
             print 'run "%s" not found' % job
             return None
 
-        from info import scalar, vector, value, total, len
+        from info import scalar, vector, value, values, total, len
         stat.system = self[job.system]
         if scalar(stat):
             return value(stat, run.run)
diff --git a/util/stats/info.py b/util/stats/info.py
index 9932d7922..889af6d53 100644
--- a/util/stats/info.py
+++ b/util/stats/info.py
@@ -51,7 +51,7 @@ def values(stat, run):
     stat = unproxy(stat)
     result = []
     for i in xrange(len(stat)):
-        val = value(stat, run.run, i)
+        val = value(stat, run, i)
         if val is None:
             return None
         result.append(val)
-- 
cgit v1.2.3