Merge zizzer:/bk/m5

into  zeep.eecs.umich.edu:/z/saidi/work/m5

--HG--
extra : convert_revision : 3cc23080d19cc464a8ba7c1c93b6e5d45af7d463

9 files changed, 507 insertions, 519 deletions

diff --git a/util/pbs/pbs.py b/util/pbs/pbs.py
index 292ddf1f9..21c2cb89d 100755
--- a/util/pbs/pbs.py
+++ b/util/pbs/pbs.py
@@ -147,6 +147,8 @@ class qsub:
                 flags.append('e')
             if len(flags):
                 self.cmd.append('-m ' + flags)
+        else:
+            self.cmd.append('-mn')
 
         if self.name:
             self.cmd.append("-N%s" % self.name)
@@ -158,7 +160,7 @@ class qsub:
             self.cmd.append('-q' + self.queue)
 
         if self.afterok:
-            self.cmd.append('-Wdepend=afterok:%s' % self.after)
+            self.cmd.append('-Wdepend=afterok:%s' % self.afterok)
 
         self.cmd.extend(args)
         self.script = script
diff --git a/util/pbs/send.py b/util/pbs/send.py
index 1f48b2b0e..54bba931d 100755
--- a/util/pbs/send.py
+++ b/util/pbs/send.py
@@ -98,7 +98,7 @@ Usage:
 
 try:
     import getopt
-    opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lq:Rt:v')
+    opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v')
 except getopt.GetoptError:
     sys.exit(usage)
 
@@ -115,6 +115,7 @@ docpts = False
 doruns = True
 runflag = False
 node_type = 'FAST'
+update = True
 
 for opt,arg in opts:
     if opt == '-C':
@@ -134,6 +135,8 @@ for opt,arg in opts:
         jfile = arg
     if opt == '-l':
         listonly = True
+    if opt == '-n':
+        update = False
     if opt == '-q':
         queue = arg
     if opt == '-R':
@@ -154,7 +157,7 @@ from job import JobDir, date
 
 conf = jobfile.JobFile(jfile)
 
-if not listonly and not onlyecho and isdir(conf.linkdir):
+if update and not listonly and not onlyecho and isdir(conf.linkdir):
     if verbose:
         print 'Checking for outdated files in Link directory'
     if not isdir(conf.basedir):
diff --git a/util/qdo b/util/qdo
index 3a475b420..339d944a7 100755
--- a/util/qdo
+++ b/util/qdo
@@ -45,7 +45,11 @@ optparser.add_option('-o', dest='stdout_file',
                      help='command stdout output file')
 optparser.add_option('-l', dest='save_log', action='store_true',
                      help='save qsub output log file')
-optparser.add_option('-q', dest='qsub_timeout', type='int',
+optparser.add_option('-N', dest='job_name',
+                     help='qsub job name')
+optparser.add_option('-q', dest='dest_queue',
+                     help='qsub destination queue')
+optparser.add_option('--qwait', dest='qsub_timeout', type='int',
                      help='qsub queue wait timeout', default=30*60)
 optparser.add_option('-t', dest='cmd_timeout', type='int',
                      help='command execution timeout', default=600*60)
@@ -56,6 +60,12 @@ if cmd == []:
     print >>sys.stderr, "%s: missing command" % progname
     sys.exit(1)
 
+# If we want to do this, need to add check here to make sure cmd[0] is
+# a valid PBS job name, else qsub will die on us.
+#
+#if not options.job_name:
+#    options.job_name = cmd[0]
+
 cwd = os.getcwd()
 
 # Deal with systems where /n is a symlink to /.automount
@@ -137,6 +147,10 @@ if False and len(cmd) > 50:
     print "%s: running %s on poolfs" % (progname, cmd[0])
 else:
     shell_cmd = 'qsub -I -S /bin/sh'
+    if options.job_name:
+        shell_cmd += ' -N "%s"' % options.job_name
+    if options.dest_queue:
+        shell_cmd += ' -q ' + options.dest_queue
 
 shell = Shell(shell_cmd)
 
diff --git a/util/stats/db.py b/util/stats/db.py
index ab005e67b..1ece6df88 100644
--- a/util/stats/db.py
+++ b/util/stats/db.py
@@ -103,6 +103,20 @@ class Node(object):
     def __str__(self):
         return self.name
 
+class Result(object):
+    def __init__(self, x, y):
+        self.data = {}
+        self.x = x
+        self.y = y
+
+    def __contains__(self, run):
+        return run in self.data
+
+    def __getitem__(self, run):
+        if run not in self.data:
+            self.data[run] = [ [ 0.0 ] * self.y for i in xrange(self.x) ]
+        return self.data[run]
+
 class Database(object):
     def __init__(self):
         self.host = 'zizzer.pool'
@@ -135,7 +149,23 @@ class Database(object):
         self.runs = None
         self.bins = None
         self.ticks = None
-        self.__dict__['get'] = type(self).sum
+        self.method = 'sum'
+        self._method = type(self).sum
+
+    def get(self, job, stat):
+        run = self.allRunNames.get(job.name, None)
+        if run is None:
+            print 'run "%s" not found' % job
+            return None
+
+        from info import scalar, vector, value, values, total, len
+        stat.system = self[job.system]
+        if scalar(stat):
+            return value(stat, run.run)
+        if vector(stat):
+            return values(stat, run.run)
+
+        return None
 
     def query(self, sql):
         self.cursor.execute(sql)
@@ -203,7 +233,7 @@ class Database(object):
         self.query('select * from stats')
         import info
         for result in self.cursor.fetchall():
-            stat = info.NewStat(StatData(result))
+            stat = info.NewStat(self, StatData(result))
             self.append(stat)
             self.allStats.append(stat)
             self.allStatIds[stat.stat] = stat
@@ -421,30 +451,17 @@ class Database(object):
     def stdev(self, stat, bins, ticks):
         return self.outer('stddev', 'sum', stat, bins, ticks)
 
-    def __getattribute__(self, attr):
-        if attr != 'get':
-            return super(Database, self).__getattribute__(attr)
-
-        if self.__dict__['get'] == type(self).sum:
-            return 'sum'
-        elif self.__dict__['get'] == type(self).avg:
-            return 'avg'
-        elif self.__dict__['get'] == type(self).stdev:
-            return 'stdev'
-        else:
-            return ''
-
     def __setattr__(self, attr, value):
-        if attr != 'get':
-            super(Database, self).__setattr__(attr, value)
+        super(Database, self).__setattr__(attr, value)
+        if attr != 'method':
             return
 
         if value == 'sum':
-            self.__dict__['get'] = type(self).sum
+            self._method = self.sum
         elif value == 'avg':
-            self.__dict__['get'] = type(self).avg
+            self._method = self.avg
         elif value == 'stdev':
-            self.__dict__['get'] = type(self).stdev
+            self._method = self.stdev
         else:
             raise AttributeError, "can only set get to: sum | avg | stdev"
 
@@ -453,10 +470,12 @@ class Database(object):
             bins = self.bins
         if ticks is None:
             ticks = self.ticks
-        sql = self.__dict__['get'](self, stat, bins, ticks)
+        sql = self._method(self, stat, bins, ticks)
         self.query(sql)
 
         runs = {}
+        xmax = 0
+        ymax = 0
         for x in self.cursor.fetchall():
             data = Data(x)
             if not runs.has_key(data.run):
@@ -464,8 +483,17 @@ class Database(object):
             if not runs[data.run].has_key(data.x):
                 runs[data.run][data.x] = {}
 
+            xmax = max(xmax, data.x)
+            ymax = max(ymax, data.y)
             runs[data.run][data.x][data.y] = data.data
-        return runs
+
+        results = Result(xmax + 1, ymax + 1)
+        for run,data in runs.iteritems():
+            result = results[run]
+            for x,ydata in data.iteritems():
+                for y,data in ydata.iteritems():
+                    result[x][y] = data
+        return results
 
     def __getitem__(self, key):
         return self.stattop[key]
diff --git a/util/stats/display.py b/util/stats/display.py
index 629684ca4..fbcff5c70 100644
--- a/util/stats/display.py
+++ b/util/stats/display.py
@@ -26,7 +26,7 @@
 
 class Value:
     def __init__(self, value, precision, percent = False):
-        self.value = value
+        self.value = float(value)
         self.precision = precision
         self.percent = percent
     def __str__(self):
@@ -90,61 +90,60 @@ class Print:
 
 class VectorDisplay:
     def display(self):
+        if not self.value:
+            return
+
         p = Print()
         p.flags = self.flags
         p.precision = self.precision
 
-        if isinstance(self.value, (list, tuple)):
-            if not len(self.value):
-                return
-
-            mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
-            mycdf = 0.0
+        if not isinstance(self.value, (list, tuple)):
+            p.name = self.name
+            p.desc = self.desc
+            p.value = self.value
+            p.display()
+            return
 
-            value = self.value
+        mytotal = reduce(lambda x,y: float(x) + float(y), self.value)
+        mycdf = 0.0
 
-            if display_all:
-                subnames = [ '[%d]' % i for i in range(len(value)) ]
-            else:
-                subnames = [''] * len(value)
-
-            if self.__dict__.has_key('subnames'):
-                for i,each in enumerate(self.subnames):
-                    if len(each) > 0:
-                        subnames[i] = '.%s' % each
-
-            subdescs = [self.desc]*len(value)
-            if self.__dict__.has_key('subdescs'):
-                for i in xrange(min(len(value), len(self.subdescs))):
-                    subdescs[i] = self.subdescs[i]
-
-            for val,sname,sdesc in map(None, value, subnames, subdescs):
-                if mytotal > 0.0:
-                    mypdf = float(val) / float(mytotal)
-                    mycdf += mypdf
-                    if (self.flags & flags_pdf):
-                        p.pdf = mypdf
-                        p.cdf = mycdf
-
-                if len(sname) == 0:
-                    continue
-
-                p.name = self.name + sname
-                p.desc = sdesc
-                p.value = val
-                p.display()
-
-            if (self.flags & flags_total):
-                if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
-                if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
-                p.name = self.name + '.total'
-                p.desc = self.desc
-                p.value = mytotal
-                p.display()
+        value = self.value
 
+        if display_all:
+            subnames = [ '[%d]' % i for i in range(len(value)) ]
         else:
-            p.name = self.name
-            p.desc = self.desc
-            p.value = self.value
+            subnames = [''] * len(value)
+
+        if self.__dict__.has_key('subnames'):
+            for i,each in enumerate(self.subnames):
+                if len(each) > 0:
+                    subnames[i] = '.%s' % each
+
+        subdescs = [self.desc]*len(value)
+        if self.__dict__.has_key('subdescs'):
+            for i in xrange(min(len(value), len(self.subdescs))):
+                subdescs[i] = self.subdescs[i]
+
+        for val,sname,sdesc in map(None, value, subnames, subdescs):
+            if mytotal > 0.0:
+                mypdf = float(val) / float(mytotal)
+                mycdf += mypdf
+                if (self.flags & flags_pdf):
+                    p.pdf = mypdf
+                    p.cdf = mycdf
+
+            if len(sname) == 0:
+                continue
+
+            p.name = self.name + sname
+            p.desc = sdesc
+            p.value = val
             p.display()
 
+        if (self.flags & flags_total):
+            if (p.__dict__.has_key('pdf')): del p.__dict__['pdf']
+            if (p.__dict__.has_key('cdf')): del p.__dict__['cdf']
+            p.name = self.name + '.total'
+            p.desc = self.desc
+            p.value = mytotal
+            p.display()
diff --git a/util/stats/info.py b/util/stats/info.py
index ae5d3211f..889af6d53 100644
--- a/util/stats/info.py
+++ b/util/stats/info.py
@@ -27,391 +27,347 @@
 from __future__ import division
 import operator, re, types
 
-source = None
-display_run = 0
-global globalTicks
-globalTicks = None
-
-def total(f):
-    if isinstance(f, FormulaStat):
-        v = f.value
-    else:
-        v = f
-
-    f = FormulaStat()
-    if isinstance(v, (list, tuple)):
-        f.value = reduce(operator.add, v)
-    else:
-        f.value = v
-
-    return f
-
-def unaryop(op, f):
-    if isinstance(f, FormulaStat):
-        v = f.value
-    else:
-        v = f
-
-    if isinstance(v, (list, tuple)):
-        return map(op, v)
-    else:
-        return op(v)
-
-def zerodiv(lv, rv):
-    if rv == 0.0:
-        return 0.0
-    else:
-        return operator.truediv(lv, rv)
-
-def wrapop(op, lv, rv):
-    if isinstance(lv, str):
-        return lv
-
-    if isinstance(rv, str):
-        return rv
-
-    return op(lv, rv)
-
-def same(lrun, rrun):
-    for lx,rx in zip(lrun.keys(),rrun.keys()):
-        if lx != rx:
-            print 'lx != rx'
-            print lx, rx
-            print lrun.keys()
-            print rrun.keys()
-            return False
-        for ly,ry in zip(lrun[lx].keys(),rrun[rx].keys()):
-            if ly != ry:
-                print 'ly != ry'
-                print ly, ry
-                print lrun[lx].keys()
-                print rrun[rx].keys()
-                return False
-    return True
-
-
-def binaryop(op, lf, rf):
-    result = {}
-
-    if isinstance(lf, FormulaStat) and isinstance(rf, FormulaStat):
-        lv = lf.value
-        rv = rf.value
-
-        theruns = []
-        for r in lv.keys():
-            if rv.has_key(r):
-                if same(lv[r], rv[r]):
-                    theruns.append(r)
-                else:
-                    raise AttributeError
-
-        for run in theruns:
-            result[run] = {}
-            for x in lv[run].keys():
-                result[run][x] = {}
-                for y in lv[run][x].keys():
-                    result[run][x][y] = wrapop(op, lv[run][x][y],
-                                               rv[run][x][y])
-    elif isinstance(lf, FormulaStat):
-        lv = lf.value
-        for run in lv.keys():
-            result[run] = {}
-            for x in lv[run].keys():
-                result[run][x] = {}
-                for y in lv[run][x].keys():
-                    result[run][x][y] = wrapop(op, lv[run][x][y], rf)
-    elif isinstance(rf, FormulaStat):
-        rv = rf.value
-        for run in rv.keys():
-            result[run] = {}
-            for x in rv[run].keys():
-                result[run][x] = {}
-                for y in rv[run][x].keys():
-                    result[run][x][y] = wrapop(op, lf, rv[run][x][y])
-
+def unproxy(proxy):
+    if hasattr(proxy, '__unproxy__'):
+        return proxy.__unproxy__()
+
+    return proxy
+
+def scalar(stat):
+    stat = unproxy(stat)
+    assert(stat.__scalar__() != stat.__vector__())
+    return stat.__scalar__()
+
+def vector(stat):
+    stat = unproxy(stat)
+    assert(stat.__scalar__() != stat.__vector__())
+    return stat.__vector__()
+
+def value(stat, *args):
+    stat = unproxy(stat)
+    return stat.__value__(*args)
+
+def values(stat, run):
+    stat = unproxy(stat)
+    result = []
+    for i in xrange(len(stat)):
+        val = value(stat, run, i)
+        if val is None:
+            return None
+        result.append(val)
     return result
 
-def sums(x, y):
-    if isinstance(x, (list, tuple)):
-        return map(lambda x, y: x + y, x, y)
-    else:
-        return x + y
-
-def alltrue(seq):
-    return reduce(lambda x, y: x and y, seq)
-
-def allfalse(seq):
-    return not reduce(lambda x, y: x or y, seq)
-
-def enumerate(seq):
-    return map(None, range(len(seq)), seq)
-
-def cmp(a, b):
-    if a < b:
-        return -1
-    elif a == b:
-        return 0
-    else:
-        return 1
-
-class Statistic(object):
-
-    def __init__(self, data):
-        self.__dict__.update(data.__dict__)
-        if not self.__dict__.has_key('value'):
-            self.__dict__['value'] = None
-        if not self.__dict__.has_key('bins'):
-            self.__dict__['bins'] = None
-        if not self.__dict__.has_key('ticks'):
-            self.__dict__['ticks'] = None
-        if 'vc' not in self.__dict__:
-            self.vc = {}
-
-    def __getattribute__(self, attr):
-        if attr == 'ticks':
-            if self.__dict__['ticks'] != globalTicks:
-                self.__dict__['value'] = None
-                self.__dict__['ticks'] = globalTicks
-            return self.__dict__['ticks']
-        if attr == 'value':
-            if self.__dict__['ticks'] != globalTicks:
-                if self.__dict__['ticks'] != None and \
-                                    len(self.__dict__['ticks']) == 1:
-                    self.vc[self.__dict__['ticks'][0]] = self.__dict__['value']
-                self.__dict__['ticks'] = globalTicks
-                if len(globalTicks) == 1 and self.vc.has_key(globalTicks[0]):
-                    self.__dict__['value'] = self.vc[globalTicks[0]]
-                else:
-                    self.__dict__['value'] = None
-            if self.__dict__['value'] == None:
-                self.__dict__['value'] = self.getValue()
-            return self.__dict__['value']
-        else:
-            return super(Statistic, self).__getattribute__(attr)
-
-    def __setattr__(self, attr, value):
-        if attr == 'bins' or attr == 'ticks':
-            if attr == 'bins':
-                if value is not None:
-                    value = source.getBin(value)
-            #elif attr == 'ticks' and type(value) is str:
-            #    value = [ int(x) for x in value.split() ]
-
-            self.__dict__[attr] = value
-            self.__dict__['value'] = None
-            self.vc = {}
-        else:
-            super(Statistic, self).__setattr__(attr, value)
+def total(stat, run):
+    return sum(values(stat, run))
 
-    def getValue(self):
-        raise AttributeError, 'getValue() must be defined'
-
-    def zero(self):
-        return False
+def len(stat):
+    stat = unproxy(stat)
+    return stat.__len__()
 
-    def __ne__(self, other):
-        return not (self == other)
+class Value(object):
+    def __scalar__(self):
+        raise AttributeError, "must define __scalar__ for %s" % (type (self))
+    def __vector__(self):
+        raise AttributeError, "must define __vector__ for %s" % (type (self))
 
-    def __str__(self):
-        return '%f' % (float(self))
-
-class FormulaStat(object):
     def __add__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.add, self, other)
-        return f
+        return BinaryProxy(operator.__add__, self, other)
     def __sub__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.sub, self, other)
-        return f
+        return BinaryProxy(operator.__sub__, self, other)
     def __mul__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.mul, self, other)
-        return f
+        return BinaryProxy(operator.__mul__, self, other)
+    def __div__(self, other):
+        return BinaryProxy(operator.__div__, self, other)
     def __truediv__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(zerodiv, self, other)
-        return f
-    def __mod__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.mod, self, other)
-        return f
+        return BinaryProxy(operator.__truediv__, self, other)
+    def __floordiv__(self, other):
+        return BinaryProxy(operator.__floordiv__, self, other)
+
     def __radd__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.add, other, self)
-        return f
+        return BinaryProxy(operator.__add__, other, self)
     def __rsub__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.sub, other, self)
-        return f
+        return BinaryProxy(operator.__sub__, other, self)
     def __rmul__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.mul, other, self)
-        return f
+        return BinaryProxy(operator.__mul__, other, self)
+    def __rdiv__(self, other):
+        return BinaryProxy(operator.__div__, other, self)
     def __rtruediv__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(zerodiv, other, self)
-        return f
-    def __rmod__(self, other):
-        f = FormulaStat()
-        f.value = binaryop(operator.mod, other, self)
-        return f
+        return BinaryProxy(operator.__truediv__, other, self)
+    def __rfloordiv__(self, other):
+        return BinaryProxy(operator.__floordiv__, other, self)
+
     def __neg__(self):
-        f = FormulaStat()
-        f.value = unaryop(operator.neg, self)
-        return f
-    def __getitem__(self, idx):
-        f = FormulaStat()
-        f.value = {}
-        for key in self.value.keys():
-            f.value[key] = {}
-            f.value[key][0] = {}
-            f.value[key][0][0] = self.value[key][idx][0]
-        return f
-
-    def __float__(self):
-        if isinstance(self.value, FormulaStat):
-            return float(self.value)
-        if not self.value.has_key(display_run):
-            return (1e300*1e300)
-        if len(self.value[display_run]) == 1:
-            return self.value[display_run][0][0]
-        else:
-            #print self.value[display_run]
-            return self.value[display_run][4][0]
-            #raise ValueError
+        return UnaryProxy(operator.__neg__, self)
+    def __pos__(self):
+        return UnaryProxy(operator.__pos__, self)
+    def __abs__(self):
+        return UnaryProxy(operator.__abs__, self)
+
+class Scalar(Value):
+    def __scalar__(self):
+        return True
 
-    def display(self):
-        import display
-        d = display.VectorDisplay()
-        d.flags = 0
-        d.precision = 1
-        d.name = 'formula'
-        d.desc = 'formula'
-        val = self.value[display_run]
-        d.value = [ val[x][0] for x in val.keys() ]
-        d.display()
+    def __vector__(self):
+        return False
 
+    def __value__(self, run):
+        raise AttributeError, '__value__ must be defined'
 
-class Scalar(Statistic,FormulaStat):
-    def getValue(self):
-        return source.data(self, self.bins, self.ticks)
+class VectorItemProxy(Value):
+    def __init__(self, proxy, index):
+        self.proxy = proxy
+        self.index = index
 
-    def display(self):
-        import display
-        p = display.Print()
-        p.name = self.name
-        p.desc = self.desc
-        p.value = float(self)
-        p.flags = self.flags
-        p.precision = self.precision
-        if display.all or (self.flags & flags.printable):
-            p.display()
+    def __scalar__(self):
+        return True
 
-    def comparable(self, other):
-        return self.name == other.name
+    def __vector__(self):
+        return False
 
-    def __eq__(self, other):
-        return self.value == other.value
+    def __value__(self, run):
+        return value(self.proxy, run, self.index)
 
-    def __isub__(self, other):
-        self.value -= other.value
-        return self
+class Vector(Value):
+    def __scalar__(self):
+        return False
 
-    def __iadd__(self, other):
-        self.value += other.value
-        return self
+    def __vector__(self):
+        return True
 
-    def __itruediv__(self, other):
-        if not other:
-            return self
-        self.value /= other
-        return self
+    def __value__(self, run, index):
+        raise AttributeError, '__value__ must be defined'
 
-class Vector(Statistic,FormulaStat):
-    def getValue(self):
-        return source.data(self, self.bins, self.ticks);
+    def __getitem__(self, index):
+        return VectorItemProxy(self, index)
 
-    def display(self):
-        import display
-        if not display.all and not (self.flags & flags.printable):
-            return
+class ScalarConstant(Scalar):
+    def __init__(self, constant):
+        self.constant = constant
+    def __value__(self, run):
+        return self.constant
 
-        d = display.VectorDisplay()
-        d.__dict__.update(self.__dict__)
-        d.display()
+class VectorConstant(Vector):
+    def __init__(self, constant):
+        self.constant = constant
+    def __value__(self, run, index):
+        return self.constant[index]
+    def __len__(self):
+        return len(self.constant)
 
-    def comparable(self, other):
-        return self.name == other.name and \
-               len(self.value) == len(other.value)
+def WrapValue(value):
+    if isinstance(value, (int, long, float)):
+        return ScalarConstant(value)
+    if isinstance(value, (list, tuple)):
+        return VectorConstant(value)
+    if isinstance(value, Value):
+        return value
 
-    def __eq__(self, other):
-        if isinstance(self.value, (list, tuple)) != \
-               isinstance(other.value, (list, tuple)):
-            return False
+    raise AttributeError, 'Only values can be wrapped'
 
-        if isinstance(self.value, (list, tuple)):
-            if len(self.value) != len(other.value):
-                return False
-            else:
-                for v1,v2 in zip(self.value, other.value):
-                    if v1 != v2:
-                        return False
-                return True
-        else:
-            return self.value == other.value
+class Statistic(object):
+    def __getattr__(self, attr):
+        if attr in ('data', 'x', 'y'):
+            result = self.source.data(self, self.bins, self.ticks)
+            self.data = result.data
+            self.x = result.x
+            self.y = result.y
+        return super(Statistic, self).__getattribute__(attr)
 
-    def __isub__(self, other):
-        self.value = binaryop(operator.sub, self.value, other.value)
-        return self
+    def __setattr__(self, attr, value):
+        if attr == 'stat':
+            raise AttributeError, '%s is read only' % stat
+        if attr in ('source', 'bins', 'ticks'):
+            if getattr(self, attr) != value:
+                if hasattr(self, 'data'):
+                    delattr(self, 'data')
+
+        super(Statistic, self).__setattr__(attr, value)
+
+class ValueProxy(Value):
+    def __getattr__(self, attr):
+        if attr == '__value__':
+            if scalar(self):
+                return self.__scalarvalue__
+            if vector(self):
+                return self.__vectorvalue__
+        if attr == '__len__':
+            if vector(self):
+                return self.__vectorlen__
+        return super(ValueProxy, self).__getattribute__(attr)
+
+class UnaryProxy(ValueProxy):
+    def __init__(self, op, arg):
+        self.op = op
+        self.arg = WrapValue(arg)
+
+    def __scalar__(self):
+        return scalar(self.arg)
+
+    def __vector__(self):
+        return vector(self.arg)
+
+    def __scalarvalue__(self, run):
+        val = value(self.arg, run)
+        if val is None:
+            return None
+        return self.op(val)
+
+    def __vectorvalue__(self, run, index):
+        val = value(self.arg, run, index)
+        if val is None:
+            return None
+        return self.op(val)
+
+    def __vectorlen__(self):
+        return len(unproxy(self.arg))
+
+class BinaryProxy(ValueProxy):
+    def __init__(self, op, arg0, arg1):
+        super(BinaryProxy, self).__init__()
+        self.op = op
+        self.arg0 = WrapValue(arg0)
+        self.arg1 = WrapValue(arg1)
+
+    def __scalar__(self):
+        return scalar(self.arg0) and scalar(self.arg1)
+
+    def __vector__(self):
+        return vector(self.arg0) or vector(self.arg1)
+
+    def __scalarvalue__(self, run):
+        val0 = value(self.arg0, run)
+        val1 = value(self.arg1, run)
+        if val0 is None or val1 is None:
+            return None
+        return self.op(val0, val1)
+
+    def __vectorvalue__(self, run, index):
+        if scalar(self.arg0):
+            val0 = value(self.arg0, run)
+        if vector(self.arg0):
+            val0 = value(self.arg0, run, index)
+        if scalar(self.arg1):
+            val1 = value(self.arg1, run)
+        if vector(self.arg1):
+            val1 = value(self.arg1, run, index)
+
+        if val0 is None or val1 is None:
+            return None
+
+        return self.op(val0, val1)
+
+    def __vectorlen__(self):
+        if vector(self.arg0) and scalar(self.arg1):
+            return len(self.arg0)
+        if scalar(self.arg0) and vector(self.arg1):
+            return len(self.arg1)
+
+        len0 = len(self.arg0)
+        len1 = len(self.arg1)
+
+        if len0 != len1:
+            raise AttributeError, \
+                  "vectors of different lengths %d != %d" % (len0, len1)
+
+        return len0
+
+class Proxy(Value):
+    def __init__(self, name, dict):
+        self.name = name
+        self.dict = dict
+
+    def __unproxy__(self):
+        return unproxy(self.dict[self.name])
+
+    def __getitem__(self, index):
+        return ItemProxy(self, index)
+
+    def __getattr__(self, attr):
+        return AttrProxy(self, attr)
+
+class ItemProxy(Proxy):
+    def __init__(self, proxy, index):
+        self.proxy = proxy
+        self.index = index
+
+    def __unproxy__(self):
+        return unproxy(unproxy(self.proxy)[self.index])
 
-    def __iadd__(self, other):
-        self.value = binaryop(operator.add, self.value, other.value)
-        return self
+class AttrProxy(Proxy):
+    def __init__(self, proxy, attr):
+        self.proxy = proxy
+        self.attr = attr
 
-    def __itruediv__(self, other):
-        if not other:
-            return self
-        if isinstance(self.value, (list, tuple)):
-            for i in xrange(len(self.value)):
-                self.value[i] /= other
-        else:
-            self.value /= other
-        return self
+    def __unproxy__(self):
+        return unproxy(getattr(unproxy(self.proxy), self.attr))
 
-class Formula(Vector):
-    def getValue(self):
-        formula = re.sub(':', '__', self.formula)
-        x = eval(formula, source.stattop)
-        return x.value
+class ProxyGroup(object):
+    def __init__(self, dict=None, **kwargs):
+        self.__dict__['dict'] = {}
 
-    def comparable(self, other):
-        return self.name == other.name and \
-               compare(self.dist, other.dist)
+        if dict is not None:
+            self.dict.update(dict)
 
-    def __eq__(self, other):
-        return self.value == other.value
+        if kwargs:
+            self.dict.update(kwargs)
 
-    def __isub__(self, other):
-        return self
+    def __getattr__(self, name):
+        return Proxy(name, self.dict)
 
-    def __iadd__(self, other):
-        return self
+    def __setattr__(self, attr, value):
+        self.dict[attr] = value
 
-    def __itruediv__(self, other):
-        if not other:
-            return self
-        return self
+class ScalarStat(Statistic,Scalar):
+    def __value__(self, run):
+        if run not in self.data:
+            return None
+        return self.data[run][0][0]
+
+    def display(self, run=None):
+        import display
+        p = display.Print()
+        p.name = self.name
+        p.desc = self.desc
+        p.value = value(self, run)
+        p.flags = self.flags
+        p.precision = self.precision
+        if display.all or (self.flags & flags.printable):
+            p.display()
+
+class VectorStat(Statistic,Vector):
+    def __value__(self, run, item):
+        if run not in self.data:
+            return None
+        return self.data[run][item][0]
+
+    def __len__(self):
+        return self.x
 
-class SimpleDist(object):
+    def display(self, run=None):
+        import display
+        d = display.VectorDisplay()
+        d.name = self.name
+        d.desc = self.desc
+        d.value = [ value(self, run, i) for i in xrange(len(self)) ]
+        d.flags = self.flags
+        d.precision = self.precision
+        d.display()
+
+class Formula(Value):
+    def __getattribute__(self, attr):
+        if attr not in ( '__scalar__', '__vector__', '__value__', '__len__' ):
+            return super(Formula, self).__getattribute__(attr)
+
+        formula = re.sub(':', '__', self.formula)
+        value = eval(formula, self.source.stattop)
+        return getattr(value, attr)
+
+class SimpleDist(Statistic):
     def __init__(self, sums, squares, samples):
         self.sums = sums
         self.squares = squares
         self.samples = samples
 
-    def getValue(self):
-        return 0.0
-
     def display(self, name, desc, flags, precision):
         import display
         p = display.Print()
@@ -482,9 +438,6 @@ class FullDist(SimpleDist):
         self.bsize = bsize
         self.size = size
 
-    def getValue(self):
-        return 0.0
-
     def display(self, name, desc, flags, precision):
         import display
         p = display.Print()
@@ -574,9 +527,6 @@ class FullDist(SimpleDist):
         return self
 
 class Dist(Statistic):
-    def getValue(self):
-        return 0.0
-
     def display(self):
         import display
         if not display.all and not (self.flags & flags.printable):
@@ -606,9 +556,6 @@ class Dist(Statistic):
         return self
 
 class VectorDist(Statistic):
-    def getValue(self):
-        return 0.0
-
     def display(self):
         import display
         if not display.all and not (self.flags & flags.printable):
@@ -694,9 +641,6 @@ class VectorDist(Statistic):
         return self
 
 class Vector2d(Statistic):
-    def getValue(self):
-        return 0.0
-
     def display(self):
         import display
         if not display.all and not (self.flags & flags.printable):
@@ -748,20 +692,25 @@ class Vector2d(Statistic):
             return self
         return self
 
-def NewStat(data):
+def NewStat(source, data):
     stat = None
     if data.type == 'SCALAR':
-        stat = Scalar(data)
+        stat = ScalarStat()
     elif data.type == 'VECTOR':
-        stat = Vector(data)
+        stat = VectorStat()
     elif data.type == 'DIST':
-        stat = Dist(data)
+        stat = Dist()
     elif data.type == 'VECTORDIST':
-        stat = VectorDist(data)
+        stat = VectorDist()
     elif data.type == 'VECTOR2D':
-        stat = Vector2d(data)
+        stat = Vector2d()
     elif data.type == 'FORMULA':
-        stat = Formula(data)
+        stat = Formula()
+
+    stat.__dict__['source'] = source
+    stat.__dict__['bins'] = None
+    stat.__dict__['ticks'] = None
+    stat.__dict__.update(data.__dict__)
 
     return stat
 
diff --git a/util/stats/output.py b/util/stats/output.py
index f793749f7..cf76f291e 100644
--- a/util/stats/output.py
+++ b/util/stats/output.py
@@ -26,24 +26,8 @@
 #
 # Authors: Nathan Binkert
 
-class dbinfo(object):
-    def get(self, job, stat):
-        import info
-
-        run = info.source.allRunNames.get(job.name, None)
-        if run is None:
-            print 'run "%s" not found' % job
-            return None
-
-        stat.system = info.source[job.system]
-        info.display_run = run.run;
-        val = float(stat)
-        if val == 1e300*1e300:
-            return None
-        return val
-
 class StatOutput(object):
-    def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None):
+    def __init__(self, name, jobfile, info, stat=None, binstats=None):
         self.name = name
         self.jobfile = jobfile
         self.stat = stat
@@ -141,7 +125,7 @@ class StatOutput(object):
             data = zeros((len(groupopts), len(baropts)), Float)
             data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ]
             enabled = False
-            stacked = None
+            stacked = 0
             for g,gopt in enumerate(groupopts):
                 for b,bopt in enumerate(baropts):
                     job = self.jobfile.job(options + [ gopt, bopt ])
@@ -149,17 +133,28 @@ class StatOutput(object):
                         continue
 
                     val = self.info.get(job, self.stat)
-                    if val is None:
-                        val = 0.0
-                    curstacked = isinstance(val, (list, tuple))
-                    if stacked is None:
-                        stacked = curstacked
-                    else:
-                        if stacked != curstacked:
-                            raise ValueError, "some stats stacked, some not"
+                    if isinstance(val, (list, tuple)):
+                        if len(val) == 1:
+                            val = val[0]
+                        else:
+                            stacked = len(val)
 
                     data[g][b] = val
 
+            if stacked == 0:
+                for i in xrange(len(groupopts)):
+                    for j in xrange(len(baropts)):
+                        if data[i][j] is None:
+                            data[i][j] = 0.0
+            else:
+                for i in xrange(len(groupopts)):
+                    for j in xrange(len(baropts)):
+                        val = data[i][j]
+                        if val is None:
+                            data[i][j] = [] * stacked
+                        elif len(val) != stacked:
+                            raise ValueError, "some stats stacked, some not"
+
             data = array(data)
             if data.sum() == 0:
                 continue
@@ -167,7 +162,10 @@ class StatOutput(object):
             bar_descs = [ opt.desc for opt in baropts ]
             group_descs = [ opt.desc for opt in groupopts ]
             if stacked:
-                legend = self.info.rcategories
+                try:
+                    legend = self.info.rcategories
+                except:
+                    legend = [ str(i) for i in xrange(stacked) ]
             else:
                 legend = bar_descs
 
diff --git a/util/stats/profile.py b/util/stats/profile.py
index cf946452b..57b854a30 100644
--- a/util/stats/profile.py
+++ b/util/stats/profile.py
@@ -316,7 +316,7 @@ class Profile(object):
             symbols.tree.dot(dot, threshold=threshold)
             dot.write(symbols.filename[:-3] + 'dot')
 
-    def write_txt(self, jobfile=None, jobs=None):
+    def write_txt(self, jobfile=None, jobs=None, limit=None):
         if jobs is None:
             jobs = [ job for job in jobfile.jobs() ]
 
@@ -327,7 +327,7 @@ class Profile(object):
                 continue
 
             output = file(symbols.filename[:-3] + 'txt', 'w')
-            symbols.display(output)
+            symbols.display(output, limit)
 
     def display(self, jobfile=None, jobs=None, limit=None):
         if jobs is None:
@@ -453,6 +453,8 @@ if __name__ == '__main__':
         else:
             profile = PCProfile()
 
+        if not categorize:
+            profile.categorize = None
         profile.inputdir(jobfile.rootdir)
 
         if graph:
@@ -470,11 +472,6 @@ if __name__ == '__main__':
                 profile.cpu = cpu
                 profile.write_dot(jobfile=jobfile, threshold=threshold)
 
-        if not categorize:
-            for cpu in cpus:
-                profile.cpu = cpu
-                profile.categorize = None
-
         if textout:
             for cpu in cpus:
                 profile.cpu = cpu
@@ -482,5 +479,7 @@ if __name__ == '__main__':
 
         if not graph and not textout and not dodot:
             for cpu in cpus:
+                if not categorize:
+                    profile.categorize = None
                 profile.cpu = cpu
                 profile.display(jobfile=jobfile, limit=numsyms)
diff --git a/util/stats/stats.py b/util/stats/stats.py
index 3f6d162bc..61f54dede 100755
--- a/util/stats/stats.py
+++ b/util/stats/stats.py
@@ -98,29 +98,27 @@ def commands(options, command, args):
 
         raise CommandException
 
-    import db, info
-    info.source = db.Database()
-    info.source.host = options.host
-    info.source.db = options.db
-    info.source.passwd = options.passwd
-    info.source.user = options.user
-    info.source.connect()
-    #info.source.update_dict(globals())
-
-    if type(options.get) is str:
-        info.source.get = options.get
+    import db
+    source = db.Database()
+    source.host = options.host
+    source.db = options.db
+    source.passwd = options.passwd
+    source.user = options.user
+    source.connect()
+    #source.update_dict(globals())
+
+    if type(options.method) is str:
+        source.method = options.method
 
     if options.runs is None:
-        runs = info.source.allRuns
+        runs = source.allRuns
     else:
         rx = re.compile(options.runs)
         runs = []
-        for run in info.source.allRuns:
+        for run in source.allRuns:
             if rx.match(run.name):
                 runs.append(run)
 
-    info.display_run = runs[0].run
-
     if command == 'runs':
         user = None
         opts, args = getopts(args, '-u')
@@ -129,14 +127,14 @@ def commands(options, command, args):
         for o,a in opts:
             if o == '-u':
                 user = a
-        info.source.listRuns(user)
+        source.listRuns(user)
         return
 
     if command == 'stats':
         if len(args) == 0:
-            info.source.listStats()
+            source.listStats()
         elif len(args) == 1:
-            info.source.listStats(args[0])
+            source.listStats(args[0])
         else:
             raise CommandException
 
@@ -144,9 +142,9 @@ def commands(options, command, args):
 
     if command == 'bins':
         if len(args) == 0:
-            info.source.listBins()
+            source.listBins()
         elif len(args) == 1:
-            info.source.listBins(args[0])
+            source.listBins(args[0])
         else:
             raise CommandException
 
@@ -154,9 +152,9 @@ def commands(options, command, args):
 
     if command == 'formulas':
         if len(args) == 0:
-            info.source.listFormulas()
+            source.listFormulas()
         elif len(args) == 1:
-            info.source.listFormulas(args[0])
+            source.listFormulas(args[0])
         else:
             raise CommandException
 
@@ -166,7 +164,7 @@ def commands(options, command, args):
         if len(args):
             raise CommandException
 
-        info.source.listTicks(runs)
+        source.listTicks(runs)
         return
 
     if command == 'stability':
@@ -177,8 +175,8 @@ def commands(options, command, args):
             merge = int(args[0])
         except ValueError:
             usage()
-        stats = info.source.getStat(args[1])
-        info.source.get = "sum"
+        stats = source.getStat(args[1])
+        source.method = 'sum'
 
         def disp(*args):
             print "%-35s %12s %12s %4s %5s %5s %5s %10s" % args
@@ -195,18 +193,17 @@ def commands(options, command, args):
 
             #loop through all the selected runs
             for run in runs:
-                info.display_run = run.run;
-                runTicks = info.source.retTicks([ run ])
+                runTicks = source.retTicks([ run ])
                 #throw away the first one, it's 0
                 runTicks.pop(0)
-                info.globalTicks = runTicks
+                source.ticks = runTicks
                 avg = 0
                 stdev = 0
                 numoutsideavg  = 0
                 numoutside1std = 0
                 numoutside2std = 0
                 pairRunTicks = []
-                if float(stat) == 1e300*1e300:
+                if value(stat, run.run) == 1e300*1e300:
                     continue
                 for t in range(0, len(runTicks)-(merge-1), merge):
                     tempPair = []
@@ -215,17 +212,17 @@ def commands(options, command, args):
                     pairRunTicks.append(tempPair)
                 #loop through all the various ticks for each run
                 for tick in pairRunTicks:
-                    info.globalTicks = tick
-                    avg += float(stat)
+                    source.ticks = tick
+                    avg += value(stat, run.run)
                 avg /= len(pairRunTicks)
                 for tick in pairRunTicks:
-                    info.globalTicks = tick
-                    val = float(stat)
+                    source.ticks = tick
+                    val = value(stat, run.run)
                     stdev += pow((val-avg),2)
                 stdev = math.sqrt(stdev / len(pairRunTicks))
                 for tick in pairRunTicks:
-                    info.globalTicks = tick
-                    val = float(stat)
+                    source.ticks = tick
+                    val = value(stat, run.run)
                     if (val < (avg * .9)) or (val > (avg * 1.1)):
                         numoutsideavg += 1
                     if (val < (avg - stdev)) or (val > (avg + stdev)):
@@ -264,9 +261,9 @@ def commands(options, command, args):
     if options.ticks:
         if not options.graph:
             print 'only displaying sample %s' % options.ticks
-        info.globalTicks = [ int(x) for x in options.ticks.split() ]
+        source.ticks = [ int(x) for x in options.ticks.split() ]
 
-    from output import StatOutput
+    import output
 
     def display():
         if options.graph:
@@ -280,12 +277,12 @@ def commands(options, command, args):
             raise CommandException
 
         if command == 'stat':
-            stats = info.source.getStat(args[0])
+            stats = source.getStat(args[0])
         if command == 'formula':
             stats = eval(args[0])
 
         for stat in stats:
-            output = StatOutput(stat.name, options.jobfile)
+            output = output.StatOutput(stat.name, options.jobfile, source)
             output.stat = stat
             output.label = stat.name
             display()
@@ -295,12 +292,11 @@ def commands(options, command, args):
     if len(args):
         raise CommandException
 
-    system = info.source.__dict__[options.system]
-
-    from proxy import ProxyGroup
-    sim_ticks = info.source['sim_ticks']
-    sim_seconds = info.source['sim_seconds']
-    proxy = ProxyGroup(system = info.source[options.system])
+    system = source.__dict__[options.system]
+    from info import ProxyGroup
+    sim_ticks = source['sim_ticks']
+    sim_seconds = source['sim_seconds']
+    proxy = ProxyGroup(system = source[options.system])
     system = proxy.system
 
     etherdev = system.tsunami.etherdev0
@@ -309,7 +305,7 @@ def commands(options, command, args):
     packets = etherdev.rxPackets + etherdev.txPackets
     bps = etherdev.rxBandwidth + etherdev.txBandwidth
 
-    output = StatOutput(command, options.jobfile)
+    output = output.StatOutput(command, options.jobfile, source)
 
     if command == 'usertime':
         import copy
@@ -460,7 +456,7 @@ if __name__ == '__main__':
     options.user = getpass.getuser()
     options.runs = None
     options.system = 'client'
-    options.get = None
+    options.method = None
     options.binned = False
     options.graph = False
     options.ticks = False
@@ -469,7 +465,7 @@ if __name__ == '__main__':
     options.jobfile = None
     options.all = False
 
-    opts, args = getopts(sys.argv[1:], '-BEFG:Jad:g:h:j:pr:s:u:T:')
+    opts, args = getopts(sys.argv[1:], '-BEFJad:g:h:j:m:pr:s:u:T:')
     for o,a in opts:
         if o == '-B':
             options.binned = True
@@ -477,8 +473,6 @@ if __name__ == '__main__':
             options.printmode = 'E'
         if o == '-F':
             options.printmode = 'F'
-        if o == '-G':
-            options.get = a
         if o == '-a':
             options.all = True
         if o == '-d':
@@ -492,6 +486,8 @@ if __name__ == '__main__':
             jobfilename = None
         if o == '-j':
             jobfilename = a
+        if o == '-m':
+            options.method = a
         if o == '-p':
             options.passwd = getpass.getpass()
         if o == '-r':