8 files changed, 801 insertions, 17 deletions

diff --git a/util/pbs/job.py b/util/pbs/job.py
new file mode 100755
index 000000000..5eed0cd75
--- /dev/null
+++ b/util/pbs/job.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+#          Steve Reinhardt
+#          Ali Saidi
+
+import os, os.path, shutil, signal, socket, sys, time
+from os import environ as env
+from os.path import join as joinpath, expanduser
+
+class rsync:
+    def __init__(self):
+        self.sudo = False
+        self.rsync = 'rsync'
+        self.compress = False
+        self.archive = True
+        self.delete = False
+        self.options = ''
+
+    def do(self, src, dst):
+        args = []
+        if self.sudo:
+            args.append('sudo')
+
+        args.append(self.rsync)
+        if (self.archive):
+            args.append('-a')
+        if (self.compress):
+            args.append('-z')
+        if (self.delete):
+            args.append('--delete')
+        if len(self.options):
+            args.append(self.options)
+        args.append(src)
+        args.append(dst)
+
+        return os.spawnvp(os.P_WAIT, args[0], args)
+
+def cleandir(dir):
+    for root, dirs, files in os.walk(dir, False):
+        for name in files:
+            os.remove(joinpath(root, name))
+        for name in dirs:
+            os.rmdir(joinpath(root, name))
+
+def date():
+    return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime())
+
+def remfile(file):
+    if os.path.isfile(file):
+        os.unlink(file)
+
+def readval(filename):
+    file = open(filename, 'r')
+    value = file.readline().strip()
+    file.close()
+    return value
+
+if __name__ == '__main__':
+    rootdir = env.setdefault('ROOTDIR', os.getcwd())
+    jobid = env['PBS_JOBID']
+    jobname = env['PBS_JOBNAME']
+    jobdir = joinpath(rootdir, jobname)
+    basedir = joinpath(rootdir, 'Base')
+    user = env['USER']
+
+    env['POOLJOB'] = 'True'
+    env['OUTPUT_DIR'] = jobdir
+    env['JOBFILE'] = joinpath(basedir, 'test.py')
+    env['JOBNAME'] = jobname
+
+    def echofile(filename, string):
+        try:
+            f = file(joinpath(jobdir, filename), 'w')
+            print >>f, string
+            f.flush()
+            f.close()
+        except IOError,e:
+            sys.exit(e)
+
+    if os.path.isdir("/work"):
+        workbase = "/work"
+    else:
+        workbase = "/tmp/"
+
+    workdir = joinpath(workbase, '%s.%s' % (user, jobid))
+
+    os.umask(0022)
+
+    echofile('.start', date())
+    echofile('.jobid', jobid)
+    echofile('.host', socket.gethostname())
+
+    if os.path.isdir(workdir):
+        cleandir(workdir)
+    else:
+        os.mkdir(workdir)
+
+    if os.path.isdir('/z/dist'):
+        sync = rsync()
+        sync.delete = True
+        sync.sudo = True
+        sync.do('poolfs::dist/m5/', '/z/dist/m5/')
+
+    try:
+        os.chdir(workdir)
+    except OSError,e:
+        sys.exit(e)
+
+    os.symlink(joinpath(jobdir, 'output'), 'status.out')
+
+    args = [ joinpath(basedir, 'm5'), joinpath(basedir, 'run.mpy') ]
+    if not len(args):
+        sys.exit("no arguments")
+
+    print 'starting job... %s' % date()
+    print ' '.join(args)
+    print
+    sys.stdout.flush()
+
+    childpid = os.fork()
+    if not childpid:
+        # Execute command
+        sys.stdin.close()
+        fd = os.open(joinpath(jobdir, "output"),
+                     os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
+        os.dup2(fd, sys.stdout.fileno())
+        os.dup2(fd, sys.stderr.fileno())
+        os.execvp(args[0], args)
+
+    def handler(signum, frame):
+        if childpid != 0:
+            os.kill(childpid, signum)
+
+    signal.signal(signal.SIGHUP, handler)
+    signal.signal(signal.SIGINT, handler)
+    signal.signal(signal.SIGQUIT, handler)
+    signal.signal(signal.SIGTERM, handler)
+    signal.signal(signal.SIGSTOP, handler)
+    signal.signal(signal.SIGCONT, handler)
+    signal.signal(signal.SIGUSR1, handler)
+    signal.signal(signal.SIGUSR2, handler)
+
+    done = 0
+    while not done:
+        try:
+            thepid,ec = os.waitpid(childpid, 0)
+            if ec:
+                print 'Exit code ', ec
+                echofile('.failure', date())
+            else:
+                echofile('.success', date())
+            done = 1
+        except OSError:
+            pass
+
+    print '\njob complete... %s' % date()
+    echofile('.stop', date())
diff --git a/util/pbs/jobfile.py b/util/pbs/jobfile.py
new file mode 100644
index 000000000..83eb81358
--- /dev/null
+++ b/util/pbs/jobfile.py
@@ -0,0 +1,92 @@
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+
+from os.path import expanduser, isfile, join as joinpath
+import sys
+
+def crossproduct(options):
+    number = len(options)
+    indexes = [ 0 ] * number
+    maxes = [ len(opt) for opt in options ]
+    def next():
+        for i in xrange(number - 1, -1, -1):
+            indexes[i] += 1
+            if indexes[i] < maxes[i]:
+                return False
+
+            indexes[i] = 0
+        return True
+
+    done = False
+    while not done:
+        result = []
+        for i in xrange(number):
+            result.append(options[i][indexes[i]])
+        yield result
+        done = next()
+
+class JobFile(object):
+    def __init__(self, jfile):
+        self.data = {}
+        jfile = expanduser(jfile)
+        if not isfile(jfile):
+            for p in sys.path:
+                if isfile(joinpath(p, jfile)):
+                    jfile = joinpath(p, jfile)
+                    break
+
+        execfile(jfile, self.data)
+        self.options = self.data['options']
+        self.environment = self.data['environment']
+        self.jobinfo = {}
+        self.jobs = []
+        for job in crossproduct(self.options):
+            jobname = '.'.join([ id[0] for id in job ])
+            self.jobs.append(jobname)
+            list = []
+            for info in job:
+                for item in info[1:]:
+                    list.append(item)
+            self.jobinfo[jobname] = list
+
+    def env(self, jobname):
+        env = {}
+        for key,val in self.jobinfo[jobname]:
+            env[key] = val
+
+        for key,val in self.environment:
+            env[key] = val
+        return env
+
+    def printinfo(self, jobname):
+        print '%s:' % jobname
+        for key,val in self.jobinfo[jobname]:
+            print '    %s = %s' % (key, val)
+
+        for key,val in self.environment:
+            print '    %s = %s' % (key, val)
diff --git a/util/pbs/pbs.py b/util/pbs/pbs.py
new file mode 100755
index 000000000..ecacbeba2
--- /dev/null
+++ b/util/pbs/pbs.py
@@ -0,0 +1,176 @@
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+
+import os, popen2, re, sys
+
+class MyPOpen(object):
+    def __init__(self, cmd, input = None, output = None, bufsize = -1):
+        self.status = -1
+
+        if input is None:
+            p2c_read, p2c_write = os.pipe()
+            self.tochild = os.fdopen(p2c_write, 'w', bufsize)
+        else:
+            p2c_write = None
+            if isinstance(input, file):
+                p2c_read = input.fileno()
+            elif isinstance(input, str):
+                input = file(input, 'r')
+                p2c_read = input.fileno()
+            elif isinstance(input, int):
+                p2c_read = input
+            else:
+                raise AttributeError
+
+        if output is None:
+            c2p_read, c2p_write = os.pipe()
+            self.fromchild = os.fdopen(c2p_read, 'r', bufsize)
+        else:
+            c2p_read = None
+            if isinstance(output, file):
+                c2p_write = output.fileno()
+            elif isinstance(output, str):
+                output = file(output, 'w')
+                c2p_write = output.fileno()
+            elif isinstance(output, int):
+                c2p_write = output
+            else:
+                raise AttributeError
+
+        self.pid = os.fork()
+        if self.pid == 0:
+            os.dup2(p2c_read, sys.stdin.fileno())
+            os.dup2(c2p_write, sys.stdout.fileno())
+            os.dup2(c2p_write, sys.stderr.fileno())
+            try:
+                os.execvp(cmd[0], cmd)
+            finally:
+                os._exit(1)
+
+        os.close(p2c_read)
+        os.close(c2p_write)
+
+    def poll(self):
+        if self.status < 0:
+            pid, status = os.waitpid(self.pid, os.WNOHANG)
+            if pid == self.pid:
+                self.status = status
+        return self.status
+
+    def wait(self):
+        if self.status < 0:
+            pid, status = os.waitpid(self.pid, 0)
+            if pid == self.pid:
+                self.status = status
+        return self.status
+
+class qsub:
+    def __init__(self):
+        self.hold = False
+        self.join = False
+        self.keep_stdout = False
+        self.keep_stderr = False
+        self.node_type = ''
+        self.mail_abort = False
+        self.mail_begin = False
+        self.mail_end = False
+        self.name = ''
+        self.stdout = ''
+        self.priority = 0
+        self.queue = ''
+        self.pbshost = ''
+        self.qsub = 'qsub'
+        self.env = {}
+
+    def build(self, script, args = []):
+        self.cmd = [ self.qsub ]
+
+        if self.env:
+            arg = '-v'
+            arg += ','.join([ '%s=%s' % i for i in self.env.iteritems() ])
+            self.cmd.append(arg)
+
+        if self.hold:
+            self.cmd.append('-h')
+
+        if len(self.stdout):
+            self.cmd.append('-olocalhost:' + self.stdout)
+
+        if self.keep_stdout and self.keep_stderr:
+            self.cmd.append('-koe')
+        elif self.keep_stdout:
+            self.cmd.append('-ko')
+        elif self.keep_stderr:
+            self.cmd.append('-ke')
+        else:
+            self.cmd.append('-kn')
+
+        if self.join:
+            self.cmd.append('-joe')
+
+        if len(self.node_type):
+            self.cmd.append('-lnodes=' + self.node_type)
+
+        if self.mail_abort or self.mail_begin or self.mail_end:
+            flags = ''
+            if self.mail_abort:
+                flags.append('a')
+            if self.mail_begin:
+                flags.append('b')
+            if self.mail_end:
+                flags.append('e')
+            if len(flags):
+                self.cmd.append('-m ' + flags)
+
+        if len(self.name):
+            self.cmd.append("-N%s" % self.name)
+
+        if self.priority != 0:
+            self.cmd.append('-p' + self.priority)
+
+        if len(self.queue):
+            self.cmd.append('-q' + self.queue)
+
+        self.cmd.extend(args)
+        self.script = script
+        self.command = ' '.join(self.cmd + [ self.script ])
+
+    def do(self):
+        pbs = MyPOpen(self.cmd + [ self.script ])
+        self.result = pbs.fromchild.read()
+        ec = pbs.wait()
+
+        if ec != 0 and self.pbshost:
+            cmd = ' '.join(self.cmd + [ '-' ])
+            cmd = [ 'ssh', '-x', self.pbshost, cmd ]
+            self.command = ' '.join(cmd)
+            ssh = MyPOpen(cmd, input = self.script)
+            self.result = ssh.fromchild.read()
+            ec = ssh.wait()
+
+        return ec
diff --git a/util/pbs/send.py b/util/pbs/send.py
new file mode 100755
index 000000000..b796cadbd
--- /dev/null
+++ b/util/pbs/send.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Ali Saidi
+#          Nathan Binkert
+
+import os, os.path, re, socket, sys
+from os import environ as env, listdir
+from os.path import basename, isdir, isfile, islink, join as joinpath
+from filecmp import cmp as filecmp
+from shutil import copyfile
+
+def nfspath(dir):
+    if dir.startswith('/.automount/'):
+        dir = '/n/%s' % dir[12:]
+    elif not dir.startswith('/n/'):
+        dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir)
+    return dir
+
+progpath = nfspath(sys.path[0])
+progname = basename(sys.argv[0])
+usage = """\
+Usage:
+    %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp>
+    -c           clean directory if job can be run
+    -e           only echo pbs command info, don't actually send the job
+    -f           force the job to run regardless of state
+    -q <queue>   submit job to the named queue
+    -v           be verbose
+
+    %(progname)s -l [-v] <regexp>
+    -l           list job names, don't submit
+    -v           be verbose (list job parameters)
+
+    %(progname)s -h
+    -h           display this help
+""" % locals()
+
+try:
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], '-cd:efhlq:v')
+except getopt.GetoptError:
+    sys.exit(usage)
+
+clean = False
+onlyecho = False
+exprs = []
+force = False
+listonly = False
+queue = ''
+verbose = False
+rootdir = nfspath(os.getcwd())
+for opt,arg in opts:
+    if opt == '-c':
+        clean = True
+    if opt == '-d':
+        rootdir = arg
+    if opt == '-e':
+        onlyecho = True
+    if opt == '-f':
+        force = True
+    if opt == '-h':
+        print usage
+        sys.exit(0)
+    if opt == '-l':
+        listonly = True
+    if opt == '-q':
+        queue = arg
+    if opt == '-v':
+        verbose = True
+
+basedir = joinpath(rootdir, 'Base')
+linkdir = joinpath(rootdir, 'Link')
+
+for arg in args:
+    exprs.append(re.compile(arg))
+
+if not listonly and not onlyecho and isdir(linkdir):
+    if verbose:
+        print 'Checking for outdated files in Link directory'
+    entries = listdir(linkdir)
+    for entry in entries:
+        link = joinpath(linkdir, entry)
+        if not islink(link) or not isfile(link):
+            continue
+
+        base = joinpath(basedir, entry)
+        if not isfile(base) or not filecmp(link, base):
+            print 'Base/%s is different than Link/%s: copying' % (entry, entry)
+            copyfile(link, base)
+
+import job, jobfile, pbs
+
+test = jobfile.JobFile(joinpath(basedir, 'test.py'))
+
+joblist = []
+for jobname in test.jobs:
+    if not exprs:
+        joblist.append(jobname)
+        continue
+
+    for expr in exprs:
+        if expr.match(jobname):
+            joblist.append(jobname)
+            break
+
+if listonly:
+    if verbose:
+        for jobname in joblist:
+            test.printinfo(jobname)
+    else:
+        for jobname in joblist:
+            print jobname
+    sys.exit(0)
+
+if not onlyecho:
+    jl = []
+    for jobname in joblist:
+        jobdir = joinpath(rootdir, jobname)
+        if os.path.exists(jobname):
+            if not force:
+                if os.path.isfile(joinpath(jobdir, '.success')):
+                    continue
+
+                if os.path.isfile(joinpath(jobdir, '.start')) and \
+                       not os.path.isfile(joinpath(jobdir, '.stop')):
+                    continue
+
+            if not clean:
+                sys.exit('job directory not clean!')
+
+            job.cleandir(jobdir)
+        else:
+            os.mkdir(jobdir)
+        jl.append(jobname)
+    joblist = jl
+
+for jobname in joblist:
+    jobdir = joinpath(rootdir, jobname)
+
+    if not onlyecho and not os.path.isdir(jobdir):
+        sys.exit('%s is not a directory.  Cannot build job' % jobdir)
+
+    print 'Job name:       %s' % jobname
+    print 'Job directory:  %s' % jobdir
+
+    qsub = pbs.qsub()
+    qsub.pbshost = 'simpool.eecs.umich.edu'
+    qsub.stdout = joinpath(jobdir, 'jobout')
+    qsub.name = jobname
+    qsub.join = True
+    qsub.node_type = 'FAST'
+    qsub.env['ROOTDIR'] = rootdir
+    if len(queue):
+        qsub.queue = queue
+    qsub.build(joinpath(progpath, 'job.py'))
+
+    if verbose:
+        print 'PBS Command:    %s' % qsub.command
+
+    if not onlyecho:
+        ec = qsub.do()
+        if ec == 0:
+            print 'PBS Jobid:      %s' % qsub.result
+        else:
+            print 'PBS Failed'
diff --git a/util/stats/db.py b/util/stats/db.py
index 495cdb5b5..ed5d10bc2 100644
--- a/util/stats/db.py
+++ b/util/stats/db.py
@@ -207,16 +207,43 @@ class Database(object):
 
     # Name: listTicks
     # Desc: Prints all samples for a given run
-    def listTicks(self, run=None):
+    def listTicks(self, runs=None):
         print "tick"
         print "----------------------------------------"
-        sql = 'select distinct dt_tick from data where dt_stat=1950'
-        #if run != None:
-        #    sql += ' where dt_run=%d' % run
+        sql = 'select distinct dt_tick from data where dt_stat=1180 and ('
+        if runs != None:
+            first = True
+            for run in runs:
+               if first:
+            #       sql += ' where'
+                   first = False
+               else:
+                   sql += ' or'
+               sql += ' dt_run=%s' % run.run
+            sql += ')'
         self.query(sql)
         for r in self.cursor.fetchall():
             print r[0]
 
+    # Name: retTicks
+    # Desc: Prints all samples for a given run
+    def retTicks(self, runs=None):
+        sql = 'select distinct dt_tick from data where dt_stat=1180 and ('
+        if runs != None:
+            first = True
+            for run in runs:
+               if first:
+                   first = False
+               else:
+                   sql += ' or'
+               sql += ' dt_run=%s' % run.run
+            sql += ')'
+        self.query(sql)
+        ret = []
+        for r in self.cursor.fetchall():
+            ret.append(r[0])
+        return ret
+
     # Name: liststats
     # Desc: Prints all statistics that appear in the database,
     #         the optional argument is a regular expression that can
diff --git a/util/stats/info.py b/util/stats/info.py
index d11619765..01d7bdb0f 100644
--- a/util/stats/info.py
+++ b/util/stats/info.py
@@ -3,6 +3,8 @@ import operator, re, types
 
 source = None
 display_run = 0
+global globalTicks
+globalTicks = None
 
 def issequence(t):
     return isinstance(t, types.TupleType) or isinstance(t, types.ListType)
@@ -130,6 +132,7 @@ def cmp(a, b):
         return 1
 
 class Statistic(object):
+
     def __init__(self, data):
         self.__dict__.update(data.__dict__)
         if not self.__dict__.has_key('value'):
@@ -138,9 +141,25 @@ class Statistic(object):
             self.__dict__['bins'] = None
         if not self.__dict__.has_key('ticks'):
             self.__dict__['ticks'] = None
+        if 'vc' not in self.__dict__:
+            self.vc = {}
 
     def __getattribute__(self, attr):
+        if attr == 'ticks':
+            if self.__dict__['ticks'] != globalTicks:
+                self.__dict__['value'] = None
+                self.__dict__['ticks'] = globalTicks
+            return self.__dict__['ticks']
         if attr == 'value':
+            if self.__dict__['ticks'] != globalTicks:
+                if self.__dict__['ticks'] != None and \
+                                    len(self.__dict__['ticks']) == 1:
+                    self.vc[self.__dict__['ticks'][0]] = self.__dict__['value']
+                self.__dict__['ticks'] = globalTicks
+                if len(globalTicks) == 1 and self.vc.has_key(globalTicks[0]):
+                    self.__dict__['value'] = self.vc[globalTicks[0]]
+                else:
+                    self.__dict__['value'] = None
             if self.__dict__['value'] == None:
                 self.__dict__['value'] = self.getValue()
             return self.__dict__['value']
@@ -152,11 +171,12 @@ class Statistic(object):
             if attr == 'bins':
                 if value is not None:
                     value = source.getBin(value)
-            elif attr == 'ticks' and type(value) is str:
-                value = [ int(x) for x in value.split() ]
+            #elif attr == 'ticks' and type(value) is str:
+            #    value = [ int(x) for x in value.split() ]
 
             self.__dict__[attr] = value
             self.__dict__['value'] = None
+            self.vc = {}
         else:
             super(Statistic, self).__setattr__(attr, value)
 
@@ -287,7 +307,7 @@ class Scalar(Statistic,FormulaStat):
 
 class Vector(Statistic,FormulaStat):
     def getValue(self):
-        return source.data(self, self.bins);
+        return source.data(self, self.bins, self.ticks);
 
     def display(self):
         import display
@@ -304,7 +324,7 @@ class Vector(Statistic,FormulaStat):
 
     def __eq__(self, other):
         if issequence(self.value) != issequence(other.value):
-            return false
+            return False
 
         if issequence(self.value):
             if len(self.value) != len(other.value):
diff --git a/util/stats/stats.py b/util/stats/stats.py
index b2b0ff8ad..c9b7ab2ac 100755
--- a/util/stats/stats.py
+++ b/util/stats/stats.py
@@ -1,11 +1,24 @@
 #!/usr/bin/env python
 from __future__ import division
-import re, sys
+import re, sys, math
+
 
 def usage():
     print '''\
 Usage: %s [-E] [-F] [-d <db> ] [-g <get> ] [-h <host>] [-p]
-       [-s <system>] [-r <runs> ] [-u <username>] <command> [command args]
+       [-s <system>] [-r <runs> ] [-T <samples>] [-u <username>]
+       <command> [command args]
+
+       commands    extra parameters   description
+       ----------- ------------------ ---------------------------------------
+       bins        [regex]            List bins (only matching regex)
+       formula     <formula>          Evaluated formula specified
+       formulas    [regex]            List formulas (only matching regex)
+       runs        none               List all runs in database
+       samples     none               List samples present in database
+       stability   <pairnum> <stats>  Calculated statistical info about stats
+       stat        <regex>            Show stat data (only matching regex)
+       stats       [regex]            List all stats (only matching regex)
 ''' % sys.argv[0]
     sys.exit(1)
 
@@ -249,6 +262,86 @@ def commands(options, command, args):
         info.source.listRuns(user)
         return
 
+    if command == 'stability':
+        if len(args) < 2:
+            raise CommandException
+
+        try:
+            merge = int(args[0])
+        except ValueError:
+            usage()
+        stats = info.source.getStat(args[1])
+        info.source.get = "sum"
+
+
+        #loop through all the stats selected
+        for stat in stats:
+
+            print "%s:" % stat.name
+            print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
+                  ("run name", "average", "stdev", ">10%", ">1SDV", ">2SDV", "SAMP", "CV")
+            print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
+                  ("--------------------", "------------",
+                   "------------", "----", "-----", "-----", "-----", "----------")
+            #loop through all the selected runs
+            for run in runs:
+                info.display_run = run.run;
+                runTicks = info.source.retTicks([ run ])
+                #throw away the first one, it's 0
+                runTicks.pop(0)
+                info.globalTicks = runTicks
+                avg = 0
+                stdev = 0
+                numoutsideavg  = 0
+                numoutside1std = 0
+                numoutside2std = 0
+                pairRunTicks = []
+                if float(stat) == 1e300*1e300:
+                    continue
+                for t in range(0, len(runTicks)-(merge-1), merge):
+                    tempPair = []
+                    for p in range(0,merge):
+                        tempPair.append(runTicks[t+p])
+                    pairRunTicks.append(tempPair)
+                #loop through all the various ticks for each run
+                for tick in pairRunTicks:
+                    info.globalTicks = tick
+                    avg += float(stat)
+                avg /= len(pairRunTicks)
+                for tick in pairRunTicks:
+                    info.globalTicks = tick
+                    val = float(stat)
+                    stdev += pow((val-avg),2)
+                stdev = math.sqrt(stdev / len(pairRunTicks))
+                for tick in pairRunTicks:
+                    info.globalTicks = tick
+                    val = float(stat)
+                    if (val < (avg * .9)) or (val > (avg * 1.1)):
+                        numoutsideavg += 1
+                    if (val < (avg - stdev)) or (val > (avg + stdev)):
+                        numoutside1std += 1
+                    if (val < (avg - (2*stdev))) or (val > (avg + (2*stdev))):
+                        numoutside2std += 1
+                if avg > 1000:
+                    print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
+                          (run.name, "%.1f" % avg, "%.1f" % stdev,
+                           "%d" % numoutsideavg, "%d" % numoutside1std,
+                           "%d" % numoutside2std, "%d" % len(pairRunTicks),
+                           "%.3f" % (stdev/avg*100))
+                elif avg > 100:
+                    print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
+                          (run.name, "%.1f" % avg, "%.1f" % stdev,
+                           "%d" % numoutsideavg, "%d" % numoutside1std,
+                           "%d" % numoutside2std, "%d" % len(pairRunTicks),
+                           "%.5f" % (stdev/avg*100))
+                else:
+                    print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
+                          (run.name, "%.5f" % avg, "%.5f" % stdev,
+                           "%d" % numoutsideavg, "%d" % numoutside1std,
+                           "%d" % numoutside2std, "%d" % len(pairRunTicks),
+                           "%.7f" % (stdev/avg*100))
+        return
+
     if command == 'stats':
         if len(args) == 0:
             info.source.listStats()
@@ -270,7 +363,7 @@ def commands(options, command, args):
             else:
                 if options.ticks:
                    print 'only displaying sample %s' % options.ticks
-                   stat.ticks = options.ticks
+                   info.globalTicks = [ int(x) for x in options.ticks.split() ]
 
                 if options.binned:
                     print 'kernel ticks'
diff --git a/util/tracediff b/util/tracediff
index a95ce8b82..87210f1ed 100755
--- a/util/tracediff
+++ b/util/tracediff
@@ -1,5 +1,5 @@
 #! /usr/bin/env perl
-# Copyright (c) 2003-2004 The Regents of The University of Michigan
+# Copyright (c) 2003-2005 The Regents of The University of Michigan
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -51,12 +51,15 @@ $sim2 = shift;
 # be given to both invocations
 $simargs = '"' . join('" "', @ARGV) . '"';
 
-# Redirect config output to cout so that gets diffed too (in case
-# that's the source of the problem).
-$simargs += " --Universe:config_output_file=cout";
+# Run individual invocations in separate dirs so output and intermediate
+# files (particularly config.py and config.ini) don't conflict.
+$dir1 = "tracediff-$$-1";
+$dir2 = "tracediff-$$-2";
+mkdir($dir1) or die "Can't create dir $dir1\n";
+mkdir($dir2) or die "Can't create dir $dir2\n";
 
-$cmd1 = "$sim1 $simargs --stats:text_file=tracediff-$$-1.stats 2>&1 |";
-$cmd2 = "$sim2 $simargs --stats:text_file=tracediff-$$-2.stats 2>&1 |";
+$cmd1 = "$sim1 $simargs -d $dir1 2>&1 |";
+$cmd2 = "$sim2 $simargs -d $dir2 2>&1 |";
 
 # This only works if you have rundiff in your path.  I just edit it
 # with an explicit path if necessary.