summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xutil/pbs/job.py139
-rw-r--r--util/pbs/jobfile.py534
-rwxr-xr-xutil/pbs/send.py164
-rw-r--r--util/stats/__init__.py28
-rw-r--r--util/stats/barchart.py246
-rw-r--r--util/stats/categories.py (renamed from util/categories.py)28
-rw-r--r--util/stats/db.py5
-rw-r--r--util/stats/orderdict.py78
-rw-r--r--util/stats/output.py182
-rw-r--r--util/stats/profile.py146
-rwxr-xr-xutil/stats/stats.py680
11 files changed, 1596 insertions, 634 deletions
diff --git a/util/pbs/job.py b/util/pbs/job.py
index f370862de..e2636c111 100755
--- a/util/pbs/job.py
+++ b/util/pbs/job.py
@@ -29,10 +29,21 @@
# Steve Reinhardt
# Ali Saidi
-import os, os.path, shutil, signal, socket, sys, time
+import os, os.path, shutil, signal, socket, sys
from os import environ as env
from os.path import join as joinpath, expanduser
+def date():
+ import time
+ return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime())
+
+def cleandir(dir):
+ for root, dirs, files in os.walk(dir, False):
+ for name in files:
+ os.remove(joinpath(root, name))
+ for name in dirs:
+ os.rmdir(joinpath(root, name))
+
class rsync:
def __init__(self):
self.sudo = False
@@ -61,25 +72,76 @@ class rsync:
return os.spawnvp(os.P_WAIT, args[0], args)
-def cleandir(dir):
- for root, dirs, files in os.walk(dir, False):
- for name in files:
- os.remove(joinpath(root, name))
- for name in dirs:
- os.rmdir(joinpath(root, name))
+class JobDir(object):
+ def __init__(self, dir):
+ self.dir = dir
-def date():
- return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime())
+ def file(self, filename):
+ return joinpath(self.dir, filename)
+
+ def create(self):
+ if os.path.exists(self.dir):
+ if not os.path.isdir(self.dir):
+ sys.exit('%s is not a directory. Cannot build job' % self.dir)
+ else:
+ os.mkdir(self.dir)
+
+ def exists(self):
+ return os.path.isdir(self.dir)
+
+ def clean(self):
+ cleandir(self.dir)
+
+ def hasfile(self, filename):
+ return os.path.isfile(self.file(filename))
+
+ def echofile(self, filename, string):
+ filename = self.file(filename)
+ try:
+ f = file(filename, 'w')
+ print >>f, string
+ f.flush()
+ f.close()
+ except IOError,e:
+ sys.exit(e)
-def remfile(file):
- if os.path.isfile(file):
- os.unlink(file)
+ def rmfile(self, filename):
+ filename = self.file(filename)
+ if os.path.isfile(filename):
+ os.unlink(filename)
-def readval(filename):
- file = open(filename, 'r')
- value = file.readline().strip()
- file.close()
- return value
+ def readval(self, filename):
+ filename = self.file(filename)
+ f = file(filename, 'r')
+ value = f.readline().strip()
+ f.close()
+ return value
+
+ def setstatus(self, string):
+ filename = self.file('.status')
+ try:
+ f = file(filename, 'a')
+ print >>f, string
+ f.flush()
+ f.close()
+ except IOError,e:
+ sys.exit(e)
+
+ def getstatus(self):
+ filename = self.file('.status')
+ try:
+ f = file(filename, 'r')
+ except IOError, e:
+ return 'none'
+
+ # fast forward to the end
+ for line in f: pass
+
+ # the first word on the last line is the status
+ return line.split(' ')[0]
+
+ def __str__(self):
+ return self.dir
if __name__ == '__main__':
rootdir = env.setdefault('ROOTDIR', os.getcwd())
@@ -97,29 +159,27 @@ if __name__ == '__main__':
workbase = "/tmp/"
workdir = joinpath(workbase, '%s.%s' % (env['USER'], pbs_jobid))
-
- def echofile(filename, string):
- try:
- f = file(joinpath(outdir, filename), 'w')
- print >>f, string
- f.flush()
- f.close()
- except IOError,e:
- sys.exit(e)
+ host = socket.gethostname()
os.umask(0022)
- echofile('.start', date())
- echofile('.pbs_jobid', pbs_jobid)
- echofile('.pbs_jobname', pbs_jobid)
- echofile('.host', socket.gethostname())
+ jobdir = JobDir(outdir)
+
+ started = date()
+ jobdir.echofile('.running', started)
+ jobdir.rmfile('.queued')
+ jobdir.echofile('.pbs_jobid', pbs_jobid)
+ jobdir.echofile('.pbs_jobname', pbs_jobid)
+ jobdir.echofile('.host', host)
+
+ jobdir.setstatus('running on %s on %s' % (host, started))
if os.path.isdir(workdir):
cleandir(workdir)
else:
os.mkdir(workdir)
- if os.path.isdir('/z/dist'):
+ if False and os.path.isdir('/z/dist'):
sync = rsync()
sync.delete = True
sync.sudo = True
@@ -130,13 +190,13 @@ if __name__ == '__main__':
except OSError,e:
sys.exit(e)
- os.symlink(joinpath(outdir, 'output'), 'status.out')
+ os.symlink(jobdir.file('output'), 'status.out')
args = [ joinpath(basedir, 'm5'), joinpath(basedir, 'run.py') ]
if not len(args):
sys.exit("no arguments")
- print 'starting job... %s' % date()
+ print 'starting job... %s' % started
print ' '.join(args)
print
sys.stdout.flush()
@@ -145,7 +205,7 @@ if __name__ == '__main__':
if not childpid:
# Execute command
sys.stdin.close()
- fd = os.open(joinpath(outdir, "output"),
+ fd = os.open(jobdir.file("output"),
os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
os.dup2(fd, sys.stdout.fileno())
os.dup2(fd, sys.stderr.fileno())
@@ -170,12 +230,15 @@ if __name__ == '__main__':
thepid,ec = os.waitpid(childpid, 0)
if ec:
print 'Exit code ', ec
- echofile('.failure', date())
+ status = 'failure'
else:
- echofile('.success', date())
+ status = 'success'
done = 1
except OSError:
pass
- print '\njob complete... %s' % date()
- echofile('.stop', date())
+ complete = date()
+ print '\njob complete... %s' % complete
+ jobdir.echofile('.%s' % status, complete)
+ jobdir.rmfile('.running')
+ jobdir.setstatus('%s on %s' % (status, complete))
diff --git a/util/pbs/jobfile.py b/util/pbs/jobfile.py
index 83eb81358..d36b5ee6d 100644
--- a/util/pbs/jobfile.py
+++ b/util/pbs/jobfile.py
@@ -26,67 +26,485 @@
#
# Authors: Nathan Binkert
-from os.path import expanduser, isfile, join as joinpath
import sys
-def crossproduct(options):
- number = len(options)
- indexes = [ 0 ] * number
- maxes = [ len(opt) for opt in options ]
- def next():
- for i in xrange(number - 1, -1, -1):
- indexes[i] += 1
- if indexes[i] < maxes[i]:
+class ternary(object):
+ def __new__(cls, *args):
+ if len(args) > 1:
+ raise TypeError, \
+ '%s() takes at most 1 argument (%d given)' % \
+ (cls.__name__, len(args))
+
+ if args:
+ if not isinstance(args[0], (bool, ternary)):
+ raise TypeError, \
+ '%s() argument must be True, False, or Any' % \
+ cls.__name__
+ return args[0]
+ return super(ternary, cls).__new__(cls)
+
+ def __bool__(self):
+ return True
+
+ def __neg__(self):
+ return self
+
+ def __eq__(self, other):
+ return True
+
+ def __ne__(self, other):
+ return False
+
+ def __str__(self):
+ return 'Any'
+
+ def __repr__(self):
+ return 'Any'
+
+Any = ternary()
+
+class Flags(dict):
+ def __init__(self, *args, **kwargs):
+ super(Flags, self).__init__()
+ self.update(*args, **kwargs)
+
+ def __getattr__(self, attr):
+ return self[attr]
+
+ def __setattr__(self, attr, value):
+ self[attr] = value
+
+ def __setitem__(self, item, value):
+ return super(Flags, self).__setitem__(item, ternary(value))
+
+ def __getitem__(self, item):
+ if item not in self:
+ return False
+ return super(Flags, self).__getitem__(item)
+
+ def update(self, *args, **kwargs):
+ for arg in args:
+ if isinstance(arg, Flags):
+ super(Flags, self).update(arg)
+ elif isinstance(arg, dict):
+ for key,val in kwargs.iteritems():
+ self[key] = val
+ else:
+ raise AttributeError, \
+ 'flags not of type %s or %s, but %s' % \
+ (Flags, dict, type(arg))
+
+ for key,val in kwargs.iteritems():
+ self[key] = val
+
+ def match(self, *args, **kwargs):
+ match = Flags(*args, **kwargs)
+
+ for key,value in match.iteritems():
+ if self[key] != value:
return False
- indexes[i] = 0
return True
- done = False
- while not done:
- result = []
- for i in xrange(number):
- result.append(options[i][indexes[i]])
- yield result
- done = next()
-
-class JobFile(object):
- def __init__(self, jfile):
- self.data = {}
- jfile = expanduser(jfile)
- if not isfile(jfile):
- for p in sys.path:
- if isfile(joinpath(p, jfile)):
- jfile = joinpath(p, jfile)
- break
-
- execfile(jfile, self.data)
- self.options = self.data['options']
- self.environment = self.data['environment']
- self.jobinfo = {}
- self.jobs = []
- for job in crossproduct(self.options):
- jobname = '.'.join([ id[0] for id in job ])
- self.jobs.append(jobname)
- list = []
- for info in job:
- for item in info[1:]:
- list.append(item)
- self.jobinfo[jobname] = list
-
- def env(self, jobname):
- env = {}
- for key,val in self.jobinfo[jobname]:
- env[key] = val
-
- for key,val in self.environment:
- env[key] = val
- return env
-
- def printinfo(self, jobname):
- print '%s:' % jobname
- for key,val in self.jobinfo[jobname]:
- print ' %s = %s' % (key, val)
-
- for key,val in self.environment:
- print ' %s = %s' % (key, val)
+def crossproduct(items):
+ if not isinstance(items, (list, tuple)):
+ raise AttributeError, 'crossproduct works only on sequences'
+
+ if not items:
+ yield None
+ return
+
+ current = items[0]
+ remainder = items[1:]
+
+ if not hasattr(current, '__iter__'):
+ current = [ current ]
+
+ for item in current:
+ for rem in crossproduct(remainder):
+ data = [ item ]
+ if rem:
+ data += rem
+ yield data
+
+def flatten(items):
+ if not isinstance(items, (list, tuple)):
+ yield items
+ return
+
+ for item in items:
+ for flat in flatten(item):
+ yield flat
+
+class Data(object):
+ def __init__(self, name, desc, **kwargs):
+ self.name = name
+ self.desc = desc
+ self.system = None
+ self.flags = Flags()
+ self.env = {}
+ for k,v in kwargs.iteritems():
+ setattr(self, k, v)
+
+ def update(self, obj):
+ if not isinstance(obj, Data):
+ raise AttributeError, "can only update from Data object"
+
+ self.env.update(obj.env)
+ self.flags.update(obj.flags)
+ if obj.system:
+ if self.system and self.system != obj.system:
+ raise AttributeError, \
+ "conflicting values for system: '%s'/'%s'" % \
+ (self.system, obj.system)
+ self.system = obj.system
+
+ def printinfo(self):
+ if self.name:
+ print 'name: %s' % self.name
+ if self.desc:
+ print 'desc: %s' % self.desc
+ if self.system:
+ print 'system: %s' % self.system
+
+ def printverbose(self):
+ print 'flags:'
+ keys = self.flags.keys()
+ keys.sort()
+ for key in keys:
+ print ' %s = %s' % (key, self.flags[key])
+ print 'env:'
+ keys = self.env.keys()
+ keys.sort()
+ for key in keys:
+ print ' %s = %s' % (key, self.env[key])
+ print
+
+ def __str__(self):
+ return self.name
+
+class Job(Data):
+ def __init__(self, options):
+ super(Job, self).__init__('', '')
+ self.setoptions(options)
+
+ self.checkpoint = False
+ opts = []
+ for opt in options:
+ cpt = opt.group.checkpoint
+ if not cpt:
+ self.checkpoint = True
+ continue
+ if isinstance(cpt, Option):
+ opt = cpt.clone(suboptions=False)
+ else:
+ opt = opt.clone(suboptions=False)
+
+ opts.append(opt)
+
+ if not opts:
+ self.checkpoint = False
+
+ if self.checkpoint:
+ self.checkpoint = Job(opts)
+
+ def clone(self):
+ return Job(self.options)
+
+ def __getattribute__(self, attr):
+ if attr == 'name':
+ names = [ ]
+ for opt in self.options:
+ if opt.name:
+ names.append(opt.name)
+ return ':'.join(names)
+
+ if attr == 'desc':
+ descs = [ ]
+ for opt in self.options:
+ if opt.desc:
+ descs.append(opt.desc)
+ return ', '.join(descs)
+
+ return super(Job, self).__getattribute__(attr)
+
+ def setoptions(self, options):
+ config = options[0].config
+ for opt in options:
+ if opt.config != config:
+ raise AttributeError, \
+ "All options are not from the same Configuration"
+
+ self.config = config
+ self.groups = [ opt.group for opt in options ]
+ self.options = options
+
+ self.update(self.config)
+ for group in self.groups:
+ self.update(group)
+
+ for option in self.options:
+ self.update(option)
+ if option._suboption:
+ self.update(option._suboption)
+
+ def printinfo(self):
+ super(Job, self).printinfo()
+ if self.checkpoint:
+ print 'checkpoint: %s' % self.checkpoint.name
+ print 'config: %s' % self.config.name
+ print 'groups: %s' % [ g.name for g in self.groups ]
+ print 'options: %s' % [ o.name for o in self.options ]
+ super(Job, self).printverbose()
+
+class SubOption(Data):
+ def __init__(self, name, desc, **kwargs):
+ super(SubOption, self).__init__(name, desc, **kwargs)
+ self.number = None
+
+class Option(Data):
+ def __init__(self, name, desc, **kwargs):
+ super(Option, self).__init__(name, desc, **kwargs)
+ self._suboptions = []
+ self._suboption = None
+ self.number = None
+
+ def __getattribute__(self, attr):
+ if attr == 'name':
+ name = self.__dict__[attr]
+ if self._suboption is not None:
+ name = '%s:%s' % (name, self._suboption.name)
+ return name
+
+ if attr == 'desc':
+ desc = self.__dict__[attr]
+ if self._suboption is not None:
+ desc = '%s, %s' % (desc, self._suboption.desc)
+ return desc
+
+ return super(Option, self).__getattribute__(attr)
+
+ def suboption(self, name, desc, **kwargs):
+ subo = SubOption(name, desc, **kwargs)
+ subo.config = self.config
+ subo.group = self.group
+ subo.option = self
+ subo.number = len(self._suboptions)
+ self._suboptions.append(subo)
+ return subo
+
+ def clone(self, suboptions=True):
+ option = Option(self.__dict__['name'], self.__dict__['desc'])
+ option.update(self)
+ option.group = self.group
+ option.config = self.config
+ option.number = self.number
+ if suboptions:
+ option._suboptions.extend(self._suboptions)
+ option._suboption = self._suboption
+ return option
+
+ def subopts(self):
+ if not self._suboptions:
+ return [ self ]
+
+ subopts = []
+ for subo in self._suboptions:
+ option = self.clone()
+ option._suboption = subo
+ subopts.append(option)
+
+ return subopts
+
+ def printinfo(self):
+ super(Option, self).printinfo()
+ print 'config: %s' % self.config.name
+ super(Option, self).printverbose()
+
+class Group(Data):
+ def __init__(self, name, desc, **kwargs):
+ super(Group, self).__init__(name, desc, **kwargs)
+ self._options = []
+ self.checkpoint = False
+ self.number = None
+
+ def option(self, name, desc, **kwargs):
+ opt = Option(name, desc, **kwargs)
+ opt.config = self.config
+ opt.group = self
+ opt.number = len(self._options)
+ self._options.append(opt)
+ return opt
+
+ def options(self):
+ return self._options
+
+ def subopts(self):
+ subopts = []
+ for opt in self._options:
+ for subo in opt.subopts():
+ subopts.append(subo)
+ return subopts
+
+ def printinfo(self):
+ super(Group, self).printinfo()
+ print 'config: %s' % self.config.name
+ print 'options: %s' % [ o.name for o in self._options ]
+ super(Group, self).printverbose()
+
+class Configuration(Data):
+ def __init__(self, name, desc, **kwargs):
+ super(Configuration, self).__init__(name, desc, **kwargs)
+ self._groups = []
+
+ def group(self, name, desc, **kwargs):
+ grp = Group(name, desc, **kwargs)
+ grp.config = self
+ grp.number = len(self._groups)
+ self._groups.append(grp)
+ return grp
+
+ def groups(self, flags=Flags(), sign=True):
+ if not flags:
+ return self._groups
+
+ return [ grp for grp in self._groups if sign ^ grp.flags.match(flags) ]
+
+ def checkchildren(self, kids):
+ for kid in kids:
+ if kid.config != self:
+ raise AttributeError, "child from the wrong configuration"
+
+ def sortgroups(self, groups):
+ groups = [ (grp.number, grp) for grp in groups ]
+ groups.sort()
+ return [ grp[1] for grp in groups ]
+
+ def options(self, groups = None, checkpoint = False):
+ if groups is None:
+ groups = self._groups
+ self.checkchildren(groups)
+ groups = self.sortgroups(groups)
+ if checkpoint:
+ groups = [ grp for grp in groups if grp.checkpoint ]
+ optgroups = [ g.options() for g in groups ]
+ else:
+ optgroups = [ g.subopts() for g in groups ]
+ for options in crossproduct(optgroups):
+ for opt in options:
+ cpt = opt.group.checkpoint
+ if not isinstance(cpt, bool) and cpt != opt:
+ if checkpoint:
+ break
+ else:
+ yield options
+ else:
+ if checkpoint:
+ yield options
+
+ def checkpoints(self, groups = None):
+ for options in self.options(groups, True):
+ yield Job(options)
+
+ def jobs(self, groups = None):
+ for options in self.options(groups, False):
+ yield Job(options)
+
+ def alljobs(self, groups = None):
+ for options in self.options(groups, True):
+ yield Job(options)
+ for options in self.options(groups, False):
+ yield Job(options)
+
+ def find(self, jobname):
+ for job in self.alljobs():
+ if job.name == jobname:
+ return job
+ else:
+ raise AttributeError, "job '%s' not found" % jobname
+
+ def job(self, options):
+ self.checkchildren(options)
+ options = [ (opt.group.number, opt) for opt in options ]
+ options.sort()
+ options = [ opt[1] for opt in options ]
+ job = Job(options)
+ return job
+
+ def printinfo(self):
+ super(Configuration, self).printinfo()
+ print 'groups: %s' % [ g.name for g in self._grouips ]
+ super(Configuration, self).printverbose()
+
+def JobFile(jobfile):
+ from os.path import expanduser, isfile, join as joinpath
+ filename = expanduser(jobfile)
+
+ # Can't find filename in the current path, search sys.path
+ if not isfile(filename):
+ for path in sys.path:
+ testname = joinpath(path, filename)
+ if isfile(testname):
+ filename = testname
+ break
+ else:
+ raise AttributeError, \
+ "Could not find file '%s'" % jobfile
+
+ data = {}
+ execfile(filename, data)
+ if 'conf' not in data:
+ raise ImportError, 'cannot import name conf from %s' % jobfile
+ conf = data['conf']
+ import jobfile
+ if not isinstance(conf, Configuration):
+ raise AttributeError, \
+ 'conf in jobfile: %s (%s) is not type %s' % \
+ (jobfile, type(conf), Configuration)
+ return conf
+
+if __name__ == '__main__':
+ from jobfile import *
+ import sys
+
+ usage = 'Usage: %s [-b] [-c] [-v] <jobfile>' % sys.argv[0]
+
+ try:
+ import getopt
+ opts, args = getopt.getopt(sys.argv[1:], '-bcv')
+ except getopt.GetoptError:
+ sys.exit(usage)
+
+ if len(args) != 1:
+ raise AttributeError, usage
+
+ both = False
+ checkpoint = False
+ verbose = False
+ for opt,arg in opts:
+ if opt == '-b':
+ both = True
+ checkpoint = True
+ if opt == '-c':
+ checkpoint = True
+ if opt == '-v':
+ verbose = True
+
+ jobfile = args[0]
+ conf = JobFile(jobfile)
+
+ if both:
+ gen = conf.alljobs()
+ elif checkpoint:
+ gen = conf.checkpoints()
+ else:
+ gen = conf.jobs()
+
+ for job in gen:
+ if not verbose:
+ cpt = ''
+ if job.checkpoint:
+ cpt = job.checkpoint.name
+ print job.name, cpt
+ else:
+ job.printinfo()
diff --git a/util/pbs/send.py b/util/pbs/send.py
index ecb0be0ec..c66fb1c05 100755
--- a/util/pbs/send.py
+++ b/util/pbs/send.py
@@ -96,7 +96,7 @@ Usage:
try:
import getopt
- opts, args = getopt.getopt(sys.argv[1:], '-cd:efhj:lq:v')
+ opts, args = getopt.getopt(sys.argv[1:], '-CRcd:efhj:lq:v')
except getopt.GetoptError:
sys.exit(usage)
@@ -107,13 +107,18 @@ force = False
listonly = False
queue = ''
verbose = False
-rootdir = nfspath(os.getcwd())
-jfile = 'test.py'
+jfile = 'Base/test.py'
+docpts = False
+doruns = True
+runflag = False
+
for opt,arg in opts:
+ if opt == '-C':
+ docpts = True
+ if opt == '-R':
+ runflag = True
if opt == '-c':
clean = True
- if opt == '-d':
- rootdir = arg
if opt == '-e':
onlyecho = True
if opt == '-f':
@@ -130,95 +135,123 @@ for opt,arg in opts:
if opt == '-v':
verbose = True
-basedir = joinpath(rootdir, 'Base')
-linkdir = joinpath(rootdir, 'Link')
+if docpts:
+ doruns = runflag
for arg in args:
exprs.append(re.compile(arg))
-if not listonly and not onlyecho and isdir(linkdir):
+import jobfile, pbs
+from job import JobDir, date
+
+conf = jobfile.JobFile(jfile)
+
+if not listonly and not onlyecho and isdir(conf.linkdir):
if verbose:
print 'Checking for outdated files in Link directory'
- syncdir(linkdir, basedir)
+ syncdir(conf.linkdir, conf.basedir)
-import job, jobfile, pbs
+jobnames = {}
+joblist = []
-test = jobfile.JobFile(joinpath(basedir, jfile))
+if docpts and doruns:
+ gen = conf.alljobs()
+elif docpts:
+ gen = conf.checkpoints()
+elif doruns:
+ gen = conf.jobs()
-joblist = []
-for jobname in test.jobs:
- if not exprs:
- joblist.append(jobname)
+for job in gen:
+ if job.name in jobnames:
continue
- for expr in exprs:
- if expr.match(jobname):
- joblist.append(jobname)
- break
+ if exprs:
+ for expr in exprs:
+ if expr.match(job.name):
+ joblist.append(job)
+ break
+ else:
+ joblist.append(job)
if listonly:
if verbose:
- for jobname in joblist:
- test.printinfo(jobname)
+ for job in joblist:
+ job.printinfo()
else:
- for jobname in joblist:
- print jobname
+ for job in joblist:
+ print job.name
sys.exit(0)
if not onlyecho:
- jl = []
- for jobname in joblist:
- jobdir = joinpath(rootdir, jobname)
- if os.path.exists(jobname):
+ newlist = []
+ for job in joblist:
+ jobdir = JobDir(joinpath(conf.rootdir, job.name))
+ if jobdir.exists():
if not force:
- if os.path.isfile(joinpath(jobdir, '.success')):
+ status = jobdir.getstatus()
+ if status == 'queued':
+ continue
+
+ if status == 'running':
continue
- if os.path.isfile(joinpath(jobdir, '.start')) and \
- not os.path.isfile(joinpath(jobdir, '.stop')):
+ if status == 'success':
continue
if not clean:
- sys.exit('job directory not clean!')
+ sys.exit('job directory %s not clean!' % jobdir)
- job.cleandir(jobdir)
- else:
- os.mkdir(jobdir)
- jl.append(jobname)
- joblist = jl
-
-def setname(jobid, jobname):
- # since pbs can handle jobnames of 15 characters or less, don't
- # use the raj hack.
- if len(jobname) <= 15:
- return
-
- import socket
- s = socket.socket()
- # Connect to pbs.pool and send the jobid/jobname pair to port
- # 24465 (Raj didn't realize that there are only 64k ports and
- # setup inetd to point to port 90001)
- s.connect(("pbs.pool", 24465))
- s.send("%s %s\n" % (jobid, jobname))
- s.close()
-
-for jobname in joblist:
- jobdir = joinpath(rootdir, jobname)
-
- if not onlyecho and not os.path.isdir(jobdir):
- sys.exit('%s is not a directory. Cannot build job' % jobdir)
-
- print 'Job name: %s' % jobname
+ jobdir.clean()
+ newlist.append(job)
+ joblist = newlist
+
+class NameHack(object):
+ def __init__(self, host='pbs.pool', port=24465):
+ self.host = host
+ self.port = port
+ self.socket = None
+
+ def setname(self, jobid, jobname):
+ try:
+ jobid = int(jobid)
+ except ValueError:
+ jobid = int(jobid.strip().split('.')[0])
+
+ jobname = jobname.strip()
+ # since pbs can handle jobnames of 15 characters or less,
+ # don't use the raj hack.
+ if len(jobname) <= 15:
+ return
+
+ if self.socket is None:
+ import socket
+ self.socket = socket.socket()
+ # Connect to pbs.pool and send the jobid/jobname pair to port
+ # 24465 (Raj didn't realize that there are only 64k ports and
+ # setup inetd to point to port 90001)
+ self.socket.connect((self.host, self.port))
+
+ self.socket.send("%s %s\n" % (jobid, jobname))
+
+namehack = NameHack()
+
+for job in joblist:
+ jobdir = JobDir(joinpath(conf.rootdir, job.name))
+
+ if not onlyecho:
+ jobdir.create()
+
+ print 'Job name: %s' % job.name
print 'Job directory: %s' % jobdir
qsub = pbs.qsub()
qsub.pbshost = 'simpool.eecs.umich.edu'
- qsub.stdout = joinpath(jobdir, 'jobout')
- qsub.name = jobname[:15]
+ qsub.stdout = jobdir.file('jobout')
+ qsub.name = job.name[:15]
qsub.join = True
qsub.node_type = 'FAST'
- qsub.env['ROOTDIR'] = rootdir
- qsub.env['JOBNAME'] = jobname
+ qsub.env['ROOTDIR'] = conf.rootdir
+ qsub.env['JOBNAME'] = job.name
if len(queue):
qsub.queue = queue
qsub.build(joinpath(progpath, 'job.py'))
@@ -231,6 +264,9 @@ for jobname in joblist:
if ec == 0:
jobid = qsub.result
print 'PBS Jobid: %s' % jobid
- setname(jobid, jobname)
+ namehack.setname(jobid, job.name)
+ queued = date()
+ jobdir.echofile('.queued', queued)
+ jobdir.setstatus('queued on %s' % queued)
else:
print 'PBS Failed'
diff --git a/util/stats/__init__.py b/util/stats/__init__.py
new file mode 100644
index 000000000..b9968bf48
--- /dev/null
+++ b/util/stats/__init__.py
@@ -0,0 +1,28 @@
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+
diff --git a/util/stats/barchart.py b/util/stats/barchart.py
new file mode 100644
index 000000000..a2cbea816
--- /dev/null
+++ b/util/stats/barchart.py
@@ -0,0 +1,246 @@
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+# Lisa Hsu
+
+import matplotlib, pylab
+from matplotlib.numerix import array, arange, reshape, shape, transpose, zeros
+from matplotlib.numerix import Float
+
+matplotlib.interactive(False)
+
+class BarChart(object):
+ def __init__(self, **kwargs):
+ self.init(**kwargs)
+
+ def init(self, **kwargs):
+ self.colormap = 'jet'
+ self.inputdata = None
+ self.chartdata = None
+ self.xlabel = None
+ self.ylabel = None
+ self.legend = None
+ self.xticks = None
+ self.yticks = None
+ self.title = None
+
+ for key,value in kwargs.iteritems():
+ self.__setattr__(key, value)
+
+ def gen_colors(self, count):
+ cmap = matplotlib.cm.get_cmap(self.colormap)
+ if count == 1:
+ return cmap([ 0.5 ])
+ else:
+ return cmap(arange(count) / float(count - 1))
+
+ # The input data format does not match the data format that the
+ # graph function takes because it is intuitive. The conversion
+ # from input data format to chart data format depends on the
+ # dimensionality of the input data. Check here for the
+ # dimensionality and correctness of the input data
+ def set_data(self, data):
+ if data is None:
+ self.inputdata = None
+ self.chartdata = None
+ return
+
+ data = array(data)
+ dim = len(shape(data))
+ if dim not in (1, 2, 3):
+ raise AttributeError, "Input data must be a 1, 2, or 3d matrix"
+ self.inputdata = data
+
+ # If the input data is a 1d matrix, then it describes a
+ # standard bar chart.
+ if dim == 1:
+ self.chartdata = array([[data]])
+
+ # If the input data is a 2d matrix, then it describes a bar
+ # chart with groups. The matrix being an array of groups of
+ # bars.
+ if dim == 2:
+ self.chartdata = transpose([data], axes=(2,0,1))
+
+ # If the input data is a 3d matrix, then it describes an array
+ # of groups of bars with each bar being an array of stacked
+ # values.
+ if dim == 3:
+ self.chartdata = transpose(data, axes=(1,2,0))
+
+ def get_data(self):
+ return self.inputdata
+
+ data = property(get_data, set_data)
+
+ # Graph the chart data.
+ # Input is a 3d matrix that describes a plot that has multiple
+ # groups, multiple bars in each group, and multiple values stacked
+ # in each bar. The underlying bar() function expects a sequence of
+ # bars in the same stack location and same group location, so the
+ # organization of the matrix is that the inner most sequence
+ # represents one of these bar groups, then those are grouped
+ # together to make one full stack of bars in each group, and then
+ # the outer most layer describes the groups. Here is an example
+ # data set and how it gets plotted as a result.
+ #
+ # e.g. data = [[[10,11,12], [13,14,15], [16,17,18], [19,20,21]],
+ # [[22,23,24], [25,26,27], [28,29,30], [31,32,33]]]
+ #
+ # will plot like this:
+ #
+ # 19 31 20 32 21 33
+ # 16 28 17 29 18 30
+ # 13 25 14 26 15 27
+ # 10 22 11 23 12 24
+ #
+ # Because this arrangement is rather conterintuitive, the rearrange
+ # function takes various matricies and arranges them to fit this
+ # profile.
+ #
+ # This code deals with one of the dimensions in the matrix being
+ # one wide.
+ #
+ def graph(self):
+ if self.chartdata is None:
+ raise AttributeError, "Data not set for bar chart!"
+
+ self.figure = pylab.figure()
+ self.axes = self.figure.add_subplot(111)
+
+ dim = len(shape(self.inputdata))
+ cshape = shape(self.chartdata)
+ if dim == 1:
+ colors = self.gen_colors(cshape[2])
+ colors = [ [ colors ] * cshape[1] ] * cshape[0]
+
+ if dim == 2:
+ colors = self.gen_colors(cshape[0])
+ colors = [ [ [ c ] * cshape[2] ] * cshape[1] for c in colors ]
+
+ if dim == 3:
+ colors = self.gen_colors(cshape[1])
+ colors = [ [ [ c ] * cshape[2] for c in colors ] ] * cshape[0]
+
+ colors = array(colors)
+
+ bars_in_group = len(self.chartdata)
+ if bars_in_group < 5:
+ width = 1.0 / ( bars_in_group + 1)
+ center = width / 2
+ else:
+ width = .8 / bars_in_group
+ center = .1
+
+ bars = []
+ for i,stackdata in enumerate(self.chartdata):
+ bottom = array([0] * len(stackdata[0]))
+ stack = []
+ for j,bardata in enumerate(stackdata):
+ bardata = array(bardata)
+ ind = arange(len(bardata)) + i * width + center
+ bar = self.axes.bar(ind, bardata, width, bottom=bottom,
+ color=colors[i][j])
+ stack.append(bar)
+ bottom += bardata
+ bars.append(stack)
+
+ if self.xlabel is not None:
+ self.axes.set_xlabel(self.xlabel)
+
+ if self.ylabel is not None:
+ self.axes.set_ylabel(self.ylabel)
+
+ if self.yticks is not None:
+ ymin, ymax = self.axes.get_ylim()
+ nticks = float(len(self.yticks))
+ ticks = arange(nticks) / (nticks - 1) * (ymax - ymin) + ymin
+ self.axes.set_yticks(ticks)
+ self.axes.set_yticklabels(self.yticks)
+
+ if self.xticks is not None:
+ self.axes.set_xticks(arange(cshape[2]) + .5)
+ self.axes.set_xticklabels(self.xticks)
+
+ if self.legend is not None:
+ if dim == 1:
+ lbars = bars[0][0]
+ if dim == 2:
+ lbars = [ bars[i][0][0] for i in xrange(len(bars))]
+ if dim == 3:
+ number = len(bars[0])
+ lbars = [ bars[0][number - j - 1][0] for j in xrange(number)]
+
+ self.axes.legend(lbars, self.legend, loc='best')
+
+ if self.title is not None:
+ self.axes.set_title(self.title)
+
+ def savefig(self, name):
+ self.figure.savefig(name)
+
+if __name__ == '__main__':
+ import random, sys
+
+ dim = 3
+ number = 5
+
+ args = sys.argv[1:]
+ if len(args) > 3:
+ sys.exit("invalid number of arguments")
+ elif len(args) > 0:
+ myshape = [ int(x) for x in args ]
+ else:
+ myshape = [ 3, 4, 8 ]
+
+ # generate a data matrix of the given shape
+ size = reduce(lambda x,y: x*y, myshape)
+ #data = [ random.randrange(size - i) + 10 for i in xrange(size) ]
+ data = [ float(i)/100.0 for i in xrange(size) ]
+ data = reshape(data, myshape)
+
+ # setup some test bar charts
+ if True:
+ chart1 = BarChart()
+ chart1.data = data
+
+ chart1.xlabel = 'Benchmark'
+ chart1.ylabel = 'Bandwidth (GBps)'
+ chart1.legend = [ 'x%d' % x for x in xrange(myshape[-1]) ]
+ chart1.xticks = [ 'xtick%d' % x for x in xrange(myshape[0]) ]
+ chart1.title = 'this is the title'
+ chart1.graph()
+ #chart1.savefig('/tmp/test1.png')
+
+ if False:
+ chart2 = BarChart()
+ chart2.data = data
+ chart2.colormap = 'gray'
+ chart2.graph()
+ #chart2.savefig('/tmp/test2.png')
+
+ pylab.show()
diff --git a/util/categories.py b/util/stats/categories.py
index 877e548ed..0c359b856 100644
--- a/util/categories.py
+++ b/util/stats/categories.py
@@ -1,3 +1,31 @@
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+
categories = {
'CALL_PALrdunique_' : 'interrupt', #
'Call_Pal_Callsys' : 'interrupt', #
diff --git a/util/stats/db.py b/util/stats/db.py
index 6e1ccec95..ab005e67b 100644
--- a/util/stats/db.py
+++ b/util/stats/db.py
@@ -101,7 +101,7 @@ class Node(object):
def __init__(self, name):
self.name = name
def __str__(self):
- return name
+ return self.name
class Database(object):
def __init__(self):
@@ -466,3 +466,6 @@ class Database(object):
runs[data.run][data.x][data.y] = data.data
return runs
+
+ def __getitem__(self, key):
+ return self.stattop[key]
diff --git a/util/stats/orderdict.py b/util/stats/orderdict.py
new file mode 100644
index 000000000..816355ae2
--- /dev/null
+++ b/util/stats/orderdict.py
@@ -0,0 +1,78 @@
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+__all__ = [ 'orderdict' ]
+
+class orderdict(dict):
+ def __init__(self, d = {}):
+ self._keys = d.keys()
+ super(orderdict, self).__init__(d)
+
+ def __setitem__(self, key, item):
+ super(orderdict, self).__setitem__(key, item)
+ if not hasattr(self, '_keys'):
+ self._keys = [key,]
+ if key not in self._keys:
+ self._keys.append(key)
+
+ def __delitem__(self, key):
+ super(orderdict, self).__delitem__(key)
+ self._keys.remove(key)
+
+ def clear(self):
+ super(orderdict, self).clear()
+ self._keys = []
+
+ def items(self):
+ for i in self._keys:
+ yield i, self[i]
+
+ def keys(self):
+ return self._keys
+
+ def popitem(self):
+ if len(self._keys) == 0:
+ raise KeyError('dictionary is empty')
+ else:
+ key = self._keys[-1]
+ val = self[key]
+ del self[key]
+ return key, val
+
+ def setdefault(self, key, failobj = None):
+ super(orderdict, self).setdefault(key, failobj)
+ if key not in self._keys:
+ self._keys.append(key)
+
+ def update(self, d):
+ for key in d.keys():
+ if not self.has_key(key):
+ self._keys.append(key)
+ super(orderdict, self).update(d)
+
+ def values(self):
+ for i in self._keys:
+ yield self[i]
diff --git a/util/stats/output.py b/util/stats/output.py
new file mode 100644
index 000000000..44dba5d15
--- /dev/null
+++ b/util/stats/output.py
@@ -0,0 +1,182 @@
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+
+class dbinfo(object):
+ def get(self, job, stat):
+ import info
+
+ run = info.source.allRunNames.get(job.name, None)
+ if run is None:
+ print 'run "%s" not found' % job
+ return None
+
+ stat.system = info.source[job.system]
+ info.display_run = run.run;
+ val = float(stat)
+ if val == 1e300*1e300:
+ return None
+ return val
+
+class StatOutput(object):
+ def __init__(self, name, jobfile, stat=None, info=dbinfo(), binstats=None):
+ self.name = name
+ self.jobfile = jobfile
+ self.stat = stat
+ self.binstats = None
+ self.label = self.name
+ self.invert = False
+ self.info = info
+
+ def printdata(self, bin = None, printmode = 'G'):
+ import info
+
+ if bin:
+ print '%s %s stats' % (self.name, bin)
+
+ if self.binstats:
+ for stat in self.binstats:
+ stat.bins = bin
+
+ if printmode == 'G':
+ valformat = '%g'
+ elif printmode != 'F' and value > 1e6:
+ valformat = '%0.5e'
+ else:
+ valformat = '%f'
+
+ for job in self.jobfile.jobs():
+ value = self.info.get(job, self.stat)
+ if value is None:
+ return
+
+ if not isinstance(value, list):
+ value = [ value ]
+
+ if self.invert:
+ for i,val in enumerate(value):
+ if val != 0.0:
+ value[i] = 1 / val
+
+ valstring = ', '.join([ valformat % val for val in value ])
+ print '%-50s %s' % (job.name + ':', valstring)
+
+ def display(self, binned = False, printmode = 'G'):
+ if binned and self.binstats:
+ self.printdata('kernel', printmode)
+ self.printdata('idle', printmode)
+ self.printdata('user', printmode)
+ self.printdata('interrupt', printmode)
+
+ print '%s total stats' % self.name
+ self.printdata(printmode=printmode)
+
+ def graph(self, graphdir):
+ from os.path import expanduser, join as joinpath
+ from barchart import BarChart
+ from matplotlib.numerix import Float, zeros
+ import re
+
+ confgroups = self.jobfile.groups()
+ ngroups = len(confgroups)
+ skiplist = [ False ] * ngroups
+ groupopts = None
+ baropts = None
+ groups = []
+ for i,group in enumerate(confgroups):
+ if group.flags.graph_group:
+ if groupopts is not None:
+ raise AttributeError, \
+ 'Two groups selected for graph group'
+ groupopts = group.subopts()
+ skiplist[i] = True
+ elif group.flags.graph_bars:
+ if baropts is not None:
+ raise AttributeError, \
+ 'Two groups selected for graph bars'
+ baropts = group.subopts()
+ skiplist[i] = True
+ else:
+ groups.append(group)
+
+ if groupopts is None:
+ raise AttributeError, 'No group selected for graph group'
+
+ if baropts is None:
+ raise AttributeError, 'No group selected for graph bars'
+
+ directory = expanduser(graphdir)
+ html = file(joinpath(directory, '%s.html' % self.name), 'w')
+ print >>html, '<html>'
+ print >>html, '<title>Graphs for %s</title>' % self.name
+ print >>html, '<body>'
+
+ for options in self.jobfile.options(groups):
+ data = zeros((len(groupopts), len(baropts)), Float)
+ data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ]
+ enabled = False
+ stacked = None
+ for g,gopt in enumerate(groupopts):
+ for b,bopt in enumerate(baropts):
+ job = self.jobfile.job(options + [ gopt, bopt ])
+
+ val = self.info.get(job, self.stat)
+ if val is None:
+ val = 0.0
+ curstacked = isinstance(val, (list, tuple))
+ if stacked is None:
+ stacked = curstacked
+ else:
+ if stacked != curstacked:
+ raise ValueError, "some stats stacked, some not"
+
+ data[g][b] = val
+
+ bar_descs = [ opt.desc for opt in baropts ]
+ group_descs = [ opt.desc for opt in groupopts ]
+ if stacked:
+ legend = self.info.rcategories
+ else:
+ legend = bar_descs
+
+ chart = BarChart(data=data, xlabel='Benchmark', ylabel=self.label,
+ legend=legend, xticks=group_descs)
+ chart.graph()
+
+ names = [ opt.name for opt in options ]
+ descs = [ opt.desc for opt in options ]
+
+ filename = '%s-%s.png' % (self.name, ':'.join(names))
+ desc = ' '.join(descs)
+ filepath = joinpath(directory, filename)
+ chart.savefig(filepath)
+ filename = re.sub(':', '%3A', filename)
+ print >>html, '''%s<br><img src="%s"><br>''' % (desc, filename)
+
+ print >>html, '</body>'
+ print >>html, '</html>'
+ html.close()
diff --git a/util/stats/profile.py b/util/stats/profile.py
new file mode 100644
index 000000000..65a03e9aa
--- /dev/null
+++ b/util/stats/profile.py
@@ -0,0 +1,146 @@
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from orderdict import orderdict
+import output
+
+class ProfileData(object):
+ def __init__(self):
+ self.data = {}
+ self.total = {}
+ self.runs = orderdict()
+ self.runlist = []
+
+ def addvalue(self, run, cpu, symbol, value):
+ value = float(value)
+ self.data[run, cpu, symbol] = self.getvalue(run, cpu, symbol) + value
+ self.total[run, cpu] = self.gettotal(run, cpu) + value
+ if run not in self.runs:
+ self.runs[run] = orderdict()
+
+ if cpu not in self.runs[run]:
+ self.runs[run][cpu] = {}
+
+ if symbol not in self.runs[run][cpu]:
+ self.runs[run][cpu][symbol] = 0
+
+ self.runs[run][cpu][symbol] += value
+
+ def getvalue(self, run, cpu, symbol):
+ return self.data.get((run, cpu, symbol), 0)
+
+ def gettotal(self, run, cpu):
+ return self.total.get((run, cpu), 0)
+
+class Profile(object):
+ default_order = ['ste', 'hte', 'htd', 'ocm', 'occ', 'ocp']
+
+ # This list controls the order of values in stacked bar data output
+ default_categories = [ 'interrupt',
+ 'driver',
+ 'stack',
+ 'bufmgt',
+ 'copy',
+ 'user',
+ 'other',
+ 'idle']
+
+ def __init__(self, run_order=[], categories=[], stacknames=[]):
+ if not run_order:
+ run_order = Profile.default_order
+ if not categories:
+ categories = Profile.default_categories
+
+ self.run_order = run_order
+ self.categories = categories
+ self.rcategories = []
+ self.rcategories.extend(categories)
+ self.rcategories.reverse()
+ self.stacknames = stacknames
+ self.prof = ProfileData()
+ self.categorize = True
+ self.showidle = True
+ self.maxsymlen = 0
+
+ def category(self, symbol):
+ from categories import categories, categories_re
+ if categories.has_key(symbol):
+ return categories[symbol]
+ for regexp, cat in categories_re:
+ if regexp.match(symbol):
+ return cat
+ return 'other'
+
+ # Parse input file and put the results in the given run and cpu
+ def parsefile(self, run, cpu, filename):
+ fd = file(filename)
+
+ for line in fd:
+ (symbol, count) = line.split()
+ if symbol == "0x0":
+ continue
+ count = int(count)
+
+ if self.categorize:
+ symbol = self.category(symbol)
+ if symbol == 'idle' and not self.showidle:
+ continue
+
+ if symbol not in self.categories:
+ symbol = 'other'
+
+ self.maxsymlen = max(self.maxsymlen, len(symbol))
+ self.prof.addvalue(run, cpu, symbol, count)
+
+ fd.close()
+
+ # Read in files
+ def inputdir(self, directory):
+ import os, os.path, re
+ from os.path import expanduser, join as joinpath
+
+ directory = expanduser(directory)
+ label_ex = re.compile(r'm5prof\.(.*)')
+ for root,dirs,files in os.walk(directory):
+ for name in files:
+ match = label_ex.match(name)
+ if not match:
+ continue
+
+ filename = joinpath(root, name)
+ prefix = os.path.commonprefix([root, directory])
+ dirname = root[len(prefix)+1:]
+ self.parsefile(dirname, match.group(1), filename)
+
+ def get(self, job, stat):
+ if job.system is None:
+ raise AttributeError, 'The job must have a system set'
+
+ cpu = '%s.full0' % job.system
+ values = []
+ for cat in self.categories:
+ values.append(self.prof.getvalue(job.name, cpu, cat))
+ return values
diff --git a/util/stats/stats.py b/util/stats/stats.py
index be9767d6e..c53d77d54 100755
--- a/util/stats/stats.py
+++ b/util/stats/stats.py
@@ -29,10 +29,9 @@
from __future__ import division
import re, sys, math
-
def usage():
print '''\
-Usage: %s [-E] [-F] [-d <db> ] [-g <get> ] [-h <host>] [-p]
+Usage: %s [-E] [-F] [ -G <get> ] [-d <db> ] [-g <graphdir> ] [-h <host>] [-p]
[-s <system>] [-r <runs> ] [-T <samples>] [-u <username>]
<command> [command args]
@@ -61,143 +60,6 @@ def getopts(list, flags):
return opts, args
-def printval(name, value, invert = False):
- if invert and value != 0.0:
- value = 1 / value
-
- if value == (1e300*1e300):
- return
-
- if printval.mode == 'G':
- print '%s: %g' % (name, value)
- elif printval.mode != 'F' and value > 1e6:
- print '%s: %0.5e' % (name, value)
- else:
- print '%s: %f' % (name, value)
-
-printval.mode = 'G'
-
-def unique(list):
- set = {}
- map(set.__setitem__, list, [])
- return set.keys()
-
-#benchmarks = [ 'm', 's', 'snt', 'nb1', 'w1', 'w2', 'w3', 'w4', 'nm', 'ns', 'nw1', 'nw2', 'nw3' ]
-
-def graphdata(runs, options, tag, label, value):
- import info
-
- bench_system = {
- 'm' : 'client',
- 's' : 'client',
- 'snt' : 'client',
- 'nb1' : 'server',
- 'nb2' : 'server',
- 'nt1' : 'server',
- 'nt2' : 'server',
- 'w1' : 'server',
- 'w2' : 'server',
- 'w3' : 'server',
- 'w4' : 'server',
- 'w1s' : 'server',
- 'w2s' : 'server',
- 'w3s' : 'server',
- 'ns' : 'natbox',
- 'nm' : 'natbox',
- 'nw1' : 'natbox',
- 'nw2' : 'natbox',
- 'nw3' : 'natbox'
- }
-
- system_configs = {
- 's1' : 'Uni 4GHz',
- 'm1' : 'Uni 6GHz',
- 'f1' : 'Uni 8GHz',
- 'q1' : 'Uni 10GHz',
- 's2' : 'Dual 4GHz',
- 'm2' : 'Dual 6GHz',
- 's4' : 'Quad 4GHz',
- 'm4' : 'Quad 6GHz' }
-
- configs = ['ste', 'hte', 'htd', 'ocm', 'occ', 'ocp' ]
- benchmarks = [ 'm', 'snt', 'w2', 'nm', 'nw2' ]
- caches = [ '0', '2', '4' ]
-
- names = []
- for bench in benchmarks:
- if bench_system[bench] != options.system:
- continue
-
- for cache in caches:
- names.append([bench, cache])
-
- for bench,cache in names:
- base = '%s.%s' % (bench, cache)
- fname = 'data/uni.%s.%s.dat' % (tag, base)
- f = open(fname, 'w')
- print >>f, '#set TITLE = '
- print >>f, '#set ylbl = %s' % label
- #print >>f, '#set sublabels = %s' % ' '.join(configs)
- print >>f, '#set sublabels = ste hte htd ocm occ ocs'
-
- for speed in ('s1', 'm1', 'f1', 'q1'):
- label = system_configs[speed]
- print >>f, '"%s"' % label,
- for conf in configs:
- name = '%s.%s.%s.%s' % (conf, bench, cache, speed)
- run = info.source.allRunNames[name]
- info.display_run = run.run;
- val = float(value)
- if val == 1e300*1e300:
- print >>f, 0.0,
- else:
- print >>f, "%f" % val,
- print >>f
- f.close()
-
- configs = ['ste', 'hte', 'htd', 'ocm', 'occ', 'ocp' ]
- benchmarks = [ 'w2']
- caches = [ '0', '2', '4' ]
-
- names = []
- for bench in benchmarks:
- if bench_system[bench] != options.system:
- continue
-
- for cache in caches:
- names.append([bench, cache])
-
- for bench,cache in names:
- base = '%s.%s' % (bench, cache)
- fname = 'data/mp.%s.%s.dat' % (tag, base)
- f = open(fname, 'w')
- print >>f, '#set TITLE = '
- print >>f, '#set ylbl = %s' % label
- #print >>f, '#set sublabels = %s' % ' '.join(configs)
- print >>f, '#set sublabels = ste hte htd ocm occ ocs'
-
- for speed in ('s2', 'm2', 's4', 'm4'):
- label = system_configs[speed]
- print >>f, '"%s"' % label,
- for conf in configs:
- name = '%s.%s.%s.%s' % (conf, bench, cache, speed)
- run = info.source.allRunNames[name]
- info.display_run = run.run;
- val = float(value)
- if val == 1e300*1e300:
- print >>f, 0.0,
- else:
- print >>f, "%f" % val,
- print >>f
- f.close()
-
-def printdata(runs, value, invert = False):
- import info
- for run in runs:
- info.display_run = run.run;
- val = float(value)
- printval(run.name, val)
-
class CommandException(Exception):
pass
@@ -243,7 +105,7 @@ def commands(options, command, args):
info.source.passwd = options.passwd
info.source.user = options.user
info.source.connect()
- info.source.update_dict(globals())
+ #info.source.update_dict(globals())
if type(options.get) is str:
info.source.get = options.get
@@ -270,6 +132,43 @@ def commands(options, command, args):
info.source.listRuns(user)
return
+ if command == 'stats':
+ if len(args) == 0:
+ info.source.listStats()
+ elif len(args) == 1:
+ info.source.listStats(args[0])
+ else:
+ raise CommandException
+
+ return
+
+ if command == 'bins':
+ if len(args) == 0:
+ info.source.listBins()
+ elif len(args) == 1:
+ info.source.listBins(args[0])
+ else:
+ raise CommandException
+
+ return
+
+ if command == 'formulas':
+ if len(args) == 0:
+ info.source.listFormulas()
+ elif len(args) == 1:
+ info.source.listFormulas(args[0])
+ else:
+ raise CommandException
+
+ return
+
+ if command == 'samples':
+ if len(args):
+ raise CommandException
+
+ info.source.listTicks(runs)
+ return
+
if command == 'stability':
if len(args) < 2:
raise CommandException
@@ -281,16 +180,19 @@ def commands(options, command, args):
stats = info.source.getStat(args[1])
info.source.get = "sum"
+ def disp(*args):
+ print "%-20s %12s %12s %4s %5s %5s %5s %10s" % args
+
+ # temporary variable containing a bunch of dashes
+ d = '-' * 100
#loop through all the stats selected
for stat in stats:
-
print "%s:" % stat.name
- print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
- ("run name", "average", "stdev", ">10%", ">1SDV", ">2SDV", "SAMP", "CV")
- print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
- ("--------------------", "------------",
- "------------", "----", "-----", "-----", "-----", "----------")
+ disp("run name", "average", "stdev", ">10%", ">1SDV", ">2SDV",
+ "SAMP", "CV")
+ disp(d[:20], d[:12], d[:12], d[:4], d[:5], d[:5], d[:5], d[:10])
+
#loop through all the selected runs
for run in runs:
info.display_run = run.run;
@@ -331,130 +233,63 @@ def commands(options, command, args):
if (val < (avg - (2*stdev))) or (val > (avg + (2*stdev))):
numoutside2std += 1
if avg > 1000:
- print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
- (run.name, "%.1f" % avg, "%.1f" % stdev,
- "%d" % numoutsideavg, "%d" % numoutside1std,
- "%d" % numoutside2std, "%d" % len(pairRunTicks),
- "%.3f" % (stdev/avg*100))
+ disp(run.name, "%.1f" % avg, "%.1f" % stdev,
+ "%d" % numoutsideavg, "%d" % numoutside1std,
+ "%d" % numoutside2std, "%d" % len(pairRunTicks),
+ "%.3f" % (stdev/avg*100))
elif avg > 100:
- print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
- (run.name, "%.1f" % avg, "%.1f" % stdev,
- "%d" % numoutsideavg, "%d" % numoutside1std,
- "%d" % numoutside2std, "%d" % len(pairRunTicks),
- "%.5f" % (stdev/avg*100))
+ disp(run.name, "%.1f" % avg, "%.1f" % stdev,
+ "%d" % numoutsideavg, "%d" % numoutside1std,
+ "%d" % numoutside2std, "%d" % len(pairRunTicks),
+ "%.5f" % (stdev/avg*100))
else:
- print "%-20s %12s %12s %4s %5s %5s %5s %10s" % \
- (run.name, "%.5f" % avg, "%.5f" % stdev,
- "%d" % numoutsideavg, "%d" % numoutside1std,
- "%d" % numoutside2std, "%d" % len(pairRunTicks),
- "%.7f" % (stdev/avg*100))
+ disp(run.name, "%.5f" % avg, "%.5f" % stdev,
+ "%d" % numoutsideavg, "%d" % numoutside1std,
+ "%d" % numoutside2std, "%d" % len(pairRunTicks),
+ "%.7f" % (stdev/avg*100))
return
- if command == 'stats':
- if len(args) == 0:
- info.source.listStats()
- elif len(args) == 1:
- info.source.listStats(args[0])
- else:
- raise CommandException
-
- return
-
- if command == 'stat':
- if len(args) != 1:
+ if command == 'all':
+ if len(args):
raise CommandException
- stats = info.source.getStat(args[0])
- for stat in stats:
- if options.graph:
- graphdata(runs, options, stat.name, stat.name, stat)
- else:
- if options.ticks:
- print 'only displaying sample %s' % options.ticks
- info.globalTicks = [ int(x) for x in options.ticks.split() ]
-
- if options.binned:
- print 'kernel ticks'
- stat.bins = 'kernel'
- printdata(runs, stat)
-
- print 'idle ticks'
- stat.bins = 'idle'
- printdata(runs, stat)
-
- print 'user ticks'
- stat.bins = 'user'
- printdata(runs, stat)
-
- print 'interrupt ticks'
- stat.bins = 'interrupt'
- printdata(runs, stat)
-
- print 'total ticks'
-
- stat.bins = None
- print stat.name
- printdata(runs, stat)
- return
+ all = [ 'bps', 'rxbps', 'txbps', 'bpt',
+ 'misses', 'mpkb',
+ 'ipkb',
+ 'pps', 'bpp', 'txbpp', 'rxbpp',
+ 'rtp', 'rtb' ]
+ for command in all:
+ commands(options, command, args)
- if command == 'formula':
- if len(args) != 1:
- raise CommandException
+ if options.ticks:
+ if not options.graph:
+ print 'only displaying sample %s' % options.ticks
+ info.globalTicks = [ int(x) for x in options.ticks.split() ]
- stats = eval(args[0])
- for stat in stats:
- if options.graph:
- graphdata(runs, options, stat.name, stat.name, stat)
- else:
- if options.binned:
- print 'kernel ticks'
- stat.bins = 'kernel'
- printdata(runs, stat)
-
- print 'idle ticks'
- stat.bins = 'idle'
- printdata(runs, stat)
-
- print 'user ticks'
- stat.bins = 'user'
- printdata(runs, stat)
-
- print 'interrupt ticks'
- stat.bins = 'interrupt'
- printdata(runs, stat)
-
- print 'total ticks'
-
- stat.bins = None
- print args[0]
- printdata(runs, stat)
- return
+ from output import StatOutput
- if command == 'bins':
- if len(args) == 0:
- info.source.listBins()
- elif len(args) == 1:
- info.source.listBins(args[0])
+ def display():
+ if options.graph:
+ output.graph(options.graphdir)
else:
- raise CommandException
+ output.display(options.binned, options.printmode)
- return
- if command == 'formulas':
- if len(args) == 0:
- info.source.listFormulas()
- elif len(args) == 1:
- info.source.listFormulas(args[0])
- else:
+ if command == 'stat' or command == 'formula':
+ if len(args) != 1:
raise CommandException
- return
+ if command == 'stat':
+ stats = info.source.getStat(args[0])
+ if command == 'formula':
+ stats = eval(args[0])
- if command == 'samples':
- if len(args):
- raise CommandException
+ for stat in stats:
+ output = StatOutput(stat.name, options.jobfile)
+ output.stat = stat
+ output.label = stat.name
+ display()
- info.source.listTicks(runs)
return
if len(args):
@@ -462,273 +297,151 @@ def commands(options, command, args):
system = info.source.__dict__[options.system]
+ from proxy import ProxyGroup
+ sim_ticks = info.source['sim_ticks']
+ sim_seconds = info.source['sim_seconds']
+ proxy = ProxyGroup(system = info.source[options.system])
+ system = proxy.system
+
+ etherdev = system.tsunami.etherdev0
+ bytes = etherdev.rxBytes + etherdev.txBytes
+ kbytes = bytes / 1024
+ packets = etherdev.rxPackets + etherdev.txPackets
+ bps = etherdev.rxBandwidth + etherdev.txBandwidth
+
+ output = StatOutput(command, options.jobfile)
+
if command == 'usertime':
import copy
- kernel = copy.copy(system.full0.numCycles)
- kernel.bins = 'kernel'
-
user = copy.copy(system.full0.numCycles)
user.bins = 'user'
- if options.graph:
- graphdata(runs, options, 'usertime', 'User Fraction',
- user / system.full0.numCycles)
- else:
- printdata(runs, user / system.full0.numCycles)
+ output.stat = user / system.full0.numCycles
+ output.label = 'User Fraction'
+
+ display()
return
if command == 'ticks':
- if options.binned:
- print 'kernel ticks'
- system.full0.numCycles.bins = 'kernel'
- printdata(runs, system.full0.numCycles)
+ output.stat = system.full0.numCycles
+ output.binstats = [ system.full0.numCycles ]
- print 'idle ticks'
- system.full0.numCycles.bins = 'idle'
- printdata(runs, system.full0.numCycles)
-
- print 'user ticks'
- system.full0.numCycles.bins = 'user'
- printdata(runs, system.full0.numCycles)
-
- print 'total ticks'
+ display()
+ return
- system.full0.numCycles.bins = None
- printdata(runs, system.full0.numCycles)
+ if command == 'bytes':
+ output.stat = bytes
+ display()
return
if command == 'packets':
- packets = system.tsunami.etherdev0.rxPackets
- if options.graph:
- graphdata(runs, options, 'packets', 'Packets', packets)
- else:
- printdata(runs, packets)
+ output.stat = packets
+ display()
return
if command == 'ppt' or command == 'tpp':
- ppt = system.tsunami.etherdev0.rxPackets / sim_ticks
- printdata(runs, ppt, command == 'tpp')
+ output.stat = packets / sim_ticks
+ output.invert = command == 'tpp'
+ display()
return
if command == 'pps':
- pps = system.tsunami.etherdev0.rxPackets / sim_seconds
- if options.graph:
- graphdata(runs, options, 'pps', 'Packets/s', pps)
- else:
- printdata(runs, pps)
+ output.stat = packets / sim_seconds
+ output.label = 'Packets/s'
+ display()
return
if command == 'bpt' or command == 'tpb':
- bytes = system.tsunami.etherdev0.rxBytes + system.tsunami.etherdev0.txBytes
- bpt = bytes / sim_ticks * 8
- if options.graph:
- graphdata(runs, options, 'bpt', 'bps / Hz', bpt)
- else:
- printdata(runs, bpt, command == 'tpb')
+ output.stat = bytes / sim_ticks * 8
+ output.label = 'bps / Hz'
+ output.invert = command == 'tpb'
+ display()
return
- if command == 'bptb' or command == 'tpbb':
- bytes = system.tsunami.etherdev0.rxBytes + system.tsunami.etherdev0.txBytes
-
- print 'kernel stats'
- bytes.bins = 'kernel'
- printdata(runs, bytes / ticks)
-
- print 'idle stats'
- bytes.bins = 'idle'
- printdata(runs, bytes / ticks)
-
- print 'user stats'
- bytes.bins = 'user'
- printdata(runs, bytes / ticks)
-
+ if command == 'rxbps':
+ output.stat = etherdev.rxBandwidth / 1e9
+ output.label = 'Bandwidth (Gbps)'
+ display()
return
- if command == 'bytes':
- stat = system.tsunami.etherdev0.rxBytes + system.tsunami.etherdev0.txBytes
-
- if options.binned:
- print '%s kernel stats' % stat.name
- stat.bins = 'kernel'
- printdata(runs, stat)
-
- print '%s idle stats' % stat.name
- stat.bins = 'idle'
- printdata(runs, stat)
-
- print '%s user stats' % stat.name
- stat.bins = 'user'
- printdata(runs, stat)
-
- print '%s total stats' % stat.name
- stat.bins = None
-
- printdata(runs, stat)
+ if command == 'txbps':
+ output.stat = etherdev.txBandwidth / 1e9
+ output.label = 'Bandwidth (Gbps)'
+ display()
return
- if command == 'rxbps':
- gbps = system.tsunami.etherdev0.rxBandwidth / 1e9
- if options.graph:
- graphdata(runs, options, 'rxbps', 'Bandwidth (Gbps)', gbps)
- else:
- printdata(runs, gbps)
+ if command == 'bps':
+ output.stat = bps / 1e9
+ output.label = 'Bandwidth (Gbps)'
+ display()
return
- if command == 'txbps':
- gbps = system.tsunami.etherdev0.txBandwidth / 1e9
- if options.graph:
- graphdata(runs, options, 'txbps', 'Bandwidth (Gbps)', gbps)
- else:
- printdata(runs, gbps)
+ if command == 'bpp':
+ output.stat = bytes / packets
+ output.label = 'Bytes / Packet'
+ display()
return
- if command == 'bps':
- rxbps = system.tsunami.etherdev0.rxBandwidth
- txbps = system.tsunami.etherdev0.txBandwidth
- gbps = (rxbps + txbps) / 1e9
- if options.graph:
- graphdata(runs, options, 'bps', 'Bandwidth (Gbps)', gbps)
- else:
- printdata(runs, gbps)
+ if command == 'rxbpp':
+ output.stat = etherdev.rxBytes / etherdev.rxPackets
+ output.label = 'Receive Bytes / Packet'
+ display()
return
- if command == 'misses':
- stat = system.l2.overall_mshr_misses
- if options.binned:
- print '%s kernel stats' % stat.name
- stat.bins = 'kernel'
- printdata(runs, stat)
+ if command == 'txbpp':
+ output.stat = etherdev.txBytes / etherdev.txPackets
+ output.label = 'Transmit Bytes / Packet'
+ display()
+ return
- print '%s idle stats' % stat.name
- stat.bins = 'idle'
- printdata(runs, stat)
+ if command == 'rtp':
+ output.stat = etherdev.rxPackets / etherdev.txPackets
+ output.label = 'rxPackets / txPackets'
+ display()
+ return
- print '%s user stats' % stat.name
- stat.bins = 'user'
- printdata(runs, stat)
+ if command == 'rtb':
+ output.stat = etherdev.rxBytes / etherdev.txBytes
+ output.label = 'rxBytes / txBytes'
+ display()
+ return
- print '%s total stats' % stat.name
+ misses = system.l2.overall_mshr_misses
- stat.bins = None
- if options.graph:
- graphdata(runs, options, 'misses', 'Overall MSHR Misses', stat)
- else:
- printdata(runs, stat)
+ if command == 'misses':
+ output.stat = misses
+ output.label = 'Overall MSHR Misses'
+ display()
return
if command == 'mpkb':
- misses = system.l2.overall_mshr_misses
- rxbytes = system.tsunami.etherdev0.rxBytes
- txbytes = system.tsunami.etherdev0.txBytes
-
- if options.binned:
- print 'mpkb kernel stats'
- misses.bins = 'kernel'
- mpkb = misses / ((rxbytes + txbytes) / 1024)
- printdata(runs, mpkb)
-
- print 'mpkb idle stats'
- misses.bins = 'idle'
- mpkb = misses / ((rxbytes + txbytes) / 1024)
- printdata(runs, mpkb)
-
- print 'mpkb user stats'
- misses.bins = 'user'
- mpkb = misses / ((rxbytes + txbytes) / 1024)
- printdata(runs, mpkb)
-
- print 'mpkb total stats'
-
- mpkb = misses / ((rxbytes + txbytes) / 1024)
- misses.bins = None
- if options.graph:
- graphdata(runs, options, 'mpkb', 'Misses / KB', mpkb)
- else:
- printdata(runs, mpkb)
+ output.stat = misses / (bytes / 1024)
+ output.binstats = [ misses ]
+ output.label = 'Misses / KB'
+ display()
return
if command == 'ipkb':
interrupts = system.full0.kern.faults[4]
- rxbytes = system.tsunami.etherdev0.rxBytes
- txbytes = system.tsunami.etherdev0.txBytes
-
- if options.binned:
- print 'ipkb kernel stats'
- interrupts.bins = 'kernel'
- ipkb = interrupts / ((rxbytes + txbytes) / 1024)
- printdata(runs, ipkb)
-
- print 'ipkb idle stats'
- interrupts.bins = 'idle'
- ipkb = interrupts / ((rxbytes + txbytes) / 1024)
- printdata(runs, ipkb)
-
- print 'ipkb user stats'
- interrupts.bins = 'user'
- ipkb = interrupts / ((rxbytes + txbytes) / 1024)
- printdata(runs, ipkb)
-
- print 'ipkb total stats'
-
- ipkb = interrupts / ((rxbytes + txbytes) / 1024)
- interrupts.bins = None
- if options.graph:
- graphdata(runs, options, 'ipkb', 'Interrupts / KB', ipkb)
- else:
- printdata(runs, ipkb)
+ output.stat = interrupts / kbytes
+ output.binstats = [ interrupts ]
+ output.label = 'Interrupts / KB'
+ display()
return
if command == 'execute':
- printdata(runs, system.full0.ISSUE__count)
+ output.stat = system.full0.ISSUE__count
+ display()
return
if command == 'commit':
- printdata(runs, system.full0.COM__count)
+ output.stat = system.full0.COM__count
+ display()
return
if command == 'fetch':
- printdata(runs, system.full0.FETCH__count)
- return
-
- if command == 'bpp':
- ed = system.tsunami.etherdev0
- bpp = (ed.rxBytes + ed.txBytes) / (ed.rxPackets + ed.txPackets)
- if options.graph:
- graphdata(runs, options, 'bpp', 'Bytes / Packet', bpp)
- else:
- printdata(runs, bpp)
- return
-
- if command == 'rxbpp':
- bpp = system.tsunami.etherdev0.rxBytes / system.tsunami.etherdev0.rxPackets
- if options.graph:
- graphdata(runs, options, 'rxbpp', 'Receive Bytes / Packet', bpp)
- else:
- printdata(runs, bpp)
- return
-
- if command == 'txbpp':
- bpp = system.tsunami.etherdev0.txBytes / system.tsunami.etherdev0.txPackets
- if options.graph:
- graphdata(runs, options, 'txbpp', 'Transmit Bytes / Packet', bpp)
- else:
- printdata(runs, bpp)
- return
-
- if command == 'rtp':
- rtp = system.tsunami.etherdev0.rxPackets / system.tsunami.etherdev0.txPackets
- if options.graph:
- graphdata(runs, options, 'rtp', 'rxPackets / txPackets', rtp)
- else:
- printdata(runs, rtp)
- return
-
- if command == 'rtb':
- rtb = system.tsunami.etherdev0.rxBytes / system.tsunami.etherdev0.txBytes
- if options.graph:
- graphdata(runs, options, 'rtb', 'rxBytes / txBytes', rtb)
- else:
- printdata(runs, rtb)
+ output.stat = system.full0.FETCH__count
+ display()
return
raise CommandException
@@ -738,9 +451,10 @@ class Options: pass
if __name__ == '__main__':
import getpass
+ from jobfile import JobFile
options = Options()
- options.host = 'zizzer.pool'
+ options.host = None
options.db = None
options.passwd = ''
options.user = getpass.getuser()
@@ -750,23 +464,31 @@ if __name__ == '__main__':
options.binned = False
options.graph = False
options.ticks = False
+ options.printmode = 'G'
+ options.jobfile = None
+ options.all = False
- opts, args = getopts(sys.argv[1:], '-6BEFGd:g:h:pr:s:u:T:')
+ opts, args = getopts(sys.argv[1:], '-BEFG:ad:g:h:j:pr:s:u:T:')
for o,a in opts:
if o == '-B':
options.binned = True
if o == '-E':
- printval.mode = 'E'
+ options.printmode = 'E'
if o == '-F':
- printval.mode = 'F'
+ options.printmode = 'F'
if o == '-G':
- options.graph = True;
+ options.get = a
+ if o == '-a':
+ options.all = True
if o == '-d':
options.db = a
if o == '-g':
- options.get = a
+ options.graph = True;
+ options.graphdir = a
if o == '-h':
options.host = a
+ if o == '-j':
+ options.jobfile = JobFile(a)
if o == '-p':
options.passwd = getpass.getpass()
if o == '-r':
@@ -778,6 +500,18 @@ if __name__ == '__main__':
if o == '-T':
options.ticks = a
+ if options.jobfile:
+ if not options.host:
+ options.host = options.jobfile.dbhost
+ if not options.db:
+ options.db = options.jobfile.statdb
+
+ if not options.host:
+ sys.exit('Database server must be provided from a jobfile or -h')
+
+ if not options.db:
+ sys.exit('Database name must be provided from a jobfile or -d')
+
if len(args) == 0:
usage()
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-10 23:50:00 +0000