From c0a4836077425e03cc39dfba88bec7da21af950b Mon Sep 17 00:00:00 2001 From: Nathan Binkert Date: Tue, 22 Nov 2005 21:50:34 -0500 Subject: Major improvements in the graph output code. Mostly adding more options, making existing options more visible and dealing with holes in data better. util/stats/barchart.py: - move the options for BarChart to a base class ChartOptions so they can be more easily set and copied. - add an option to set the chart size (so you can adjust the aspect ratio) - don't do the add_subplot thing, use add_axes directly so we can affect the size of the figure itself to make room for the legend - make the initial array bottom floating point so we don't lose precision - add an option to set the limits on the y axis - use a figure legend instead of an axes legend so we can put the legend outside of the actual chart. Also add an option to set the fontsize of the legend. - initial hack at outputting csv files util/stats/db.py: don't print out an error when the run is missing from the database just return None, the error will be print elsewhere. util/stats/output.py: - make StatOutput derive from ChartOptions so that it's easier to set default chart options. - make the various output functions (graph, display, etc.) take the name of the data as a parameter instead of making it a parameter to __init__. This allows me to create the StatOutput object with generic parameters while still being able to specialize the name after the fact - add support for graph_group and graph_bars to be applied to multiple configuration groups. This results in a cross product of the groups to be generated and used. - flush the html file output as we go so that we can load the file while graphs are still being generated. - make the proxy a parameter to the graph function so the proper system's data can be graphed - for any groups or bars that are completely missing, remove them from the graph. This way, if we decide not to do a set of runs, there won't be holes in the data. - output eps and ps by default in addition to the png. util/stats/profile.py: - clean up the data structures that are used to store the function profile information and try our best to avoid keeping extra data around that isn't used. - make get() return None if a job is missing so we know it was missing rather than the all zeroes thing. - make the function profile categorization stuff total up to 100% - Fixup the x-axis and y-axis labels. - fix the dot file output stuff. util/stats/stats.py: support the new options stuff for StatOutput --HG-- extra : convert_revision : fae35df8c57a36257ea93bc3e0a0e617edc46bb7 --- util/stats/output.py | 128 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 48 deletions(-) (limited to 'util/stats/output.py') diff --git a/util/stats/output.py b/util/stats/output.py index cf76f291e..e67751bbc 100644 --- a/util/stats/output.py +++ b/util/stats/output.py @@ -26,21 +26,22 @@ # # Authors: Nathan Binkert -class StatOutput(object): - def __init__(self, name, jobfile, info, stat=None, binstats=None): - self.name = name +from chart import ChartOptions + +class StatOutput(ChartOptions): + def __init__(self, jobfile, info, stat=None, binstats=None): + super(StatOutput, self).__init__() self.jobfile = jobfile self.stat = stat self.binstats = None - self.label = self.name self.invert = False self.info = info - def printdata(self, bin = None, printmode = 'G'): + def printdata(self, name, bin = None, printmode = 'G'): import info if bin: - print '%s %s stats' % (self.name, bin) + print '%s %s stats' % (name, bin) if self.binstats: for stat in self.binstats: @@ -69,70 +70,78 @@ class StatOutput(object): valstring = ', '.join([ valformat % val for val in value ]) print '%-50s %s' % (job.name + ':', valstring) - def display(self, binned = False, printmode = 'G'): + def display(self, name, binned = False, printmode = 'G'): if binned and self.binstats: - self.printdata('kernel', printmode) - self.printdata('idle', printmode) - self.printdata('user', printmode) - self.printdata('interrupt', printmode) + self.printdata(name, 'kernel', printmode) + self.printdata(name, 'idle', printmode) + self.printdata(name, 'user', printmode) + self.printdata(name, 'interrupt', printmode) - print '%s total stats' % self.name - self.printdata(printmode=printmode) + print '%s total stats' % name + self.printdata(name, printmode=printmode) - def graph(self, graphdir): + def graph(self, name, graphdir, proxy=None): from os.path import expanduser, isdir, join as joinpath from barchart import BarChart from matplotlib.numerix import Float, array, zeros - import os, re + import os, re, urllib + from jobfile import crossproduct confgroups = self.jobfile.groups() ngroups = len(confgroups) skiplist = [ False ] * ngroups - groupopts = None - baropts = None + groupopts = [] + baropts = [] groups = [] for i,group in enumerate(confgroups): if group.flags.graph_group: - if groupopts is not None: - raise AttributeError, \ - 'Two groups selected for graph group' - groupopts = group.subopts() + groupopts.append(group.subopts()) skiplist[i] = True elif group.flags.graph_bars: - if baropts is not None: - raise AttributeError, \ - 'Two groups selected for graph bars' - baropts = group.subopts() + baropts.append(group.subopts()) skiplist[i] = True else: groups.append(group) - if groupopts is None: + if not groupopts: raise AttributeError, 'No group selected for graph group' - if baropts is None: + if not baropts: raise AttributeError, 'No group selected for graph bars' + groupopts = [ group for group in crossproduct(groupopts) ] + baropts = [ bar for bar in crossproduct(baropts) ] + directory = expanduser(graphdir) if not isdir(directory): os.mkdir(directory) - html = file(joinpath(directory, '%s.html' % self.name), 'w') + html = file(joinpath(directory, '%s.html' % name), 'w') print >>html, '' - print >>html, 'Graphs for %s' % self.name + print >>html, 'Graphs for %s' % name print >>html, '' + html.flush() for options in self.jobfile.options(groups): + chart = BarChart(self) + data = zeros((len(groupopts), len(baropts)), Float) data = [ [ None ] * len(baropts) for i in xrange(len(groupopts)) ] enabled = False stacked = 0 for g,gopt in enumerate(groupopts): for b,bopt in enumerate(baropts): - job = self.jobfile.job(options + [ gopt, bopt ]) + job = self.jobfile.job(options + gopt + bopt) if not job: continue + if proxy: + import db + proxy.dict['system'] = self.info[job.system] val = self.info.get(job, self.stat) + if val is None: + print 'stat "%s" for job "%s" not found' % \ + (self.stat, job) + if isinstance(val, (list, tuple)): if len(val) == 1: val = val[0] @@ -151,7 +160,7 @@ class StatOutput(object): for j in xrange(len(baropts)): val = data[i][j] if val is None: - data[i][j] = [] * stacked + data[i][j] = [ 0.0 ] * stacked elif len(val) != stacked: raise ValueError, "some stats stacked, some not" @@ -159,29 +168,52 @@ class StatOutput(object): if data.sum() == 0: continue - bar_descs = [ opt.desc for opt in baropts ] - group_descs = [ opt.desc for opt in groupopts ] - if stacked: - try: - legend = self.info.rcategories - except: - legend = [ str(i) for i in xrange(stacked) ] - else: - legend = bar_descs - - chart = BarChart(data=data, xlabel='Benchmark', ylabel=self.label, - legend=legend, xticks=group_descs) + x = data.shape[0] + y = data.shape[1] + xkeep = [ i for i in xrange(x) if data[i].sum() != 0 ] + ykeep = [ i for i in xrange(y) if data[:,i].sum() != 0 ] + data = data.take(xkeep, axis=0) + data = data.take(ykeep, axis=1) + chart.data = data + + gopts = [ groupopts[i] for i in xkeep ] + bopts = [ baropts[i] for i in ykeep ] + + bdescs = [ ' '.join([o.desc for o in opt]) for opt in bopts] + gdescs = [ ' '.join([o.desc for o in opt]) for opt in gopts] + + if chart.legend is None: + if stacked: + try: + chart.legend = self.info.rcategories + except: + chart.legend = [ str(i) for i in xrange(stacked) ] + else: + chart.legend = bdescs + + if chart.xticks is None: + chart.xticks = gdescs chart.graph() names = [ opt.name for opt in options ] descs = [ opt.desc for opt in options ] - filename = '%s-%s.png' % (self.name, ':'.join(names)) + if names[0] == 'run': + names = names[1:] + descs = descs[1:] + + basename = '%s-%s' % (name, ':'.join(names)) desc = ' '.join(descs) - filepath = joinpath(directory, filename) - chart.savefig(filepath) - filename = re.sub(':', '%3A', filename) - print >>html, '''%s

''' % (desc, filename) + + pngname = '%s.png' % basename + psname = '%s.eps' % re.sub(':', '-', basename) + epsname = '%s.ps' % re.sub(':', '-', basename) + chart.savefig(joinpath(directory, pngname)) + chart.savefig(joinpath(directory, epsname)) + chart.savefig(joinpath(directory, psname)) + html_name = urllib.quote(pngname) + print >>html, '''%s

''' % (desc, html_name) + html.flush() print >>html, '' print >>html, '' -- cgit v1.2.3