Major improvements in the graph output code. Mostly adding more

options, making existing options more visible and dealing with
holes in data better.

util/stats/barchart.py:
    - move the options for BarChart to a base class ChartOptions so
    they can be more easily set and copied.
    - add an option to set the chart size (so you can adjust the aspect ratio)
    - don't do the add_subplot thing, use add_axes directly so we can
    affect the size of the figure itself to make room for the legend
    - make the initial array bottom floating point so we don't lose precision
    - add an option to set the limits on the y axis
    - use a figure legend instead of an axes legend so we can put the legend
    outside of the actual chart.  Also add an option to set the fontsize of
    the legend.
    - initial hack at outputting csv files
util/stats/db.py:
    don't print out an error when the run is missing from the database
    just return None, the error will be print elsewhere.
util/stats/output.py:
    - make StatOutput derive from ChartOptions so that it's easier to
    set default chart options.
    - make the various output functions (graph, display, etc.) take the
    name of the data as a parameter instead of making it a parameter to
    __init__.  This allows me to create the StatOutput object with
    generic parameters while still being able to specialize the name
    after the fact
    - add support for graph_group and graph_bars to be applied to multiple
    configuration groups.  This results in a cross product of the groups
    to be generated and used.
    - flush the html file output as we go so that we can load the file
    while graphs are still being generated.
    - make the proxy a parameter to the graph function so the proper system's
    data can be graphed
    - for any groups or bars that are completely missing, remove them from
    the graph.  This way, if we decide not to do a set of runs, there won't
    be holes in the data.
    - output eps and ps by default in addition to the png.
util/stats/profile.py:
    - clean up the data structures that are used to store the function
    profile information and try our best to avoid keeping extra data
    around that isn't used.
    - make get() return None if a job is missing so we know it was
    missing rather than the all zeroes thing.
    - make the function profile categorization stuff total up to 100%
    - Fixup the x-axis and y-axis labels.
    - fix the dot file output stuff.
util/stats/stats.py:
    support the new options stuff for StatOutput

--HG--
extra : convert_revision : fae35df8c57a36257ea93bc3e0a0e617edc46bb7

1 files changed, 64 insertions, 52 deletions

diff --git a/util/stats/profile.py b/util/stats/profile.py
index 0f51d643e..151170280 100644
--- a/util/stats/profile.py
+++ b/util/stats/profile.py
@@ -27,19 +27,39 @@
 from orderdict import orderdict
 import output
 
+class FileData(dict):
+    def __init__(self, filename):
+        self.filename = filename
+        fd = file(filename)
+        current = []
+        for line in fd:
+            line = line.strip()
+            if line.startswith('>>>'):
+                current = []
+                self[line[3:]] = current
+            else:
+                current.append(line)
+        fd.close()
+
 class RunData(dict):
-    def __init__(self, filename=None):
+    def __init__(self, filename):
         self.filename = filename
 
-    def __getattr__(self, attr):
+    def __getattribute__(self, attr):
         if attr == 'total':
             total = 0.0
             for value in self.itervalues():
                 total += value
             return total
+
+        if attr == 'filedata':
+            return FileData(self.filename)
+
         if attr == 'maxsymlen':
             return max([ len(sym) for sym in self.iterkeys() ])
 
+        return super(RunData, self).__getattribute__(attr)
+
     def display(self, output=None, limit=None, maxsymlen=None):
         if not output:
             import sys
@@ -62,24 +82,12 @@ class RunData(dict):
         for number,name in symbols:
             print >>output, symbolf % (name, 100.0 * (float(number) / total))
 
-
-
 class PCData(RunData):
     def __init__(self, filename=None, categorize=None, showidle=True):
         super(PCData, self).__init__(self, filename)
-        if filename is None:
-            return
-
-        fd = file(filename)
-
-        for line in fd:
-            if line.strip() == '>>>PC data':
-                break
-
-        for line in fd:
-            if line.startswith('>>>'):
-                break
 
+        filedata = self.filedata['PC data']
+        for line in filedata:
             (symbol, count) = line.split()
             if symbol == "0x0":
                 continue
@@ -94,30 +102,21 @@ class PCData(RunData):
 
                 self[category] = count
 
-        fd.close()
-
 class FuncNode(object):
-    def __new__(cls, filename = None):
-        if filename is None:
+    def __new__(cls, filedata=None):
+        if filedata is None:
             return super(FuncNode, cls).__new__(cls)
 
-        fd = file(filename, 'r')
-        fditer = iter(fd)
         nodes = {}
-        for line in fditer:
-            if line.strip() == '>>>function data':
-                break
-
-        for line in fditer:
-            if line.startswith('>>>'):
-                break
-
-            data = line.split()
-            node_id = int(data[0], 16)
+        for line in filedata['function data']:
+            data = line.split(' ')
+            node_id = long(data[0], 16)
             node = FuncNode()
             node.symbol = data[1]
-            node.count = int(data[2])
-            node.children = [ int(child, 16) for child in data[3:] ]
+            if node.symbol == '':
+                node.symbol = 'unknown'
+            node.count = long(data[2])
+            node.children = [ long(child, 16) for child in data[3:] ]
             nodes[node_id] = node
 
         for node in nodes.itervalues():
@@ -128,13 +127,10 @@ class FuncNode(object):
                 child.parent = node
             node.children = tuple(children)
         if not nodes:
-            print filename
+            print filedata.filename
             print nodes
         return nodes[0]
 
-    def __init__(self, filename=None):
-        pass
-
     def total(self):
         total = self.count
         for child in self.children:
@@ -198,9 +194,14 @@ class FuncNode(object):
 class FuncData(RunData):
     def __init__(self, filename, categorize=None):
         super(FuncData, self).__init__(filename)
-        self.tree = FuncNode(filename)
-        self.tree.aggregate(self, categorize, incategory=False)
-        self.total = self.tree.total()
+        tree = self.tree
+        tree.aggregate(self, categorize, incategory=False)
+        self.total = tree.total()
+
+    def __getattribute__(self, attr):
+        if attr == 'tree':
+            return FuncNode(self.filedata)
+        return super(FuncData, self).__getattribute__(attr)
 
     def displayx(self, output=None, maxcount=None):
         if output is None:
@@ -274,6 +275,7 @@ class Profile(object):
         try:
             return self.data[run][cpu]
         except KeyError:
+            print run, cpu
             return None
 
     def alldata(self):
@@ -289,12 +291,16 @@ class Profile(object):
         cpu = '%s.run%d' % (job.system, self.cpu)
         data = self.getdata(run, cpu)
         if not data:
-            return [ 0.0 for c in self.categories ]
+            return None
 
         values = []
         for category in self.categories:
-            values.append(data.get(category, 0.0))
-        return values
+            val = float(data.get(category, 0.0))
+            if val < 0.0:
+                raise ValueError, 'value is %f' % val
+            values.append(val)
+        total = sum(values)
+        return [ v / total * 100.0 for v in values ]
 
     def dump(self):
         for run,cpu,data in self.alldata():
@@ -382,7 +388,6 @@ if __name__ == '__main__':
     import getopt, re, sys
     from os.path import expanduser
     from output import StatOutput
-    from jobfile import JobFile
 
     # default option values
     numsyms = 10
@@ -393,7 +398,7 @@ if __name__ == '__main__':
     funcdata = True
     jobfilename = 'Test.py'
     dodot = False
-    dotformat = 'raw'
+    dotfile = None
     textout = False
     threshold = 0.01
     inputfile = None
@@ -409,7 +414,7 @@ if __name__ == '__main__':
         elif o == '-c':
             categorize = True
         elif o == '-D':
-            dotformat = a
+            dotfile = a
         elif o == '-d':
             dodot = True
         elif o == '-f':
@@ -434,20 +439,24 @@ if __name__ == '__main__':
         usage(1)
 
     if inputfile:
-        data = FuncData(inputfile)
+        catfunc = None
+        if categorize:
+            catfunc = func_categorize
+        data = FuncData(inputfile, categorize=catfunc)
 
         if dodot:
             import pydot
             dot = pydot.Dot()
-            data.dot(dot, threshold=threshold)
+            data.tree.dot(dot, threshold=threshold)
             #dot.orientation = 'landscape'
             #dot.ranksep='equally'
             #dot.rank='samerank'
-            dot.write(dotfile, format=dotformat)
+            dot.write(dotfile, format='png')
         else:
             data.display(limit=numsyms)
 
     else:
+        from jobfile import JobFile
         jobfile = JobFile(jobfilename)
 
         if funcdata:
@@ -466,8 +475,11 @@ if __name__ == '__main__':
                     name = 'funcstacks%d' % cpu
                 else:
                     name = 'stacks%d' % cpu
-                output = StatOutput(name, jobfile, info=profile)
-                output.graph(graph)
+                output = StatOutput(jobfile, info=profile)
+                output.xlabel = 'System Configuration'
+                output.ylabel = '% CPU utilization'
+                output.stat = name
+                output.graph(name, graph)
 
         if dodot:
             for cpu in cpus: