Completely re-work how the scons framework incorporates swig

and python code into m5 to allow swig an python code to
easily added by any SConscript instead of just the one in
src/python.  This provides SwigSource and PySource for
adding new files to m5 (similar to Source for C++).  Also
provides SimObject for including files that contain SimObject
information and build the m5.objects __init__.py file.

--HG--
extra : convert_revision : 38b50a0629846ef451ed02f96fe3633947df23eb

4 files changed, 294 insertions, 135 deletions

diff --git a/src/SConscript b/src/SConscript
index da940c97a..34c5453b7 100644
--- a/src/SConscript
+++ b/src/SConscript
@@ -30,7 +30,9 @@
 
 import os
 import sys
+import zipfile
 
+from os.path import basename
 from os.path import join as joinpath
 
 import SCons
@@ -40,6 +42,12 @@ import SCons
 
 Import('*')
 
+# Children need to see the environment
+Export('env')
+
+########################################################################
+# Code for adding source files
+#
 sources = []
 def Source(source):
     if isinstance(source, SCons.Node.FS.File):
@@ -47,9 +55,46 @@ def Source(source):
     else:
         sources.append(File(source))
 
-Export('env')
+# Children should have access
 Export('Source')
 
+########################################################################
+# Code for adding python objects
+#
+py_sources = []
+py_source_packages = {}
+def PySource(package, source):
+    if not isinstance(source, SCons.Node.FS.File):
+        source = File(source)
+    py_source_packages[source] = package
+    py_sources.append(source)
+
+sim_objects = []
+def SimObject(source):
+    if not isinstance(source, SCons.Node.FS.File):
+        source = File(source)
+    PySource('m5.objects', source)
+    modname = basename(str(source))
+    sim_objects.append(modname)
+
+swig_sources = []
+swig_source_packages = {}
+def SwigSource(package, source):
+    if not isinstance(source, SCons.Node.FS.File):
+        source = File(source)
+    swig_source_packages[source] = package
+    swig_sources.append(source)
+
+# Children should have access
+Export('PySource')
+Export('SimObject')
+Export('SwigSource')
+
+########################################################################
+#
+# Set some compiler variables
+#
+
 # Include file paths are rooted in this directory.  SCons will
 # automatically expand '.' to refer to both the source directory and
 # the corresponding build directory to pick up generated include
@@ -59,7 +104,9 @@ env.Append(CPPPATH=Dir('.'))
 # Add a flag defining what THE_ISA should be for all compilation
 env.Append(CPPDEFINES=[('THE_ISA','%s_ISA' % env['TARGET_ISA'].upper())])
 
+########################################################################
 # Walk the tree and execute all SConscripts
+#
 scripts = []
 srcdir = env['SRCDIR']
 for root, dirs, files in os.walk(srcdir, topdown=True):
@@ -76,6 +123,132 @@ for root, dirs, files in os.walk(srcdir, topdown=True):
 for opt in env.ExportOptions:
     env.ConfigFile(opt)
 
+########################################################################
+#
+# Deal with python/swig, object code.  Collect .py files and
+# generating a zip archive that is appended to the m5 binary.
+#
+
+# Generate Python file that contains a dict specifying the current
+# build_env flags.
+def MakeDefinesPyFile(target, source, env):
+    f = file(str(target[0]), 'w')
+    print >>f, "m5_build_env = ", source[0]
+    f.close()
+
+optionDict = dict([(opt, env[opt]) for opt in env.ExportOptions])
+env.Command('python/m5/defines.py', Value(optionDict), MakeDefinesPyFile)
+PySource('m5', 'python/m5/defines.py')
+
+def MakeInfoPyFile(target, source, env):
+    f = file(str(target[0]), 'w')
+    for src in source:
+        data = ''.join(file(src.srcnode().abspath, 'r').xreadlines())
+        print >>f, "%s = %s" % (src, repr(data))
+    f.close()
+
+env.Command('python/m5/info.py',
+            [ '#/AUTHORS', '#/LICENSE', '#/README', '#/RELEASE_NOTES' ],
+            MakeInfoPyFile)
+PySource('m5', 'python/m5/info.py')
+
+def MakeObjectsInitFile(target, source, env):
+    f = file(str(target[0]), 'w')
+    print >>f, 'from m5.SimObject import *'
+    for src_path in source:
+        src_file = basename(src_path.get_contents())
+        assert(src_file.endswith('.py'))
+        src_module = src_file[:-3]
+        print >>f, 'from %s import *' % src_module
+    f.close()
+
+env.Command('python/m5/objects/__init__.py',
+            [ Value(o) for o in sim_objects],
+            MakeObjectsInitFile)
+PySource('m5.objects', 'python/m5/objects/__init__.py')
+
+swig_modules = []
+for source in swig_sources:
+    source.rfile() # Hack to cause the symlink to the .i file to be created
+    package = swig_source_packages[source]
+    filename = str(source)
+    module = basename(filename)
+
+    assert(module.endswith('.i'))
+    module = module[:-2]
+    cc_file = 'swig/%s_wrap.cc' % module
+    py_file = 'm5/internal/%s.py' % module
+
+    env.Command([cc_file, py_file], source,
+                '$SWIG $SWIGFLAGS -outdir ${TARGETS[1].dir} '
+                '-o ${TARGETS[0]} $SOURCES')
+    env.Depends(py_file, source)
+    env.Depends(cc_file, source)
+                
+    swig_modules.append(Value(module))
+    Source(cc_file)
+    PySource(package, py_file)
+
+def MakeSwigInit(target, source, env):
+    f = file(str(target[0]), 'w')
+    print >>f, 'extern "C" {'
+    for module in source:
+        print >>f, '    void init_%s();' % module.get_contents()
+    print >>f, '}'
+    print >>f, 'void init_swig() {'
+    for module in source:
+        print >>f, '    init_%s();' % module.get_contents()
+    print >>f, '}'
+    f.close()
+env.Command('python/swig/init.cc', swig_modules, MakeSwigInit)
+
+def CompilePyFile(target, source, env):
+    import py_compile
+    py_compile.compile(str(source[0]), str(target[0]))
+
+py_compiled = []
+py_arcname = {}
+py_zip_depends = []
+for source in py_sources:
+    filename = str(source)
+    package = py_source_packages[source]
+    arc_path = package.split('.') + [ basename(filename) + 'c' ]
+    zip_path = [ 'zip' ] + arc_path
+    arcname = joinpath(*arc_path)
+    zipname = joinpath(*zip_path)
+    f = File(zipname)
+
+    env.Command(f, source, CompilePyFile)
+    py_compiled.append(f)
+    py_arcname[f] = arcname
+
+    # make the zipfile depend on the archive name so that the archive
+    # is rebuilt if the name changes
+    py_zip_depends.append(Value(arcname))
+
+# Action function to build the zip archive.  Uses the PyZipFile module
+# included in the standard Python library.
+def buildPyZip(target, source, env):
+    zf = zipfile.ZipFile(str(target[0]), 'w')
+    for s in source:
+        arcname = py_arcname[s]
+        zipname = str(s)
+        zf.write(zipname, arcname)
+    zf.close()
+
+# Add the zip file target to the environment.
+env.Command('m5py.zip', py_compiled, buildPyZip)
+env.Depends('m5py.zip', py_zip_depends)
+
+########################################################################
+#
+# Define binaries.  Each different build type (debug, opt, etc.) gets
+# a slightly different build environment.
+#
+
+# List of constructed environments to pass back to SConstruct
+envList = []
+
 # This function adds the specified sources to the given build
 # environment, and returns a list of all the corresponding SCons
 # Object nodes (including an extra one for date.cc).  We explicitly
@@ -90,16 +263,6 @@ def make_objs(sources, env):
     objs.append(date_obj)
     return objs
 
-###################################################
-#
-# Define binaries.  Each different build type (debug, opt, etc.) gets
-# a slightly different build environment.
-#
-###################################################
-
-# List of constructed environments to pass back to SConstruct
-envList = []
-
 # Function to create a new build environment as clone of current
 # environment 'env' with modified object suffix and optional stripped
 # binary.  Additional keyword arguments are appended to corresponding
@@ -118,7 +281,7 @@ def makeEnv(label, objsfx, strip = False, **kwargs):
         else:
             newEnv.Command(stripped_bin, bin, 'strip $SOURCE -o $TARGET')
         bin = stripped_bin
-    targets = newEnv.Concat(exe, [bin, 'python/m5py.zip'])
+    targets = newEnv.Concat(exe, [bin, 'm5py.zip'])
     newEnv.M5Binary = targets[0]
     envList.append(newEnv)
 
diff --git a/src/base/SConscript b/src/base/SConscript
index 6fc140145..5e4aaafc2 100644
--- a/src/base/SConscript
+++ b/src/base/SConscript
@@ -97,3 +97,5 @@ Source('stats/visit.cc')
 if env['USE_MYSQL']:
     Source('mysql.cc')
     Source('stats/mysql.cc')
+
+PySource('m5', 'traceflags.py')
diff --git a/src/python/SConscript b/src/python/SConscript
index fc0b12f85..e1095eabe 100644
--- a/src/python/SConscript
+++ b/src/python/SConscript
@@ -30,132 +30,91 @@
 #          Nathan Binkert
 
 import os
-import zipfile
-
-# handy function for path joins
-def join(*args):
-    return os.path.normpath(os.path.join(*args))
-
 Import('*')
 
-# This SConscript is in charge of collecting .py files and generating
-# a zip archive that is appended to the m5 binary.
-
-# List of files & directories to include in the zip file.  To include
-# a package, list only the root directory of the package, not any
-# internal .py files (else they will get the path stripped off when
-# they are imported into the zip file).
-pyzip_files = []
-
-# List of additional files on which the zip archive depends, but which
-# are not included in pyzip_files... i.e. individual .py files within
-# a package.
-pyzip_dep_files = []
-
-# Add the specified package to the zip archive.  Adds the directory to
-# pyzip_files and all included .py files to pyzip_dep_files.
-def addPkg(pkgdir):
-    pyzip_files.append(pkgdir)
-    origdir = os.getcwd()
-    srcdir = join(Dir('.').srcnode().abspath, pkgdir)
-    os.chdir(srcdir)
-    for path, dirs, files in os.walk('.'):
-        for i,dir in enumerate(dirs):
-            if dir == 'SCCS':
-                del dirs[i]
-                break
-
-        for f in files:
-            if f.endswith('.py'):
-                pyzip_dep_files.append(join(pkgdir, path, f))
-
-    os.chdir(origdir)
-
-# Generate Python file that contains a dict specifying the current
-# build_env flags.
-def MakeDefinesPyFile(target, source, env):
-    f = file(str(target[0]), 'w')
-    print >>f, "m5_build_env = ", source[0]
-    f.close()
-
-optionDict = dict([(opt, env[opt]) for opt in env.ExportOptions])
-env.Command('m5/defines.py', Value(optionDict), MakeDefinesPyFile)
-
-def MakeInfoPyFile(target, source, env):
-    f = file(str(target[0]), 'w')
-    for src in source:
-        data = ''.join(file(src.srcnode().abspath, 'r').xreadlines())
-        print >>f, "%s = %s" % (src, repr(data))
-    f.close()
-
-env.Command('m5/info.py',
-            [ '#/AUTHORS', '#/LICENSE', '#/README', '#/RELEASE_NOTES' ],
-            MakeInfoPyFile)
-
-# Now specify the packages & files for the zip archive.
-addPkg('m5')
-pyzip_files.append('m5/defines.py')
-pyzip_files.append('m5/info.py')
-pyzip_files.append(join(env['ROOT'], 'util/pbs/jobfile.py'))
-pyzip_files.append(join(env['ROOT'], 'src/base/traceflags.py'))
-
-swig_modules = []
-def swig_it(module):
-    cc_file = 'swig/%s_wrap.cc' % module
-    py_file = 'm5/internal/%s.py' % module
-    source = File('swig/%s.i' % module)
-    source.rfile() # Hack to cause the symlink to the .i file to be created
-    env.Command([cc_file, py_file], source,
-                '$SWIG $SWIGFLAGS -outdir ${TARGETS[1].dir} '
-                '-o ${TARGETS[0]} $SOURCES')
-    swig_modules.append(module)
-    Source('swig/%s_wrap.cc' % module)
-
 Source('swig/init.cc')
 Source('swig/pyevent.cc')
 Source('swig/pyobject.cc')
 
-swig_it('core')
-swig_it('debug')
-swig_it('event')
-swig_it('random')
-swig_it('sim_object')
-swig_it('stats')
-swig_it('trace')
-
-# Automatically generate m5/internals/__init__.py
-def MakeInternalsInit(target, source, env):
-    f = file(str(target[0]), 'w')
-    for m in swig_modules:
-        print >>f, 'import %s' % m
-    f.close()
-
-swig_py_files = [ 'm5/internal/%s.py' % m for m in swig_modules ]
-env.Command('m5/internal/__init__.py', swig_py_files, MakeInternalsInit)
-pyzip_dep_files.append('m5/internal/__init__.py')
-
-def MakeSwigInit(target, source, env):
-    f = file(str(target[0]), 'w')
-    print >>f, 'extern "C" {'
-    for m in swig_modules:
-        print >>f, '    void init_%s();' % m
-    print >>f, '}'
-    print >>f, 'void init_swig() {'
-    for m in swig_modules:
-        print >>f, '    init_%s();' % m
-    print >>f, '}'
-    f.close()
-
-swig_cc_files = [ 'swig/%s_wrap.cc' % m for m in swig_modules ]
-env.Command('swig/init.cc', swig_cc_files, MakeSwigInit)
-
-# Action function to build the zip archive.  Uses the PyZipFile module
-# included in the standard Python library.
-def buildPyZip(target, source, env):
-    pzf = zipfile.PyZipFile(str(target[0]), 'w')
-    for s in source:
-        pzf.writepy(str(s))
-
-# Add the zip file target to the environment.
-env.Command('m5py.zip', pyzip_files, buildPyZip)
-env.Depends('m5py.zip', pyzip_dep_files)
+PySource('m5', 'm5/__init__.py')
+PySource('m5', 'm5/SimObject.py')
+PySource('m5', 'm5/attrdict.py')
+PySource('m5', 'm5/convert.py')
+PySource('m5', 'm5/event.py')
+PySource('m5', 'm5/main.py')
+PySource('m5', 'm5/multidict.py')
+PySource('m5', 'm5/params.py')
+PySource('m5', 'm5/proxy.py')
+PySource('m5', 'm5/smartdict.py')
+PySource('m5', 'm5/stats.py')
+PySource('m5', 'm5/ticks.py')
+PySource('m5', 'm5/util.py')
+
+PySource('m5', os.path.join(env['ROOT'], 'util/pbs/jobfile.py'))
+
+SwigSource('m5.internal', 'swig/core.i')
+SwigSource('m5.internal', 'swig/debug.i')
+SwigSource('m5.internal', 'swig/event.i')
+SwigSource('m5.internal', 'swig/random.i')
+SwigSource('m5.internal', 'swig/sim_object.i')
+SwigSource('m5.internal', 'swig/stats.i')
+SwigSource('m5.internal', 'swig/trace.i')
+PySource('m5.internal', 'm5/internal/__init__.py')
+
+SimObject('m5/objects/AlphaConsole.py')
+SimObject('m5/objects/AlphaTLB.py')
+SimObject('m5/objects/BadDevice.py')
+SimObject('m5/objects/BaseCPU.py')
+SimObject('m5/objects/BaseCache.py')
+SimObject('m5/objects/BaseHier.py')
+SimObject('m5/objects/BaseMem.py')
+SimObject('m5/objects/BaseMemory.py')
+SimObject('m5/objects/BranchPred.py')
+SimObject('m5/objects/Bridge.py')
+SimObject('m5/objects/Bus.py')
+SimObject('m5/objects/Checker.py')
+SimObject('m5/objects/CoherenceProtocol.py')
+SimObject('m5/objects/DRAMMemory.py')
+SimObject('m5/objects/Device.py')
+SimObject('m5/objects/DiskImage.py')
+SimObject('m5/objects/Ethernet.py')
+SimObject('m5/objects/FUPool.py')
+SimObject('m5/objects/FastCPU.py')
+#SimObject('m5/objects/FreebsdSystem.py')
+SimObject('m5/objects/FullCPU.py')
+SimObject('m5/objects/FuncUnit.py')
+SimObject('m5/objects/FuncUnitConfig.py')
+SimObject('m5/objects/FunctionalMemory.py')
+SimObject('m5/objects/HierParams.py')
+SimObject('m5/objects/Ide.py')
+SimObject('m5/objects/IntrControl.py')
+SimObject('m5/objects/LinuxSystem.py')
+SimObject('m5/objects/MainMemory.py')
+SimObject('m5/objects/MemObject.py')
+SimObject('m5/objects/MemTest.py')
+SimObject('m5/objects/MemoryController.py')
+SimObject('m5/objects/O3CPU.py')
+SimObject('m5/objects/OzoneCPU.py')
+SimObject('m5/objects/Pci.py')
+SimObject('m5/objects/PhysicalMemory.py')
+SimObject('m5/objects/PipeTrace.py')
+SimObject('m5/objects/Platform.py')
+SimObject('m5/objects/Process.py')
+SimObject('m5/objects/Repl.py')
+SimObject('m5/objects/Root.py')
+SimObject('m5/objects/Sampler.py')
+SimObject('m5/objects/Scsi.py')
+SimObject('m5/objects/SimConsole.py')
+SimObject('m5/objects/SimpleCPU.py')
+SimObject('m5/objects/SimpleDisk.py')
+#SimObject('m5/objects/SimpleOzoneCPU.py')
+SimObject('m5/objects/SparcTLB.py')
+SimObject('m5/objects/System.py')
+SimObject('m5/objects/T1000.py')
+#SimObject('m5/objects/Tru64System.py')
+SimObject('m5/objects/Tsunami.py')
+SimObject('m5/objects/Uart.py')
+
+if env['ALPHA_TLASER']:
+    SimObject('m5/objects/DmaEngine.py')
+    SimObject('m5/objects/Turbolaser.py')
diff --git a/src/python/m5/internal/__init__.py b/src/python/m5/internal/__init__.py
new file mode 100644
index 000000000..6b7859cd7
--- /dev/null
+++ b/src/python/m5/internal/__init__.py
@@ -0,0 +1,35 @@
+# Copyright (c) 2006 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+
+import core
+import debug
+import event
+import random
+import sim_object
+import stats
+import trace