From 0337db3388db335ea23f02f3aa00bca9d483ef1c Mon Sep 17 00:00:00 2001 From: Steve Reinhardt Date: Tue, 30 May 2006 13:11:34 -0400 Subject: Link in Python interpreter. Use embedded zip archive to carry Python code instead of homegrown embedded string/file mechanism. Do argument parsing in Python instead of C++. SConstruct: Add Python interpreter include path & library. Define two new simple builders which copy & concatenate files, respectively, for use by the Python embedded zipfile code. src/SConscript: Encapsulate environment creation in a function. Add code to append Python zip archive to final executable. Eliminate references to obsolete files. src/python/SConscript: Rewrite to generate embedded zip archive of Python code (replacing old "embedded string" mechanism). src/python/m5/__init__.py: Move main arg-parsing loop here (out of C++ main()). src/python/m5/config.py: Minor fix (version incompatibility?). src/sim/main.cc: Invoke embedded Python interpreter to parse args and generate config.ini, replacing C++ arg parsing code. --HG-- extra : convert_revision : 72d21236b2bee139ff39ba4cf031a4a1f8560029 --- src/python/SConscript | 226 ++++++++++++++------------------------------------ 1 file changed, 61 insertions(+), 165 deletions(-) (limited to 'src/python/SConscript') diff --git a/src/python/SConscript b/src/python/SConscript index 4407e403d..1f2d7fe0e 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -27,126 +27,55 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os, os.path, re, sys +from zipfile import PyZipFile -Import('env') +# handy function for path joins +def join(*args): + return os.path.normpath(os.path.join(*args)) -import scons_helper - -def WriteEmbeddedPyFile(target, source, path, name, ext, filename): - if isinstance(source, str): - source = file(source, 'r') - - if isinstance(target, str): - target = file(target, 'w') - - print >>target, "AddModule(%s, %s, %s, %s, '''\\" % \ - (`path`, `name`, `ext`, `filename`) - - for line in source: - line = line - # escape existing backslashes - line = line.replace('\\', '\\\\') - # escape existing triple quotes - line = line.replace("'''", r"\'\'\'") - - print >>target, line, - - print >>target, "''')" - print >>target - -def WriteCFile(target, source, name): - if isinstance(source, str): - source = file(source, 'r') - - if isinstance(target, str): - target = file(target, 'w') - - print >>target, 'const char %s_string[] = {' % name - - count = 0 - from array import array - try: - while True: - foo = array('B') - foo.fromfile(source, 10000) - l = [ str(i) for i in foo.tolist() ] - count += len(l) - for i in xrange(0,9999,20): - print >>target, ','.join(l[i:i+20]) + ',' - except EOFError: - l = [ str(i) for i in foo.tolist() ] - count += len(l) - for i in xrange(0,len(l),20): - print >>target, ','.join(l[i:i+20]) + ',' - print >>target, ','.join(l[i:]) + ',' - - print >>target, '};' - print >>target, 'const int %s_length = %d;' % (name, count) - print >>target - -def splitpath(path): - dir,file = os.path.split(path) - path = [] - assert(file) - while dir: - dir,base = os.path.split(dir) - path.insert(0, base) - return path, file - -def MakeEmbeddedPyFile(target, source, env): - target = file(str(target[0]), 'w') - - tree = {} - for src in source: - src = str(src) - path,pyfile = splitpath(src) - node = tree - for dir in path: - if not node.has_key(dir): - node[dir] = { } - node = node[dir] - - name,ext = pyfile.split('.') - if name == '__init__': - node['.hasinit'] = True - node[pyfile] = (src,name,ext,src) - - done = False - while not done: - done = True - for name,entry in tree.items(): - if not isinstance(entry, dict): continue - if entry.has_key('.hasinit'): continue - - done = False - del tree[name] - for key,val in entry.iteritems(): - if tree.has_key(key): - raise NameError, \ - "dir already has %s can't add it again" % key - tree[key] = val - - files = [] - def populate(node, path = []): - names = node.keys() - names.sort() - for name in names: - if name == '.hasinit': - continue - - entry = node[name] - if isinstance(entry, dict): - if not entry.has_key('.hasinit'): - raise NameError, 'package directory missing __init__.py' - populate(entry, path + [ name ]) - else: - pyfile,name,ext,filename = entry - files.append((pyfile, path, name, ext, filename)) - populate(tree) - - for pyfile, path, name, ext, filename in files: - WriteEmbeddedPyFile(target, pyfile, path, name, ext, filename) +Import('env') +# This SConscript is in charge of collecting .py files and generating a zip archive that is appended to the m5 binary. + +# Copy .py source files here (relative to src/python in the build +# directory). +pyzip_root = 'zip' + +# List of files & directories to include in the zip file. To include +# a package, list only the root directory of the package, not any +# internal .py files (else they will get the path stripped off when +# they are imported into the zip file). +pyzip_files = [] + +# List of additional files on which the zip archive depends, but which +# are not included in pyzip_files... i.e. individual .py files within +# a package. +pyzip_dep_files = [] + +# Add the specified package to the zip archive. Adds the directory to +# pyzip_files and all included .py files to pyzip_dep_files. +def addPkg(pkgdir): + pyzip_files.append(join(pyzip_root, pkgdir)) + origdir = os.getcwd() + srcdir = join(Dir('.').srcnode().abspath, pkgdir) + os.chdir(srcdir) + for path, dirs, files in os.walk('.'): + for i,dir in enumerate(dirs): + if dir == 'SCCS': + del dirs[i] + break + + for f in files: + if f.endswith('.py'): + source = join(pkgdir, path, f) + target = join(pyzip_root, source) + pyzip_dep_files.append(target) + env.CopyFile(target, source) + + os.chdir(origdir) + +# Generate Python file that contains a dict specifying the current +# build_env flags. def MakeDefinesPyFile(target, source, env): f = file(str(target[0]), 'w') print >>f, "import __main__" @@ -154,54 +83,21 @@ def MakeDefinesPyFile(target, source, env): print >>f, source[0] f.close() -CFileCounter = 0 -def MakePythonCFile(target, source, env): - global CFileCounter - target = file(str(target[0]), 'w') - - print >>target, '''\ -#include "base/embedfile.hh" - -namespace { -''' - for src in source: - src = str(src) - fname = os.path.basename(src) - name = 'embedded_file%d' % CFileCounter - CFileCounter += 1 - WriteCFile(target, src, name) - print >>target, '''\ -EmbedMap %(name)s("%(fname)s", - %(name)s_string, %(name)s_length); - -''' % locals() - print >>target, '''\ - -/* namespace */ } -''' - -# base list of .py files to embed -embedded_py_files = [ os.path.join(env['ROOT'], 'util/pbs/jobfile.py') ] -# add all .py files in python/m5 -objpath = os.path.join(env['SRCDIR'], 'python', 'm5') -for root, dirs, files in os.walk(objpath, topdown=True): - for i,dir in enumerate(dirs): - if dir == 'SCCS': - del dirs[i] - break - - assert(root.startswith(objpath)) - for f in files: - if f.endswith('.py'): - embedded_py_files.append(os.path.join(root, f)) - -embedfile_hh = os.path.join(env['SRCDIR'], 'base/embedfile.hh') - optionDict = dict([(opt, env[opt]) for opt in env.ExportOptions]) env.Command('defines.py', Value(optionDict), MakeDefinesPyFile) -env.Command('embedded_py.py', embedded_py_files, MakeEmbeddedPyFile) -env.Depends('embedded_py.cc', embedfile_hh) -env.Command('embedded_py.cc', - ['string_importer.py', 'defines.py', 'embedded_py.py'], - MakePythonCFile) +# Now specify the packages & files for the zip archive. +addPkg('m5') +pyzip_files.append('defines.py') +pyzip_files.append(join(env['ROOT'], 'util/pbs/jobfile.py')) + +# Action function to build the zip archive. Uses the PyZipFile module +# included in the standard Python library. +def buildPyZip(target, source, env): + pzf = PyZipFile(str(target[0]), 'w') + for s in source: + pzf.writepy(str(s)) + +# Add the zip file target to the environment. +env.Command('m5py.zip', pyzip_files, buildPyZip) +env.Depends('m5py.zip', pyzip_dep_files) -- cgit v1.2.3 From cb0cf2dd8ab1cd60ef13de925ac862268c07297f Mon Sep 17 00:00:00 2001 From: Ali Saidi Date: Wed, 31 May 2006 19:26:56 -0400 Subject: Updated Authors from bk prs info --HG-- extra : convert_revision : 77f475b156d81c03a2811818fa23593d5615c685 --- src/python/SConscript | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/python/SConscript') diff --git a/src/python/SConscript b/src/python/SConscript index 1f2d7fe0e..5a787cfdf 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -25,6 +25,9 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Steve Reinhardt +# Nathan Binkert import os, os.path, re, sys from zipfile import PyZipFile -- cgit v1.2.3 From 29e34a739b991af8d8e1eafe75ecb0904c324dc8 Mon Sep 17 00:00:00 2001 From: Steve Reinhardt Date: Fri, 9 Jun 2006 23:01:31 -0400 Subject: Move main control from C++ into Python. User script now invokes initialization and simulation loop after building configuration. These functions are exported from C++ to Python using SWIG. SConstruct: Set up SWIG builder & scanner. Set up symlinking of source files into build directory (by not disabling the default behavior). configs/test/test.py: Rewrite to use new script-driven interface. Include a sample option. src/SConscript: Set up symlinking of source files into build directory (by not disabling the default behavior). Add SWIG-generated main_wrap.cc to source list. src/arch/SConscript: Set up symlinking of source files into build directory (by not disabling the default behavior). src/arch/alpha/ev5.cc: src/arch/alpha/isa/decoder.isa: src/cpu/o3/alpha_cpu_impl.hh: src/cpu/trace/opt_cpu.cc: src/cpu/trace/trace_cpu.cc: src/sim/pseudo_inst.cc: src/sim/root.cc: src/sim/serialize.cc: src/sim/syscall_emul.cc: SimExit() is now exitSimLoop(). src/cpu/base.cc: SimExitEvent is now SimLoopExitEvent src/python/SConscript: Add SWIG build command for main.i. Use python/m5 in build dir as source for zip archive... easy now with file duplication enabled. src/python/m5/__init__.py: - Move copyright notice back to C++ so we can print it right away, even for interactive sessions. - Get rid of argument parsing code; just provide default option descriptors for user script to call optparse with. - Don't clutter m5 namespace by sucking in all of m5.config and m5.objects. - Move instantiate() function here from config.py. src/python/m5/config.py: - Move instantiate() function to __init__.py. - Param.Foo deferred type lookups must use m5.objects namespace now (not m5). src/python/m5/objects/AlphaConsole.py: src/python/m5/objects/AlphaFullCPU.py: src/python/m5/objects/AlphaTLB.py: src/python/m5/objects/BadDevice.py: src/python/m5/objects/BaseCPU.py: src/python/m5/objects/BaseCache.py: src/python/m5/objects/Bridge.py: src/python/m5/objects/Bus.py: src/python/m5/objects/CoherenceProtocol.py: src/python/m5/objects/Device.py: src/python/m5/objects/DiskImage.py: src/python/m5/objects/Ethernet.py: src/python/m5/objects/Ide.py: src/python/m5/objects/IntrControl.py: src/python/m5/objects/MemObject.py: src/python/m5/objects/MemTest.py: src/python/m5/objects/Pci.py: src/python/m5/objects/PhysicalMemory.py: src/python/m5/objects/Platform.py: src/python/m5/objects/Process.py: src/python/m5/objects/Repl.py: src/python/m5/objects/Root.py: src/python/m5/objects/SimConsole.py: src/python/m5/objects/SimpleDisk.py: src/python/m5/objects/System.py: src/python/m5/objects/Tsunami.py: src/python/m5/objects/Uart.py: Fix up imports (m5 namespace no longer includes m5.config). src/sim/eventq.cc: src/sim/eventq.hh: Support for Python-called simulate() function: - Use IsExitEvent flag to signal events that want to exit the simulation loop gracefully (instead of calling exit() to terminate the process). - Modify interface to hand exit event object back to caller so it can be inspected for cause. src/sim/host.hh: Add MaxTick constant. src/sim/main.cc: Move copyright notice back to C++ so we can print it right away, even for interactive sessions. Use PYTHONPATH environment var to set module path (instead of clunky code injection method). Move main control from here into Python: - Separate initialization code and simulation loop into separate functions callable from Python. - Make Python interpreter invocation more pure (more like directly invoking interpreter). Add -i and -p flags (only options on binary itself; other options processed by Python). Import readline package when using interactive mode. src/sim/sim_events.cc: SimExitEvent is now SimLoopExitEvent, and uses IsSimExit flag to terminate loop (instead of exiting simulator process). src/sim/sim_events.hh: SimExitEvent is now SimLoopExitEvent, and uses IsSimExit flag to terminate loop (instead of exiting simulator process). Get rid of a few unused constructors. src/sim/sim_exit.hh: SimExit() is now exitSimLoop(). Get rid of unused functions. Add comments. --HG-- extra : convert_revision : 280b0d671516b25545a6f24cefa64a68319ff3d4 --- src/python/SConscript | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'src/python/SConscript') diff --git a/src/python/SConscript b/src/python/SConscript index 5a787cfdf..7b0f591eb 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -38,11 +38,8 @@ def join(*args): Import('env') -# This SConscript is in charge of collecting .py files and generating a zip archive that is appended to the m5 binary. - -# Copy .py source files here (relative to src/python in the build -# directory). -pyzip_root = 'zip' +# This SConscript is in charge of collecting .py files and generating +# a zip archive that is appended to the m5 binary. # List of files & directories to include in the zip file. To include # a package, list only the root directory of the package, not any @@ -58,7 +55,7 @@ pyzip_dep_files = [] # Add the specified package to the zip archive. Adds the directory to # pyzip_files and all included .py files to pyzip_dep_files. def addPkg(pkgdir): - pyzip_files.append(join(pyzip_root, pkgdir)) + pyzip_files.append(pkgdir) origdir = os.getcwd() srcdir = join(Dir('.').srcnode().abspath, pkgdir) os.chdir(srcdir) @@ -70,10 +67,7 @@ def addPkg(pkgdir): for f in files: if f.endswith('.py'): - source = join(pkgdir, path, f) - target = join(pyzip_root, source) - pyzip_dep_files.append(target) - env.CopyFile(target, source) + pyzip_dep_files.append(join(pkgdir, path, f)) os.chdir(origdir) @@ -81,19 +75,25 @@ def addPkg(pkgdir): # build_env flags. def MakeDefinesPyFile(target, source, env): f = file(str(target[0]), 'w') - print >>f, "import __main__" - print >>f, "__main__.m5_build_env = ", + print >>f, "m5_build_env = ", print >>f, source[0] f.close() optionDict = dict([(opt, env[opt]) for opt in env.ExportOptions]) -env.Command('defines.py', Value(optionDict), MakeDefinesPyFile) +env.Command('m5/defines.py', Value(optionDict), MakeDefinesPyFile) # Now specify the packages & files for the zip archive. addPkg('m5') -pyzip_files.append('defines.py') +pyzip_files.append('m5/defines.py') pyzip_files.append(join(env['ROOT'], 'util/pbs/jobfile.py')) +env.Command(['swig/main_wrap.cc', 'm5/main.py'], + 'swig/main.i', + '$SWIG $SWIGFLAGS -outdir ${TARGETS[1].dir} ' + '-o ${TARGETS[0]} $SOURCES') + +pyzip_dep_files.append('m5/main.py') + # Action function to build the zip archive. Uses the PyZipFile module # included in the standard Python library. def buildPyZip(target, source, env): -- cgit v1.2.3 From d96d28e56d39eec0baa1377779119495cfbf4701 Mon Sep 17 00:00:00 2001 From: Steve Reinhardt Date: Sat, 17 Jun 2006 12:08:19 -0400 Subject: Rename SWIG "main" module to "cc_main" so it's clear from the Python side that this is the interface to C++. src/SConscript: main_wrap.cc -> cc_main_wrap.cc src/python/SConscript: src/python/m5/__init__.py: src/sim/main.cc: s/main/cc_main/ src/python/m5/config.py: s/main/cc_main/ Also directly import cc_main so we don't need to put the "m5." in front all the time. --HG-- extra : convert_revision : 755552f70cf671881ff31e476c677b95ef12950d --- src/python/SConscript | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/python/SConscript') diff --git a/src/python/SConscript b/src/python/SConscript index 7b0f591eb..3a9def9a8 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -87,12 +87,12 @@ addPkg('m5') pyzip_files.append('m5/defines.py') pyzip_files.append(join(env['ROOT'], 'util/pbs/jobfile.py')) -env.Command(['swig/main_wrap.cc', 'm5/main.py'], - 'swig/main.i', +env.Command(['swig/cc_main_wrap.cc', 'm5/cc_main.py'], + 'swig/cc_main.i', '$SWIG $SWIGFLAGS -outdir ${TARGETS[1].dir} ' '-o ${TARGETS[0]} $SOURCES') -pyzip_dep_files.append('m5/main.py') +pyzip_dep_files.append('m5/cc_main.py') # Action function to build the zip archive. Uses the PyZipFile module # included in the standard Python library. -- cgit v1.2.3