scons: Track swig packages when loading embedded swig code

This patch changes how the embedded swig code is loaded to ensure that
gem5 works with swig 3.0.9. For Python 2.7 and above, swig 3.0.9 now
relies on importlib, and actually looks in the appropriate packages,
even for the wrapped C code. However, the swig wrapper does not
explicitly place the module in the right package (it just calls
Py_InitModule), and we have to take explicit action to ensure that the
swig code can be loaded. This patch adds the information to the
generated wrappers and the appropriate calls to set the context as
part of the swig initialisation.

Previous versions of swig used to fall back on looking in the global
namespace for the wrappers (and still do for Python 2.6), but
technically things should not work without the functionality in this
patch.

3 files changed, 47 insertions, 24 deletions

diff --git a/src/SConscript b/src/SConscript
index bb6f26fc1..02b3c28d4 100755
--- a/src/SConscript
+++ b/src/SConscript
@@ -236,6 +236,7 @@ class SwigSource(SourceFile):
         modname,ext = self.extname
         assert ext == 'i'
 
+        self.package = package
         self.module = modname
         cc_file = joinpath(self.dirname, modname + '_wrap.cc')
         py_file = joinpath(self.dirname, modname + '.py')
@@ -816,19 +817,27 @@ for name,simobj in sorted(sim_objects.iteritems()):
     SwigSource('m5.internal', i_file)
 
 # Generate the main swig init file
-def makeEmbeddedSwigInit(target, source, env):
-    code = code_formatter()
-    module = source[0].get_contents()
-    code('''\
-#include "sim/init.hh"
-
-extern "C" {
-    void init_${module}();
-}
+def makeEmbeddedSwigInit(package):
+    def body(target, source, env):
+        assert len(target) == 1 and len(source) == 1
 
-EmbeddedSwig embed_swig_${module}(init_${module});
-''')
-    code.write(str(target[0]))
+        code = code_formatter()
+        module = source[0].get_contents()
+        # Provide the full context so that the swig-generated call to
+        # Py_InitModule ends up placing the embedded module in the
+        # right package.
+        context = str(package) + "._" + str(module)
+        code('''\
+        #include "sim/init.hh"
+
+        extern "C" {
+            void init_${module}();
+        }
+
+        EmbeddedSwig embed_swig_${module}(init_${module}, "${context}");
+        ''')
+        code.write(str(target[0]))
+    return body
 
 # Build all swig modules
 for swig in SwigSource.all:
@@ -838,7 +847,8 @@ for swig in SwigSource.all:
     cc_file = str(swig.tnode)
     init_file = '%s/%s_init.cc' % (dirname(cc_file), basename(cc_file))
     env.Command(init_file, Value(swig.module),
-                MakeAction(makeEmbeddedSwigInit, Transform("EMBED SW")))
+                MakeAction(makeEmbeddedSwigInit(swig.package),
+                           Transform("EMBED SW")))
     env.Depends(SWIG, init_file)
     Source(init_file, **swig.guards)
 
diff --git a/src/sim/init.cc b/src/sim/init.cc
index 0a15c384d..2e1dd629c 100644
--- a/src/sim/init.cc
+++ b/src/sim/init.cc
@@ -148,8 +148,8 @@ EmbeddedPython::initAll()
     return 0;
 }
 
-EmbeddedSwig::EmbeddedSwig(void (*init_func)())
-    : initFunc(init_func)
+EmbeddedSwig::EmbeddedSwig(void (*init_func)(), const string& _context)
+    : initFunc(init_func), context(_context)
 {
     getList().push_back(this);
 }
@@ -164,12 +164,22 @@ EmbeddedSwig::getList()
 void
 EmbeddedSwig::initAll()
 {
-    // initialize SWIG modules.  initSwig() is autogenerated and calls
+    char* old_context = _Py_PackageContext;
+    // initialize SWIG modules. initFunc() is autogenerated and calls
     // all of the individual swig initialization functions.
-    list<EmbeddedSwig *>::iterator i = getList().begin();
-    list<EmbeddedSwig *>::iterator end = getList().end();
-    for (; i != end; ++i)
-        (*i)->initFunc();
+    for (auto i : getList()) {
+        // to ensure that the loaded modules are placed in the right
+        // package we have to be a bit unorthodox and directly
+        // manipulate the package context since swig simply calls
+        // Py_InitModule with nothing but the module name of the
+        // wrapper
+        char* cstr = new char[i->context.size() + 1];
+        strcpy(cstr, i->context.c_str());
+        _Py_PackageContext = cstr;
+        i->initFunc();
+        delete[] cstr;
+    }
+    _Py_PackageContext = old_context;
 }
 
 int
diff --git a/src/sim/init.hh b/src/sim/init.hh
index 766cb4365..2bbcd23da 100644
--- a/src/sim/init.hh
+++ b/src/sim/init.hh
@@ -33,10 +33,8 @@
 
 #include <Python.h>
 
-/*
- * Data structure describing an embedded python file.
- */
 #include <list>
+#include <string>
 
 #include <inttypes.h>
 
@@ -45,6 +43,9 @@ struct _object;
 typedef _object PyObject;
 #endif
 
+/*
+ * Data structure describing an embedded python file.
+ */
 struct EmbeddedPython
 {
     const char *filename;
@@ -70,7 +71,9 @@ struct EmbeddedSwig
 {
     void (*initFunc)();
 
-    EmbeddedSwig(void (*init_func)());
+    std::string context;
+
+    EmbeddedSwig(void (*init_func)(), const std::string& _context);
 
     static std::list<EmbeddedSwig *> &getList();
     static void initAll();