ext: Updated Pybind11 to version 2.4.1.

This updates Pybind11 from version 2.2.1 to version 2.4.1. This fixes
warning/error received when "<experiment/optional>" is used when
compiling using c++14 with clang. It should be noted that
"ext/pybind11/include/pybind11/std.h" has been changed to include a fix
added by commit ba42457254cc362eddc099f22b60d469cc6369e0. This is
necessary to avoid build errors.

Built: Linux (gcc, c++11) and MacOS (clang, c++14).
Tested: Ran quick tests for X86, ARM, and RISC-V.
Deprecates: https://gem5-review.googlesource.com/c/public/gem5/+/21019
Change-Id: Ie9783511cb6be50136076a55330e645f4f36d075
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/21119
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Reviewed-by: Andreas Sandberg <andreas.sandberg@arm.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>
Maintainer: Andreas Sandberg <andreas.sandberg@arm.com>
Tested-by: kokoro <noreply+kokoro@google.com>

4 files changed, 54 insertions, 3 deletions

diff --git a/ext/pybind11/tests/test_embed/CMakeLists.txt b/ext/pybind11/tests/test_embed/CMakeLists.txt
index 0a43e0e22..8b4f1f843 100644
--- a/ext/pybind11/tests/test_embed/CMakeLists.txt
+++ b/ext/pybind11/tests/test_embed/CMakeLists.txt
@@ -5,7 +5,9 @@ if(${PYTHON_MODULE_EXTENSION} MATCHES "pypy")
 endif()
 
 find_package(Catch 1.9.3)
-if(NOT CATCH_FOUND)
+if(CATCH_FOUND)
+  message(STATUS "Building interpreter tests using Catch v${CATCH_VERSION}")
+else()
   message(STATUS "Catch not detected. Interpreter tests will be skipped. Install Catch headers"
                  " manually or use `cmake -DDOWNLOAD_CATCH=1` to fetch them automatically.")
   return()
@@ -31,4 +33,9 @@ target_link_libraries(test_embed PUBLIC ${CMAKE_THREAD_LIBS_INIT})
 
 add_custom_target(cpptest COMMAND $<TARGET_FILE:test_embed>
                   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
+pybind11_add_module(external_module THIN_LTO external_module.cpp)
+set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+add_dependencies(cpptest external_module)
+
 add_dependencies(check cpptest)
diff --git a/ext/pybind11/tests/test_embed/catch.cpp b/ext/pybind11/tests/test_embed/catch.cpp
index cface485d..dd137385c 100644
--- a/ext/pybind11/tests/test_embed/catch.cpp
+++ b/ext/pybind11/tests/test_embed/catch.cpp
@@ -3,12 +3,18 @@
 
 #include <pybind11/embed.h>
 
+#ifdef _MSC_VER
+// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to catch
+// 2.0.1; this should be fixed in the next catch release after 2.0.1).
+#  pragma warning(disable: 4996)
+#endif
+
 #define CATCH_CONFIG_RUNNER
 #include <catch.hpp>
 
 namespace py = pybind11;
 
-int main(int argc, const char *argv[]) {
+int main(int argc, char *argv[]) {
     py::scoped_interpreter guard{};
     auto result = Catch::Session().run(argc, argv);
 
diff --git a/ext/pybind11/tests/test_embed/external_module.cpp b/ext/pybind11/tests/test_embed/external_module.cpp
new file mode 100644
index 000000000..e9a6058b1
--- /dev/null
+++ b/ext/pybind11/tests/test_embed/external_module.cpp
@@ -0,0 +1,23 @@
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+
+/* Simple test module/test class to check that the referenced internals data of external pybind11
+ * modules aren't preserved over a finalize/initialize.
+ */
+
+PYBIND11_MODULE(external_module, m) {
+    class A {
+    public:
+        A(int value) : v{value} {};
+        int v;
+    };
+
+    py::class_<A>(m, "A")
+        .def(py::init<int>())
+        .def_readwrite("value", &A::v);
+
+    m.def("internals_at", []() {
+        return reinterpret_cast<uintptr_t>(&py::detail::get_internals());
+    });
+}
diff --git a/ext/pybind11/tests/test_embed/test_interpreter.cpp b/ext/pybind11/tests/test_embed/test_interpreter.cpp
index 6b5f051f2..222bd565f 100644
--- a/ext/pybind11/tests/test_embed/test_interpreter.cpp
+++ b/ext/pybind11/tests/test_embed/test_interpreter.cpp
@@ -1,4 +1,11 @@
 #include <pybind11/embed.h>
+
+#ifdef _MSC_VER
+// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to catch
+// 2.0.1; this should be fixed in the next catch release after 2.0.1).
+#  pragma warning(disable: 4996)
+#endif
+
 #include <catch.hpp>
 
 #include <thread>
@@ -94,7 +101,8 @@ bool has_pybind11_internals_builtin() {
 };
 
 bool has_pybind11_internals_static() {
-    return py::detail::get_internals_ptr() != nullptr;
+    auto **&ipp = py::detail::get_internals_pp();
+    return ipp && *ipp;
 }
 
 TEST_CASE("Restart the interpreter") {
@@ -102,6 +110,11 @@ TEST_CASE("Restart the interpreter") {
     REQUIRE(py::module::import("widget_module").attr("add")(1, 2).cast<int>() == 3);
     REQUIRE(has_pybind11_internals_builtin());
     REQUIRE(has_pybind11_internals_static());
+    REQUIRE(py::module::import("external_module").attr("A")(123).attr("value").cast<int>() == 123);
+
+    // local and foreign module internals should point to the same internals:
+    REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) ==
+            py::module::import("external_module").attr("internals_at")().cast<uintptr_t>());
 
     // Restart the interpreter.
     py::finalize_interpreter();
@@ -116,6 +129,8 @@ TEST_CASE("Restart the interpreter") {
     pybind11::detail::get_internals();
     REQUIRE(has_pybind11_internals_builtin());
     REQUIRE(has_pybind11_internals_static());
+    REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) ==
+            py::module::import("external_module").attr("internals_at")().cast<uintptr_t>());
 
     // Make sure that an interpreter with no get_internals() created until finalize still gets the
     // internals destroyed