ext: Updated Pybind11 to version 2.4.1.

This updates Pybind11 from version 2.2.1 to version 2.4.1. This fixes
warning/error received when "<experiment/optional>" is used when
compiling using c++14 with clang. It should be noted that
"ext/pybind11/include/pybind11/std.h" has been changed to include a fix
added by commit ba42457254cc362eddc099f22b60d469cc6369e0. This is
necessary to avoid build errors.

Built: Linux (gcc, c++11) and MacOS (clang, c++14).
Tested: Ran quick tests for X86, ARM, and RISC-V.
Deprecates: https://gem5-review.googlesource.com/c/public/gem5/+/21019
Change-Id: Ie9783511cb6be50136076a55330e645f4f36d075
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/21119
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Reviewed-by: Andreas Sandberg <andreas.sandberg@arm.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>
Maintainer: Andreas Sandberg <andreas.sandberg@arm.com>
Tested-by: kokoro <noreply+kokoro@google.com>

5 files changed, 147 insertions, 36 deletions

diff --git a/ext/pybind11/tools/FindPythonLibsNew.cmake b/ext/pybind11/tools/FindPythonLibsNew.cmake
index ad3ed48fa..e660c5f3e 100644
--- a/ext/pybind11/tools/FindPythonLibsNew.cmake
+++ b/ext/pybind11/tools/FindPythonLibsNew.cmake
@@ -1,5 +1,5 @@
 # - Find python libraries
-# This module finds the libraries corresponding to the Python interpeter
+# This module finds the libraries corresponding to the Python interpreter
 # FindPythonInterp provides.
 # This code sets the following variables:
 #
@@ -64,6 +64,7 @@ endif()
 
 if(NOT PYTHONINTERP_FOUND)
     set(PYTHONLIBS_FOUND FALSE)
+    set(PythonLibsNew_FOUND FALSE)
     return()
 endif()
 
@@ -96,10 +97,14 @@ if(NOT _PYTHON_SUCCESS MATCHES 0)
             "Python config failure:\n${_PYTHON_ERROR_VALUE}")
     endif()
     set(PYTHONLIBS_FOUND FALSE)
+    set(PythonLibsNew_FOUND FALSE)
     return()
 endif()
 
 # Convert the process output into a list
+if(WIN32)
+    string(REGEX REPLACE "\\\\" "/" _PYTHON_VALUES ${_PYTHON_VALUES})
+endif()
 string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES})
 string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES})
 list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST)
@@ -124,6 +129,7 @@ if(CMAKE_SIZEOF_VOID_P AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZE
             "chosen compiler is  ${_CMAKE_BITS}-bit")
     endif()
     set(PYTHONLIBS_FOUND FALSE)
+    set(PythonLibsNew_FOUND FALSE)
     return()
 endif()
 
@@ -138,7 +144,7 @@ string(REGEX REPLACE "\\\\" "/" PYTHON_PREFIX ${PYTHON_PREFIX})
 string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR ${PYTHON_INCLUDE_DIR})
 string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES ${PYTHON_SITE_PACKAGES})
 
-if(CMAKE_HOST_WIN32)
+if(CMAKE_HOST_WIN32 AND NOT (MSYS OR MINGW))
     set(PYTHON_LIBRARY
         "${PYTHON_PREFIX}/libs/Python${PYTHON_LIBRARY_SUFFIX}.lib")
 
@@ -193,3 +199,4 @@ find_package_message(PYTHON
     "${PYTHON_EXECUTABLE}${PYTHON_VERSION}")
 
 set(PYTHONLIBS_FOUND TRUE)
+set(PythonLibsNew_FOUND TRUE)
diff --git a/ext/pybind11/tools/check-style.sh b/ext/pybind11/tools/check-style.sh
index a9eeb170b..0a9f7d24f 100755
--- a/ext/pybind11/tools/check-style.sh
+++ b/ext/pybind11/tools/check-style.sh
@@ -10,7 +10,7 @@
 # 4. missing space between keyword and parenthesis, e.g.: for(, if(, while(
 # 5. Missing space between right parenthesis and brace, e.g. 'for (...){'
 # 6. opening brace on its own line. It should always be on the same line as the
-#    if/while/for/do statment.
+#    if/while/for/do statement.
 #
 # Invoke as: tools/check-style.sh
 #
diff --git a/ext/pybind11/tools/mkdoc.py b/ext/pybind11/tools/mkdoc.py
index 1fd8cceed..44164af3d 100644..100755
--- a/ext/pybind11/tools/mkdoc.py
+++ b/ext/pybind11/tools/mkdoc.py
@@ -14,6 +14,7 @@ import textwrap
 from clang import cindex
 from clang.cindex import CursorKind
 from collections import OrderedDict
+from glob import glob
 from threading import Thread, Semaphore
 from multiprocessing import cpu_count
 
@@ -40,6 +41,10 @@ PRINT_LIST = [
     CursorKind.FIELD_DECL
 ]
 
+PREFIX_BLACKLIST = [
+    CursorKind.TRANSLATION_UNIT
+]
+
 CPP_OPERATORS = {
     '<=': 'le', '>=': 'ge', '==': 'eq', '!=': 'ne', '[]': 'array',
     '+=': 'iadd', '-=': 'isub', '*=': 'imul', '/=': 'idiv', '%=':
@@ -56,10 +61,13 @@ CPP_OPERATORS = OrderedDict(
 job_count = cpu_count()
 job_semaphore = Semaphore(job_count)
 
-output = []
+
+class NoFilenamesError(ValueError):
+    pass
+
 
 def d(s):
-    return s.decode('utf8')
+    return s if isinstance(s, str) else s.decode('utf8')
 
 
 def sanitize_name(name):
@@ -182,18 +190,18 @@ def process_comment(comment):
     return result.rstrip().lstrip('\n')
 
 
-def extract(filename, node, prefix):
+def extract(filename, node, prefix, output):
     if not (node.location.file is None or
             os.path.samefile(d(node.location.file.name), filename)):
         return 0
     if node.kind in RECURSE_LIST:
         sub_prefix = prefix
-        if node.kind != CursorKind.TRANSLATION_UNIT:
+        if node.kind not in PREFIX_BLACKLIST:
             if len(sub_prefix) > 0:
                 sub_prefix += '_'
             sub_prefix += d(node.spelling)
         for i in node.get_children():
-            extract(filename, i, sub_prefix)
+            extract(filename, i, sub_prefix, output)
     if node.kind in PRINT_LIST:
         comment = d(node.raw_comment) if node.raw_comment is not None else ''
         comment = process_comment(comment)
@@ -202,15 +210,15 @@ def extract(filename, node, prefix):
             sub_prefix += '_'
         if len(node.spelling) > 0:
             name = sanitize_name(sub_prefix + d(node.spelling))
-            global output
             output.append((name, filename, comment))
 
 
 class ExtractionThread(Thread):
-    def __init__(self, filename, parameters):
+    def __init__(self, filename, parameters, output):
         Thread.__init__(self)
         self.filename = filename
         self.parameters = parameters
+        self.output = output
         job_semaphore.acquire()
 
     def run(self):
@@ -219,13 +227,18 @@ class ExtractionThread(Thread):
             index = cindex.Index(
                 cindex.conf.lib.clang_createIndex(False, True))
             tu = index.parse(self.filename, self.parameters)
-            extract(self.filename, tu.cursor, '')
+            extract(self.filename, tu.cursor, '', self.output)
         finally:
             job_semaphore.release()
 
-if __name__ == '__main__':
-    parameters = ['-x', 'c++', '-std=c++11']
+
+def read_args(args):
+    parameters = []
     filenames = []
+    if "-x" not in args:
+        parameters.extend(['-x', 'c++'])
+    if not any(it.startswith("-std=") for it in args):
+        parameters.append('-std=c++11')
 
     if platform.system() == 'Darwin':
         dev_path = '/Applications/Xcode.app/Contents/Developer/'
@@ -240,17 +253,48 @@ if __name__ == '__main__':
             sysroot_dir = os.path.join(sdk_dir, next(os.walk(sdk_dir))[1][0])
             parameters.append('-isysroot')
             parameters.append(sysroot_dir)
-
-    for item in sys.argv[1:]:
+    elif platform.system() == 'Linux':
+        # clang doesn't find its own base includes by default on Linux,
+        # but different distros install them in different paths.
+        # Try to autodetect, preferring the highest numbered version.
+        def clang_folder_version(d):
+            return [int(ver) for ver in re.findall(r'(?<!lib)(?<!\d)\d+', d)]
+        clang_include_dir = max((
+            path
+            for libdir in ['lib64', 'lib', 'lib32']
+            for path in glob('/usr/%s/clang/*/include' % libdir)
+            if os.path.isdir(path)
+        ), default=None, key=clang_folder_version)
+        if clang_include_dir:
+            parameters.extend(['-isystem', clang_include_dir])
+
+    for item in args:
         if item.startswith('-'):
             parameters.append(item)
         else:
             filenames.append(item)
 
     if len(filenames) == 0:
-        print('Syntax: %s [.. a list of header files ..]' % sys.argv[0])
-        exit(-1)
+        raise NoFilenamesError("args parameter did not contain any filenames")
+
+    return parameters, filenames
+
+
+def extract_all(args):
+    parameters, filenames = read_args(args)
+    output = []
+    for filename in filenames:
+        thr = ExtractionThread(filename, parameters, output)
+        thr.start()
+
+    print('Waiting for jobs to finish ..', file=sys.stderr)
+    for i in range(job_count):
+        job_semaphore.acquire()
 
+    return output
+
+
+def write_header(comments, out_file=sys.stdout):
     print('''/*
   This file contains docstrings for the Python bindings.
   Do not edit! These were automatically extracted by mkdoc.py
@@ -274,20 +318,12 @@ if __name__ == '__main__':
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-variable"
 #endif
-''')
+''', file=out_file)
 
-    output.clear()
-    for filename in filenames:
-        thr = ExtractionThread(filename, parameters)
-        thr.start()
-
-    print('Waiting for jobs to finish ..', file=sys.stderr)
-    for i in range(job_count):
-        job_semaphore.acquire()
 
     name_ctr = 1
     name_prev = None
-    for name, _, comment in list(sorted(output, key=lambda x: (x[0], x[1]))):
+    for name, _, comment in list(sorted(comments, key=lambda x: (x[0], x[1]))):
         if name == name_prev:
             name_ctr += 1
             name = name + "_%i" % name_ctr
@@ -295,10 +331,49 @@ if __name__ == '__main__':
             name_prev = name
             name_ctr = 1
         print('\nstatic const char *%s =%sR"doc(%s)doc";' %
-              (name, '\n' if '\n' in comment else ' ', comment))
+              (name, '\n' if '\n' in comment else ' ', comment), file=out_file)
 
     print('''
 #if defined(__GNUG__)
 #pragma GCC diagnostic pop
 #endif
-''')
+''', file=out_file)
+
+
+def mkdoc(args):
+    args = list(args)
+    out_path = None
+    for idx, arg in enumerate(args):
+        if arg.startswith("-o"):
+            args.remove(arg)
+            try:
+                out_path = arg[2:] or args.pop(idx)
+            except IndexError:
+                print("-o flag requires an argument")
+                exit(-1)
+            break
+
+    comments = extract_all(args)
+
+    if out_path:
+        try:
+            with open(out_path, 'w') as out_file:
+                write_header(comments, out_file)
+        except:
+            # In the event of an error, don't leave a partially-written
+            # output file.
+            try:
+                os.unlink(out_path)
+            except:
+                pass
+            raise
+    else:
+        write_header(comments)
+
+
+if __name__ == '__main__':
+    try:
+        mkdoc(sys.argv[1:])
+    except NoFilenamesError:
+        print('Syntax: %s [.. a list of header files ..]' % sys.argv[0])
+        exit(-1)
diff --git a/ext/pybind11/tools/pybind11Config.cmake.in b/ext/pybind11/tools/pybind11Config.cmake.in
index 3dd1b2c1a..8a7272ff9 100644
--- a/ext/pybind11/tools/pybind11Config.cmake.in
+++ b/ext/pybind11/tools/pybind11Config.cmake.in
@@ -90,7 +90,11 @@ if(NOT TARGET ${PN}::pybind11)
       set_property(TARGET ${PN}::module APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${PYTHON_LIBRARIES})
     endif()
 
-    set_property(TARGET ${PN}::pybind11 APPEND PROPERTY INTERFACE_COMPILE_OPTIONS "${PYBIND11_CPP_STANDARD}")
+    if(CMAKE_VERSION VERSION_LESS 3.3)
+      set_property(TARGET ${PN}::pybind11 APPEND PROPERTY INTERFACE_COMPILE_OPTIONS "${PYBIND11_CPP_STANDARD}")
+    else()
+      set_property(TARGET ${PN}::pybind11 APPEND PROPERTY INTERFACE_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:CXX>:${PYBIND11_CPP_STANDARD}>)
+    endif()
 
     get_property(_iid TARGET ${PN}::pybind11 PROPERTY INTERFACE_INCLUDE_DIRECTORIES)
     get_property(_ill TARGET ${PN}::module PROPERTY INTERFACE_LINK_LIBRARIES)
diff --git a/ext/pybind11/tools/pybind11Tools.cmake b/ext/pybind11/tools/pybind11Tools.cmake
index a7c471a07..c7156c020 100644
--- a/ext/pybind11/tools/pybind11Tools.cmake
+++ b/ext/pybind11/tools/pybind11Tools.cmake
@@ -110,10 +110,10 @@ endfunction()
 
 # Build a Python extension module:
 # pybind11_add_module(<name> [MODULE | SHARED] [EXCLUDE_FROM_ALL]
-#                     [NO_EXTRAS] [THIN_LTO] source1 [source2 ...])
+#                     [NO_EXTRAS] [SYSTEM] [THIN_LTO] source1 [source2 ...])
 #
 function(pybind11_add_module target_name)
-  set(options MODULE SHARED EXCLUDE_FROM_ALL NO_EXTRAS THIN_LTO)
+  set(options MODULE SHARED EXCLUDE_FROM_ALL NO_EXTRAS SYSTEM THIN_LTO)
   cmake_parse_arguments(ARG "${options}" "" "" ${ARGN})
 
   if(ARG_MODULE AND ARG_SHARED)
@@ -130,11 +130,22 @@ function(pybind11_add_module target_name)
 
   add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS})
 
-  target_include_directories(${target_name}
+  if(ARG_SYSTEM)
+    set(inc_isystem SYSTEM)
+  endif()
+
+  target_include_directories(${target_name} ${inc_isystem}
     PRIVATE ${PYBIND11_INCLUDE_DIR}  # from project CMakeLists.txt
     PRIVATE ${pybind11_INCLUDE_DIR}  # from pybind11Config
     PRIVATE ${PYTHON_INCLUDE_DIRS})
 
+  # Python debug libraries expose slightly different objects
+  # https://docs.python.org/3.6/c-api/intro.html#debugging-builds
+  # https://stackoverflow.com/questions/39161202/how-to-work-around-missing-pymodule-create2-in-amd64-win-python35-d-lib
+  if(PYTHON_IS_DEBUG)
+    target_compile_definitions(${target_name} PRIVATE Py_DEBUG)
+  endif()
+
   # The prefix and extension are provided by FindPythonLibsNew.cmake
   set_target_properties(${target_name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}")
   set_target_properties(${target_name} PROPERTIES SUFFIX "${PYTHON_MODULE_EXTENSION}")
@@ -145,6 +156,7 @@ function(pybind11_add_module target_name)
   # namespace; also turning it on for a pybind module compilation here avoids
   # potential warnings or issues from having mixed hidden/non-hidden types.
   set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET "hidden")
+  set_target_properties(${target_name} PROPERTIES CUDA_VISIBILITY_PRESET "hidden")
 
   if(WIN32 OR CYGWIN)
     # Link against the Python shared library on Windows
@@ -173,7 +185,11 @@ function(pybind11_add_module target_name)
   endif()
 
   # Make sure C++11/14 are enabled
-  target_compile_options(${target_name} PUBLIC ${PYBIND11_CPP_STANDARD})
+  if(CMAKE_VERSION VERSION_LESS 3.3)
+    target_compile_options(${target_name} PUBLIC ${PYBIND11_CPP_STANDARD})
+  else()
+    target_compile_options(${target_name} PUBLIC $<$<COMPILE_LANGUAGE:CXX>:${PYBIND11_CPP_STANDARD}>)
+  endif()
 
   if(ARG_NO_EXTRAS)
     return()
@@ -181,7 +197,7 @@ function(pybind11_add_module target_name)
 
   _pybind11_add_lto_flags(${target_name} ${ARG_THIN_LTO})
 
-  if (NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug)
+  if (NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo)
     # Strip unnecessary sections of the binary on Linux/Mac OS
     if(CMAKE_STRIP)
       if(APPLE)
@@ -197,6 +213,15 @@ function(pybind11_add_module target_name)
   if(MSVC)
     # /MP enables multithreaded builds (relevant when there are many files), /bigobj is
     # needed for bigger binding projects due to the limit to 64k addressable sections
-    target_compile_options(${target_name} PRIVATE /MP /bigobj)
+    target_compile_options(${target_name} PRIVATE /bigobj)
+    if(CMAKE_VERSION VERSION_LESS 3.11)
+      target_compile_options(${target_name} PRIVATE $<$<NOT:$<CONFIG:Debug>>:/MP>)
+    else()
+      # Only set these options for C++ files.  This is important so that, for
+      # instance, projects that include other types of source files like CUDA
+      # .cu files don't get these options propagated to nvcc since that would
+      # cause the build to fail.
+      target_compile_options(${target_name} PRIVATE $<$<NOT:$<CONFIG:Debug>>:$<$<COMPILE_LANGUAGE:CXX>:/MP>>)
+    endif()
   endif()
 endfunction()