diff options
| author | Andreas Sandberg <andreas.sandberg@arm.com> | 2017-05-09 19:22:53 +0100 |
|---|---|---|
| committer | Andreas Sandberg <andreas.sandberg@arm.com> | 2017-05-22 17:15:09 +0000 |
| commit | 6914a229a038206341ae1fea46393965a555ca9a (patch) | |
| tree | 4a11cfaed46dabc827c5ee17cd976f42b5f53d49 /ext/pybind11 | |
| parent | ca1d18d599dcc620bf526fb22042af95b1b60b68 (diff) | |
| download | gem5-6914a229a038206341ae1fea46393965a555ca9a.tar.xz | |
ext: Upgrade PyBind11 to version 2.1.1
Change-Id: I16870dec402d661295f9d013dc23e362b2b2c169
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-by: Curtis Dunham <curtis.dunham@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/3225
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Diffstat (limited to 'ext/pybind11')
108 files changed, 7388 insertions, 2025 deletions
diff --git a/ext/pybind11/.appveyor.yml b/ext/pybind11/.appveyor.yml index 740bd5c62..b908d0763 100644 --- a/ext/pybind11/.appveyor.yml +++ b/ext/pybind11/.appveyor.yml @@ -1,16 +1,22 @@ version: 1.0.{build} -os: Visual Studio 2015 +image: +- Visual Studio 2017 +- Visual Studio 2015 test: off platform: -- x86 - x64 +- x86 environment: matrix: + - CONDA: 36 - CONDA: 27 - - CONDA: 35 +matrix: + fast_finish: true # Stop remaining jobs after a job failure install: - ps: | if ($env:PLATFORM -eq "x64") { $env:CMAKE_ARCH = "x64" } + if ($env:APPVEYOR_JOB_NAME -like "*Visual Studio 2017*") { $env:CMAKE_GENERATOR = "Visual Studio 15 2017" } + else { $env:CMAKE_GENERATOR = "Visual Studio 14 2015" } if ($env:PYTHON) { if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" } $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH" @@ -27,7 +33,8 @@ install: 7z x 3.3.0.zip -y > $null $env:CMAKE_INCLUDE_PATH = "eigen-eigen-26667be4f70b" build_script: -- cmake -A "%CMAKE_ARCH%" -DPYBIND11_WERROR=ON +- cmake -G "%CMAKE_GENERATOR%" -A "%CMAKE_ARCH%" -DPYBIND11_WERROR=ON - set MSBuildLogger="C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - cmake --build . --config Release --target pytest -- /v:m /logger:%MSBuildLogger% -- cmake --build . --config Release --target test_install -- /v:m /logger:%MSBuildLogger% +- cmake --build . --config Release --target test_cmake_build -- /v:m /logger:%MSBuildLogger% +on_failure: if exist "tests\test_cmake_build" type tests\test_cmake_build\*.log diff --git a/ext/pybind11/.readthedocs.yml b/ext/pybind11/.readthedocs.yml new file mode 100644 index 000000000..c9c61617c --- /dev/null +++ b/ext/pybind11/.readthedocs.yml @@ -0,0 +1,3 @@ +python: + version: 3 +requirements_file: docs/requirements.txt diff --git a/ext/pybind11/.travis.yml b/ext/pybind11/.travis.yml index 28d3e840b..a3e99a003 100644 --- a/ext/pybind11/.travis.yml +++ b/ext/pybind11/.travis.yml @@ -20,12 +20,41 @@ matrix: - sudo: true services: docker env: PYTHON=3.5 CPP=14 GCC=6 DEBUG=1 + - sudo: true + services: docker + env: PYTHON=3.5 CPP=17 GCC=7 + - sudo: true + services: docker + env: PYTHON=3.5 CPP=17 CLANG=4.0 - os: osx osx_image: xcode7.3 env: PYTHON=2.7 CPP=14 CLANG - os: osx osx_image: xcode7.3 - env: PYTHON=3.5 CPP=14 CLANG + env: PYTHON=3.6 CPP=14 CLANG + # Test a PyPy 2.7 build + - os: linux + dist: trusty + env: PYPY=5.7 PYTHON=2.7 CPP=11 GCC=4.8 + addons: + apt: + packages: [g++-4.8, cmake] + - sudo: true + services: docker + env: ARCH=i386 PYTHON=3.5 CPP=14 GCC=6 + # This next one does a make install *before* testing, then builds the tests against the installed version: + - sudo: true + services: docker + env: PYTHON=3.5 CPP=14 CLANG=3.9 INSTALL=1 + script: + - | + $SCRIPT_RUN_PREFIX sh -c "set -e + cmake ${CMAKE_EXTRA_ARGS} -DPYBIND11_INSTALL=1 -DPYBIND11_TEST=0 + make install + cp -a tests /pybind11-tests + mkdir /build-tests && cd /build-tests + cmake ../pybind11-tests ${CMAKE_EXTRA_ARGS} -DPYBIND11_WERROR=ON + make pytest -j 2" # A barebones build makes sure everything still works without optional deps (numpy/scipy/eigen) # and also tests the automatic discovery functions in CMake (Python version, C++ standard). - os: linux @@ -41,11 +70,17 @@ matrix: env: DOCS STYLE LINT install: - pip install --upgrade sphinx sphinx_rtd_theme flake8 pep8-naming - - pip install docutils==0.12 + - | + curl -fsSL ftp://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.12.linux.bin.tar.gz | tar xz + export PATH="$PWD/doxygen-1.8.12/bin:$PATH" + pip install https://github.com/michaeljones/breathe/archive/master.zip script: - make -C docs html SPHINX_OPTIONS=-W - tools/check-style.sh - flake8 + allow_failures: + - env: PYTHON=3.5 CPP=17 GCC=7 + - env: PYTHON=3.5 CPP=17 CLANG=4.0 cache: directories: - $HOME/.cache/pip @@ -54,26 +89,50 @@ before_install: - | # Configure build variables if [ "$TRAVIS_OS_NAME" = "linux" ]; then - if [ -z "$GCC" ]; then export GCC=4.8; fi - export CXX=g++-$GCC CC=gcc-$GCC; - if [ "$GCC" = "6" ]; then export DOCKER=debian:testing CXX=g++ CC=gcc; fi + if [ -n "$CLANG" ]; then + export CXX=clang++-$CLANG CC=clang-$CLANG COMPILER_PACKAGES="clang-$CLANG llvm-$CLANG-dev" + if [ "$CLANG" = "4.0" ]; then export CXXFLAGS="-stdlib=libc++"; fi + else + if [ -z "$GCC" ]; then export GCC=4.8 + else export COMPILER_PACKAGES=g++-$GCC + fi + export CXX=g++-$GCC CC=gcc-$GCC + fi + if [ "$CLANG" = "4.0" ]; then export DOCKER=debian:sid + elif [ "$GCC" = "6" ] || [ -n "$CLANG" ]; then export DOCKER=${ARCH:+$ARCH/}debian:testing + elif [ "$GCC" = "7" ]; then export DOCKER=debian:experimental APT_GET_EXTRA="-t experimental" + fi elif [ "$TRAVIS_OS_NAME" = "osx" ]; then export CXX=clang++ CC=clang; fi if [ -n "$CPP" ]; then export CPP=-std=c++$CPP; fi if [ "${PYTHON:0:1}" = "3" ]; then export PY=3; fi + if [ "$PYPY" = "5.7" ]; then + curl -fSL https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-linux64.tar.bz2 | tar -xj + export PYPY_BINARY=$(echo `pwd`/pypy2-v5.7.0-linux64/bin/pypy) + export CMAKE_EXTRA_ARGS="-DPYTHON_EXECUTABLE:FILEPATH=$PYPY_BINARY" + fi if [ -n "$DEBUG" ]; then export CMAKE_EXTRA_ARGS="-DCMAKE_BUILD_TYPE=Debug"; fi - | - # Initialize enviornment - if [ -n "$DOCKER" ]; then + # Initialize environment + if [ -n "$PYPY" ]; then + $PYPY_BINARY -m ensurepip + $PYPY_BINARY -m pip install pytest + elif [ -n "$DOCKER" ]; then docker pull $DOCKER + # Disable LTO with gcc until gcc 79296 is fixed: + if [ -n "$GCC" ]; then export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DPYBIND11_LTO_CXX_FLAGS="; fi + export containerid=$(docker run --detach --tty \ --volume="$PWD":/pybind11 --workdir=/pybind11 \ + --env="CXXFLAGS=$CXXFLAGS" \ --env="CC=$CC" --env="CXX=$CXX" --env="DEBIAN_FRONTEND=$DEBIAN_FRONTEND" \ --env=GCC_COLORS=\ \ $DOCKER) - docker exec --tty "$containerid" sh -c 'for s in 0 15; do sleep $s; apt-get update && apt-get -qy dist-upgrade && break; done' export SCRIPT_RUN_PREFIX="docker exec --tty $containerid" + $SCRIPT_RUN_PREFIX sh -c 'for s in 0 15; do sleep $s; apt-get update && apt-get -qy dist-upgrade && break; done' + # gcc-7 currently generates warnings; some are upstream bugs, so just turn off -Werror for now + if [ "$GCC" = "7" ]; then WERROR=off; fi else if [ "$TRAVIS_OS_NAME" = "linux" ]; then pip install --user --upgrade pip virtualenv @@ -94,10 +153,29 @@ install: - | # Install dependencies if [ -n "$DOCKER" ]; then - docker exec --tty "$containerid" sh -c "for s in 0 15; do sleep \$s; apt-get -qy --no-install-recommends install \ - python$PYTHON-dev python$PY-pytest python$PY-scipy \ - libeigen3-dev cmake make g++ && break; done" - else + if [ -n "$DEBUG" ]; then + PY_DEBUG="python$PY-dbg python$PY-scipy-dbg" + export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DPYTHON_EXECUTABLE=/usr/bin/python${PYTHON}dm" + fi + $SCRIPT_RUN_PREFIX sh -c "for s in 0 15; do sleep \$s; \ + apt-get -qy --no-install-recommends $APT_GET_EXTRA install \ + $PY_DEBUG python$PY-dev python$PY-pytest python$PY-scipy \ + libeigen3-dev cmake make ${COMPILER_PACKAGES} && break; done" + + if [ "$CLANG" = "4.0" ]; then + # Neither debian nor llvm provide a libc++ deb; luckily it's fairly quick + # to build and install, so do it ourselves: + git clone --depth=1 https://github.com/llvm-mirror/llvm.git llvm-source + git clone https://github.com/llvm-mirror/libcxx.git llvm-source/projects/libcxx -b release_40 + git clone https://github.com/llvm-mirror/libcxxabi.git llvm-source/projects/libcxxabi -b release_40 + $SCRIPT_RUN_PREFIX sh -c "mkdir llvm-build && cd llvm-build && \ + CXXFLAGS= cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr ../llvm-source && \ + make -j2 install-cxxabi install-cxx && \ + cp -a include/c++/v1/*cxxabi*.h /usr/include/c++/v1" + + if [ "$CPP" = "-std=c++17" ]; then export CPP="-std=c++1z"; fi + fi + elif [ -z "$PYPY" ]; then pip install numpy scipy pytest wget -q -O eigen.tar.gz https://bitbucket.org/eigen/eigen/get/3.3.0.tar.gz @@ -108,8 +186,9 @@ script: - $SCRIPT_RUN_PREFIX cmake ${CMAKE_EXTRA_ARGS} -DPYBIND11_PYTHON_VERSION=$PYTHON -DPYBIND11_CPP_STANDARD=$CPP - -DPYBIND11_WERROR=ON + -DPYBIND11_WERROR=${WERROR:-ON} - $SCRIPT_RUN_PREFIX make pytest -j 2 -- $SCRIPT_RUN_PREFIX make test_install +- $SCRIPT_RUN_PREFIX make test_cmake_build +after_failure: cat tests/test_cmake_build/*.log after_script: - if [ -n "$DOCKER" ]; then docker stop "$containerid"; docker rm "$containerid"; fi diff --git a/ext/pybind11/CMakeLists.txt b/ext/pybind11/CMakeLists.txt index 28012b8da..16f1d67f1 100644 --- a/ext/pybind11/CMakeLists.txt +++ b/ext/pybind11/CMakeLists.txt @@ -7,6 +7,11 @@ cmake_minimum_required(VERSION 2.8.12) +if (POLICY CMP0048) + # cmake warns if loaded from a min-3.0-required parent dir, so silence the warning: + cmake_policy(SET CMP0048 NEW) +endif() + project(pybind11) # Check if pybind11 is being used directly or via add_subdirectory @@ -17,7 +22,6 @@ endif() option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) -option(PYBIND11_WERROR "Report all warnings as errors" OFF) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/tools") @@ -30,27 +34,11 @@ set(PYTHON_LIBRARIES ${PYTHON_LIBRARIES} CACHE INTERNAL "") set(PYTHON_MODULE_PREFIX ${PYTHON_MODULE_PREFIX} CACHE INTERNAL "") set(PYTHON_MODULE_EXTENSION ${PYTHON_MODULE_EXTENSION} CACHE INTERNAL "") -# Compile with compiler warnings turned on -function(pybind11_enable_warnings target_name) - if(MSVC) - target_compile_options(${target_name} PRIVATE /W4) - else() - target_compile_options(${target_name} PRIVATE -Wall -Wextra -Wconversion) - endif() - - if(PYBIND11_WERROR) - if(MSVC) - target_compile_options(${target_name} PRIVATE /WX) - else() - target_compile_options(${target_name} PRIVATE -Werror) - endif() - endif() -endfunction() - set(PYBIND11_HEADERS include/pybind11/attr.h include/pybind11/cast.h include/pybind11/chrono.h + include/pybind11/class_support.h include/pybind11/common.h include/pybind11/complex.h include/pybind11/descr.h @@ -85,15 +73,27 @@ foreach(ver ${pybind11_version_defines}) endif() endforeach() set(${PROJECT_NAME}_VERSION ${PYBIND11_VERSION_MAJOR}.${PYBIND11_VERSION_MINOR}.${PYBIND11_VERSION_PATCH}) +message(STATUS "pybind11 v${${PROJECT_NAME}_VERSION}") + +option (USE_PYTHON_INCLUDE_DIR "Install pybind11 headers in Python include directory instead of default installation prefix" OFF) +if (USE_PYTHON_INCLUDE_DIR) + file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${PYTHON_INCLUDE_DIRS}) +endif() if(NOT (CMAKE_VERSION VERSION_LESS 3.0)) # CMake >= 3.0 # Build an interface library target: - add_library(pybind11 INTERFACE) - target_include_directories(pybind11 INTERFACE $<BUILD_INTERFACE:${PYBIND11_INCLUDE_DIR}> - $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>) - if(APPLE) - target_link_libraries(pybind11 INTERFACE "-undefined dynamic_lookup") + add_library(module INTERFACE) + target_include_directories(module INTERFACE $<BUILD_INTERFACE:${PYBIND11_INCLUDE_DIR}> + $<BUILD_INTERFACE:${PYTHON_INCLUDE_DIRS}> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>) + if(WIN32 OR CYGWIN) + target_link_libraries(module INTERFACE $<BUILD_INTERFACE:${PYTHON_LIBRARIES}>) + elseif(APPLE) + target_link_libraries(module INTERFACE "-undefined dynamic_lookup") endif() + target_compile_options(module INTERFACE $<BUILD_INTERFACE:${PYBIND11_CPP_STANDARD}>) + + add_library(pybind11::module ALIAS module) # to match exported target endif() if (PYBIND11_INSTALL) @@ -115,11 +115,10 @@ if (PYBIND11_INSTALL) DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) if(NOT (CMAKE_VERSION VERSION_LESS 3.0)) - install(TARGETS pybind11 + install(TARGETS module EXPORT "${PROJECT_NAME}Targets") install(EXPORT "${PROJECT_NAME}Targets" NAMESPACE "${PROJECT_NAME}::" DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) - message(STATUS "Exporting ${PROJECT_NAME}::pybind11 interface library target version ${${PROJECT_NAME}_VERSION}") endif() endif() diff --git a/ext/pybind11/ISSUE_TEMPLATE.md b/ext/pybind11/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..2e5f920f8 --- /dev/null +++ b/ext/pybind11/ISSUE_TEMPLATE.md @@ -0,0 +1,17 @@ +Make sure you've completed the following steps before submitting your issue -- thank you! +You can remove this template text afterward. + +1. Check if your question has already been answered in the [FAQ](http://pybind11.readthedocs.io/en/latest/faq.html) section. +2. Make sure you've read the [documentation](http://pybind11.readthedocs.io/en/latest/). Your issue may be addressed there. +3. If those resources didn't help and you only have a short question (not a bug report), consider asking in the [Gitter chat room](https://gitter.im/pybind/Lobby). +4. If you have a genuine bug report or a more complex question which is not answered in the previous items (or not suitable for chat), please fill in the details below. +5. Include a self-contained and minimal piece of code that reproduces the problem. If that's not possible, try to make the description as clear as possible. + + +#### Issue description + +(Provide a short description, state the expected behavior and what actually happens.) + +#### Reproducible example code + +(The code should be minimal, have no external dependencies, isolate the function(s) that cause breakage. Submit matched and complete C++ and Python snippets that can be easily compiled and run to diagnose the issue.) diff --git a/ext/pybind11/README.md b/ext/pybind11/README.md index cfe3f1519..447788240 100644 --- a/ext/pybind11/README.md +++ b/ext/pybind11/README.md @@ -4,6 +4,7 @@ [](http://pybind11.readthedocs.org/en/master/?badge=master) [](http://pybind11.readthedocs.org/en/stable/?badge=stable) +[](https://gitter.im/pybind/Lobby) [](https://travis-ci.org/pybind/pybind11) [](https://ci.appveyor.com/project/wjakob/pybind11) @@ -24,12 +25,12 @@ become an excessively large and unnecessary dependency. Think of this library as a tiny self-contained version of Boost.Python with everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~2.5K lines of code and depend on -Python (2.7 or 3.x) and the C++ standard library. This compact implementation -was possible thanks to some of the new C++11 language features (specifically: -tuples, lambda functions and variadic templates). Since its creation, this -library has grown beyond Boost.Python in many ways, leading to dramatically -simpler binding code in many common situations. +comments, the core header files only require ~4K lines of code and depend on +Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This +compact implementation was possible thanks to some of the new C++11 language +features (specifically: tuples, lambda functions and variadic templates). Since +its creation, this library has grown beyond Boost.Python in many ways, leading +to dramatically simpler binding code in many common situations. Tutorial and reference documentation is provided at [http://pybind11.readthedocs.org/en/master](http://pybind11.readthedocs.org/en/master). @@ -58,12 +59,15 @@ pybind11 can map the following core C++ features to Python ## Goodies In addition to the core functionality, pybind11 provides some extra goodies: -- pybind11 uses C++11 move constructors and move assignment operators whenever - possible to efficiently transfer custom data types. +- Python 2.7, 3.x, and PyPy (PyPy2.7 >= 5.7) are supported with an + implementation-agnostic interface. - It is possible to bind C++11 lambda functions with captured variables. The lambda capture data is stored inside the resulting Python function object. +- pybind11 uses C++11 move constructors and move assignment operators whenever + possible to efficiently transfer custom data types. + - It's easy to expose the internal storage of custom data types through Pythons' buffer protocols. This is handy e.g. for fast conversion between C++ matrix classes like Eigen and NumPy without expensive copy operations. @@ -92,15 +96,15 @@ In addition to the core functionality, pybind11 provides some extra goodies: ## Supported compilers -1. Clang/LLVM (any non-ancient version with C++11 support) -2. GCC (any non-ancient version with C++11 support) -3. Microsoft Visual Studio 2015 or newer +1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or newer) +2. GCC 4.8 or newer +3. Microsoft Visual Studio 2015 Update 3 or newer 4. Intel C++ compiler 16 or newer (15 with a [workaround](https://github.com/pybind/pybind11/issues/276)) 5. Cygwin/GCC (tested on 2.5.1) ## About -This project was created by [Wenzel Jakob](https://www.mitsuba-renderer.org/~wenzel/). +This project was created by [Wenzel Jakob](http://rgl.epfl.ch/people/wjakob). Significant features and/or improvements to the code were contributed by Jonas Adler, Sylvain Corlay, @@ -114,8 +118,9 @@ Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris SchΓ€ling, -Pim Schellart, and -Ivan Smirnov. +Pim Schellart, +Ivan Smirnov, and +Patrick Stewart. ### License diff --git a/ext/pybind11/docs/Doxyfile b/ext/pybind11/docs/Doxyfile new file mode 100644 index 000000000..4dc8bf059 --- /dev/null +++ b/ext/pybind11/docs/Doxyfile @@ -0,0 +1,19 @@ +PROJECT_NAME = pybind11 +INPUT = ../include/pybind11/ + +GENERATE_HTML = NO +GENERATE_LATEX = NO +GENERATE_XML = YES +XML_OUTPUT = .build/doxygenxml +XML_PROGRAMLISTING = YES + +MACRO_EXPANSION = YES +EXPAND_ONLY_PREDEF = YES +EXPAND_AS_DEFINED = PYBIND11_RUNTIME_EXCEPTION + +ALIASES = "rst=\verbatim embed:rst" +ALIASES += "endrst=\endverbatim" + +QUIET = YES +WARNINGS = YES +WARN_IF_UNDOCUMENTED = NO diff --git a/ext/pybind11/docs/advanced/cast/chrono.rst b/ext/pybind11/docs/advanced/cast/chrono.rst index 6d4a5ee55..8c6b3d7e5 100644 --- a/ext/pybind11/docs/advanced/cast/chrono.rst +++ b/ext/pybind11/docs/advanced/cast/chrono.rst @@ -4,8 +4,8 @@ Chrono When including the additional header file :file:`pybind11/chrono.h` conversions from C++11 chrono datatypes to python datetime objects are automatically enabled. This header also enables conversions of python floats (often from sources such -as `time.monotonic()`, `time.perf_counter()` and `time.process_time()`) into -durations. +as ``time.monotonic()``, ``time.perf_counter()`` and ``time.process_time()``) +into durations. An overview of clocks in C++11 ------------------------------ diff --git a/ext/pybind11/docs/advanced/cast/eigen.rst b/ext/pybind11/docs/advanced/cast/eigen.rst index b83ca9af9..5b0b08ca6 100644 --- a/ext/pybind11/docs/advanced/cast/eigen.rst +++ b/ext/pybind11/docs/advanced/cast/eigen.rst @@ -1,48 +1,308 @@ Eigen -===== +##### `Eigen <http://eigen.tuxfamily.org>`_ is C++ header-based library for dense and sparse linear algebra. Due to its popularity and widespread adoption, pybind11 -provides transparent conversion support between Eigen and Scientific Python linear -algebra data types. +provides transparent conversion and limited mapping support between Eigen and +Scientific Python linear algebra data types. -Specifically, when including the optional header file :file:`pybind11/eigen.h`, -pybind11 will automatically and transparently convert +To enable the built-in Eigen support you must include the optional header file +:file:`pybind11/eigen.h`. -1. Static and dynamic Eigen dense vectors and matrices to instances of - ``numpy.ndarray`` (and vice versa). +Pass-by-value +============= -2. Returned matrix expressions such as blocks (including columns or rows) and - diagonals will be converted to ``numpy.ndarray`` of the expression - values. +When binding a function with ordinary Eigen dense object arguments (for +example, ``Eigen::MatrixXd``), pybind11 will accept any input value that is +already (or convertible to) a ``numpy.ndarray`` with dimensions compatible with +the Eigen type, copy its values into a temporary Eigen variable of the +appropriate type, then call the function with this temporary variable. -3. Returned matrix-like objects such as Eigen::DiagonalMatrix or - Eigen::SelfAdjointView will be converted to ``numpy.ndarray`` containing the - expressed value. +Sparse matrices are similarly copied to or from +``scipy.sparse.csr_matrix``/``scipy.sparse.csc_matrix`` objects. -4. Eigen sparse vectors and matrices to instances of - ``scipy.sparse.csr_matrix``/``scipy.sparse.csc_matrix`` (and vice versa). +Pass-by-reference +================= -This makes it possible to bind most kinds of functions that rely on these types. -One major caveat are functions that take Eigen matrices *by reference* and modify -them somehow, in which case the information won't be propagated to the caller. +One major limitation of the above is that every data conversion implicitly +involves a copy, which can be both expensive (for large matrices) and disallows +binding functions that change their (Matrix) arguments. Pybind11 allows you to +work around this by using Eigen's ``Eigen::Ref<MatrixType>`` class much as you +would when writing a function taking a generic type in Eigen itself (subject to +some limitations discussed below). + +When calling a bound function accepting a ``Eigen::Ref<const MatrixType>`` +type, pybind11 will attempt to avoid copying by using an ``Eigen::Map`` object +that maps into the source ``numpy.ndarray`` data: this requires both that the +data types are the same (e.g. ``dtype='float64'`` and ``MatrixType::Scalar`` is +``double``); and that the storage is layout compatible. The latter limitation +is discussed in detail in the section below, and requires careful +consideration: by default, numpy matrices and eigen matrices are *not* storage +compatible. + +If the numpy matrix cannot be used as is (either because its types differ, e.g. +passing an array of integers to an Eigen paramater requiring doubles, or +because the storage is incompatible), pybind11 makes a temporary copy and +passes the copy instead. + +When a bound function parameter is instead ``Eigen::Ref<MatrixType>`` (note the +lack of ``const``), pybind11 will only allow the function to be called if it +can be mapped *and* if the numpy array is writeable (that is +``a.flags.writeable`` is true). Any access (including modification) made to +the passed variable will be transparently carried out directly on the +``numpy.ndarray``. + +This means you can can write code such as the following and have it work as +expected: .. code-block:: cpp - /* The Python bindings of these functions won't replicate - the intended effect of modifying the function arguments */ - void scale_by_2(Eigen::Vector3f &v) { + void scale_by_2(Eigen::Ref<Eigen::VectorXd> m) { v *= 2; } - void scale_by_2(Eigen::Ref<Eigen::MatrixXd> &v) { - v *= 2; + +Note, however, that you will likely run into limitations due to numpy and +Eigen's difference default storage order for data; see the below section on +:ref:`storage_orders` for details on how to bind code that won't run into such +limitations. + +.. note:: + + Passing by reference is not supported for sparse types. + +Returning values to Python +========================== + +When returning an ordinary dense Eigen matrix type to numpy (e.g. +``Eigen::MatrixXd`` or ``Eigen::RowVectorXf``) pybind11 keeps the matrix and +returns a numpy array that directly references the Eigen matrix: no copy of the +data is performed. The numpy array will have ``array.flags.owndata`` set to +``False`` to indicate that it does not own the data, and the lifetime of the +stored Eigen matrix will be tied to the returned ``array``. + +If you bind a function with a non-reference, ``const`` return type (e.g. +``const Eigen::MatrixXd``), the same thing happens except that pybind11 also +sets the numpy array's ``writeable`` flag to false. + +If you return an lvalue reference or pointer, the usual pybind11 rules apply, +as dictated by the binding function's return value policy (see the +documentation on :ref:`return_value_policies` for full details). That means, +without an explicit return value policy, lvalue references will be copied and +pointers will be managed by pybind11. In order to avoid copying, you should +explictly specify an appropriate return value policy, as in the following +example: + +.. code-block:: cpp + + class MyClass { + Eigen::MatrixXd big_mat = Eigen::MatrixXd::Zero(10000, 10000); + public: + Eigen::MatrixXd &getMatrix() { return big_mat; } + const Eigen::MatrixXd &viewMatrix() { return big_mat; } + }; + + // Later, in binding code: + py::class_<MyClass>(m, "MyClass") + .def(py::init<>()) + .def("copy_matrix", &MyClass::getMatrix) // Makes a copy! + .def("get_matrix", &MyClass::getMatrix, py::return_value_policy::reference_internal) + .def("view_matrix", &MyClass::viewMatrix, py::return_value_policy::reference_internal) + ; + +.. code-block:: python + + a = MyClass() + m = a.get_matrix() # flags.writeable = True, flags.owndata = False + v = a.view_matrix() # flags.writeable = False, flags.owndata = False + c = a.copy_matrix() # flags.writeable = True, flags.owndata = True + # m[5,6] and v[5,6] refer to the same element, c[5,6] does not. + +Note in this example that ``py::return_value_policy::reference_internal`` is +used to tie the life of the MyClass object to the life of the returned arrays. + +You may also return an ``Eigen::Ref``, ``Eigen::Map`` or other map-like Eigen +object (for example, the return value of ``matrix.block()`` and related +methods) that map into a dense Eigen type. When doing so, the default +behaviour of pybind11 is to simply reference the returned data: you must take +care to ensure that this data remains valid! You may ask pybind11 to +explicitly *copy* such a return value by using the +``py::return_value_policy::copy`` policy when binding the function. You may +also use ``py::return_value_policy::reference_internal`` or a +``py::keep_alive`` to ensure the data stays valid as long as the returned numpy +array does. + +When returning such a reference of map, pybind11 additionally respects the +readonly-status of the returned value, marking the numpy array as non-writeable +if the reference or map was itself read-only. + +.. note:: + + Sparse types are always copied when returned. + +.. _storage_orders: + +Storage orders +============== + +Passing arguments via ``Eigen::Ref`` has some limitations that you must be +aware of in order to effectively pass matrices by reference. First and +foremost is that the default ``Eigen::Ref<MatrixType>`` class requires +contiguous storage along columns (for column-major types, the default in Eigen) +or rows if ``MatrixType`` is specifically an ``Eigen::RowMajor`` storage type. +The former, Eigen's default, is incompatible with ``numpy``'s default row-major +storage, and so you will not be able to pass numpy arrays to Eigen by reference +without making one of two changes. + +(Note that this does not apply to vectors (or column or row matrices): for such +types the "row-major" and "column-major" distinction is meaningless). + +The first approach is to change the use of ``Eigen::Ref<MatrixType>`` to the +more general ``Eigen::Ref<MatrixType, 0, Eigen::Stride<Eigen::Dynamic, +Eigen::Dynamic>>`` (or similar type with a fully dynamic stride type in the +third template argument). Since this is a rather cumbersome type, pybind11 +provides a ``py::EigenDRef<MatrixType>`` type alias for your convenience (along +with EigenDMap for the equivalent Map, and EigenDStride for just the stride +type). + +This type allows Eigen to map into any arbitrary storage order. This is not +the default in Eigen for performance reasons: contiguous storage allows +vectorization that cannot be done when storage is not known to be contiguous at +compile time. The default ``Eigen::Ref`` stride type allows non-contiguous +storage along the outer dimension (that is, the rows of a column-major matrix +or columns of a row-major matrix), but not along the inner dimension. + +This type, however, has the added benefit of also being able to map numpy array +slices. For example, the following (contrived) example uses Eigen with a numpy +slice to multiply by 2 all coefficients that are both on even rows (0, 2, 4, +...) and in columns 2, 5, or 8: + +.. code-block:: cpp + + m.def("scale", [](py::EigenDRef<Eigen::MatrixXd> m, double c) { m *= c; }); + +.. code-block:: python + + # a = np.array(...) + scale_by_2(myarray[0::2, 2:9:3]) + +The second approach to avoid copying is more intrusive: rearranging the +underlying data types to not run into the non-contiguous storage problem in the +first place. In particular, that means using matrices with ``Eigen::RowMajor`` +storage, where appropriate, such as: + +.. code-block:: cpp + + using RowMatrixXd = Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>; + // Use RowMatrixXd instead of MatrixXd + +Now bound functions accepting ``Eigen::Ref<RowMatrixXd>`` arguments will be +callable with numpy's (default) arrays without involving a copying. + +You can, alternatively, change the storage order that numpy arrays use by +adding the ``order='F'`` option when creating an array: + +.. code-block:: python + + myarray = np.array(source, order='F') + +Such an object will be passable to a bound function accepting an +``Eigen::Ref<MatrixXd>`` (or similar column-major Eigen type). + +One major caveat with this approach, however, is that it is not entirely as +easy as simply flipping all Eigen or numpy usage from one to the other: some +operations may alter the storage order of a numpy array. For example, ``a2 = +array.transpose()`` results in ``a2`` being a view of ``array`` that references +the same data, but in the opposite storage order! + +While this approach allows fully optimized vectorized calculations in Eigen, it +cannot be used with array slices, unlike the first approach. + +When *returning* a matrix to Python (either a regular matrix, a reference via +``Eigen::Ref<>``, or a map/block into a matrix), no special storage +consideration is required: the created numpy array will have the required +stride that allows numpy to properly interpret the array, whatever its storage +order. + +Failing rather than copying +=========================== + +The default behaviour when binding ``Eigen::Ref<const MatrixType>`` eigen +references is to copy matrix values when passed a numpy array that does not +conform to the element type of ``MatrixType`` or does not have a compatible +stride layout. If you want to explicitly avoid copying in such a case, you +should bind arguments using the ``py::arg().noconvert()`` annotation (as +described in the :ref:`nonconverting_arguments` documentation). + +The following example shows an example of arguments that don't allow data +copying to take place: + +.. code-block:: cpp + + // The method and function to be bound: + class MyClass { + // ... + double some_method(const Eigen::Ref<const MatrixXd> &matrix) { /* ... */ } + }; + float some_function(const Eigen::Ref<const MatrixXf> &big, + const Eigen::Ref<const MatrixXf> &small) { + // ... } -To see why this is, refer to the section on :ref:`opaque` (although that -section specifically covers STL data types, the underlying issue is the same). -The :ref:`numpy` sections discuss an efficient alternative for exposing the -underlying native Eigen types as opaque objects in a way that still integrates -with NumPy and SciPy. + // The associated binding code: + using namespace pybind11::literals; // for "arg"_a + py::class_<MyClass>(m, "MyClass") + // ... other class definitions + .def("some_method", &MyClass::some_method, py::arg().nocopy()); + + m.def("some_function", &some_function, + "big"_a.nocopy(), // <- Don't allow copying for this arg + "small"_a // <- This one can be copied if needed + ); + +With the above binding code, attempting to call the the ``some_method(m)`` +method on a ``MyClass`` object, or attempting to call ``some_function(m, m2)`` +will raise a ``RuntimeError`` rather than making a temporary copy of the array. +It will, however, allow the ``m2`` argument to be copied into a temporary if +necessary. + +Note that explicitly specifying ``.noconvert()`` is not required for *mutable* +Eigen references (e.g. ``Eigen::Ref<MatrixXd>`` without ``const`` on the +``MatrixXd``): mutable references will never be called with a temporary copy. + +Vectors versus column/row matrices +================================== + +Eigen and numpy have fundamentally different notions of a vector. In Eigen, a +vector is simply a matrix with the number of columns or rows set to 1 at +compile time (for a column vector or row vector, respectively). Numpy, in +contast, has comparable 2-dimensional 1xN and Nx1 arrays, but *also* has +1-dimensional arrays of size N. + +When passing a 2-dimensional 1xN or Nx1 array to Eigen, the Eigen type must +have matching dimensions: That is, you cannot pass a 2-dimensional Nx1 numpy +array to an Eigen value expecting a row vector, or a 1xN numpy array as a +column vector argument. + +On the other hand, pybind11 allows you to pass 1-dimensional arrays of length N +as Eigen parameters. If the Eigen type can hold a column vector of length N it +will be passed as such a column vector. If not, but the Eigen type constraints +will accept a row vector, it will be passed as a row vector. (The column +vector takes precendence when both are supported, for example, when passing a +1D numpy array to a MatrixXd argument). Note that the type need not be +expicitly a vector: it is permitted to pass a 1D numpy array of size 5 to an +Eigen ``Matrix<double, Dynamic, 5>``: you would end up with a 1x5 Eigen matrix. +Passing the same to an ``Eigen::MatrixXd`` would result in a 5x1 Eigen matrix. + +When returning an eigen vector to numpy, the conversion is ambiguous: a row +vector of length 4 could be returned as either a 1D array of length 4, or as a +2D array of size 1x4. When encoutering such a situation, pybind11 compromises +by considering the returned Eigen type: if it is a compile-time vector--that +is, the type has either the number of rows or columns set to 1 at compile +time--pybind11 converts to a 1D numpy array when returning the value. For +instances that are a vector only at run-time (e.g. ``MatrixXd``, +``Matrix<float, Dynamic, 4>``), pybind11 returns the vector as a 2D array to +numpy. If this isn't want you want, you can use ``array.reshape(...)`` to get +a view of the same data in the desired dimensions. .. seealso:: diff --git a/ext/pybind11/docs/advanced/cast/index.rst b/ext/pybind11/docs/advanced/cast/index.rst index 36586af5c..54c10570b 100644 --- a/ext/pybind11/docs/advanced/cast/index.rst +++ b/ext/pybind11/docs/advanced/cast/index.rst @@ -33,6 +33,7 @@ the last case of the above list. :maxdepth: 1 overview + strings stl functional chrono diff --git a/ext/pybind11/docs/advanced/cast/overview.rst b/ext/pybind11/docs/advanced/cast/overview.rst index ab37b90be..54c11a90a 100644 --- a/ext/pybind11/docs/advanced/cast/overview.rst +++ b/ext/pybind11/docs/advanced/cast/overview.rst @@ -94,14 +94,26 @@ as arguments and return values, refer to the section on binding :ref:`classes`. +------------------------------------+---------------------------+-------------------------------+ | ``char`` | Character literal | :file:`pybind11/pybind11.h` | +------------------------------------+---------------------------+-------------------------------+ +| ``char16_t`` | UTF-16 character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-------------------------------+ +| ``char32_t`` | UTF-32 character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-------------------------------+ | ``wchar_t`` | Wide character literal | :file:`pybind11/pybind11.h` | +------------------------------------+---------------------------+-------------------------------+ | ``const char *`` | UTF-8 string literal | :file:`pybind11/pybind11.h` | +------------------------------------+---------------------------+-------------------------------+ +| ``const char16_t *`` | UTF-16 string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-------------------------------+ +| ``const char32_t *`` | UTF-32 string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-------------------------------+ | ``const wchar_t *`` | Wide string literal | :file:`pybind11/pybind11.h` | +------------------------------------+---------------------------+-------------------------------+ | ``std::string`` | STL dynamic UTF-8 string | :file:`pybind11/pybind11.h` | +------------------------------------+---------------------------+-------------------------------+ +| ``std::u16string`` | STL dynamic UTF-16 string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-------------------------------+ +| ``std::u32string`` | STL dynamic UTF-32 string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-------------------------------+ | ``std::wstring`` | STL dynamic wide string | :file:`pybind11/pybind11.h` | +------------------------------------+---------------------------+-------------------------------+ | ``std::pair<T1, T2>`` | Pair of two custom types | :file:`pybind11/pybind11.h` | diff --git a/ext/pybind11/docs/advanced/cast/stl.rst b/ext/pybind11/docs/advanced/cast/stl.rst index bbd23732b..c76da5ca1 100644 --- a/ext/pybind11/docs/advanced/cast/stl.rst +++ b/ext/pybind11/docs/advanced/cast/stl.rst @@ -5,10 +5,12 @@ Automatic conversion ==================== When including the additional header file :file:`pybind11/stl.h`, conversions -between ``std::vector<>``, ``std::list<>``, ``std::set<>``, and ``std::map<>`` -and the Python ``list``, ``set`` and ``dict`` data structures are automatically -enabled. The types ``std::pair<>`` and ``std::tuple<>`` are already supported -out of the box with just the core :file:`pybind11/pybind11.h` header. +between ``std::vector<>``/``std::list<>``/``std::array<>``, +``std::set<>``/``std::unordered_set<>``, and +``std::map<>``/``std::unordered_map<>`` and the Python ``list``, ``set`` and +``dict`` data structures are automatically enabled. The types ``std::pair<>`` +and ``std::tuple<>`` are already supported out of the box with just the core +:file:`pybind11/pybind11.h` header. The major downside of these implicit conversions is that containers must be converted (i.e. copied) on every Python->C++ and C++->Python transition, which @@ -72,7 +74,7 @@ functions: /* ... binding code ... */ py::class_<MyClass>(m, "MyClass") - .def(py::init<>) + .def(py::init<>()) .def_readwrite("contents", &MyClass::contents); In this case, properties can be read and written in their entirety. However, an diff --git a/ext/pybind11/docs/advanced/cast/strings.rst b/ext/pybind11/docs/advanced/cast/strings.rst new file mode 100644 index 000000000..c70fb0bec --- /dev/null +++ b/ext/pybind11/docs/advanced/cast/strings.rst @@ -0,0 +1,243 @@ +Strings, bytes and Unicode conversions +###################################### + +.. note:: + + This section discusses string handling in terms of Python 3 strings. For Python 2.7, replace all occurrences of ``str`` with ``unicode`` and ``bytes`` with ``str``. Python 2.7 users may find it best to use ``from __future__ import unicode_literals`` to avoid unintentionally using ``str`` instead of ``unicode``. + +Passing Python strings to C++ +============================= + +When a Python ``str`` is passed from Python to a C++ function that accepts ``std::string`` or ``char *`` as arguments, pybind11 will encode the Python string to UTF-8. All Python ``str`` can be encoded in UTF-8, so this operation does not fail. + +The C++ language is encoding agnostic. It is the responsibility of the programmer to track encodings. It's often easiest to simply `use UTF-8 everywhere <http://utf8everywhere.org/>`_. + +.. code-block:: c++ + + m.def("utf8_test", + [](const std::string &s) { + cout << "utf-8 is icing on the cake.\n"; + cout << s; + } + ); + m.def("utf8_charptr", + [](const char *s) { + cout << "My favorite food is\n"; + cout << s; + } + ); + +.. code-block:: python + + >>> utf8_test('π') + utf-8 is icing on the cake. + π + + >>> utf8_charptr('π') + My favorite food is + π + +.. note:: + + Some terminal emulators do not support UTF-8 or emoji fonts and may not display the example above correctly. + +The results are the same whether the C++ function accepts arguments by value or reference, and whether or not ``const`` is used. + +Passing bytes to C++ +-------------------- + +A Python ``bytes`` object will be passed to C++ functions that accept ``std::string`` or ``char*`` *without* conversion. + + +Returning C++ strings to Python +=============================== + +When a C++ function returns a ``std::string`` or ``char*`` to a Python caller, **pybind11 will assume that the string is valid UTF-8** and will decode it to a native Python ``str``, using the same API as Python uses to perform ``bytes.decode('utf-8')``. If this implicit conversion fails, pybind11 will raise a ``UnicodeDecodeError``. + +.. code-block:: c++ + + m.def("std_string_return", + []() { + return std::string("This string needs to be UTF-8 encoded"); + } + ); + +.. code-block:: python + + >>> isinstance(example.std_string_return(), str) + True + + +Because UTF-8 is inclusive of pure ASCII, there is never any issue with returning a pure ASCII string to Python. If there is any possibility that the string is not pure ASCII, it is necessary to ensure the encoding is valid UTF-8. + +.. warning:: + + Implicit conversion assumes that a returned ``char *`` is null-terminated. If there is no null terminator a buffer overrun will occur. + +Explicit conversions +-------------------- + +If some C++ code constructs a ``std::string`` that is not a UTF-8 string, one can perform a explicit conversion and return a ``py::str`` object. Explicit conversion has the same overhead as implicit conversion. + +.. code-block:: c++ + + // This uses the Python C API to convert Latin-1 to Unicode + m.def("str_output", + []() { + std::string s = "Send your r\xe9sum\xe9 to Alice in HR"; // Latin-1 + py::str py_s = PyUnicode_DecodeLatin1(s.data(), s.length()); + return py_s; + } + ); + +.. code-block:: python + + >>> str_output() + 'Send your rΓ©sumΓ© to Alice in HR' + +The `Python C API <https://docs.python.org/3/c-api/unicode.html#built-in-codecs>`_ provides several built-in codecs. + + +One could also use a third party encoding library such as libiconv to transcode to UTF-8. + +Return C++ strings without conversion +------------------------------------- + +If the data in a C++ ``std::string`` does not represent text and should be returned to Python as ``bytes``, then one can return the data as a ``py::bytes`` object. + +.. code-block:: c++ + + m.def("return_bytes", + []() { + std::string s("\xba\xd0\xba\xd0"); // Not valid UTF-8 + return py::bytes(s); // Return the data without transcoding + } + ); + +.. code-block:: python + + >>> example.return_bytes() + b'\xba\xd0\xba\xd0' + + +Note the asymmetry: pybind11 will convert ``bytes`` to ``std::string`` without encoding, but cannot convert ``std::string`` back to ``bytes`` implicitly. + +.. code-block:: c++ + + m.def("asymmetry", + [](std::string s) { // Accepts str or bytes from Python + return s; // Looks harmless, but implicitly converts to str + } + ); + +.. code-block:: python + + >>> isinstance(example.asymmetry(b"have some bytes"), str) + True + + >>> example.asymmetry(b"\xba\xd0\xba\xd0") # invalid utf-8 as bytes + UnicodeDecodeError: 'utf-8' codec can't decode byte 0xba in position 0: invalid start byte + + +Wide character strings +====================== + +When a Python ``str`` is passed to a C++ function expecting ``std::wstring``, ``wchar_t*``, ``std::u16string`` or ``std::u32string``, the ``str`` will be encoded to UTF-16 or UTF-32 depending on how the C++ compiler implements each type, in the platform's endian. When strings of these types are returned, they are assumed to contain valid UTF-16 or UTF-32, and will be decoded to Python ``str``. + +.. code-block:: c++ + + #define UNICODE + #include <windows.h> + + m.def("set_window_text", + [](HWND hwnd, std::wstring s) { + // Call SetWindowText with null-terminated UTF-16 string + ::SetWindowText(hwnd, s.c_str()); + } + ); + m.def("get_window_text", + [](HWND hwnd) { + const int buffer_size = ::GetWindowTextLength(hwnd) + 1; + auto buffer = std::make_unique< wchar_t[] >(buffer_size); + + ::GetWindowText(hwnd, buffer.data(), buffer_size); + + std::wstring text(buffer.get()); + + // wstring will be converted to Python str + return text; + } + ); + +.. warning:: + + Wide character strings may not work as described on Python 2.7 or Python 3.3 compiled with ``--enable-unicode=ucs2``. + +Strings in multibyte encodings such as Shift-JIS must transcoded to a UTF-8/16/32 before being returned to Python. + + +Character literals +================== + +C++ functions that accept character literals as input will receive the first character of a Python ``str`` as their input. If the string is longer than one Unicode character, trailing characters will be ignored. + +When a character literal is returned from C++ (such as a ``char`` or a ``wchar_t``), it will be converted to a ``str`` that represents the single character. + +.. code-block:: c++ + + m.def("pass_char", [](char c) { return c; }); + m.def("pass_wchar", [](wchar_t w) { return w; }); + +.. code-block:: python + + >>> example.pass_char('A') + 'A' + +While C++ will cast integers to character types (``char c = 0x65;``), pybind11 does not convert Python integers to characters implicitly. The Python function ``chr()`` can be used to convert integers to characters. + +.. code-block:: python + + >>> example.pass_char(0x65) + TypeError + + >>> example.pass_char(chr(0x65)) + 'A' + +If the desire is to work with an 8-bit integer, use ``int8_t`` or ``uint8_t`` as the argument type. + +Grapheme clusters +----------------- + +A single grapheme may be represented by two or more Unicode characters. For example 'Γ©' is usually represented as U+00E9 but can also be expressed as the combining character sequence U+0065 U+0301 (that is, the letter 'e' followed by a combining acute accent). The combining character will be lost if the two-character sequence is passed as an argument, even though it renders as a single grapheme. + +.. code-block:: python + + >>> example.pass_wchar('Γ©') + 'Γ©' + + >>> combining_e_acute = 'e' + '\u0301' + + >>> combining_e_acute + 'eΜ' + + >>> combining_e_acute == 'Γ©' + False + + >>> example.pass_wchar(combining_e_acute) + 'e' + +Normalizing combining characters before passing the character literal to C++ may resolve *some* of these issues: + +.. code-block:: python + + >>> example.pass_wchar(unicodedata.normalize('NFC', combining_e_acute)) + 'Γ©' + +In some languages (Thai for example), there are `graphemes that cannot be expressed as a single Unicode code point <http://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries>`_, so there is no way to capture them in a C++ character type. + + +References +========== + +* `The Absolute Minimum Every Software Developer Absolutely, Positively Must Know About Unicode and Character Sets (No Excuses!) <https://www.joelonsoftware.com/2003/10/08/the-absolute-minimum-every-software-developer-absolutely-positively-must-know-about-unicode-and-character-sets-no-excuses/>`_ +* `C++ - Using STL Strings at Win32 API Boundaries <https://msdn.microsoft.com/en-ca/magazine/mt238407.aspx>`_
\ No newline at end of file diff --git a/ext/pybind11/docs/advanced/classes.rst b/ext/pybind11/docs/advanced/classes.rst index 4a423b578..8896441b6 100644 --- a/ext/pybind11/docs/advanced/classes.rst +++ b/ext/pybind11/docs/advanced/classes.rst @@ -79,7 +79,7 @@ helper class that is defined as follows: PYBIND11_OVERLOAD_PURE( std::string, /* Return type */ Animal, /* Parent class */ - go, /* Name of function */ + go, /* Name of function in C++ (must match Python name) */ n_times /* Argument(s) */ ); } @@ -90,7 +90,8 @@ functions, and :func:`PYBIND11_OVERLOAD` should be used for functions which have a default implementation. There are also two alternate macros :func:`PYBIND11_OVERLOAD_PURE_NAME` and :func:`PYBIND11_OVERLOAD_NAME` which take a string-valued name argument between the *Parent class* and *Name of the -function* slots. This is useful when the C++ and Python versions of the +function* slots, which defines the name of function in Python. This is required +when the C++ and Python versions of the function have different names, e.g. ``operator()`` vs ``__call__``. The binding code also needs a few minor adaptations (highlighted): @@ -115,11 +116,20 @@ The binding code also needs a few minor adaptations (highlighted): } Importantly, pybind11 is made aware of the trampoline helper class by -specifying it as an extra template argument to :class:`class_`. (This can also +specifying it as an extra template argument to :class:`class_`. (This can also be combined with other template arguments such as a custom holder type; the order of template types does not matter). Following this, we are able to define a constructor as usual. +Bindings should be made against the actual class, not the trampoline helper class. + +.. code-block:: cpp + + py::class_<Animal, PyAnimal /* <--- trampoline*/> animal(m, "Animal"); + animal + .def(py::init<>()) + .def("go", &PyAnimal::go); /* <--- THIS IS WRONG, use &Animal::go */ + Note, however, that the above is sufficient for allowing python classes to extend ``Animal``, but not ``Dog``: see ref:`virtual_and_inheritance` for the necessary steps required to providing proper overload support for inherited @@ -186,7 +196,7 @@ example as follows: virtual std::string go(int n_times) = 0; virtual std::string name() { return "unknown"; } }; - class Dog : public class Animal { + class Dog : public Animal { public: std::string go(int n_times) override { std::string result; @@ -220,6 +230,13 @@ override the ``name()`` method): std::string bark() override { PYBIND11_OVERLOAD(std::string, Dog, bark, ); } }; +.. note:: + + Note the trailing commas in the ``PYBIND11_OVERLOAD`` calls to ``name()`` + and ``bark()``. These are needed to portably implement a trampoline for a + function that does not take any arguments. For functions that take + a nonzero number of arguments, the trailing comma must be omitted. + A registered class derived from a pybind11-registered class with virtual methods requires a similar trampoline class, *even if* it doesn't explicitly declare or override any virtual methods itself: @@ -228,7 +245,8 @@ declare or override any virtual methods itself: class Husky : public Dog {}; class PyHusky : public Husky { - using Dog::Dog; // Inherit constructors + public: + using Husky::Husky; // Inherit constructors std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, Husky, go, n_times); } std::string name() override { PYBIND11_OVERLOAD(std::string, Husky, name, ); } std::string bark() override { PYBIND11_OVERLOAD(std::string, Husky, bark, ); } @@ -242,11 +260,13 @@ follows: .. code-block:: cpp template <class AnimalBase = Animal> class PyAnimal : public AnimalBase { + public: using AnimalBase::AnimalBase; // Inherit constructors std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, AnimalBase, go, n_times); } std::string name() override { PYBIND11_OVERLOAD(std::string, AnimalBase, name, ); } }; template <class DogBase = Dog> class PyDog : public PyAnimal<DogBase> { + public: using PyAnimal<DogBase>::PyAnimal; // Inherit constructors // Override PyAnimal's pure virtual go() with a non-pure one: std::string go(int n_times) override { PYBIND11_OVERLOAD(std::string, DogBase, go, n_times); } @@ -373,7 +393,9 @@ crucial that instances are deallocated on the C++ side to avoid memory leaks. /* ... binding code ... */ py::class_<MyClass, std::unique_ptr<MyClass, py::nodelete>>(m, "MyClass") - .def(py::init<>) + .def(py::init<>()) + +.. _implicit_conversions: Implicit conversions ==================== @@ -422,11 +444,11 @@ The section on :ref:`properties` discussed the creation of instance properties that are implemented in terms of C++ getters and setters. Static properties can also be created in a similar way to expose getters and -setters of static class attributes. It is important to note that the implicit -``self`` argument also exists in this case and is used to pass the Python -``type`` subclass instance. This parameter will often not be needed by the C++ -side, and the following example illustrates how to instantiate a lambda getter -function that ignores it: +setters of static class attributes. Note that the implicit ``self`` argument +also exists in this case and is used to pass the Python ``type`` subclass +instance. This parameter will often not be needed by the C++ side, and the +following example illustrates how to instantiate a lambda getter function +that ignores it: .. code-block:: cpp @@ -478,6 +500,7 @@ to Python. .def(py::self += py::self) .def(py::self *= float()) .def(float() * py::self) + .def(py::self * float()) .def("__repr__", &Vector2::toString); return m.ptr(); diff --git a/ext/pybind11/docs/advanced/functions.rst b/ext/pybind11/docs/advanced/functions.rst index f291e8222..e0b0fe095 100644 --- a/ext/pybind11/docs/advanced/functions.rst +++ b/ext/pybind11/docs/advanced/functions.rst @@ -6,6 +6,8 @@ with the basics of binding functions and classes, as explained in :doc:`/basics` and :doc:`/classes`. The following guide is applicable to both free and member functions, i.e. *methods* in Python. +.. _return_value_policies: + Return value policies ===================== @@ -14,7 +16,7 @@ lifetime of objects managed by them. This can lead to issues when creating bindings for functions that return a non-trivial type. Just by looking at the type information, it is not clear whether Python should take charge of the returned value and eventually free its resources, or if this is handled on the -C++ side. For this reason, pybind11 provides a several `return value policy` +C++ side. For this reason, pybind11 provides a several *return value policy* annotations that can be passed to the :func:`module::def` and :func:`class_::def` functions. The default policy is :enum:`return_value_policy::automatic`. @@ -24,11 +26,11 @@ Just to illustrate what can go wrong, consider the following simple example: .. code-block:: cpp - /* Function declaration */ + /* Function declaration */ Data *get_data() { return _data; /* (pointer to a static data structure) */ } ... - /* Binding code */ + /* Binding code */ m.def("get_data", &get_data); // <-- KABOOM, will cause crash when called from Python What's going on here? When ``get_data()`` is called from Python, the return @@ -44,7 +46,7 @@ silent data corruption. In the above example, the policy :enum:`return_value_policy::reference` should have been specified so that the global data instance is only *referenced* without any -implied transfer of ownership, i.e.: +implied transfer of ownership, i.e.: .. code-block:: cpp @@ -88,11 +90,12 @@ The following table provides an overview of available policies: | | return value is referenced by Python. This is the default policy for | | | property getters created via ``def_property``, ``def_readwrite``, etc. | +--------------------------------------------------+----------------------------------------------------------------------------+ -| :enum:`return_value_policy::automatic` | This is the default return value policy, which falls back to the policy | +| :enum:`return_value_policy::automatic` | **Default policy.** This policy falls back to the policy | | | :enum:`return_value_policy::take_ownership` when the return value is a | -| | pointer. Otherwise, it uses :enum:`return_value::move` or | -| | :enum:`return_value::copy` for rvalue and lvalue references, respectively. | -| | See above for a description of what all of these different policies do. | +| | pointer. Otherwise, it uses :enum:`return_value_policy::move` or | +| | :enum:`return_value_policy::copy` for rvalue and lvalue references, | +| | respectively. See above for a description of what all of these different | +| | policies do. | +--------------------------------------------------+----------------------------------------------------------------------------+ | :enum:`return_value_policy::automatic_reference` | As above, but use policy :enum:`return_value_policy::reference` when the | | | return value is a pointer. This is the default conversion policy for | @@ -158,8 +161,12 @@ targeted arguments can be passed through the :class:`cpp_function` constructor: Additional call policies ======================== -In addition to the above return value policies, further `call policies` can be -specified to indicate dependencies between parameters. There is currently just +In addition to the above return value policies, further *call policies* can be +specified to indicate dependencies between parameters. In general, call policies +are required when the C++ object is any kind of container and another object is being +added to the container. + +There is currently just one policy named ``keep_alive<Nurse, Patient>``, which indicates that the argument with index ``Patient`` should be kept alive at least until the argument with index ``Nurse`` is freed by the garbage collector. Argument @@ -207,8 +214,8 @@ For instance, the following statement iterates over a Python ``dict``: void print_dict(py::dict dict) { /* Easily interact with Python types */ for (auto item : dict) - std::cout << "key=" << item.first << ", " - << "value=" << item.second << std::endl; + std::cout << "key=" << std::string(py::str(item.first)) << ", " + << "value=" << std::string(py::str(item.second)) << std::endl; } It can be exported: @@ -252,16 +259,21 @@ Such functions can also be created using pybind11: m.def("generic", &generic); The class ``py::args`` derives from ``py::tuple`` and ``py::kwargs`` derives -from ``py::dict``. Note that the ``kwargs`` argument is invalid if no keyword -arguments were actually provided. Please refer to the other examples for -details on how to iterate over these, and on how to cast their entries into -C++ objects. A demonstration is also available in -``tests/test_kwargs_and_defaults.cpp``. +from ``py::dict``. -.. warning:: +You may also use just one or the other, and may combine these with other +arguments as long as the ``py::args`` and ``py::kwargs`` arguments are the last +arguments accepted by the function. - Unlike Python, pybind11 does not allow combining normal parameters with the - ``args`` / ``kwargs`` special parameters. +Please refer to the other examples for details on how to iterate over these, +and on how to cast their entries into C++ objects. A demonstration is also +available in ``tests/test_kwargs_and_defaults.cpp``. + +.. note:: + + When combining \*args or \*\*kwargs with :ref:`keyword_args` you should + *not* include ``py::arg`` tags for the ``py::args`` and ``py::kwargs`` + arguments. Default arguments revisited =========================== @@ -309,3 +321,89 @@ like so: py::class_<MyClass>("MyClass") .def("myFunction", py::arg("arg") = (SomeType *) nullptr); + +.. _nonconverting_arguments: + +Non-converting arguments +======================== + +Certain argument types may support conversion from one type to another. Some +examples of conversions are: + +* :ref:`implicit_conversions` declared using ``py::implicitly_convertible<A,B>()`` +* Calling a method accepting a double with an integer argument +* Calling a ``std::complex<float>`` argument with a non-complex python type + (for example, with a float). (Requires the optional ``pybind11/complex.h`` + header). +* Calling a function taking an Eigen matrix reference with a numpy array of the + wrong type or of an incompatible data layout. (Requires the optional + ``pybind11/eigen.h`` header). + +This behaviour is sometimes undesirable: the binding code may prefer to raise +an error rather than convert the argument. This behaviour can be obtained +through ``py::arg`` by calling the ``.noconvert()`` method of the ``py::arg`` +object, such as: + +.. code-block:: cpp + + m.def("floats_only", [](double f) { return 0.5 * f; }, py::arg("f").noconvert()); + m.def("floats_preferred", [](double f) { return 0.5 * f; }, py::arg("f")); + +Attempting the call the second function (the one without ``.noconvert()``) with +an integer will succeed, but attempting to call the ``.noconvert()`` version +will fail with a ``TypeError``: + +.. code-block:: pycon + + >>> floats_preferred(4) + 2.0 + >>> floats_only(4) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + TypeError: floats_only(): incompatible function arguments. The following argument types are supported: + 1. (f: float) -> float + + Invoked with: 4 + +You may, of course, combine this with the :var:`_a` shorthand notation (see +:ref:`keyword_args`) and/or :ref:`default_args`. It is also permitted to omit +the argument name by using the ``py::arg()`` constructor without an argument +name, i.e. by specifying ``py::arg().noconvert()``. + +.. note:: + + When specifying ``py::arg`` options it is necessary to provide the same + number of options as the bound function has arguments. Thus if you want to + enable no-convert behaviour for just one of several arguments, you will + need to specify a ``py::arg()`` annotation for each argument with the + no-convert argument modified to ``py::arg().noconvert()``. + +Overload resolution order +========================= + +When a function or method with multiple overloads is called from Python, +pybind11 determines which overload to call in two passes. The first pass +attempts to call each overload without allowing argument conversion (as if +every argument had been specified as ``py::arg().noconvert()`` as decribed +above). + +If no overload succeeds in the no-conversion first pass, a second pass is +attempted in which argument conversion is allowed (except where prohibited via +an explicit ``py::arg().noconvert()`` attribute in the function definition). + +If the second pass also fails a ``TypeError`` is raised. + +Within each pass, overloads are tried in the order they were registered with +pybind11. + +What this means in practice is that pybind11 will prefer any overload that does +not require conversion of arguments to an overload that does, but otherwise prefers +earlier-defined overloads to later-defined ones. + +.. note:: + + pybind11 does *not* further prioritize based on the number/pattern of + overloaded arguments. That is, pybind11 does not prioritize a function + requiring one conversion over one requiring three, but only prioritizes + overloads requiring no conversion at all to overloads that require + conversion of at least one argument. diff --git a/ext/pybind11/docs/advanced/misc.rst b/ext/pybind11/docs/advanced/misc.rst index c13df7bf8..d98466512 100644 --- a/ext/pybind11/docs/advanced/misc.rst +++ b/ext/pybind11/docs/advanced/misc.rst @@ -19,6 +19,7 @@ another name and use it in the macro to avoid this problem. Global Interpreter Lock (GIL) ============================= +When calling a C++ function from Python, the GIL is always held. The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to acquire and release the global interpreter lock in the body of a C++ function call. In this way, long-running C++ code can be parallelized using @@ -169,6 +170,20 @@ would be then able to access the data behind the same pointer. .. [#f6] https://docs.python.org/3/extending/extending.html#using-capsules +Module Destructors +================== + +pybind11 does not provide an explicit mechanism to invoke cleanup code at +module destruction time. In rare cases where such functionality is required, it +is possible to emulate it using Python capsules with a destruction callback. + +.. code-block:: cpp + + auto cleanup_callback = []() { + // perform cleanup here -- this function is called with the GIL held + }; + + m.add_object("_cleanup", py::capsule(cleanup_callback)); Generating documentation using Sphinx ===================================== diff --git a/ext/pybind11/docs/advanced/pycpp/numpy.rst b/ext/pybind11/docs/advanced/pycpp/numpy.rst index 8b46b7c83..6bcc46719 100644 --- a/ext/pybind11/docs/advanced/pycpp/numpy.rst +++ b/ext/pybind11/docs/advanced/pycpp/numpy.rst @@ -33,7 +33,7 @@ completely avoid copy operations with Python expressions like .. code-block:: cpp - py::class_<Matrix>(m, "Matrix") + py::class_<Matrix>(m, "Matrix", py::buffer_protocol()) .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( m.data(), /* Pointer to buffer */ @@ -46,9 +46,12 @@ completely avoid copy operations with Python expressions like ); }); -The snippet above binds a lambda function, which can create ``py::buffer_info`` -description records on demand describing a given matrix. The contents of -``py::buffer_info`` mirror the Python buffer protocol specification. +Supporting the buffer protocol in a new type involves specifying the special +``py::buffer_protocol()`` tag in the ``py::class_`` constructor and calling the +``def_buffer()`` method with a lambda function that creates a +``py::buffer_info`` description record on demand describing a given matrix +instance. The contents of ``py::buffer_info`` mirror the Python buffer protocol +specification. .. code-block:: cpp @@ -77,7 +80,7 @@ buffer objects (e.g. a NumPy matrix). typedef Matrix::Scalar Scalar; constexpr bool rowMajor = Matrix::Flags & Eigen::RowMajorBit; - py::class_<Matrix>(m, "Matrix") + py::class_<Matrix>(m, "Matrix", py::buffer_protocol()) .def("__init__", [](Matrix &m, py::buffer b) { typedef Eigen::Stride<Eigen::Dynamic, Eigen::Dynamic> Strides; @@ -152,7 +155,7 @@ NumPy array containing double precision values. When it is invoked with a different type (e.g. an integer or a list of integers), the binding code will attempt to cast the input into a NumPy array of the requested type. Note that this feature requires the -:file:``pybind11/numpy.h`` header to be included. +:file:`pybind11/numpy.h` header to be included. Data in NumPy arrays is not guaranteed to packed in a dense manner; furthermore, entries can be separated by arbitrary column and row strides. @@ -173,9 +176,10 @@ function overload. Structured types ================ -In order for ``py::array_t`` to work with structured (record) types, we first need -to register the memory layout of the type. This can be done via ``PYBIND11_NUMPY_DTYPE`` -macro which expects the type followed by field names: +In order for ``py::array_t`` to work with structured (record) types, we first +need to register the memory layout of the type. This can be done via +``PYBIND11_NUMPY_DTYPE`` macro, called in the plugin definition code, which +expects the type followed by field names: .. code-block:: cpp @@ -189,10 +193,14 @@ macro which expects the type followed by field names: A a; }; - PYBIND11_NUMPY_DTYPE(A, x, y); - PYBIND11_NUMPY_DTYPE(B, z, a); + // ... + PYBIND11_PLUGIN(test) { + // ... - /* now both A and B can be used as template arguments to py::array_t */ + PYBIND11_NUMPY_DTYPE(A, x, y); + PYBIND11_NUMPY_DTYPE(B, z, a); + /* now both A and B can be used as template arguments to py::array_t */ + } Vectorizing functions ===================== @@ -297,3 +305,75 @@ simply using ``vectorize``). The file :file:`tests/test_numpy_vectorize.cpp` contains a complete example that demonstrates using :func:`vectorize` in more detail. + +Direct access +============= + +For performance reasons, particularly when dealing with very large arrays, it +is often desirable to directly access array elements without internal checking +of dimensions and bounds on every access when indices are known to be already +valid. To avoid such checks, the ``array`` class and ``array_t<T>`` template +class offer an unchecked proxy object that can be used for this unchecked +access through the ``unchecked<N>`` and ``mutable_unchecked<N>`` methods, +where ``N`` gives the required dimensionality of the array: + +.. code-block:: cpp + + m.def("sum_3d", [](py::array_t<double> x) { + auto r = x.unchecked<3>(); // x must have ndim = 3; can be non-writeable + double sum = 0; + for (size_t i = 0; i < r.shape(0); i++) + for (size_t j = 0; j < r.shape(1); j++) + for (size_t k = 0; k < r.shape(2); k++) + sum += r(i, j, k); + return sum; + }); + m.def("increment_3d", [](py::array_t<double> x) { + auto r = x.mutable_unchecked<3>(); // Will throw if ndim != 3 or flags.writeable is false + for (size_t i = 0; i < r.shape(0); i++) + for (size_t j = 0; j < r.shape(1); j++) + for (size_t k = 0; k < r.shape(2); k++) + r(i, j, k) += 1.0; + }, py::arg().noconvert()); + +To obtain the proxy from an ``array`` object, you must specify both the data +type and number of dimensions as template arguments, such as ``auto r = +myarray.mutable_unchecked<float, 2>()``. + +If the number of dimensions is not known at compile time, you can omit the +dimensions template parameter (i.e. calling ``arr_t.unchecked()`` or +``arr.unchecked<T>()``. This will give you a proxy object that works in the +same way, but results in less optimizable code and thus a small efficiency +loss in tight loops. + +Note that the returned proxy object directly references the array's data, and +only reads its shape, strides, and writeable flag when constructed. You must +take care to ensure that the referenced array is not destroyed or reshaped for +the duration of the returned object, typically by limiting the scope of the +returned instance. + +The returned proxy object supports some of the same methods as ``py::array`` so +that it can be used as a drop-in replacement for some existing, index-checked +uses of ``py::array``: + +- ``r.ndim()`` returns the number of dimensions + +- ``r.data(1, 2, ...)`` and ``r.mutable_data(1, 2, ...)``` returns a pointer to + the ``const T`` or ``T`` data, respectively, at the given indices. The + latter is only available to proxies obtained via ``a.mutable_unchecked()``. + +- ``itemsize()`` returns the size of an item in bytes, i.e. ``sizeof(T)``. + +- ``ndim()`` returns the number of dimensions. + +- ``shape(n)`` returns the size of dimension ``n`` + +- ``size()`` returns the total number of elements (i.e. the product of the shapes). + +- ``nbytes()`` returns the number of bytes used by the referenced elements + (i.e. ``itemsize()`` times ``size()``). + +.. seealso:: + + The file :file:`tests/test_numpy_array.cpp` contains additional examples + demonstrating the use of this feature. diff --git a/ext/pybind11/docs/advanced/pycpp/object.rst b/ext/pybind11/docs/advanced/pycpp/object.rst index 8fc165d16..ae58876de 100644 --- a/ext/pybind11/docs/advanced/pycpp/object.rst +++ b/ext/pybind11/docs/advanced/pycpp/object.rst @@ -33,6 +33,8 @@ The reverse direction uses the following syntax: When conversion fails, both directions throw the exception :class:`cast_error`. +.. _calling_python_functions: + Calling Python functions ======================== @@ -57,7 +59,7 @@ In C++, the same call can be made using: .. code-block:: cpp - using pybind11::literals; // to bring in the `_a` literal + using namespace pybind11::literals; // to bring in the `_a` literal f(1234, "say"_a="hello", "to"_a=some_instance); // keyword call in C++ Unpacking of ``*args`` and ``**kwargs`` is also possible and can be mixed with diff --git a/ext/pybind11/docs/advanced/smart_ptrs.rst b/ext/pybind11/docs/advanced/smart_ptrs.rst index 23072b6bf..e4a238603 100644 --- a/ext/pybind11/docs/advanced/smart_ptrs.rst +++ b/ext/pybind11/docs/advanced/smart_ptrs.rst @@ -123,7 +123,7 @@ Custom smart pointers pybind11 supports ``std::unique_ptr`` and ``std::shared_ptr`` right out of the box. For any other custom smart pointer, transparent conversions can be enabled using a macro invocation similar to the following. It must be declared at the -level before any binding code: +top namespace level before any binding code: .. code-block:: cpp @@ -134,8 +134,42 @@ placeholder name that is used as a template parameter of the second argument. Thus, feel free to use any identifier, but use it consistently on both sides; also, don't use the name of a type that already exists in your codebase. +The macro also accepts a third optional boolean parameter that is set to false +by default. Specify + +.. code-block:: cpp + + PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>, true); + +if ``SmartPtr<T>`` can always be initialized from a ``T*`` pointer without the +risk of inconsistencies (such as multiple independent ``SmartPtr`` instances +believing that they are the sole owner of the ``T*`` pointer). A common +situation where ``true`` should be passed is when the ``T`` instances use +*intrusive* reference counting. + Please take a look at the :ref:`macro_notes` before using this feature. +By default, pybind11 assumes that your custom smart pointer has a standard +interface, i.e. provides a ``.get()`` member function to access the underlying +raw pointer. If this is not the case, pybind11's ``holder_helper`` must be +specialized: + +.. code-block:: cpp + + // Always needed for custom holder types + PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>); + + // Only needed if the type's `.get()` goes by another name + namespace pybind11 { namespace detail { + template <typename T> + struct holder_helper<SmartPtr<T>> { // <-- specialization + static const T *get(const SmartPtr<T> &p) { return p.getPointer(); } + }; + }} + +The above specialization informs pybind11 that the custom ``SmartPtr`` class +provides ``.get()`` functionality via ``.getPointer()``. + .. seealso:: The file :file:`tests/test_smart_ptr.cpp` contains a complete example diff --git a/ext/pybind11/docs/basics.rst b/ext/pybind11/docs/basics.rst index c1dd47412..33c60049d 100644 --- a/ext/pybind11/docs/basics.rst +++ b/ext/pybind11/docs/basics.rst @@ -25,7 +25,7 @@ After installing the prerequisites, run mkdir build cd build cmake .. - make pytest -j 4 + make check -j 4 The last line will both compile and run the tests. @@ -42,7 +42,7 @@ To compile and run the tests: mkdir build cd build cmake .. - cmake --build . --config Release --target pytest + cmake --build . --config Release --target check This will create a Visual Studio project, compile and run the target, all from the command line. diff --git a/ext/pybind11/docs/changelog.rst b/ext/pybind11/docs/changelog.rst index 15cf8d959..aba8a2009 100644 --- a/ext/pybind11/docs/changelog.rst +++ b/ext/pybind11/docs/changelog.rst @@ -3,64 +3,410 @@ Changelog ######### -Starting with version 1.8, pybind11 releases use a -[semantic versioning](http://semver.org) policy. +Starting with version 1.8.0, pybind11 releases use a `semantic versioning +<http://semver.org>`_ policy. -Breaking changes queued for v2.0.0 (Not yet released) + +v2.1.1 (April 7, 2017) ----------------------------------------------------- -* Redesigned virtual call mechanism and user-facing syntax (see - https://github.com/pybind/pybind11/commit/86d825f3302701d81414ddd3d38bcd09433076bc) -* Remove ``handle.call()`` method +* Fixed minimum version requirement for MSVC 2015u3 + `#773 <https://github.com/pybind/pybind11/pull/773>`_. + +v2.1.0 (March 22, 2017) +----------------------------------------------------- + +* pybind11 now performs function overload resolution in two phases. The first + phase only considers exact type matches, while the second allows for implicit + conversions to take place. A special ``noconvert()`` syntax can be used to + completely disable implicit conversions for specific arguments. + `#643 <https://github.com/pybind/pybind11/pull/643>`_, + `#634 <https://github.com/pybind/pybind11/pull/634>`_, + `#650 <https://github.com/pybind/pybind11/pull/650>`_. + +* Fixed a regression where static properties no longer worked with classes + using multiple inheritance. The ``py::metaclass`` attribute is no longer + necessary (and deprecated as of this release) when binding classes with + static properties. + `#679 <https://github.com/pybind/pybind11/pull/679>`_, + +* Classes bound using ``pybind11`` can now use custom metaclasses. + `#679 <https://github.com/pybind/pybind11/pull/679>`_, + +* ``py::args`` and ``py::kwargs`` can now be mixed with other positional + arguments when binding functions using pybind11. + `#611 <https://github.com/pybind/pybind11/pull/611>`_. + +* Improved support for C++11 unicode string and character types; added + extensive documentation regarding pybind11's string conversion behavior. + `#624 <https://github.com/pybind/pybind11/pull/624>`_, + `#636 <https://github.com/pybind/pybind11/pull/636>`_, + `#715 <https://github.com/pybind/pybind11/pull/715>`_. + +* pybind11 can now avoid expensive copies when converting Eigen arrays to NumPy + arrays (and vice versa). `#610 <https://github.com/pybind/pybind11/pull/610>`_. + +* The "fast path" in ``py::vectorize`` now works for any full-size group of C or + F-contiguous arrays. The non-fast path is also faster since it no longer performs + copies of the input arguments (except when type conversions are necessary). + `#610 <https://github.com/pybind/pybind11/pull/610>`_. + +* Added fast, unchecked access to NumPy arrays via a proxy object. + `#746 <https://github.com/pybind/pybind11/pull/746>`_. + +* Transparent support for class-specific ``operator new`` and + ``operator delete`` implementations. + `#755 <https://github.com/pybind/pybind11/pull/755>`_. + +* Slimmer and more efficient STL-compatible iterator interface for sequence types. + `#662 <https://github.com/pybind/pybind11/pull/662>`_. + +* Improved custom holder type support. + `#607 <https://github.com/pybind/pybind11/pull/607>`_. + +* ``nullptr`` to ``None`` conversion fixed in various builtin type casters. + `#732 <https://github.com/pybind/pybind11/pull/732>`_. + +* ``enum_`` now exposes its members via a special ``__members__`` attribute. + `#666 <https://github.com/pybind/pybind11/pull/666>`_. + +* ``std::vector`` bindings created using ``stl_bind.h`` can now optionally + implement the buffer protocol. `#488 <https://github.com/pybind/pybind11/pull/488>`_. + +* Automated C++ reference documentation using doxygen and breathe. + `#598 <https://github.com/pybind/pybind11/pull/598>`_. + +* Added minimum compiler version assertions. + `#727 <https://github.com/pybind/pybind11/pull/727>`_. + +* Improved compatibility with C++1z. + `#677 <https://github.com/pybind/pybind11/pull/677>`_. + +* Improved ``py::capsule`` API. Can be used to implement cleanup + callbacks that are involved at module destruction time. + `#752 <https://github.com/pybind/pybind11/pull/752>`_. + +* Various minor improvements and fixes. + `#595 <https://github.com/pybind/pybind11/pull/595>`_, + `#588 <https://github.com/pybind/pybind11/pull/588>`_, + `#589 <https://github.com/pybind/pybind11/pull/589>`_, + `#603 <https://github.com/pybind/pybind11/pull/603>`_, + `#619 <https://github.com/pybind/pybind11/pull/619>`_, + `#648 <https://github.com/pybind/pybind11/pull/648>`_, + `#695 <https://github.com/pybind/pybind11/pull/695>`_, + `#720 <https://github.com/pybind/pybind11/pull/720>`_, + `#723 <https://github.com/pybind/pybind11/pull/723>`_, + `#729 <https://github.com/pybind/pybind11/pull/729>`_, + `#724 <https://github.com/pybind/pybind11/pull/724>`_, + `#742 <https://github.com/pybind/pybind11/pull/742>`_, + `#753 <https://github.com/pybind/pybind11/pull/753>`_. + +v2.0.1 (Jan 4, 2017) +----------------------------------------------------- + +* Fix pointer to reference error in type_caster on MSVC + `#583 <https://github.com/pybind/pybind11/pull/583>`_. + +* Fixed a segmentation in the test suite due to a typo + `cd7eac <https://github.com/pybind/pybind11/commit/cd7eac>`_. + +v2.0.0 (Jan 1, 2017) +----------------------------------------------------- + +* Fixed a reference counting regression affecting types with custom metaclasses + (introduced in v2.0.0-rc1). + `#571 <https://github.com/pybind/pybind11/pull/571>`_. + +* Quenched a CMake policy warning. + `#570 <https://github.com/pybind/pybind11/pull/570>`_. + +v2.0.0-rc1 (Dec 23, 2016) +----------------------------------------------------- + +The pybind11 developers are excited to issue a release candidate of pybind11 +with a subsequent v2.0.0 release planned in early January next year. + +An incredible amount of effort by went into pybind11 over the last ~5 months, +leading to a release that is jam-packed with exciting new features and numerous +usability improvements. The following list links PRs or individual commits +whenever applicable. + +Happy Christmas! + +* Support for binding C++ class hierarchies that make use of multiple + inheritance. `#410 <https://github.com/pybind/pybind11/pull/410>`_. + +* PyPy support: pybind11 now supports nightly builds of PyPy and will + interoperate with the future 5.7 release. No code changes are necessary, + everything "just" works as usual. Note that we only target the Python 2.7 + branch for now; support for 3.x will be added once its ``cpyext`` extension + support catches up. A few minor features remain unsupported for the time + being (notably dynamic attributes in custom types). + `#527 <https://github.com/pybind/pybind11/pull/527>`_. + +* Significant work on the documentation -- in particular, the monolitic + ``advanced.rst`` file was restructured into a easier to read hierarchical + organization. `#448 <https://github.com/pybind/pybind11/pull/448>`_. + +* Many NumPy-related improvements: + + 1. Object-oriented API to access and modify NumPy ``ndarray`` instances, + replicating much of the corresponding NumPy C API functionality. + `#402 <https://github.com/pybind/pybind11/pull/402>`_. + + 2. NumPy array ``dtype`` array descriptors are now first-class citizens and + are exposed via a new class ``py::dtype``. + + 3. Structured dtypes can be registered using the ``PYBIND11_NUMPY_DTYPE()`` + macro. Special ``array`` constructors accepting dtype objects were also + added. + + One potential caveat involving this change: format descriptor strings + should now be accessed via ``format_descriptor::format()`` (however, for + compatibility purposes, the old syntax ``format_descriptor::value`` will + still work for non-structured data types). `#308 + <https://github.com/pybind/pybind11/pull/308>`_. + + 4. Further improvements to support structured dtypes throughout the system. + `#472 <https://github.com/pybind/pybind11/pull/472>`_, + `#474 <https://github.com/pybind/pybind11/pull/474>`_, + `#459 <https://github.com/pybind/pybind11/pull/459>`_, + `#453 <https://github.com/pybind/pybind11/pull/453>`_, + `#452 <https://github.com/pybind/pybind11/pull/452>`_, and + `#505 <https://github.com/pybind/pybind11/pull/505>`_. + + 5. Fast access operators. `#497 <https://github.com/pybind/pybind11/pull/497>`_. + + 6. Constructors for arrays whose storage is owned by another object. + `#440 <https://github.com/pybind/pybind11/pull/440>`_. + + 7. Added constructors for ``array`` and ``array_t`` explicitly accepting shape + and strides; if strides are not provided, they are deduced assuming + C-contiguity. Also added simplified constructors for 1-dimensional case. + + 8. Added buffer/NumPy support for ``char[N]`` and ``std::array<char, N>`` types. + + 9. Added ``memoryview`` wrapper type which is constructible from ``buffer_info``. + +* Eigen: many additional conversions and support for non-contiguous + arrays/slices. + `#427 <https://github.com/pybind/pybind11/pull/427>`_, + `#315 <https://github.com/pybind/pybind11/pull/315>`_, + `#316 <https://github.com/pybind/pybind11/pull/316>`_, + `#312 <https://github.com/pybind/pybind11/pull/312>`_, and + `#267 <https://github.com/pybind/pybind11/pull/267>`_ + +* Incompatible changes in ``class_<...>::class_()``: + + 1. Declarations of types that provide access via the buffer protocol must + now include the ``py::buffer_protocol()`` annotation as an argument to + the ``class_`` constructor. + + 2. Declarations of types that require a custom metaclass (i.e. all classes + which include static properties via commands such as + ``def_readwrite_static()``) must now include the ``py::metaclass()`` + annotation as an argument to the ``class_`` constructor. + + These two changes were necessary to make type definitions in pybind11 + future-proof, and to support PyPy via its cpyext mechanism. `#527 + <https://github.com/pybind/pybind11/pull/527>`_. + + + 3. This version of pybind11 uses a redesigned mechnism for instantiating + trempoline classes that are used to override virtual methods from within + Python. This led to the following user-visible syntax change: instead of + + .. code-block:: cpp + + py::class_<TrampolineClass>("MyClass") + .alias<MyClass>() + .... + + write + + .. code-block:: cpp + + py::class_<MyClass, TrampolineClass>("MyClass") + .... + + Importantly, both the original and the trampoline class are now + specified as an arguments (in arbitrary order) to the ``py::class_`` + template, and the ``alias<..>()`` call is gone. The new scheme has zero + overhead in cases when Python doesn't override any functions of the + underlying C++ class. `rev. 86d825 + <https://github.com/pybind/pybind11/commit/86d825>`_. -1.9.0 (Not yet released) ------------------------- -* Queued changes: map indexing suite, documentation for indexing suites. -* Mapping a stateless C++ function to Python and back is now "for free" (i.e. no call overheads) -* Support for translation of arbitrary C++ exceptions to Python counterparts * Added ``eval`` and ``eval_file`` functions for evaluating expressions and - statements from a string or file -* eigen.h type converter fixed for non-contiguous arrays (e.g. slices) -* Print more informative error messages when ``make_tuple()`` or ``cast()`` fail -* ``std::enable_shared_from_this<>`` now also works for ``const`` values -* A return value policy can now be passed to ``handle::operator()`` -* ``make_iterator()`` improvements for better compatibility with various types - (now uses prefix increment operator); it now also accepts iterators with - different begin/end types as long as they are equality comparable. -* ``arg()`` now accepts a wider range of argument types for default values -* Added ``py::repr()`` function which is equivalent to Python's builtin ``repr()``. -* Added support for registering structured dtypes via ``PYBIND11_NUMPY_DTYPE()`` macro. -* Added ``PYBIND11_STR_TYPE`` macro which maps to the ``builtins.str`` type. -* Added a simplified ``buffer_info`` constructor for 1-dimensional buffers. -* Format descriptor strings should now be accessed via ``format_descriptor::format()`` - (for compatibility purposes, the old syntax ``format_descriptor::value`` will still - work for non-structured data types). -* Added a class wrapping NumPy array descriptors: ``dtype``. -* Added buffer/NumPy support for ``char[N]`` and ``std::array<char, N>`` types. -* ``array`` gained new constructors accepting dtype objects. -* Added constructors for ``array`` and ``array_t`` explicitly accepting shape and - strides; if strides are not provided, they are deduced assuming C-contiguity. - Also added simplified constructors for 1-dimensional case. -* Added constructors for ``str`` from ``bytes`` and for ``bytes`` from ``str``. - This will do the UTF-8 decoding/encoding as required. -* Added constructors for ``str`` and ``bytes`` from zero-terminated char pointers, - and from char pointers and length. -* Added ``memoryview`` wrapper type which is constructible from ``buffer_info``. -* New syntax to call a Python function from C++ using keyword arguments and unpacking, - e.g. ``foo(1, 2, "z"_a=3)`` or ``bar(1, *args, "z"_a=3, **kwargs)``. -* Added ``py::print()`` function which replicates Python's API and writes to Python's - ``sys.stdout`` by default (as opposed to C's ``stdout`` like ``std::cout``). -* Added ``py::dict`` keyword constructor:``auto d = dict("number"_a=42, "name"_a="World");`` -* Added ``py::str::format()`` method and ``_s`` literal: - ``py::str s = "1 + 2 = {}"_s.format(3);`` -* Attribute and item accessors now have a more complete interface which makes it possible - to chain attributes ``obj.attr("a")[key].attr("b").attr("method")(1, 2, 3)```. -* Added built-in support for ``std::shared_ptr`` holder type. There is no more need - to do it manually via ``PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>)``. -* Default return values policy changes: non-static properties now use ``reference_internal`` - and static properties use ``reference`` (previous default was ``automatic``, i.e. ``copy``). -* Support for ``std::experimental::optional<T>`` and ``std::optional<T>`` (C++17). -* Various minor improvements of library internals (no user-visible changes) + statements from a string or file. `rev. 0d3fc3 + <https://github.com/pybind/pybind11/commit/0d3fc3>`_. + +* pybind11 can now create types with a modifiable dictionary. + `#437 <https://github.com/pybind/pybind11/pull/437>`_ and + `#444 <https://github.com/pybind/pybind11/pull/444>`_. + +* Support for translation of arbitrary C++ exceptions to Python counterparts. + `#296 <https://github.com/pybind/pybind11/pull/296>`_ and + `#273 <https://github.com/pybind/pybind11/pull/273>`_. + +* Report full backtraces through mixed C++/Python code, better reporting for + import errors, fixed GIL management in exception processing. + `#537 <https://github.com/pybind/pybind11/pull/537>`_, + `#494 <https://github.com/pybind/pybind11/pull/494>`_, + `rev. e72d95 <https://github.com/pybind/pybind11/commit/e72d95>`_, and + `rev. 099d6e <https://github.com/pybind/pybind11/commit/099d6e>`_. + +* Support for bit-level operations, comparisons, and serialization of C++ + enumerations. `#503 <https://github.com/pybind/pybind11/pull/503>`_, + `#508 <https://github.com/pybind/pybind11/pull/508>`_, + `#380 <https://github.com/pybind/pybind11/pull/380>`_, + `#309 <https://github.com/pybind/pybind11/pull/309>`_. + `#311 <https://github.com/pybind/pybind11/pull/311>`_. + +* The ``class_`` constructor now accepts its template arguments in any order. + `#385 <https://github.com/pybind/pybind11/pull/385>`_. + +* Attribute and item accessors now have a more complete interface which makes + it possible to chain attributes as in + ``obj.attr("a")[key].attr("b").attr("method")(1, 2, 3)``. `#425 + <https://github.com/pybind/pybind11/pull/425>`_. + +* Major redesign of the default and conversion constructors in ``pytypes.h``. + `#464 <https://github.com/pybind/pybind11/pull/464>`_. + +* Added built-in support for ``std::shared_ptr`` holder type. It is no longer + necessary to to include a declaration of the form + ``PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>)`` (though continuing to + do so won't cause an error). + `#454 <https://github.com/pybind/pybind11/pull/454>`_. + +* New ``py::overload_cast`` casting operator to select among multiple possible + overloads of a function. An example: + + .. code-block:: cpp + + py::class_<Pet>(m, "Pet") + .def("set", py::overload_cast<int>(&Pet::set), "Set the pet's age") + .def("set", py::overload_cast<const std::string &>(&Pet::set), "Set the pet's name"); + + This feature only works on C++14-capable compilers. + `#541 <https://github.com/pybind/pybind11/pull/541>`_. + +* C++ types are automatically cast to Python types, e.g. when assigning + them as an attribute. For instance, the following is now legal: + + .. code-block:: cpp + + py::module m = /* ... */ + m.attr("constant") = 123; + + (Previously, a ``py::cast`` call was necessary to avoid a compilation error.) + `#551 <https://github.com/pybind/pybind11/pull/551>`_. + +* Redesigned ``pytest``-based test suite. `#321 <https://github.com/pybind/pybind11/pull/321>`_. + +* Instance tracking to detect reference leaks in test suite. `#324 <https://github.com/pybind/pybind11/pull/324>`_ + +* pybind11 can now distinguish between multiple different instances that are + located at the same memory address, but which have different types. + `#329 <https://github.com/pybind/pybind11/pull/329>`_. + +* Improved logic in ``move`` return value policy. + `#510 <https://github.com/pybind/pybind11/pull/510>`_, + `#297 <https://github.com/pybind/pybind11/pull/297>`_. + +* Generalized unpacking API to permit calling Python functions from C++ using + notation such as ``foo(a1, a2, *args, "ka"_a=1, "kb"_a=2, **kwargs)``. `#372 <https://github.com/pybind/pybind11/pull/372>`_. + +* ``py::print()`` function whose behavior matches that of the native Python + ``print()`` function. `#372 <https://github.com/pybind/pybind11/pull/372>`_. + +* Added ``py::dict`` keyword constructor:``auto d = dict("number"_a=42, + "name"_a="World");``. `#372 <https://github.com/pybind/pybind11/pull/372>`_. + +* Added ``py::str::format()`` method and ``_s`` literal: ``py::str s = "1 + 2 + = {}"_s.format(3);``. `#372 <https://github.com/pybind/pybind11/pull/372>`_. + +* Added ``py::repr()`` function which is equivalent to Python's builtin + ``repr()``. `#333 <https://github.com/pybind/pybind11/pull/333>`_. + +* Improved construction and destruction logic for holder types. It is now + possible to reference instances with smart pointer holder types without + constructing the holder if desired. The ``PYBIND11_DECLARE_HOLDER_TYPE`` + macro now accepts an optional second parameter to indicate whether the holder + type uses intrusive reference counting. + `#533 <https://github.com/pybind/pybind11/pull/533>`_ and + `#561 <https://github.com/pybind/pybind11/pull/561>`_. + +* Mapping a stateless C++ function to Python and back is now "for free" (i.e. + no extra indirections or argument conversion overheads). `rev. 954b79 + <https://github.com/pybind/pybind11/commit/954b79>`_. + +* Bindings for ``std::valarray<T>``. + `#545 <https://github.com/pybind/pybind11/pull/545>`_. + +* Improved support for C++17 capable compilers. + `#562 <https://github.com/pybind/pybind11/pull/562>`_. + +* Bindings for ``std::optional<t>``. + `#475 <https://github.com/pybind/pybind11/pull/475>`_, + `#476 <https://github.com/pybind/pybind11/pull/476>`_, + `#479 <https://github.com/pybind/pybind11/pull/479>`_, + `#499 <https://github.com/pybind/pybind11/pull/499>`_, and + `#501 <https://github.com/pybind/pybind11/pull/501>`_. + +* ``stl_bind.h``: general improvements and support for ``std::map`` and + ``std::unordered_map``. + `#490 <https://github.com/pybind/pybind11/pull/490>`_, + `#282 <https://github.com/pybind/pybind11/pull/282>`_, + `#235 <https://github.com/pybind/pybind11/pull/235>`_. + +* The ``std::tuple``, ``std::pair``, ``std::list``, and ``std::vector`` type + casters now accept any Python sequence type as input. `rev. 107285 + <https://github.com/pybind/pybind11/commit/107285>`_. + +* Improved CMake Python detection on multi-architecture Linux. + `#532 <https://github.com/pybind/pybind11/pull/532>`_. + +* Infrastructure to selectively disable or enable parts of the automatically + generated docstrings. `#486 <https://github.com/pybind/pybind11/pull/486>`_. + +* ``reference`` and ``reference_internal`` are now the default return value + properties for static and non-static properties, respectively. `#473 + <https://github.com/pybind/pybind11/pull/473>`_. (the previous defaults + were ``automatic``). `#473 <https://github.com/pybind/pybind11/pull/473>`_. + +* Support for ``std::unique_ptr`` with non-default deleters or no deleter at + all (``py::nodelete``). `#384 <https://github.com/pybind/pybind11/pull/384>`_. + +* Deprecated ``handle::call()`` method. The new syntax to call Python + functions is simply ``handle()``. It can also be invoked explicitly via + ``handle::operator<X>()``, where ``X`` is an optional return value policy. + +* Print more informative error messages when ``make_tuple()`` or ``cast()`` + fail. `#262 <https://github.com/pybind/pybind11/pull/262>`_. + +* Creation of holder types for classes deriving from + ``std::enable_shared_from_this<>`` now also works for ``const`` values. + `#260 <https://github.com/pybind/pybind11/pull/260>`_. + +* ``make_iterator()`` improvements for better compatibility with various + types (now uses prefix increment operator); it now also accepts iterators + with different begin/end types as long as they are equality comparable. + `#247 <https://github.com/pybind/pybind11/pull/247>`_. + +* ``arg()`` now accepts a wider range of argument types for default values. + `#244 <https://github.com/pybind/pybind11/pull/244>`_. + +* Support ``keep_alive`` where the nurse object may be ``None``. `#341 + <https://github.com/pybind/pybind11/pull/341>`_. + +* Added constructors for ``str`` and ``bytes`` from zero-terminated char + pointers, and from char pointers and length. Added constructors for ``str`` + from ``bytes`` and for ``bytes`` from ``str``, which will perform UTF-8 + decoding/encoding as required. + +* Many other improvements of library internals without user-visible changes + 1.8.1 (July 12, 2016) ---------------------- diff --git a/ext/pybind11/docs/classes.rst b/ext/pybind11/docs/classes.rst index 872977684..30fb2a2d5 100644 --- a/ext/pybind11/docs/classes.rst +++ b/ext/pybind11/docs/classes.rst @@ -38,7 +38,7 @@ The binding code for ``Pet`` looks as follows: return m.ptr(); } -:class:`class_` creates bindings for a C++ `class` or `struct`-style data +:class:`class_` creates bindings for a C++ *class* or *struct*-style data structure. :func:`init` is a convenience function that takes the types of a constructor's parameters as template arguments and wraps the corresponding constructor (see the :ref:`custom_constructors` section for details). An @@ -298,8 +298,8 @@ different kinds of input arguments: struct Pet { Pet(const std::string &name, int age) : name(name), age(age) { } - void set(int age) { age = age; } - void set(const std::string &name) { name = name; } + void set(int age_) { age = age_; } + void set(const std::string &name_) { name = name_; } std::string name; int age; @@ -423,6 +423,12 @@ typed enums. >>> int(p.type) 1L +The entries defined by the enumeration type are exposed in the ``__members__`` property: + +.. code-block:: pycon + + >>> Pet.Kind.__members__ + {'Dog': Kind.Dog, 'Cat': Kind.Cat} .. note:: diff --git a/ext/pybind11/docs/compiling.rst b/ext/pybind11/docs/compiling.rst index 35768fe45..c7053dbf9 100644 --- a/ext/pybind11/docs/compiling.rst +++ b/ext/pybind11/docs/compiling.rst @@ -39,30 +39,88 @@ extension module can be created with just a few lines of code: This assumes that the pybind11 repository is located in a subdirectory named :file:`pybind11` and that the code is located in a file named :file:`example.cpp`. -The CMake command ``add_subdirectory`` will import a function with the signature -``pybind11_add_module(<name> source1 [source2 ...])``. It will take care of all -the details needed to build a Python extension module on any platform. - -The target Python version can be selected by setting the ``PYBIND11_PYTHON_VERSION`` -variable before adding the pybind11 subdirectory. Alternatively, an exact Python -installation can be specified by setting ``PYTHON_EXECUTABLE``. +The CMake command ``add_subdirectory`` will import the pybind11 project which +provides the ``pybind11_add_module`` function. It will take care of all the +details needed to build a Python extension module on any platform. A working sample project, including a way to invoke CMake from :file:`setup.py` for PyPI integration, can be found in the [cmake_example]_ repository. .. [cmake_example] https://github.com/pybind/cmake_example -For CMake-based projects that don't include the pybind11 -repository internally, an external installation can be detected -through `find_package(pybind11 ... CONFIG ...)`. See the `Config file -<https://github.com/pybind/pybind11/blob/master/tools/pybind11Config.cmake.in>`_ -docstring for details of relevant CMake variables. +pybind11_add_module +------------------- -Once detected, and after setting any variables to guide Python and -C++ standard detection, the aforementioned ``pybind11_add_module`` -wrapper to ``add_library`` can be employed as described above (after -``include(pybind11Tools)``). This procedure is available when using CMake ->= 2.8.12. A working example can be found at [test_installed_module]_ . +To ease the creation of Python extension modules, pybind11 provides a CMake +function with the following signature: + +.. code-block:: cmake + + pybind11_add_module(<name> [MODULE | SHARED] [EXCLUDE_FROM_ALL] + [NO_EXTRAS] [THIN_LTO] source1 [source2 ...]) + +This function behaves very much like CMake's builtin ``add_library`` (in fact, +it's a wrapper function around that command). It will add a library target +called ``<name>`` to be built from the listed source files. In addition, it +will take care of all the Python-specific compiler and linker flags as well +as the OS- and Python-version-specific file extension. The produced target +``<name>`` can be further manipulated with regular CMake commands. + +``MODULE`` or ``SHARED`` may be given to specify the type of library. If no +type is given, ``MODULE`` is used by default which ensures the creation of a +Python-exclusive module. Specifying ``SHARED`` will create a more traditional +dynamic library which can also be linked from elsewhere. ``EXCLUDE_FROM_ALL`` +removes this target from the default build (see CMake docs for details). + +Since pybind11 is a template library, ``pybind11_add_module`` adds compiler +flags to ensure high quality code generation without bloat arising from long +symbol names and duplication of code in different translation units. The +additional flags enable LTO (Link Time Optimization), set default visibility +to *hidden* and strip unneeded symbols. See the :ref:`FAQ entry <faq:symhidden>` +for a more detailed explanation. These optimizations are never applied in +``Debug`` mode. If ``NO_EXTRAS`` is given, they will always be disabled, even +in ``Release`` mode. However, this will result in code bloat and is generally +not recommended. + +As stated above, LTO is enabled by default. Some newer compilers also support +different flavors of LTO such as `ThinLTO`_. Setting ``THIN_LTO`` will cause +the function to prefer this flavor if available. The function falls back to +regular LTO if ``-flto=thin`` is not available. + +.. _ThinLTO: http://clang.llvm.org/docs/ThinLTO.html + +Configuration variables +----------------------- + +By default, pybind11 will compile modules with the latest C++ standard +available on the target compiler. To override this, the standard flag can +be given explicitly in ``PYBIND11_CPP_STANDARD``: + +.. code-block:: cmake + + set(PYBIND11_CPP_STANDARD -std=c++11) + add_subdirectory(pybind11) # or find_package(pybind11) + +Note that this and all other configuration variables must be set **before** the +call to ``add_subdiretory`` or ``find_package``. The variables can also be set +when calling CMake from the command line using the ``-D<variable>=<value>`` flag. + +The target Python version can be selected by setting ``PYBIND11_PYTHON_VERSION`` +or an exact Python installation can be specified with ``PYTHON_EXECUTABLE``. +For example: + +.. code-block:: bash + + cmake -DPYBIND11_PYTHON_VERSION=3.6 .. + # or + cmake -DPYTHON_EXECUTABLE=path/to/python .. + +find_package vs. add_subdirectory +--------------------------------- + +For CMake-based projects that don't include the pybind11 repository internally, +an external installation can be detected through ``find_package(pybind11)``. +See the `Config file`_ docstring for details of relevant CMake variables. .. code-block:: cmake @@ -72,28 +130,33 @@ wrapper to ``add_library`` can be employed as described above (after find_package(pybind11 REQUIRED) pybind11_add_module(example example.cpp) -.. [test_installed_module] https://github.com/pybind/pybind11/blob/master/tests/test_installed_module/CMakeLists.txt +Once detected, the aforementioned ``pybind11_add_module`` can be employed as +before. The function usage and configuration variables are identical no matter +if pybind11 is added as a subdirectory or found as an installed package. You +can refer to the same [cmake_example]_ repository for a full sample project +-- just swap out ``add_subdirectory`` for ``find_package``. + +.. _Config file: https://github.com/pybind/pybind11/blob/master/tools/pybind11Config.cmake.in + +Advanced: interface library target +---------------------------------- -When using a version of CMake greater than 3.0, pybind11 can -additionally be used as a special *interface library* following the -call to ``find_package``. CMake variables to guide Python and C++ -standard detection should be set *before* ``find_package``. When -``find_package`` returns, the target ``pybind11::pybind11`` is -available with pybind11 headers, Python headers and libraries as -needed, and C++ compile definitions attached. This target is suitable -for linking to an independently constructed (through ``add_library``, -not ``pybind11_add_module``) target in the consuming project. A working -example can be found at [test_installed_target]_ . +When using a version of CMake greater than 3.0, pybind11 can additionally +be used as a special *interface library* . The target ``pybind11::module`` +is available with pybind11 headers, Python headers and libraries as needed, +and C++ compile definitions attached. This target is suitable for linking +to an independently constructed (through ``add_library``, not +``pybind11_add_module``) target in the consuming project. .. code-block:: cmake cmake_minimum_required(VERSION 3.0) project(example) - add_library(example MODULE main.cpp) + find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) - find_package(pybind11 REQUIRED) - target_link_libraries(example PRIVATE pybind11::pybind11) + add_library(example MODULE main.cpp) + target_link_libraries(example PRIVATE pybind11::module) set_target_properties(example PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" SUFFIX "${PYTHON_MODULE_EXTENSION}") @@ -112,5 +175,11 @@ example can be found at [test_installed_target]_ . (``/bigobj``). The :ref:`FAQ <faq:symhidden>` contains an explanation on why these are needed. -.. [test_installed_target] https://github.com/pybind/pybind11/blob/master/tests/test_installed_target/CMakeLists.txt +Generating binding code automatically +===================================== + +The ``Binder`` project is a tool for automatic generation of pybind11 binding +code by introspecting existing C++ codebases using LLVM/Clang. See the +[binder]_ documentation for details. +.. [binder] http://cppbinder.readthedocs.io/en/latest/about.html diff --git a/ext/pybind11/docs/conf.py b/ext/pybind11/docs/conf.py index 69f0cb337..09604cfeb 100644 --- a/ext/pybind11/docs/conf.py +++ b/ext/pybind11/docs/conf.py @@ -16,6 +16,7 @@ import sys import os import shlex +import subprocess # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -30,7 +31,11 @@ import shlex # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ['breathe'] + +breathe_projects = {'pybind11': '.build/doxygenxml/'} +breathe_default_project = 'pybind11' +breathe_domain_by_extension = {'h': 'cpp'} # Add any paths that contain templates here, relative to this directory. templates_path = ['.templates'] @@ -48,7 +53,7 @@ master_doc = 'index' # General information about the project. project = 'pybind11' -copyright = '2015, Wenzel Jakob' +copyright = '2016, Wenzel Jakob' author = 'Wenzel Jakob' # The version info for the project you're documenting, acts as replacement for @@ -56,9 +61,9 @@ author = 'Wenzel Jakob' # built documents. # # The short X.Y version. -version = '1.9' +version = '2.1' # The full version, including alpha/beta/rc tags. -release = '1.9.dev0' +release = '2.1.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -79,7 +84,7 @@ exclude_patterns = ['.build', 'release.rst'] # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +default_role = 'any' # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True @@ -306,3 +311,22 @@ texinfo_documents = [ primary_domain = 'cpp' highlight_language = 'cpp' + + +def generate_doxygen_xml(app): + build_dir = '.build' + if not os.path.exists(build_dir): + os.mkdir(build_dir) + + try: + subprocess.call(['doxygen', '--version']) + retcode = subprocess.call(['doxygen']) + if retcode < 0: + sys.stderr.write("doxygen error code: {}\n".format(-retcode)) + except OSError as e: + sys.stderr.write("doxygen execution failed: {}\n".format(e)) + + +def setup(app): + """Add hook for building doxygen xml when needed""" + app.connect("builder-inited", generate_doxygen_xml) diff --git a/ext/pybind11/docs/intro.rst b/ext/pybind11/docs/intro.rst index 429a01cdd..2149c18db 100644 --- a/ext/pybind11/docs/intro.rst +++ b/ext/pybind11/docs/intro.rst @@ -17,15 +17,14 @@ compatibility has its cost: arcane template tricks and workarounds are necessary to support the oldest and buggiest of compiler specimens. Now that C++11-compatible compilers are widely available, this heavy machinery has become an excessively large and unnecessary dependency. - Think of this library as a tiny self-contained version of Boost.Python with everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~2.5K lines of code and depend on -Python (2.7 or 3.x) and the C++ standard library. This compact implementation -was possible thanks to some of the new C++11 language features (specifically: -tuples, lambda functions and variadic templates). Since its creation, this -library has grown beyond Boost.Python in many ways, leading to dramatically -simpler binding code in many common situations. +comments, the core header files only require ~4K lines of code and depend on +Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This +compact implementation was possible thanks to some of the new C++11 language +features (specifically: tuples, lambda functions and variadic templates). Since +its creation, this library has grown beyond Boost.Python in many ways, leading +to dramatically simpler binding code in many common situations. Core features ************* @@ -51,6 +50,9 @@ Goodies ******* In addition to the core functionality, pybind11 provides some extra goodies: +- Python 2.7, 3.x, and PyPy (PyPy2.7 >= 5.7) are supported with an + implementation-agnostic interface. + - It is possible to bind C++11 lambda functions with captured variables. The lambda capture data is stored inside the resulting Python function object. @@ -88,6 +90,6 @@ Supported compilers ******************* 1. Clang/LLVM (any non-ancient version with C++11 support) -2. GCC (any non-ancient version with C++11 support) +2. GCC 4.8 or newer 3. Microsoft Visual Studio 2015 or newer 4. Intel C++ compiler v15 or newer diff --git a/ext/pybind11/docs/reference.rst b/ext/pybind11/docs/reference.rst index 542259eba..3d211f7e9 100644 --- a/ext/pybind11/docs/reference.rst +++ b/ext/pybind11/docs/reference.rst @@ -12,236 +12,69 @@ Reference Macros ====== -.. function:: PYBIND11_PLUGIN(const char *name) - - This macro creates the entry point that will be invoked when the Python - interpreter imports a plugin library. Please create a - :class:`module` in the function body and return the pointer to its - underlying Python object at the end. - - .. code-block:: cpp - - PYBIND11_PLUGIN(example) { - pybind11::module m("example", "pybind11 example plugin"); - /// Set up bindings here - return m.ptr(); - } +.. doxygendefine:: PYBIND11_PLUGIN .. _core_types: Convenience classes for arbitrary Python types ============================================== -Without reference counting --------------------------- - -.. class:: handle - - The :class:`handle` class is a thin wrapper around an arbitrary Python - object (i.e. a ``PyObject *`` in Python's C API). It does not perform any - automatic reference counting and merely provides a basic C++ interface to - various Python API functions. - -.. seealso:: - - The :class:`object` class inherits from :class:`handle` and adds automatic - reference counting features. - -.. function:: handle::handle() - - The default constructor creates a handle with a ``nullptr``-valued pointer. - -.. function:: handle::handle(const handle&) - - Copy constructor - -.. function:: handle::handle(PyObject *) - - Creates a :class:`handle` from the given raw Python object pointer. - -.. function:: PyObject * handle::ptr() const - - Return the ``PyObject *`` underlying a :class:`handle`. - -.. function:: const handle& handle::inc_ref() const - - Manually increase the reference count of the Python object. Usually, it is - preferable to use the :class:`object` class which derives from - :class:`handle` and calls this function automatically. Returns a reference - to itself. - -.. function:: const handle& handle::dec_ref() const - - Manually decrease the reference count of the Python object. Usually, it is - preferable to use the :class:`object` class which derives from - :class:`handle` and calls this function automatically. Returns a reference - to itself. - -.. function:: void handle::ref_count() const - - Return the object's current reference count - -.. function:: handle handle::get_type() const - - Return a handle to the Python type object underlying the instance - -.. function detail::accessor handle::operator[](handle key) const - - Return an internal functor to invoke the object's sequence protocol. - Casting the returned ``detail::accessor`` instance to a :class:`handle` or - :class:`object` subclass causes a corresponding call to ``__getitem__``. - Assigning a :class:`handle` or :class:`object` subclass causes a call to - ``__setitem__``. - -.. function detail::accessor handle::operator[](const char *key) const - - See the above function (the only difference is that they key is provided as - a string literal). - -.. function detail::accessor handle::attr(handle key) const - - Return an internal functor to access the object's attributes. - Casting the returned ``detail::accessor`` instance to a :class:`handle` or - :class:`object` subclass causes a corresponding call to ``__getattr``. - Assigning a :class:`handle` or :class:`object` subclass causes a call to - ``__setattr``. - -.. function detail::accessor handle::attr(const char *key) const - - See the above function (the only difference is that they key is provided as - a string literal). - -.. function operator handle::bool() const - - Return ``true`` when the :class:`handle` wraps a valid Python object. - -.. function str handle::str() const - - Return a string representation of the object. This is analogous to - the ``str()`` function in Python. - -.. function:: template <typename T> T handle::cast() const - - Attempt to cast the Python object into the given C++ type. A - :class:`cast_error` will be throw upon failure. +Common member functions +----------------------- -.. function:: template <typename ... Args> object handle::call(Args&&... args) const +.. doxygenclass:: object_api + :members: - Assuming the Python object is a function or implements the ``__call__`` - protocol, ``call()`` invokes the underlying function, passing an arbitrary - set of parameters. The result is returned as a :class:`object` and may need - to be converted back into a Python object using :func:`handle::cast`. +Without reference counting +-------------------------- - When some of the arguments cannot be converted to Python objects, the - function will throw a :class:`cast_error` exception. When the Python - function call fails, a :class:`error_already_set` exception is thrown. +.. doxygenclass:: handle + :members: With reference counting ----------------------- -.. class:: object : public handle - - Like :class:`handle`, the object class is a thin wrapper around an - arbitrary Python object (i.e. a ``PyObject *`` in Python's C API). In - contrast to :class:`handle`, it optionally increases the object's reference - count upon construction, and it *always* decreases the reference count when - the :class:`object` instance goes out of scope and is destructed. When - using :class:`object` instances consistently, it is much easier to get - reference counting right at the first attempt. - -.. function:: object::object(const object &o) - - Copy constructor; always increases the reference count - -.. function:: object::object(const handle &h, bool borrowed) - - Creates a :class:`object` from the given :class:`handle`. The reference - count is only increased if the ``borrowed`` parameter is set to ``true``. - -.. function:: object::object(PyObject *ptr, bool borrowed) - - Creates a :class:`object` from the given raw Python object pointer. The - reference count is only increased if the ``borrowed`` parameter is set to - ``true``. - -.. function:: object::object(object &&other) - - Move constructor; steals the object from ``other`` and preserves its - reference count. - -.. function:: handle object::release() +.. doxygenclass:: object + :members: - Resets the internal pointer to ``nullptr`` without without decreasing the - object's reference count. The function returns a raw handle to the original - Python object. +.. doxygenfunction:: reinterpret_borrow -.. function:: object::~object() - - Destructor, which automatically calls :func:`handle::dec_ref()`. +.. doxygenfunction:: reinterpret_steal Convenience classes for specific Python types ============================================= +.. doxygenclass:: module + :members: -.. class:: module : public object - -.. function:: module::module(const char *name, const char *doc = nullptr) - - Create a new top-level Python module with the given name and docstring - -.. function:: module module::def_submodule(const char *name, const char *doc = nullptr) - - Create and return a new Python submodule with the given name and docstring. - This also works recursively, i.e. - - .. code-block:: cpp - - pybind11::module m("example", "pybind11 example plugin"); - pybind11::module m2 = m.def_submodule("sub", "A submodule of 'example'"); - pybind11::module m3 = m2.def_submodule("subsub", "A submodule of 'example.sub'"); - -.. cpp:function:: template <typename Func, typename ... Extra> module& module::def(const char *name, Func && f, Extra && ... extra) - - Create Python binding for a new function within the module scope. ``Func`` - can be a plain C++ function, a function pointer, or a lambda function. For - details on the ``Extra&& ... extra`` argument, see section :ref:`extras`. +.. doxygengroup:: pytypes + :members: .. _extras: -Passing extra arguments to the def function -=========================================== - -.. class:: arg - -.. function:: arg::arg(const char *name) - -.. function:: template <typename T> arg_v arg::operator=(T &&value) - -.. class:: arg_v : public arg - - Represents a named argument with a default value - -.. class:: sibling - - Used to specify a handle to an existing sibling function; used internally - to implement function overloading in :func:`module::def` and - :func:`class_::def`. - -.. function:: sibling::sibling(handle handle) +Passing extra arguments to ``def`` or ``class_`` +================================================ -.. class doc +.. doxygengroup:: annotations + :members: - This is class is internally used by pybind11. +Python build-in functions +========================= -.. function:: doc::doc(const char *value) +.. doxygengroup:: python_builtins + :members: - Create a new docstring with the specified value +Exceptions +========== -.. class name +.. doxygenclass:: error_already_set + :members: - This is class is internally used by pybind11. +.. doxygenclass:: builtin_exception + :members: -.. function:: name::name(const char *value) - Used to specify the function name +Literals +======== +.. doxygennamespace:: literals diff --git a/ext/pybind11/docs/release.rst b/ext/pybind11/docs/release.rst index a15504ea8..30d159a6f 100644 --- a/ext/pybind11/docs/release.rst +++ b/ext/pybind11/docs/release.rst @@ -1,8 +1,10 @@ To release a new version of pybind11: - Update the version number and push to pypi - - Update ``pybind11/_version.py`` (set release version, remove 'dev') - - Update version in ``docs/conf.py`` + - Update ``pybind11/_version.py`` (set release version, remove 'dev'). + - Update ``PYBIND11_VERSION_MAJOR`` etc. in ``include/pybind11/common.h``. + - Ensure that all the information in ``setup.py`` is up-to-date. + - Update version in ``docs/conf.py``. - Tag release date in ``docs/changelog.rst``. - ``git add`` and ``git commit``. - if new minor version: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` diff --git a/ext/pybind11/docs/requirements.txt b/ext/pybind11/docs/requirements.txt new file mode 100644 index 000000000..3818fe80e --- /dev/null +++ b/ext/pybind11/docs/requirements.txt @@ -0,0 +1 @@ +breathe == 4.5.0 diff --git a/ext/pybind11/include/pybind11/attr.h b/ext/pybind11/include/pybind11/attr.h index 448612c52..e38a1a32d 100644 --- a/ext/pybind11/include/pybind11/attr.h +++ b/ext/pybind11/include/pybind11/attr.h @@ -14,6 +14,9 @@ NAMESPACE_BEGIN(pybind11) +/// \addtogroup annotations +/// @{ + /// Annotation for methods struct is_method { handle class_; is_method(const handle &c) : class_(c) { } }; @@ -39,7 +42,7 @@ template <typename T> struct base { }; /// Keep patient alive while nurse lives -template <int Nurse, int Patient> struct keep_alive { }; +template <size_t Nurse, size_t Patient> struct keep_alive { }; /// Annotation indicating that a class is involved in a multiple inheritance relationship struct multiple_inheritance { }; @@ -47,9 +50,25 @@ struct multiple_inheritance { }; /// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class struct dynamic_attr { }; +/// Annotation which enables the buffer protocol for a type +struct buffer_protocol { }; + +/// Annotation which requests that a special metaclass is created for a type +struct metaclass { + handle value; + + PYBIND11_DEPRECATED("py::metaclass() is no longer required. It's turned on by default now.") + metaclass() = default; + + /// Override pybind11's default metaclass + explicit metaclass(handle value) : value(value) { } +}; + /// Annotation to mark enums as an arithmetic type struct arithmetic { }; +/// @} annotations + NAMESPACE_BEGIN(detail) /* Forward declarations */ enum op_id : int; @@ -58,16 +77,17 @@ struct undefined_t; template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> struct op_; template <typename... Args> struct init; template <typename... Args> struct init_alias; -inline void keep_alive_impl(int Nurse, int Patient, handle args, handle ret); +inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret); /// Internal data structure which holds metadata about a keyword argument struct argument_record { const char *name; ///< Argument name const char *descr; ///< Human-readable version of the argument value handle value; ///< Associated Python object + bool convert : 1; ///< True if the argument is allowed to convert when loading - argument_record(const char *name, const char *descr, handle value) - : name(name), descr(descr), value(value) { } + argument_record(const char *name, const char *descr, handle value, bool convert) + : name(name), descr(descr), value(value), convert(convert) { } }; /// Internal data structure which holds metadata about a bound function (signature, overloads, etc.) @@ -89,7 +109,7 @@ struct function_record { std::vector<argument_record> args; /// Pointer to lambda function which converts arguments and performs the actual call - handle (*impl) (function_record *, handle, handle, handle) = nullptr; + handle (*impl) (function_call &) = nullptr; /// Storage for the wrapped function pointer and captured data, if any void *data[3] = { }; @@ -118,8 +138,8 @@ struct function_record { /// True if this is a method bool is_method : 1; - /// Number of arguments - uint16_t nargs; + /// Number of arguments (including py::args and/or py::kwargs, if present) + std::uint16_t nargs; /// Python method object PyMethodDef *def = nullptr; @@ -136,7 +156,8 @@ struct function_record { /// Special data structure which (temporarily) holds metadata about a bound class struct type_record { - PYBIND11_NOINLINE type_record() { } + PYBIND11_NOINLINE type_record() + : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false) { } /// Handle to the parent scope handle scope; @@ -153,6 +174,9 @@ struct type_record { /// How large is pybind11::instance<type>? size_t instance_size = 0; + /// The global operator new can be overridden with a class-specific variant + void *(*operator_new)(size_t) = ::operator new; + /// Function pointer to class_<..>::init_holder void (*init_holder)(PyObject *, const void *) = nullptr; @@ -165,11 +189,20 @@ struct type_record { /// Optional docstring const char *doc = nullptr; + /// Custom metaclass (optional) + handle metaclass; + /// Multiple inheritance marker - bool multiple_inheritance = false; + bool multiple_inheritance : 1; /// Does the class manage a __dict__? - bool dynamic_attr = false; + bool dynamic_attr : 1; + + /// Does the class implement the buffer protocol? + bool buffer_protocol : 1; + + /// Is the default (unique_ptr) holder type used? + bool default_holder : 1; PYBIND11_NOINLINE void add_base(const std::type_info *base, void *(*caster)(void *)) { auto base_info = detail::get_type_info(*base, false); @@ -180,6 +213,15 @@ struct type_record { "\" referenced unknown base type \"" + tname + "\""); } + if (default_holder != base_info->default_holder) { + std::string tname(base->name()); + detail::clean_type_id(tname); + pybind11_fail("generic_type: type \"" + std::string(name) + "\" " + + (default_holder ? "does not have" : "has") + + " a non-default holder type while its base \"" + tname + "\" " + + (base_info->default_holder ? "does not" : "does")); + } + bases.append((PyObject *) base_info->type); if (base_info->type->tp_dictoffset != 0) @@ -190,6 +232,12 @@ struct type_record { } }; +inline function_call::function_call(function_record &f, handle p) : + func(f), parent(p) { + args.reserve(f.nargs); + args_convert.reserve(f.nargs); +} + /** * Partial template specializations to process custom attributes provided to * cpp_function_ and class_. These are either used to initialize the respective @@ -202,8 +250,8 @@ template <typename T> struct process_attribute_default { /// Default implementation: do nothing static void init(const T &, function_record *) { } static void init(const T &, type_record *) { } - static void precall(handle) { } - static void postcall(handle, handle) { } + static void precall(function_call &) { } + static void postcall(function_call &, handle) { } }; /// Process an attribute specifying the function's name @@ -252,8 +300,8 @@ template <> struct process_attribute<is_operator> : process_attribute_default<is template <> struct process_attribute<arg> : process_attribute_default<arg> { static void init(const arg &a, function_record *r) { if (r->is_method && r->args.empty()) - r->args.emplace_back("self", nullptr, handle()); - r->args.emplace_back(a.name, nullptr, handle()); + r->args.emplace_back("self", nullptr, handle(), true /*convert*/); + r->args.emplace_back(a.name, nullptr, handle(), !a.flag_noconvert); } }; @@ -261,32 +309,34 @@ template <> struct process_attribute<arg> : process_attribute_default<arg> { template <> struct process_attribute<arg_v> : process_attribute_default<arg_v> { static void init(const arg_v &a, function_record *r) { if (r->is_method && r->args.empty()) - r->args.emplace_back("self", nullptr, handle()); + r->args.emplace_back("self", nullptr /*descr*/, handle() /*parent*/, true /*convert*/); if (!a.value) { #if !defined(NDEBUG) - auto descr = "'" + std::string(a.name) + ": " + a.type + "'"; + std::string descr("'"); + if (a.name) descr += std::string(a.name) + ": "; + descr += a.type + "'"; if (r->is_method) { if (r->name) descr += " in method '" + (std::string) str(r->scope) + "." + (std::string) r->name + "'"; else descr += " in method of '" + (std::string) str(r->scope) + "'"; } else if (r->name) { - descr += " in function named '" + (std::string) r->name + "'"; + descr += " in function '" + (std::string) r->name + "'"; } - pybind11_fail("arg(): could not convert default keyword argument " + pybind11_fail("arg(): could not convert default argument " + descr + " into a Python object (type not registered yet?)"); #else - pybind11_fail("arg(): could not convert default keyword argument " + pybind11_fail("arg(): could not convert default argument " "into a Python object (type not registered yet?). " "Compile in debug mode for more information."); #endif } - r->args.emplace_back(a.name, a.descr, a.value.inc_ref()); + r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert); } }; -/// Process a parent class attribute +/// Process a parent class attribute. Single inheritance only (class_ itself already guarantees that) template <typename T> struct process_attribute<T, enable_if_t<is_pyobject<T>::value>> : process_attribute_default<handle> { static void init(const handle &h, type_record *r) { r->bases.append(h); } @@ -309,6 +359,16 @@ struct process_attribute<dynamic_attr> : process_attribute_default<dynamic_attr> static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; } }; +template <> +struct process_attribute<buffer_protocol> : process_attribute_default<buffer_protocol> { + static void init(const buffer_protocol &, type_record *r) { r->buffer_protocol = true; } +}; + +template <> +struct process_attribute<metaclass> : process_attribute_default<metaclass> { + static void init(const metaclass &m, type_record *r) { r->metaclass = m.value; } +}; + /// Process an 'arithmetic' attribute for enums (does nothing here) template <> @@ -319,15 +379,15 @@ struct process_attribute<arithmetic> : process_attribute_default<arithmetic> {}; * pre-call handler if both Nurse, Patient != 0 and use the post-call handler * otherwise */ -template <int Nurse, int Patient> struct process_attribute<keep_alive<Nurse, Patient>> : public process_attribute_default<keep_alive<Nurse, Patient>> { - template <int N = Nurse, int P = Patient, enable_if_t<N != 0 && P != 0, int> = 0> - static void precall(handle args) { keep_alive_impl(Nurse, Patient, args, handle()); } - template <int N = Nurse, int P = Patient, enable_if_t<N != 0 && P != 0, int> = 0> - static void postcall(handle, handle) { } - template <int N = Nurse, int P = Patient, enable_if_t<N == 0 || P == 0, int> = 0> - static void precall(handle) { } - template <int N = Nurse, int P = Patient, enable_if_t<N == 0 || P == 0, int> = 0> - static void postcall(handle args, handle ret) { keep_alive_impl(Nurse, Patient, args, ret); } +template <size_t Nurse, size_t Patient> struct process_attribute<keep_alive<Nurse, Patient>> : public process_attribute_default<keep_alive<Nurse, Patient>> { + template <size_t N = Nurse, size_t P = Patient, enable_if_t<N != 0 && P != 0, int> = 0> + static void precall(function_call &call) { keep_alive_impl(Nurse, Patient, call, handle()); } + template <size_t N = Nurse, size_t P = Patient, enable_if_t<N != 0 && P != 0, int> = 0> + static void postcall(function_call &, handle) { } + template <size_t N = Nurse, size_t P = Patient, enable_if_t<N == 0 || P == 0, int> = 0> + static void precall(function_call &) { } + template <size_t N = Nurse, size_t P = Patient, enable_if_t<N == 0 || P == 0, int> = 0> + static void postcall(function_call &call, handle ret) { keep_alive_impl(Nurse, Patient, call, ret); } }; /// Recursively iterate over variadic template arguments @@ -340,12 +400,12 @@ template <typename... Args> struct process_attributes { int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::init(args, r), 0) ... }; ignore_unused(unused); } - static void precall(handle fn_args) { - int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::precall(fn_args), 0) ... }; + static void precall(function_call &call) { + int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::precall(call), 0) ... }; ignore_unused(unused); } - static void postcall(handle fn_args, handle fn_ret) { - int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::postcall(fn_args, fn_ret), 0) ... }; + static void postcall(function_call &call, handle fn_ret) { + int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::postcall(call, fn_ret), 0) ... }; ignore_unused(unused); } }; @@ -354,8 +414,8 @@ template <typename... Args> struct process_attributes { template <typename... Extra, size_t named = constexpr_sum(std::is_base_of<arg, Extra>::value...), size_t self = constexpr_sum(std::is_same<is_method, Extra>::value...)> -constexpr bool expected_num_args(size_t nargs) { - return named == 0 || (self + named) == nargs; +constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) { + return named == 0 || (self + named + has_args + has_kwargs) == nargs; } NAMESPACE_END(detail) diff --git a/ext/pybind11/include/pybind11/cast.h b/ext/pybind11/include/pybind11/cast.h index 535516b37..fe19075e4 100644 --- a/ext/pybind11/include/pybind11/cast.h +++ b/ext/pybind11/include/pybind11/cast.h @@ -18,12 +18,16 @@ NAMESPACE_BEGIN(pybind11) NAMESPACE_BEGIN(detail) +inline PyTypeObject *make_static_property_type(); +inline PyTypeObject *make_default_metaclass(); /// Additional type information which does not fit into the PyTypeObject struct type_info { PyTypeObject *type; size_t type_size; + void *(*operator_new)(size_t); void (*init_holder)(PyObject *, const void *); + void (*dealloc)(PyObject *); std::vector<PyObject *(*)(PyObject *, PyTypeObject *)> implicit_conversions; std::vector<std::pair<const std::type_info *, void *(*)(void *)>> implicit_casts; std::vector<bool (*)(PyObject *, void *&)> *direct_conversions; @@ -32,6 +36,8 @@ struct type_info { /** A simple type never occurs as a (direct or indirect) parent * of a class that makes use of multiple inheritance */ bool simple_type = true; + /* for base vs derived holder_type checks */ + bool default_holder = true; }; PYBIND11_NOINLINE inline internals &get_internals() { @@ -71,6 +77,8 @@ PYBIND11_NOINLINE inline internals &get_internals() { } } ); + internals_ptr->static_property_type = make_static_property_type(); + internals_ptr->default_metaclass = make_default_metaclass(); } return *internals_ptr; } @@ -108,14 +116,10 @@ PYBIND11_NOINLINE inline handle get_type_handle(const std::type_info &tp, bool t } PYBIND11_NOINLINE inline bool isinstance_generic(handle obj, const std::type_info &tp) { - const auto type = detail::get_type_handle(tp, false); + handle type = detail::get_type_handle(tp, false); if (!type) return false; - - const auto result = PyObject_IsInstance(obj.ptr(), type.ptr()); - if (result == -1) - throw error_already_set(); - return result != 0; + return isinstance(obj, type); } PYBIND11_NOINLINE inline std::string error_string() { @@ -141,6 +145,7 @@ PYBIND11_NOINLINE inline std::string error_string() { PyException_SetTraceback(scope.value, scope.trace); #endif +#if !defined(PYPY_VERSION) if (scope.trace) { PyTracebackObject *trace = (PyTracebackObject *) scope.trace; @@ -160,6 +165,7 @@ PYBIND11_NOINLINE inline std::string error_string() { } trace = trace->tb_next; } +#endif return errorString; } @@ -176,7 +182,9 @@ PYBIND11_NOINLINE inline handle get_object_handle(const void *ptr, const detail: } inline PyThreadState *get_thread_state_unchecked() { -#if PY_VERSION_HEX < 0x03000000 +#if defined(PYPY_VERSION) + return PyThreadState_GET(); +#elif PY_VERSION_HEX < 0x03000000 return _PyThreadState_Current; #elif PY_VERSION_HEX < 0x03050000 return (PyThreadState*) _Py_atomic_load_relaxed(&_PyThreadState_Current); @@ -224,7 +232,7 @@ public: /* If this is a python class, also check the parents recursively */ auto const &type_dict = get_internals().registered_types_py; - bool new_style_class = PyType_Check(tobj); + bool new_style_class = PyType_Check((PyObject *) tobj); if (type_dict.find(tobj) == type_dict.end() && new_style_class && tobj->tp_bases) { auto parents = reinterpret_borrow<tuple>(tobj->tp_bases); for (handle parent : parents) { @@ -400,6 +408,13 @@ public: make_copy_constructor(src), make_move_constructor(src)); } + static handle cast_holder(const itype *src, const void *holder) { + return type_caster_generic::cast( + src, return_value_policy::take_ownership, {}, + src ? &typeid(*src) : nullptr, &typeid(type), + nullptr, nullptr, holder); + } + template <typename T> using cast_op_type = pybind11::detail::cast_op_type<T>; operator itype*() { return (type *) value; } @@ -413,7 +428,7 @@ protected: template <typename T = type, typename = enable_if_t<is_copy_constructible<T>::value>> static auto make_copy_constructor(const T *value) -> decltype(new T(*value), Constructor(nullptr)) { return [](const void *arg) -> void * { return new T(*((const T *) arg)); }; } template <typename T = type> static auto make_move_constructor(const T *value) -> decltype(new T(std::move(*((T *) value))), Constructor(nullptr)) { - return [](const void *arg) -> void * { return (void *) new T(std::move(*((T *) arg))); }; } + return [](const void *arg) -> void * { return (void *) new T(std::move(*const_cast<T *>(reinterpret_cast<const T *>(arg)))); }; } #else /* Visual Studio 2015's SFINAE implementation doesn't yet handle the above robustly in all situations. Use a workaround that only tests for constructibility for now. */ @@ -455,6 +470,7 @@ public: public: \ static PYBIND11_DESCR name() { return type_descr(py_name); } \ static handle cast(const type *src, return_value_policy policy, handle parent) { \ + if (!src) return none().release(); \ return cast(*src, policy, parent); \ } \ operator type*() { return &value; } \ @@ -462,20 +478,31 @@ public: template <typename _T> using cast_op_type = pybind11::detail::cast_op_type<_T> +template <typename CharT> using is_std_char_type = any_of< + std::is_same<CharT, char>, /* std::string */ + std::is_same<CharT, char16_t>, /* std::u16string */ + std::is_same<CharT, char32_t>, /* std::u32string */ + std::is_same<CharT, wchar_t> /* std::wstring */ +>; + template <typename T> -struct type_caster<T, enable_if_t<std::is_arithmetic<T>::value>> { - typedef typename std::conditional<sizeof(T) <= sizeof(long), long, long long>::type _py_type_0; - typedef typename std::conditional<std::is_signed<T>::value, _py_type_0, typename std::make_unsigned<_py_type_0>::type>::type _py_type_1; - typedef typename std::conditional<std::is_floating_point<T>::value, double, _py_type_1>::type py_type; +struct type_caster<T, enable_if_t<std::is_arithmetic<T>::value && !is_std_char_type<T>::value>> { + using _py_type_0 = conditional_t<sizeof(T) <= sizeof(long), long, long long>; + using _py_type_1 = conditional_t<std::is_signed<T>::value, _py_type_0, typename std::make_unsigned<_py_type_0>::type>; + using py_type = conditional_t<std::is_floating_point<T>::value, double, _py_type_1>; public: - bool load(handle src, bool) { + bool load(handle src, bool convert) { py_type py_value; - if (!src) { + if (!src) return false; - } if (std::is_floating_point<T>::value) { - py_value = (py_type) PyFloat_AsDouble(src.ptr()); + + if (std::is_floating_point<T>::value) { + if (convert || PyFloat_Check(src.ptr())) + py_value = (py_type) PyFloat_AsDouble(src.ptr()); + else + return false; } else if (sizeof(T) <= sizeof(long)) { if (PyFloat_Check(src.ptr())) return false; @@ -502,7 +529,7 @@ public: bool type_error = PyErr_ExceptionMatches(PyExc_TypeError); #endif PyErr_Clear(); - if (type_error && PyNumber_Check(src.ptr())) { + if (type_error && convert && PyNumber_Check(src.ptr())) { auto tmp = reinterpret_borrow<object>(std::is_floating_point<T>::value ? PyNumber_Float(src.ptr()) : PyNumber_Long(src.ptr())); @@ -604,133 +631,161 @@ public: PYBIND11_TYPE_CASTER(bool, _("bool")); }; -template <> class type_caster<std::string> { -public: - bool load(handle src, bool) { - object temp; - handle load_src = src; - if (!src) { - return false; - } else if (PyUnicode_Check(load_src.ptr())) { - temp = reinterpret_steal<object>(PyUnicode_AsUTF8String(load_src.ptr())); - if (!temp) { PyErr_Clear(); return false; } // UnicodeEncodeError - load_src = temp; - } - char *buffer; - ssize_t length; - int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(load_src.ptr(), &buffer, &length); - if (err == -1) { PyErr_Clear(); return false; } // TypeError - value = std::string(buffer, (size_t) length); - success = true; - return true; - } +// Helper class for UTF-{8,16,32} C++ stl strings: +template <typename CharT, class Traits, class Allocator> +struct type_caster<std::basic_string<CharT, Traits, Allocator>, enable_if_t<is_std_char_type<CharT>::value>> { + // Simplify life by being able to assume standard char sizes (the standard only guarantees + // minimums), but Python requires exact sizes + static_assert(!std::is_same<CharT, char>::value || sizeof(CharT) == 1, "Unsupported char size != 1"); + static_assert(!std::is_same<CharT, char16_t>::value || sizeof(CharT) == 2, "Unsupported char16_t size != 2"); + static_assert(!std::is_same<CharT, char32_t>::value || sizeof(CharT) == 4, "Unsupported char32_t size != 4"); + // wchar_t can be either 16 bits (Windows) or 32 (everywhere else) + static_assert(!std::is_same<CharT, wchar_t>::value || sizeof(CharT) == 2 || sizeof(CharT) == 4, + "Unsupported wchar_t size != 2/4"); + static constexpr size_t UTF_N = 8 * sizeof(CharT); + + using StringType = std::basic_string<CharT, Traits, Allocator>; - static handle cast(const std::string &src, return_value_policy /* policy */, handle /* parent */) { - return PyUnicode_FromStringAndSize(src.c_str(), (ssize_t) src.length()); - } - - PYBIND11_TYPE_CASTER(std::string, _(PYBIND11_STRING_NAME)); -protected: - bool success = false; -}; - -template <typename type, typename deleter> class type_caster<std::unique_ptr<type, deleter>> { -public: - static handle cast(std::unique_ptr<type, deleter> &&src, return_value_policy policy, handle parent) { - handle result = type_caster_base<type>::cast(src.get(), policy, parent); - if (result) - src.release(); - return result; - } - static PYBIND11_DESCR name() { return type_caster_base<type>::name(); } -}; - -template <> class type_caster<std::wstring> { -public: bool load(handle src, bool) { +#if PY_MAJOR_VERSION < 3 object temp; +#endif handle load_src = src; if (!src) { return false; } else if (!PyUnicode_Check(load_src.ptr())) { +#if PY_MAJOR_VERSION >= 3 + return false; + // The below is a guaranteed failure in Python 3 when PyUnicode_Check returns false +#else + if (!PYBIND11_BYTES_CHECK(load_src.ptr())) + return false; temp = reinterpret_steal<object>(PyUnicode_FromObject(load_src.ptr())); if (!temp) { PyErr_Clear(); return false; } load_src = temp; - } - wchar_t *buffer = nullptr; - ssize_t length = -1; -#if PY_MAJOR_VERSION >= 3 - buffer = PyUnicode_AsWideCharString(load_src.ptr(), &length); -#else - temp = reinterpret_steal<object>( - sizeof(wchar_t) == sizeof(short) - ? PyUnicode_AsUTF16String(load_src.ptr()) - : PyUnicode_AsUTF32String(load_src.ptr())); - if (temp) { - int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), (char **) &buffer, &length); - if (err == -1) { buffer = nullptr; } // TypeError - length = length / (ssize_t) sizeof(wchar_t) - 1; ++buffer; // Skip BOM - } #endif - if (!buffer) { PyErr_Clear(); return false; } - value = std::wstring(buffer, (size_t) length); - success = true; + } + + object utfNbytes = reinterpret_steal<object>(PyUnicode_AsEncodedString( + load_src.ptr(), UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr)); + if (!utfNbytes) { PyErr_Clear(); return false; } + + const CharT *buffer = reinterpret_cast<const CharT *>(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); + size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); + if (UTF_N > 8) { buffer++; length--; } // Skip BOM for UTF-16/32 + value = StringType(buffer, length); return true; } - static handle cast(const std::wstring &src, return_value_policy /* policy */, handle /* parent */) { - return PyUnicode_FromWideChar(src.c_str(), (ssize_t) src.length()); + static handle cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) { + const char *buffer = reinterpret_cast<const char *>(src.c_str()); + ssize_t nbytes = ssize_t(src.size() * sizeof(CharT)); + handle s = decode_utfN(buffer, nbytes); + if (!s) throw error_already_set(); + return s; } - PYBIND11_TYPE_CASTER(std::wstring, _(PYBIND11_STRING_NAME)); -protected: - bool success = false; + PYBIND11_TYPE_CASTER(StringType, _(PYBIND11_STRING_NAME)); + +private: + static handle decode_utfN(const char *buffer, ssize_t nbytes) { +#if !defined(PYPY_VERSION) + return + UTF_N == 8 ? PyUnicode_DecodeUTF8(buffer, nbytes, nullptr) : + UTF_N == 16 ? PyUnicode_DecodeUTF16(buffer, nbytes, nullptr, nullptr) : + PyUnicode_DecodeUTF32(buffer, nbytes, nullptr, nullptr); +#else + // PyPy seems to have multiple problems related to PyUnicode_UTF*: the UTF8 version + // sometimes segfaults for unknown reasons, while the UTF16 and 32 versions require a + // non-const char * arguments, which is also a nuissance, so bypass the whole thing by just + // passing the encoding as a string value, which works properly: + return PyUnicode_Decode(buffer, nbytes, UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr); +#endif + } }; -template <> class type_caster<char> : public type_caster<std::string> { +// Type caster for C-style strings. We basically use a std::string type caster, but also add the +// ability to use None as a nullptr char* (which the string caster doesn't allow). +template <typename CharT> struct type_caster<CharT, enable_if_t<is_std_char_type<CharT>::value>> { + using StringType = std::basic_string<CharT>; + using StringCaster = type_caster<StringType>; + StringCaster str_caster; + bool none = false; public: bool load(handle src, bool convert) { - if (src.is_none()) return true; - return type_caster<std::string>::load(src, convert); - } - - static handle cast(const char *src, return_value_policy /* policy */, handle /* parent */) { - if (src == nullptr) return none().inc_ref(); - return PyUnicode_FromString(src); + if (!src) return false; + if (src.is_none()) { + // Defer accepting None to other overloads (if we aren't in convert mode): + if (!convert) return false; + none = true; + return true; + } + return str_caster.load(src, convert); } - static handle cast(char src, return_value_policy /* policy */, handle /* parent */) { - char str[2] = { src, '\0' }; - return PyUnicode_DecodeLatin1(str, 1, nullptr); + static handle cast(const CharT *src, return_value_policy policy, handle parent) { + if (src == nullptr) return pybind11::none().inc_ref(); + return StringCaster::cast(StringType(src), policy, parent); } - operator char*() { return success ? (char *) value.c_str() : nullptr; } - operator char&() { return value[0]; } - - static PYBIND11_DESCR name() { return type_descr(_(PYBIND11_STRING_NAME)); } -}; + static handle cast(CharT src, return_value_policy policy, handle parent) { + if (std::is_same<char, CharT>::value) { + handle s = PyUnicode_DecodeLatin1((const char *) &src, 1, nullptr); + if (!s) throw error_already_set(); + return s; + } + return StringCaster::cast(StringType(1, src), policy, parent); + } + + operator CharT*() { return none ? nullptr : const_cast<CharT *>(static_cast<StringType &>(str_caster).c_str()); } + operator CharT() { + if (none) + throw value_error("Cannot convert None to a character"); + + auto &value = static_cast<StringType &>(str_caster); + size_t str_len = value.size(); + if (str_len == 0) + throw value_error("Cannot convert empty string to a character"); + + // If we're in UTF-8 mode, we have two possible failures: one for a unicode character that + // is too high, and one for multiple unicode characters (caught later), so we need to figure + // out how long the first encoded character is in bytes to distinguish between these two + // errors. We also allow want to allow unicode characters U+0080 through U+00FF, as those + // can fit into a single char value. + if (StringCaster::UTF_N == 8 && str_len > 1 && str_len <= 4) { + unsigned char v0 = static_cast<unsigned char>(value[0]); + size_t char0_bytes = !(v0 & 0x80) ? 1 : // low bits only: 0-127 + (v0 & 0xE0) == 0xC0 ? 2 : // 0b110xxxxx - start of 2-byte sequence + (v0 & 0xF0) == 0xE0 ? 3 : // 0b1110xxxx - start of 3-byte sequence + 4; // 0b11110xxx - start of 4-byte sequence + + if (char0_bytes == str_len) { + // If we have a 128-255 value, we can decode it into a single char: + if (char0_bytes == 2 && (v0 & 0xFC) == 0xC0) { // 0x110000xx 0x10xxxxxx + return static_cast<CharT>(((v0 & 3) << 6) + (static_cast<unsigned char>(value[1]) & 0x3F)); + } + // Otherwise we have a single character, but it's > U+00FF + throw value_error("Character code point not in range(0x100)"); + } + } -template <> class type_caster<wchar_t> : public type_caster<std::wstring> { -public: - bool load(handle src, bool convert) { - if (src.is_none()) return true; - return type_caster<std::wstring>::load(src, convert); - } + // UTF-16 is much easier: we can only have a surrogate pair for values above U+FFFF, thus a + // surrogate pair with total length 2 instantly indicates a range error (but not a "your + // string was too long" error). + else if (StringCaster::UTF_N == 16 && str_len == 2) { + char16_t v0 = static_cast<char16_t>(value[0]); + if (v0 >= 0xD800 && v0 < 0xE000) + throw value_error("Character code point not in range(0x10000)"); + } - static handle cast(const wchar_t *src, return_value_policy /* policy */, handle /* parent */) { - if (src == nullptr) return none().inc_ref(); - return PyUnicode_FromWideChar(src, (ssize_t) wcslen(src)); - } + if (str_len != 1) + throw value_error("Expected a character, but multi-character string found"); - static handle cast(wchar_t src, return_value_policy /* policy */, handle /* parent */) { - wchar_t wstr[2] = { src, L'\0' }; - return PyUnicode_FromWideChar(wstr, 1); + return value[0]; } - operator wchar_t*() { return success ? (wchar_t *) value.c_str() : nullptr; } - operator wchar_t&() { return value[0]; } - static PYBIND11_DESCR name() { return type_descr(_(PYBIND11_STRING_NAME)); } + template <typename _T> using cast_op_type = typename std::remove_reference<pybind11::detail::cast_op_type<_T>>::type; }; template <typename T1, typename T2> class type_caster<std::pair<T1, T2>> { @@ -832,12 +887,19 @@ protected: return result.release(); } -protected: std::tuple<make_caster<Tuple>...> value; }; +/// Helper class which abstracts away certain actions. Users can provide specializations for +/// custom holders, but it's only necessary if the type has a non-standard interface. +template <typename T> +struct holder_helper { + static auto get(const T &p) -> decltype(p.get()) { return p.get(); } +}; + /// Type caster for holder types like std::shared_ptr, etc. -template <typename type, typename holder_type> class type_caster_holder : public type_caster_base<type> { +template <typename type, typename holder_type> +struct copyable_holder_caster : public type_caster_base<type> { public: using base = type_caster_base<type>; using base::base; @@ -858,6 +920,9 @@ public: return true; } + if (typeinfo->default_holder) + throw cast_error("Unable to load a custom holder type from a default-holder instance"); + if (typeinfo->simple_type) { /* Case 1: no multiple inheritance etc. involved */ /* Check if we can safely perform a reinterpret-style cast */ if (PyType_IsSubtype(tobj, typeinfo->type)) @@ -869,7 +934,7 @@ public: /* If this is a python class, also check the parents recursively */ auto const &type_dict = get_internals().registered_types_py; - bool new_style_class = PyType_Check(tobj); + bool new_style_class = PyType_Check((PyObject *) tobj); if (type_dict.find(tobj) == type_dict.end() && new_style_class && tobj->tp_bases) { auto parents = reinterpret_borrow<tuple>(tobj->tp_bases); for (handle parent : parents) { @@ -916,7 +981,7 @@ public: template <typename T = holder_type, detail::enable_if_t<std::is_constructible<T, const T &, type*>::value, int> = 0> bool try_implicit_casts(handle src, bool convert) { for (auto &cast : typeinfo->implicit_casts) { - type_caster_holder sub_caster(*cast.first); + copyable_holder_caster sub_caster(*cast.first); if (sub_caster.load(src, convert)) { value = cast.second(sub_caster.value); holder = holder_type(sub_caster.holder, (type *) value); @@ -939,10 +1004,8 @@ public: #endif static handle cast(const holder_type &src, return_value_policy, handle) { - return type_caster_generic::cast( - src.get(), return_value_policy::take_ownership, handle(), - src.get() ? &typeid(*src.get()) : nullptr, &typeid(type), - nullptr, nullptr, &src); + const auto *ptr = holder_helper<holder_type>::get(src); + return type_caster_base<type>::cast_holder(ptr, &src); } protected: @@ -951,12 +1014,34 @@ protected: /// Specialize for the common std::shared_ptr, so users don't need to template <typename T> -class type_caster<std::shared_ptr<T>> : public type_caster_holder<T, std::shared_ptr<T>> { }; +class type_caster<std::shared_ptr<T>> : public copyable_holder_caster<T, std::shared_ptr<T>> { }; + +template <typename type, typename holder_type> +struct move_only_holder_caster { + static handle cast(holder_type &&src, return_value_policy, handle) { + auto *ptr = holder_helper<holder_type>::get(src); + return type_caster_base<type>::cast_holder(ptr, &src); + } + static PYBIND11_DESCR name() { return type_caster_base<type>::name(); } +}; + +template <typename type, typename deleter> +class type_caster<std::unique_ptr<type, deleter>> + : public move_only_holder_caster<type, std::unique_ptr<type, deleter>> { }; + +template <typename type, typename holder_type> +using type_caster_holder = conditional_t<std::is_copy_constructible<holder_type>::value, + copyable_holder_caster<type, holder_type>, + move_only_holder_caster<type, holder_type>>; + +template <typename T, bool Value = false> struct always_construct_holder { static constexpr bool value = Value; }; /// Create a specialization for custom holder types (silently ignores std::shared_ptr) -#define PYBIND11_DECLARE_HOLDER_TYPE(type, holder_type) \ +#define PYBIND11_DECLARE_HOLDER_TYPE(type, holder_type, ...) \ namespace pybind11 { namespace detail { \ template <typename type> \ + struct always_construct_holder<holder_type> : always_construct_holder<void, ##__VA_ARGS__> { }; \ + template <typename type> \ class type_caster<holder_type, enable_if_t<!is_shared_ptr<holder_type>::value>> \ : public type_caster_holder<type, holder_type> { }; \ }} @@ -1004,23 +1089,24 @@ class type_caster<T, enable_if_t<is_pyobject<T>::value>> : public pyobject_caste // - if the type is non-copy-constructible, the object must be the sole owner of the type (i.e. it // must have ref_count() == 1)h // If any of the above are not satisfied, we fall back to copying. -template <typename T, typename SFINAE = void> struct move_is_plain_type : std::false_type {}; -template <typename T> struct move_is_plain_type<T, enable_if_t< - !std::is_void<T>::value && !std::is_pointer<T>::value && !std::is_reference<T>::value && !std::is_const<T>::value - >> : std::true_type { }; +template <typename T> using move_is_plain_type = satisfies_none_of<T, + std::is_void, std::is_pointer, std::is_reference, std::is_const +>; template <typename T, typename SFINAE = void> struct move_always : std::false_type {}; -template <typename T> struct move_always<T, enable_if_t< - move_is_plain_type<T>::value && - !std::is_copy_constructible<T>::value && std::is_move_constructible<T>::value && - std::is_same<decltype(std::declval<type_caster<T>>().operator T&()), T&>::value - >> : std::true_type { }; +template <typename T> struct move_always<T, enable_if_t<all_of< + move_is_plain_type<T>, + negation<std::is_copy_constructible<T>>, + std::is_move_constructible<T>, + std::is_same<decltype(std::declval<make_caster<T>>().operator T&()), T&> +>::value>> : std::true_type {}; template <typename T, typename SFINAE = void> struct move_if_unreferenced : std::false_type {}; -template <typename T> struct move_if_unreferenced<T, enable_if_t< - move_is_plain_type<T>::value && - !move_always<T>::value && std::is_move_constructible<T>::value && - std::is_same<decltype(std::declval<type_caster<T>>().operator T&()), T&>::value - >> : std::true_type { }; -template <typename T> using move_never = std::integral_constant<bool, !move_always<T>::value && !move_if_unreferenced<T>::value>; +template <typename T> struct move_if_unreferenced<T, enable_if_t<all_of< + move_is_plain_type<T>, + negation<move_always<T>>, + std::is_move_constructible<T>, + std::is_same<decltype(std::declval<make_caster<T>>().operator T&()), T&> +>::value>> : std::true_type {}; +template <typename T> using move_never = none_of<move_always<T>, move_if_unreferenced<T>>; // Detect whether returning a `type` from a cast on type's type_caster is going to result in a // reference or pointer to a local variable of the type_caster. Basically, only @@ -1031,6 +1117,17 @@ template <typename type> using cast_is_temporary_value_reference = bool_constant !std::is_base_of<type_caster_generic, make_caster<type>>::value >; +// When a value returned from a C++ function is being cast back to Python, we almost always want to +// force `policy = move`, regardless of the return value policy the function/method was declared +// with. Some classes (most notably Eigen::Ref and related) need to avoid this, and so can do so by +// specializing this struct. +template <typename Return, typename SFINAE = void> struct return_value_policy_override { + static return_value_policy policy(return_value_policy p) { + return !std::is_lvalue_reference<Return>::value && !std::is_pointer<Return>::value + ? return_value_policy::move : p; + } +}; + // Basic python -> C++ casting; throws if casting fails template <typename T, typename SFINAE> type_caster<T, SFINAE> &load_type(type_caster<T, SFINAE> &conv, const handle &handle) { if (!conv.load(handle, true)) { @@ -1080,7 +1177,7 @@ template <typename T> T handle::cast() const { return pybind11::cast<T>(*this); template <> inline void handle::cast() const { return; } template <typename T> -detail::enable_if_t<detail::move_always<T>::value || detail::move_if_unreferenced<T>::value, T> move(object &&obj) { +detail::enable_if_t<!detail::move_never<T>::value, T> move(object &&obj) { if (obj.ref_count() > 1) #if defined(NDEBUG) throw cast_error("Unable to cast Python instance to C++ rvalue: instance has multiple references" @@ -1171,19 +1268,27 @@ template <return_value_policy policy = return_value_policy::automatic_reference, return result; } -/// Annotation for keyword arguments +/// \ingroup annotations +/// Annotation for arguments struct arg { - constexpr explicit arg(const char *name) : name(name) { } + /// Constructs an argument with the name of the argument; if null or omitted, this is a positional argument. + constexpr explicit arg(const char *name = nullptr) : name(name), flag_noconvert(false) { } + /// Assign a value to this argument template <typename T> arg_v operator=(T &&value) const; + /// Indicate that the type should not be converted in the type caster + arg &noconvert(bool flag = true) { flag_noconvert = flag; return *this; } - const char *name; + const char *name; ///< If non-null, this is a named kwargs argument + bool flag_noconvert : 1; ///< If set, do not allow conversion (requires a supporting type caster!) }; -/// Annotation for keyword arguments with values +/// \ingroup annotations +/// Annotation for arguments with values struct arg_v : arg { +private: template <typename T> - arg_v(const char *name, T &&x, const char *descr = nullptr) - : arg(name), + arg_v(arg &&base, T &&x, const char *descr = nullptr) + : arg(base), value(reinterpret_steal<object>( detail::make_caster<T>::cast(x, return_value_policy::automatic, {}) )), @@ -1193,40 +1298,89 @@ struct arg_v : arg { #endif { } +public: + /// Direct construction with name, default, and description + template <typename T> + arg_v(const char *name, T &&x, const char *descr = nullptr) + : arg_v(arg(name), std::forward<T>(x), descr) { } + + /// Called internally when invoking `py::arg("a") = value` + template <typename T> + arg_v(const arg &base, T &&x, const char *descr = nullptr) + : arg_v(arg(base), std::forward<T>(x), descr) { } + + /// Same as `arg::noconvert()`, but returns *this as arg_v&, not arg& + arg_v &noconvert(bool flag = true) { arg::noconvert(flag); return *this; } + + /// The default value object value; + /// The (optional) description of the default value const char *descr; #if !defined(NDEBUG) + /// The C++ type name of the default value (only available when compiled in debug mode) std::string type; #endif }; template <typename T> -arg_v arg::operator=(T &&value) const { return {name, std::forward<T>(value)}; } +arg_v arg::operator=(T &&value) const { return {std::move(*this), std::forward<T>(value)}; } /// Alias for backward compatibility -- to be removed in version 2.0 template <typename /*unused*/> using arg_t = arg_v; inline namespace literals { -/// String literal version of arg +/** \rst + String literal version of `arg` + \endrst */ constexpr arg operator"" _a(const char *name, size_t) { return arg(name); } } NAMESPACE_BEGIN(detail) +// forward declaration +struct function_record; + +/// Internal data associated with a single function call +struct function_call { + function_call(function_record &f, handle p); // Implementation in attr.h + + /// The function data: + const function_record &func; + + /// Arguments passed to the function: + std::vector<handle> args; + + /// The `convert` value the arguments should be loaded with + std::vector<bool> args_convert; + + /// The parent, if any + handle parent; +}; + + /// Helper class which loads arguments for C++ functions called from Python template <typename... Args> class argument_loader { - using itypes = type_list<intrinsic_t<Args>...>; using indices = make_index_sequence<sizeof...(Args)>; + template <typename Arg> using argument_is_args = std::is_same<intrinsic_t<Arg>, args>; + template <typename Arg> using argument_is_kwargs = std::is_same<intrinsic_t<Arg>, kwargs>; + // Get args/kwargs argument positions relative to the end of the argument list: + static constexpr auto args_pos = constexpr_first<argument_is_args, Args...>() - (int) sizeof...(Args), + kwargs_pos = constexpr_first<argument_is_kwargs, Args...>() - (int) sizeof...(Args); + + static constexpr bool args_kwargs_are_last = kwargs_pos >= - 1 && args_pos >= kwargs_pos - 1; + + static_assert(args_kwargs_are_last, "py::args/py::kwargs are only permitted as the last argument(s) of a function"); + public: - static constexpr auto has_kwargs = std::is_same<itypes, type_list<args, kwargs>>::value; - static constexpr auto has_args = has_kwargs || std::is_same<itypes, type_list<args>>::value; + static constexpr bool has_kwargs = kwargs_pos < 0; + static constexpr bool has_args = args_pos < 0; static PYBIND11_DESCR arg_names() { return detail::concat(make_caster<Args>::name()...); } - bool load_args(handle args, handle kwargs, bool convert) { - return load_impl(args, kwargs, convert, itypes{}); + bool load_args(function_call &call) { + return load_impl_sequence(call, indices{}); } template <typename Return, typename Func> @@ -1241,26 +1395,12 @@ public: } private: - bool load_impl(handle args_, handle, bool convert, type_list<args>) { - std::get<0>(value).load(args_, convert); - return true; - } - - bool load_impl(handle args_, handle kwargs_, bool convert, type_list<args, kwargs>) { - std::get<0>(value).load(args_, convert); - std::get<1>(value).load(kwargs_, convert); - return true; - } - - bool load_impl(handle args, handle, bool convert, ... /* anything else */) { - return load_impl_sequence(args, convert, indices{}); - } - static constexpr bool load_impl_sequence(handle, bool, index_sequence<>) { return true; } + static bool load_impl_sequence(function_call &, index_sequence<>) { return true; } template <size_t... Is> - bool load_impl_sequence(handle src, bool convert, index_sequence<Is...>) { - for (bool r : {std::get<Is>(value).load(PyTuple_GET_ITEM(src.ptr(), Is), convert)...}) + bool load_impl_sequence(function_call &call, index_sequence<Is...>) { + for (bool r : {std::get<Is>(value).load(call.args[Is], call.args_convert[Is])...}) if (!r) return false; return true; @@ -1271,29 +1411,9 @@ private: return std::forward<Func>(f)(cast_op<Args>(std::get<Is>(value))...); } -private: std::tuple<make_caster<Args>...> value; }; -NAMESPACE_BEGIN(constexpr_impl) -/// Implementation details for constexpr functions -constexpr int first(int i) { return i; } -template <typename T, typename... Ts> -constexpr int first(int i, T v, Ts... vs) { return v ? i : first(i + 1, vs...); } - -constexpr int last(int /*i*/, int result) { return result; } -template <typename T, typename... Ts> -constexpr int last(int i, int result, T v, Ts... vs) { return last(i + 1, v ? i : result, vs...); } -NAMESPACE_END(constexpr_impl) - -/// Return the index of the first type in Ts which satisfies Predicate<T> -template <template<typename> class Predicate, typename... Ts> -constexpr int constexpr_first() { return constexpr_impl::first(0, Predicate<Ts>::value...); } - -/// Return the index of the last type in Ts which satisfies Predicate<T> -template <template<typename> class Predicate, typename... Ts> -constexpr int constexpr_last() { return constexpr_impl::last(0, -1, Predicate<Ts>::value...); } - /// Helper class which collects only positional arguments for a Python function call. /// A fancier version below can collect any argument, but this one is optimal for simple calls. template <return_value_policy policy> @@ -1369,6 +1489,13 @@ private: } void process(list &/*args_list*/, arg_v a) { + if (!a.name) +#if defined(NDEBUG) + nameless_argument_error(); +#else + nameless_argument_error(a.type); +#endif + if (m_kwargs.contains(a.name)) { #if defined(NDEBUG) multiple_values_error(); @@ -1401,6 +1528,15 @@ private: } } + [[noreturn]] static void nameless_argument_error() { + throw type_error("Got kwargs without a name; only named arguments " + "may be passed via py::arg() to a python function call. " + "(compile in debug mode for details)"); + } + [[noreturn]] static void nameless_argument_error(std::string type) { + throw type_error("Got kwargs without a name of type '" + type + "'; only named " + "arguments may be passed via py::arg() to a python function call. "); + } [[noreturn]] static void multiple_values_error() { throw type_error("Got multiple values for keyword argument " "(compile in debug mode for details)"); @@ -1427,14 +1563,14 @@ private: /// Collect only positional arguments for a Python function call template <return_value_policy policy, typename... Args, - typename = enable_if_t<all_of_t<is_positional, Args...>::value>> + typename = enable_if_t<all_of<is_positional<Args>...>::value>> simple_collector<policy> collect_arguments(Args &&...args) { return simple_collector<policy>(std::forward<Args>(args)...); } /// Collect all arguments, including keywords and unpacking (only instantiated when needed) template <return_value_policy policy, typename... Args, - typename = enable_if_t<!all_of_t<is_positional, Args...>::value>> + typename = enable_if_t<!all_of<is_positional<Args>...>::value>> unpacking_collector<policy> collect_arguments(Args &&...args) { // Following argument order rules for generalized unpacking according to PEP 448 static_assert( diff --git a/ext/pybind11/include/pybind11/chrono.h b/ext/pybind11/include/pybind11/chrono.h index 2b37f56f1..8a41d08b0 100644 --- a/ext/pybind11/include/pybind11/chrono.h +++ b/ext/pybind11/include/pybind11/chrono.h @@ -85,9 +85,11 @@ public: using ss_t = duration<int, std::ratio<1>>; using us_t = duration<int, std::micro>; - return PyDelta_FromDSU(duration_cast<dd_t>(d).count(), - duration_cast<ss_t>(d % days(1)).count(), - duration_cast<us_t>(d % seconds(1)).count()); + auto dd = duration_cast<dd_t>(d); + auto subd = d - dd; + auto ss = duration_cast<ss_t>(subd); + auto us = duration_cast<us_t>(subd - ss); + return PyDelta_FromDSU(dd.count(), ss.count(), us.count()); } PYBIND11_TYPE_CASTER(type, _("datetime.timedelta")); diff --git a/ext/pybind11/include/pybind11/class_support.h b/ext/pybind11/include/pybind11/class_support.h new file mode 100644 index 000000000..992703ff3 --- /dev/null +++ b/ext/pybind11/include/pybind11/class_support.h @@ -0,0 +1,504 @@ +/* + pybind11/class_support.h: Python C API implementation details for py::class_ + + Copyright (c) 2017 Wenzel Jakob <wenzel.jakob@epfl.ch> + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "attr.h" + +NAMESPACE_BEGIN(pybind11) +NAMESPACE_BEGIN(detail) + +#if !defined(PYPY_VERSION) + +/// `pybind11_static_property.__get__()`: Always pass the class instead of the instance. +extern "C" inline PyObject *pybind11_static_get(PyObject *self, PyObject * /*ob*/, PyObject *cls) { + return PyProperty_Type.tp_descr_get(self, cls, cls); +} + +/// `pybind11_static_property.__set__()`: Just like the above `__get__()`. +extern "C" inline int pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { + PyObject *cls = PyType_Check(obj) ? obj : (PyObject *) Py_TYPE(obj); + return PyProperty_Type.tp_descr_set(self, cls, value); +} + +/** A `static_property` is the same as a `property` but the `__get__()` and `__set__()` + methods are modified to always use the object type instead of a concrete instance. + Return value: New reference. */ +inline PyTypeObject *make_static_property_type() { + constexpr auto *name = "pybind11_static_property"; + auto name_obj = reinterpret_steal<object>(PYBIND11_FROM_STRING(name)); + + /* Danger zone: from now (and until PyType_Ready), make sure to + issue no Python C API calls which could potentially invoke the + garbage collector (the GC will call type_traverse(), which will in + turn find the newly constructed type in an invalid state) */ + auto heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + if (!heap_type) + pybind11_fail("make_static_property_type(): error allocating type!"); + + heap_type->ht_name = name_obj.inc_ref().ptr(); +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 + heap_type->ht_qualname = name_obj.inc_ref().ptr(); +#endif + + auto type = &heap_type->ht_type; + type->tp_name = name; + type->tp_base = &PyProperty_Type; + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; + type->tp_descr_get = pybind11_static_get; + type->tp_descr_set = pybind11_static_set; + + if (PyType_Ready(type) < 0) + pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); + + setattr((PyObject *) type, "__module__", str("pybind11_builtins")); + + return type; +} + +#else // PYPY + +/** PyPy has some issues with the above C API, so we evaluate Python code instead. + This function will only be called once so performance isn't really a concern. + Return value: New reference. */ +inline PyTypeObject *make_static_property_type() { + auto d = dict(); + PyObject *result = PyRun_String(R"(\ + class pybind11_static_property(property): + def __get__(self, obj, cls): + return property.__get__(self, cls, cls) + + def __set__(self, obj, value): + cls = obj if isinstance(obj, type) else type(obj) + property.__set__(self, cls, value) + )", Py_file_input, d.ptr(), d.ptr() + ); + if (result == nullptr) + throw error_already_set(); + Py_DECREF(result); + return (PyTypeObject *) d["pybind11_static_property"].cast<object>().release().ptr(); +} + +#endif // PYPY + +/** Inheriting from multiple C++ types in Python is not supported -- set an error instead. + A Python definition (`class C(A, B): pass`) will call `tp_new` so we check for multiple + C++ bases here. On the other hand, C++ type definitions (`py::class_<C, A, B>(m, "C")`) + don't not use `tp_new` and will not trigger this error. */ +extern "C" inline PyObject *pybind11_meta_new(PyTypeObject *metaclass, PyObject *args, + PyObject *kwargs) { + PyObject *bases = PyTuple_GetItem(args, 1); // arguments: (name, bases, dict) + if (!bases) + return nullptr; + + auto &internals = get_internals(); + auto num_cpp_bases = 0; + for (auto base : reinterpret_borrow<tuple>(bases)) { + auto base_type = (PyTypeObject *) base.ptr(); + auto instance_size = static_cast<size_t>(base_type->tp_basicsize); + if (PyObject_IsSubclass(base.ptr(), internals.get_base(instance_size))) + ++num_cpp_bases; + } + + if (num_cpp_bases > 1) { + PyErr_SetString(PyExc_TypeError, "Can't inherit from multiple C++ classes in Python." + "Use py::class_ to define the class in C++ instead."); + return nullptr; + } else { + return PyType_Type.tp_new(metaclass, args, kwargs); + } +} + +/** Types with static properties need to handle `Type.static_prop = x` in a specific way. + By default, Python replaces the `static_property` itself, but for wrapped C++ types + we need to call `static_property.__set__()` in order to propagate the new value to + the underlying C++ data structure. */ +extern "C" inline int pybind11_meta_setattro(PyObject* obj, PyObject* name, PyObject* value) { + // Use `_PyType_Lookup()` instead of `PyObject_GetAttr()` in order to get the raw + // descriptor (`property`) instead of calling `tp_descr_get` (`property.__get__()`). + PyObject *descr = _PyType_Lookup((PyTypeObject *) obj, name); + + // Call `static_property.__set__()` instead of replacing the `static_property`. + if (descr && PyObject_IsInstance(descr, (PyObject *) get_internals().static_property_type)) { +#if !defined(PYPY_VERSION) + return Py_TYPE(descr)->tp_descr_set(descr, obj, value); +#else + if (PyObject *result = PyObject_CallMethod(descr, "__set__", "OO", obj, value)) { + Py_DECREF(result); + return 0; + } else { + return -1; + } +#endif + } else { + return PyType_Type.tp_setattro(obj, name, value); + } +} + +/** This metaclass is assigned by default to all pybind11 types and is required in order + for static properties to function correctly. Users may override this using `py::metaclass`. + Return value: New reference. */ +inline PyTypeObject* make_default_metaclass() { + constexpr auto *name = "pybind11_type"; + auto name_obj = reinterpret_steal<object>(PYBIND11_FROM_STRING(name)); + + /* Danger zone: from now (and until PyType_Ready), make sure to + issue no Python C API calls which could potentially invoke the + garbage collector (the GC will call type_traverse(), which will in + turn find the newly constructed type in an invalid state) */ + auto heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + if (!heap_type) + pybind11_fail("make_default_metaclass(): error allocating metaclass!"); + + heap_type->ht_name = name_obj.inc_ref().ptr(); +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 + heap_type->ht_qualname = name_obj.inc_ref().ptr(); +#endif + + auto type = &heap_type->ht_type; + type->tp_name = name; + type->tp_base = &PyType_Type; + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; + + type->tp_new = pybind11_meta_new; + type->tp_setattro = pybind11_meta_setattro; + + if (PyType_Ready(type) < 0) + pybind11_fail("make_default_metaclass(): failure in PyType_Ready()!"); + + setattr((PyObject *) type, "__module__", str("pybind11_builtins")); + + return type; +} + +/// Instance creation function for all pybind11 types. It only allocates space for the +/// C++ object, but doesn't call the constructor -- an `__init__` function must do that. +extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *) { + PyObject *self = type->tp_alloc(type, 0); + auto instance = (instance_essentials<void> *) self; + auto tinfo = get_type_info(type); + instance->value = tinfo->operator_new(tinfo->type_size); + instance->owned = true; + instance->holder_constructed = false; + get_internals().registered_instances.emplace(instance->value, self); + return self; +} + +/// An `__init__` function constructs the C++ object. Users should provide at least one +/// of these using `py::init` or directly with `.def(__init__, ...)`. Otherwise, the +/// following default function will be used which simply throws an exception. +extern "C" inline int pybind11_object_init(PyObject *self, PyObject *, PyObject *) { + PyTypeObject *type = Py_TYPE(self); + std::string msg; +#if defined(PYPY_VERSION) + msg += handle((PyObject *) type).attr("__module__").cast<std::string>() + "."; +#endif + msg += type->tp_name; + msg += ": No constructor defined!"; + PyErr_SetString(PyExc_TypeError, msg.c_str()); + return -1; +} + +/// Instance destructor function for all pybind11 types. It calls `type_info.dealloc` +/// to destroy the C++ object itself, while the rest is Python bookkeeping. +extern "C" inline void pybind11_object_dealloc(PyObject *self) { + auto instance = (instance_essentials<void> *) self; + if (instance->value) { + auto type = Py_TYPE(self); + get_type_info(type)->dealloc(self); + + auto ®istered_instances = get_internals().registered_instances; + auto range = registered_instances.equal_range(instance->value); + bool found = false; + for (auto it = range.first; it != range.second; ++it) { + if (type == Py_TYPE(it->second)) { + registered_instances.erase(it); + found = true; + break; + } + } + if (!found) + pybind11_fail("pybind11_object_dealloc(): Tried to deallocate unregistered instance!"); + + if (instance->weakrefs) + PyObject_ClearWeakRefs(self); + + PyObject **dict_ptr = _PyObject_GetDictPtr(self); + if (dict_ptr) + Py_CLEAR(*dict_ptr); + } + Py_TYPE(self)->tp_free(self); +} + +/** Create a type which can be used as a common base for all classes with the same + instance size, i.e. all classes with the same `sizeof(holder_type)`. This is + needed in order to satisfy Python's requirements for multiple inheritance. + Return value: New reference. */ +inline PyObject *make_object_base_type(size_t instance_size) { + auto name = "pybind11_object_" + std::to_string(instance_size); + auto name_obj = reinterpret_steal<object>(PYBIND11_FROM_STRING(name.c_str())); + + /* Danger zone: from now (and until PyType_Ready), make sure to + issue no Python C API calls which could potentially invoke the + garbage collector (the GC will call type_traverse(), which will in + turn find the newly constructed type in an invalid state) */ + auto metaclass = get_internals().default_metaclass; + auto heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); + if (!heap_type) + pybind11_fail("make_object_base_type(): error allocating type!"); + + heap_type->ht_name = name_obj.inc_ref().ptr(); +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 + heap_type->ht_qualname = name_obj.inc_ref().ptr(); +#endif + + auto type = &heap_type->ht_type; + type->tp_name = strdup(name.c_str()); + type->tp_base = &PyBaseObject_Type; + type->tp_basicsize = static_cast<ssize_t>(instance_size); + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; + + type->tp_new = pybind11_object_new; + type->tp_init = pybind11_object_init; + type->tp_dealloc = pybind11_object_dealloc; + + /* Support weak references (needed for the keep_alive feature) */ + type->tp_weaklistoffset = offsetof(instance_essentials<void>, weakrefs); + + if (PyType_Ready(type) < 0) + pybind11_fail("PyType_Ready failed in make_object_base_type():" + error_string()); + + setattr((PyObject *) type, "__module__", str("pybind11_builtins")); + + assert(!PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)); + return (PyObject *) heap_type; +} + +/** Return the appropriate base type for the given instance size. The results are cached + in `internals.bases` so that only a single base is ever created for any size value. + Return value: Borrowed reference. */ +inline PyObject *internals::get_base(size_t instance_size) { + auto it = bases.find(instance_size); + if (it != bases.end()) { + return it->second; + } else { + auto base = make_object_base_type(instance_size); + bases[instance_size] = base; + return base; + } +} + +/// dynamic_attr: Support for `d = instance.__dict__`. +extern "C" inline PyObject *pybind11_get_dict(PyObject *self, void *) { + PyObject *&dict = *_PyObject_GetDictPtr(self); + if (!dict) + dict = PyDict_New(); + Py_XINCREF(dict); + return dict; +} + +/// dynamic_attr: Support for `instance.__dict__ = dict()`. +extern "C" inline int pybind11_set_dict(PyObject *self, PyObject *new_dict, void *) { + if (!PyDict_Check(new_dict)) { + PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, not a '%.200s'", + Py_TYPE(new_dict)->tp_name); + return -1; + } + PyObject *&dict = *_PyObject_GetDictPtr(self); + Py_INCREF(new_dict); + Py_CLEAR(dict); + dict = new_dict; + return 0; +} + +/// dynamic_attr: Allow the garbage collector to traverse the internal instance `__dict__`. +extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { + PyObject *&dict = *_PyObject_GetDictPtr(self); + Py_VISIT(dict); + return 0; +} + +/// dynamic_attr: Allow the GC to clear the dictionary. +extern "C" inline int pybind11_clear(PyObject *self) { + PyObject *&dict = *_PyObject_GetDictPtr(self); + Py_CLEAR(dict); + return 0; +} + +/// Give instances of this type a `__dict__` and opt into garbage collection. +inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { + auto type = &heap_type->ht_type; +#if defined(PYPY_VERSION) + pybind11_fail(std::string(type->tp_name) + ": dynamic attributes are " + "currently not supported in " + "conjunction with PyPy!"); +#endif + type->tp_flags |= Py_TPFLAGS_HAVE_GC; + type->tp_dictoffset = type->tp_basicsize; // place dict at the end + type->tp_basicsize += sizeof(PyObject *); // and allocate enough space for it + type->tp_traverse = pybind11_traverse; + type->tp_clear = pybind11_clear; + + static PyGetSetDef getset[] = { + {const_cast<char*>("__dict__"), pybind11_get_dict, pybind11_set_dict, nullptr, nullptr}, + {nullptr, nullptr, nullptr, nullptr, nullptr} + }; + type->tp_getset = getset; +} + +/// buffer_protocol: Fill in the view as specified by flags. +extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags) { + auto tinfo = get_type_info(Py_TYPE(obj)); + if (view == nullptr || obj == nullptr || !tinfo || !tinfo->get_buffer) { + if (view) + view->obj = nullptr; + PyErr_SetString(PyExc_BufferError, "generic_type::getbuffer(): Internal error"); + return -1; + } + memset(view, 0, sizeof(Py_buffer)); + buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data); + view->obj = obj; + view->ndim = 1; + view->internal = info; + view->buf = info->ptr; + view->itemsize = (ssize_t) info->itemsize; + view->len = view->itemsize; + for (auto s : info->shape) + view->len *= s; + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) + view->format = const_cast<char *>(info->format.c_str()); + if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { + view->ndim = (int) info->ndim; + view->strides = (ssize_t *) &info->strides[0]; + view->shape = (ssize_t *) &info->shape[0]; + } + Py_INCREF(view->obj); + return 0; +} + +/// buffer_protocol: Release the resources of the buffer. +extern "C" inline void pybind11_releasebuffer(PyObject *, Py_buffer *view) { + delete (buffer_info *) view->internal; +} + +/// Give this type a buffer interface. +inline void enable_buffer_protocol(PyHeapTypeObject *heap_type) { + heap_type->ht_type.tp_as_buffer = &heap_type->as_buffer; +#if PY_MAJOR_VERSION < 3 + heap_type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; +#endif + + heap_type->as_buffer.bf_getbuffer = pybind11_getbuffer; + heap_type->as_buffer.bf_releasebuffer = pybind11_releasebuffer; +} + +/** Create a brand new Python type according to the `type_record` specification. + Return value: New reference. */ +inline PyObject* make_new_python_type(const type_record &rec) { + auto name = reinterpret_steal<object>(PYBIND11_FROM_STRING(rec.name)); + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 + auto ht_qualname = name; + if (rec.scope && hasattr(rec.scope, "__qualname__")) { + ht_qualname = reinterpret_steal<object>( + PyUnicode_FromFormat("%U.%U", rec.scope.attr("__qualname__").ptr(), name.ptr())); + } +#endif + + object module; + if (rec.scope) { + if (hasattr(rec.scope, "__module__")) + module = rec.scope.attr("__module__"); + else if (hasattr(rec.scope, "__name__")) + module = rec.scope.attr("__name__"); + } + +#if !defined(PYPY_VERSION) + const auto full_name = module ? str(module).cast<std::string>() + "." + rec.name + : std::string(rec.name); +#else + const auto full_name = std::string(rec.name); +#endif + + char *tp_doc = nullptr; + if (rec.doc && options::show_user_defined_docstrings()) { + /* Allocate memory for docstring (using PyObject_MALLOC, since + Python will free this later on) */ + size_t size = strlen(rec.doc) + 1; + tp_doc = (char *) PyObject_MALLOC(size); + memcpy((void *) tp_doc, rec.doc, size); + } + + auto &internals = get_internals(); + auto bases = tuple(rec.bases); + auto base = (bases.size() == 0) ? internals.get_base(rec.instance_size) + : bases[0].ptr(); + + /* Danger zone: from now (and until PyType_Ready), make sure to + issue no Python C API calls which could potentially invoke the + garbage collector (the GC will call type_traverse(), which will in + turn find the newly constructed type in an invalid state) */ + auto metaclass = rec.metaclass.ptr() ? (PyTypeObject *) rec.metaclass.ptr() + : internals.default_metaclass; + + auto heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); + if (!heap_type) + pybind11_fail(std::string(rec.name) + ": Unable to create type object!"); + + heap_type->ht_name = name.release().ptr(); +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 + heap_type->ht_qualname = ht_qualname.release().ptr(); +#endif + + auto type = &heap_type->ht_type; + type->tp_name = strdup(full_name.c_str()); + type->tp_doc = tp_doc; + type->tp_base = (PyTypeObject *) handle(base).inc_ref().ptr(); + type->tp_basicsize = static_cast<ssize_t>(rec.instance_size); + if (bases.size() > 0) + type->tp_bases = bases.release().ptr(); + + /* Don't inherit base __init__ */ + type->tp_init = pybind11_object_init; + + /* Supported protocols */ + type->tp_as_number = &heap_type->as_number; + type->tp_as_sequence = &heap_type->as_sequence; + type->tp_as_mapping = &heap_type->as_mapping; + + /* Flags */ + type->tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; +#if PY_MAJOR_VERSION < 3 + type->tp_flags |= Py_TPFLAGS_CHECKTYPES; +#endif + + if (rec.dynamic_attr) + enable_dynamic_attributes(heap_type); + + if (rec.buffer_protocol) + enable_buffer_protocol(heap_type); + + if (PyType_Ready(type) < 0) + pybind11_fail(std::string(rec.name) + ": PyType_Ready failed (" + error_string() + ")!"); + + assert(rec.dynamic_attr ? PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) + : !PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)); + + /* Register type with the parent scope */ + if (rec.scope) + setattr(rec.scope, rec.name, (PyObject *) type); + + if (module) // Needed by pydoc + setattr((PyObject *) type, "__module__", module); + + return (PyObject *) type; +} + +NAMESPACE_END(detail) +NAMESPACE_END(pybind11) diff --git a/ext/pybind11/include/pybind11/common.h b/ext/pybind11/include/pybind11/common.h index f7a383007..ac2be7aef 100644 --- a/ext/pybind11/include/pybind11/common.h +++ b/ext/pybind11/include/pybind11/common.h @@ -28,6 +28,33 @@ # endif #endif +// Compiler version assertions +#if defined(__INTEL_COMPILER) +# if __INTEL_COMPILER < 1500 +# error pybind11 requires Intel C++ compiler v15 or newer +# endif +#elif defined(__clang__) && !defined(__apple_build_version__) +# if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3) +# error pybind11 requires clang 3.3 or newer +# endif +#elif defined(__clang__) +// Apple changes clang version macros to its Xcode version; the first Xcode release based on +// (upstream) clang 3.3 was Xcode 5: +# if __clang_major__ < 5 +# error pybind11 requires Xcode/clang 5.0 or newer +# endif +#elif defined(__GNUG__) +# if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8) +# error pybind11 requires gcc 4.8 or newer +# endif +#elif defined(_MSC_VER) +// Pybind hits various compiler bugs in 2015u2 and earlier, and also makes use of some stl features +// (e.g. std::negation) added in 2015u3: +# if _MSC_FULL_VER < 190024210 +# error pybind11 requires MSVC 2015 update 3 or newer +# endif +#endif + #if !defined(PYBIND11_EXPORT) # if defined(WIN32) || defined(_WIN32) # define PYBIND11_EXPORT __declspec(dllexport) @@ -52,16 +79,18 @@ # define PYBIND11_DEPRECATED(reason) __declspec(deprecated) #endif -#define PYBIND11_VERSION_MAJOR 1 -#define PYBIND11_VERSION_MINOR 9 -#define PYBIND11_VERSION_PATCH dev0 +#define PYBIND11_VERSION_MAJOR 2 +#define PYBIND11_VERSION_MINOR 1 +#define PYBIND11_VERSION_PATCH 1 /// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode #if defined(_MSC_VER) -# define HAVE_ROUND +# if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 4) +# define HAVE_ROUND 1 +# endif # pragma warning(push) # pragma warning(disable: 4510 4610 4512 4005) -# if _DEBUG +# if defined(_DEBUG) # define PYBIND11_DEBUG_MARKER # undef _DEBUG # endif @@ -111,6 +140,7 @@ #define PYBIND11_BYTES_FROM_STRING_AND_SIZE PyBytes_FromStringAndSize #define PYBIND11_BYTES_AS_STRING_AND_SIZE PyBytes_AsStringAndSize #define PYBIND11_BYTES_AS_STRING PyBytes_AsString +#define PYBIND11_BYTES_SIZE PyBytes_Size #define PYBIND11_LONG_CHECK(o) PyLong_Check(o) #define PYBIND11_LONG_AS_LONGLONG(o) PyLong_AsLongLong(o) #define PYBIND11_LONG_AS_UNSIGNED_LONGLONG(o) PyLong_AsUnsignedLongLong(o) @@ -119,7 +149,6 @@ #define PYBIND11_SLICE_OBJECT PyObject #define PYBIND11_FROM_STRING PyUnicode_FromString #define PYBIND11_STR_TYPE ::pybind11::str -#define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_base.ob_base.ob_type #define PYBIND11_PLUGIN_IMPL(name) \ extern "C" PYBIND11_EXPORT PyObject *PyInit_##name() #else @@ -129,6 +158,7 @@ #define PYBIND11_BYTES_FROM_STRING_AND_SIZE PyString_FromStringAndSize #define PYBIND11_BYTES_AS_STRING_AND_SIZE PyString_AsStringAndSize #define PYBIND11_BYTES_AS_STRING PyString_AsString +#define PYBIND11_BYTES_SIZE PyString_Size #define PYBIND11_LONG_CHECK(o) (PyInt_Check(o) || PyLong_Check(o)) #define PYBIND11_LONG_AS_LONGLONG(o) (PyInt_Check(o) ? (long long) PyLong_AsLong(o) : PyLong_AsLongLong(o)) #define PYBIND11_LONG_AS_UNSIGNED_LONGLONG(o) (PyInt_Check(o) ? (unsigned long long) PyLong_AsUnsignedLong(o) : PyLong_AsUnsignedLongLong(o)) @@ -137,9 +167,12 @@ #define PYBIND11_SLICE_OBJECT PySliceObject #define PYBIND11_FROM_STRING PyString_FromString #define PYBIND11_STR_TYPE ::pybind11::bytes -#define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_type #define PYBIND11_PLUGIN_IMPL(name) \ - extern "C" PYBIND11_EXPORT PyObject *init##name() + static PyObject *pybind11_init_wrapper(); \ + extern "C" PYBIND11_EXPORT void init##name() { \ + (void)pybind11_init_wrapper(); \ + } \ + PyObject *pybind11_init_wrapper() #endif #if PY_VERSION_HEX >= 0x03050000 && PY_VERSION_HEX < 0x03050200 @@ -155,6 +188,19 @@ extern "C" { #define PYBIND11_INTERNALS_ID "__pybind11_" \ PYBIND11_TOSTRING(PYBIND11_VERSION_MAJOR) "_" PYBIND11_TOSTRING(PYBIND11_VERSION_MINOR) "__" +/** \rst + This macro creates the entry point that will be invoked when the Python interpreter + imports a plugin library. Please create a `module` in the function body and return + the pointer to its underlying Python object at the end. + + .. code-block:: cpp + + PYBIND11_PLUGIN(example) { + pybind11::module m("example", "pybind11 example plugin"); + /// Set up bindings here + return m.ptr(); + } +\endrst */ #define PYBIND11_PLUGIN(name) \ static PyObject *pybind11_init(); \ PYBIND11_PLUGIN_IMPL(name) { \ @@ -172,6 +218,10 @@ extern "C" { } \ try { \ return pybind11_init(); \ + } catch (pybind11::error_already_set &e) { \ + e.clear(); \ + PyErr_SetString(PyExc_ImportError, e.what()); \ + return nullptr; \ } catch (const std::exception &e) { \ PyErr_SetString(PyExc_ImportError, e.what()); \ return nullptr; \ @@ -327,7 +377,7 @@ struct overload_hash { } }; -/// Internal data struture used to track registered instances and types +/// Internal data structure used to track registered instances and types struct internals { std::unordered_map<std::type_index, void*> registered_types_cpp; // std::type_index -> type_info std::unordered_map<const void *, void*> registered_types_py; // PyTypeObject* -> type_info @@ -336,17 +386,34 @@ struct internals { std::unordered_map<std::type_index, std::vector<bool (*)(PyObject *, void *&)>> direct_conversions; std::forward_list<void (*) (std::exception_ptr)> registered_exception_translators; std::unordered_map<std::string, void *> shared_data; // Custom data to be shared across extensions + PyTypeObject *static_property_type; + PyTypeObject *default_metaclass; + std::unordered_map<size_t, PyObject *> bases; // one base type per `instance_size` (very few) #if defined(WITH_THREAD) decltype(PyThread_create_key()) tstate = 0; // Usually an int but a long on Cygwin64 with Python 3.x PyInterpreterState *istate = nullptr; #endif + + /// Return the appropriate base type for the given instance size + PyObject *get_base(size_t instance_size); }; /// Return a reference to the current 'internals' information inline internals &get_internals(); -/// Index sequence for convenient template metaprogramming involving tuples +/// from __cpp_future__ import (convenient aliases from C++14/17) #ifdef PYBIND11_CPP14 +using std::enable_if_t; +using std::conditional_t; +using std::remove_cv_t; +#else +template <bool B, typename T = void> using enable_if_t = typename std::enable_if<B, T>::type; +template <bool B, typename T, typename F> using conditional_t = typename std::conditional<B, T, F>::type; +template <typename T> using remove_cv_t = typename std::remove_cv<T>::type; +#endif + +/// Index sequences +#if defined(PYBIND11_CPP14) || defined(_MSC_VER) using std::index_sequence; using std::make_index_sequence; #else @@ -356,6 +423,35 @@ template<size_t ...S> struct make_index_sequence_impl <0, S...> { typedef index_ template<size_t N> using make_index_sequence = typename make_index_sequence_impl<N>::type; #endif +/// Backports of std::bool_constant and std::negation to accomodate older compilers +template <bool B> using bool_constant = std::integral_constant<bool, B>; +template <typename T> struct negation : bool_constant<!T::value> { }; + +template <typename...> struct void_t_impl { using type = void; }; +template <typename... Ts> using void_t = typename void_t_impl<Ts...>::type; + +/// Compile-time all/any/none of that check the boolean value of all template types +#ifdef __cpp_fold_expressions +template <class... Ts> using all_of = bool_constant<(Ts::value && ...)>; +template <class... Ts> using any_of = bool_constant<(Ts::value || ...)>; +#elif !defined(_MSC_VER) +template <bool...> struct bools {}; +template <class... Ts> using all_of = std::is_same< + bools<Ts::value..., true>, + bools<true, Ts::value...>>; +template <class... Ts> using any_of = negation<all_of<negation<Ts>...>>; +#else +// MSVC has trouble with the above, but supports std::conjunction, which we can use instead (albeit +// at a slight loss of compilation efficiency). +template <class... Ts> using all_of = std::conjunction<Ts...>; +template <class... Ts> using any_of = std::disjunction<Ts...>; +#endif +template <class... Ts> using none_of = negation<any_of<Ts...>>; + +template <class T, template<class> class... Predicates> using satisfies_all_of = all_of<Predicates<T>...>; +template <class T, template<class> class... Predicates> using satisfies_any_of = any_of<Predicates<T>...>; +template <class T, template<class> class... Predicates> using satisfies_none_of = none_of<Predicates<T>...>; + /// Strip the class from a method type template <typename T> struct remove_class { }; template <typename C, typename R, typename... A> struct remove_class<R (C::*)(A...)> { typedef R type(A...); }; @@ -377,34 +473,34 @@ struct void_type { }; /// Helper template which holds a list of types template <typename...> struct type_list { }; -/// from __cpp_future__ import (convenient aliases from C++14/17) -template <bool B> using bool_constant = std::integral_constant<bool, B>; -template <class T> using negation = bool_constant<!T::value>; -template <bool B, typename T = void> using enable_if_t = typename std::enable_if<B, T>::type; -template <bool B, typename T, typename F> using conditional_t = typename std::conditional<B, T, F>::type; - /// Compile-time integer sum +#ifdef __cpp_fold_expressions +template <typename... Ts> constexpr size_t constexpr_sum(Ts... ns) { return (0 + ... + size_t{ns}); } +#else constexpr size_t constexpr_sum() { return 0; } template <typename T, typename... Ts> constexpr size_t constexpr_sum(T n, Ts... ns) { return size_t{n} + constexpr_sum(ns...); } - -// Counts the number of types in the template parameter pack matching the predicate -#if !defined(_MSC_VER) -template <template<typename> class Predicate, typename... Ts> -using count_t = std::integral_constant<size_t, constexpr_sum(Predicate<Ts>::value...)>; -#else -// MSVC workaround (2015 Update 3 has issues with some member type aliases and constexpr) -template <template<typename> class Predicate, typename... Ts> struct count_t; -template <template<typename> class Predicate> struct count_t<Predicate> : std::integral_constant<size_t, 0> {}; -template <template<typename> class Predicate, class T, class... Ts> -struct count_t<Predicate, T, Ts...> : std::integral_constant<size_t, Predicate<T>::value + count_t<Predicate, Ts...>::value> {}; #endif -/// Return true if all/any Ts satify Predicate<T> +NAMESPACE_BEGIN(constexpr_impl) +/// Implementation details for constexpr functions +constexpr int first(int i) { return i; } +template <typename T, typename... Ts> +constexpr int first(int i, T v, Ts... vs) { return v ? i : first(i + 1, vs...); } + +constexpr int last(int /*i*/, int result) { return result; } +template <typename T, typename... Ts> +constexpr int last(int i, int result, T v, Ts... vs) { return last(i + 1, v ? i : result, vs...); } +NAMESPACE_END(constexpr_impl) + +/// Return the index of the first type in Ts which satisfies Predicate<T>. Returns sizeof...(Ts) if +/// none match. template <template<typename> class Predicate, typename... Ts> -using all_of_t = bool_constant<(count_t<Predicate, Ts...>::value == sizeof...(Ts))>; +constexpr int constexpr_first() { return constexpr_impl::first(0, Predicate<Ts>::value...); } + +/// Return the index of the last type in Ts which satisfies Predicate<T>, or -1 if none match. template <template<typename> class Predicate, typename... Ts> -using any_of_t = bool_constant<(count_t<Predicate, Ts...>::value > 0)>; +constexpr int constexpr_last() { return constexpr_impl::last(0, -1, Predicate<Ts>::value...); } // Extracts the first type from the template parameter pack matching the predicate, or Default if none match. template <template<class> class Predicate, class Default, class... Ts> struct first_of; @@ -435,9 +531,9 @@ struct is_template_base_of_impl { /// `is_template_base_of<Base, T>` is true if `struct T : Base<U> {}` where U can be anything template <template<typename...> class Base, typename T> #if !defined(_MSC_VER) -using is_template_base_of = decltype(is_template_base_of_impl<Base>::check((T*)nullptr)); +using is_template_base_of = decltype(is_template_base_of_impl<Base>::check((remove_cv_t<T>*)nullptr)); #else // MSVC2015 has trouble with decltype in template aliases -struct is_template_base_of : decltype(is_template_base_of_impl<Base>::check((T*)nullptr)) { }; +struct is_template_base_of : decltype(is_template_base_of_impl<Base>::check((remove_cv_t<T>*)nullptr)) { }; #endif /// Check if T is std::shared_ptr<U> where U can be anything @@ -498,6 +594,9 @@ public: /// Give the error back to Python void restore() { PyErr_Restore(type, value, trace); type = value = trace = nullptr; } + /// Clear the held Python error state (the C++ `what()` message remains intact) + void clear() { restore(); PyErr_Clear(); } + private: PyObject *type, *value, *trace; }; @@ -506,7 +605,8 @@ private: class builtin_exception : public std::runtime_error { public: using std::runtime_error::runtime_error; - virtual void set_error() const = 0; /// Set the error using the Python C API + /// Set the error using the Python C API + virtual void set_error() const = 0; }; #define PYBIND11_RUNTIME_EXCEPTION(name, type) \ @@ -527,21 +627,49 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used in [[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const char *reason) { throw std::runtime_error(reason); } [[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const std::string &reason) { throw std::runtime_error(reason); } -/// Format strings for basic number types -#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor<t> \ - { static constexpr const char* value = v; /* for backwards compatibility */ \ - static std::string format() { return value; } } - template <typename T, typename SFINAE = void> struct format_descriptor { }; -template <typename T> struct format_descriptor<T, detail::enable_if_t<std::is_integral<T>::value>> { - static constexpr const char c = "bBhHiIqQ"[detail::log2(sizeof(T))*2 + std::is_unsigned<T>::value]; +NAMESPACE_BEGIN(detail) +// Returns the index of the given type in the type char array below, and in the list in numpy.h +// The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double; +// complex float,double,long double. Note that the long double types only participate when long +// double is actually longer than double (it isn't under MSVC). +// NB: not only the string below but also complex.h and numpy.h rely on this order. +template <typename T, typename SFINAE = void> struct is_fmt_numeric { static constexpr bool value = false; }; +template <typename T> struct is_fmt_numeric<T, enable_if_t<std::is_arithmetic<T>::value>> { + static constexpr bool value = true; + static constexpr int index = std::is_same<T, bool>::value ? 0 : 1 + ( + std::is_integral<T>::value ? detail::log2(sizeof(T))*2 + std::is_unsigned<T>::value : 8 + ( + std::is_same<T, double>::value ? 1 : std::is_same<T, long double>::value ? 2 : 0)); +}; +NAMESPACE_END(detail) + +template <typename T> struct format_descriptor<T, detail::enable_if_t<detail::is_fmt_numeric<T>::value>> { + static constexpr const char c = "?bBhHiIqQfdgFDG"[detail::is_fmt_numeric<T>::index]; static constexpr const char value[2] = { c, '\0' }; static std::string format() { return std::string(1, c); } }; template <typename T> constexpr const char format_descriptor< - T, detail::enable_if_t<std::is_integral<T>::value>>::value[2]; + T, detail::enable_if_t<detail::is_fmt_numeric<T>::value>>::value[2]; + +NAMESPACE_BEGIN(detail) + +template <typename T, typename SFINAE = void> struct compare_buffer_info { + static bool compare(const buffer_info& b) { + return b.format == format_descriptor<T>::format() && b.itemsize == sizeof(T); + } +}; + +template <typename T> struct compare_buffer_info<T, detail::enable_if_t<std::is_integral<T>::value>> { + static bool compare(const buffer_info& b) { + return b.itemsize == sizeof(T) && (b.format == format_descriptor<T>::value || + ((sizeof(T) == sizeof(long)) && b.format == (std::is_unsigned<T>::value ? "L" : "l")) || + ((sizeof(T) == sizeof(size_t)) && b.format == (std::is_unsigned<T>::value ? "N" : "n"))); + } +}; + +NAMESPACE_END(detail) /// RAII wrapper that temporarily clears any Python error state struct error_scope { @@ -550,16 +678,11 @@ struct error_scope { ~error_scope() { PyErr_Restore(type, value, trace); } }; -PYBIND11_DECL_FMT(float, "f"); -PYBIND11_DECL_FMT(double, "d"); -PYBIND11_DECL_FMT(bool, "?"); - /// Dummy destructor wrapper that can be used to expose classes with a private destructor struct nodelete { template <typename T> void operator()(T*) { } }; -// overload_cast requires variable templates: C++14 or MSVC 2015 Update 2 -#if defined(PYBIND11_CPP14) || ( \ - defined(_MSC_FULL_VER) &&_MSC_FULL_VER >= 190023918) +// overload_cast requires variable templates: C++14 or MSVC +#if defined(PYBIND11_CPP14) || defined(_MSC_VER) #define PYBIND11_OVERLOAD_CAST 1 NAMESPACE_BEGIN(detail) diff --git a/ext/pybind11/include/pybind11/complex.h b/ext/pybind11/include/pybind11/complex.h index f767f354c..945ca0710 100644 --- a/ext/pybind11/include/pybind11/complex.h +++ b/ext/pybind11/include/pybind11/complex.h @@ -18,16 +18,21 @@ #endif NAMESPACE_BEGIN(pybind11) +NAMESPACE_BEGIN(detail) -PYBIND11_DECL_FMT(std::complex<float>, "Zf"); -PYBIND11_DECL_FMT(std::complex<double>, "Zd"); +// The format codes are already in the string in common.h, we just need to provide a specialization +template <typename T> struct is_fmt_numeric<std::complex<T>> { + static constexpr bool value = true; + static constexpr int index = is_fmt_numeric<T>::index + 3; +}; -NAMESPACE_BEGIN(detail) template <typename T> class type_caster<std::complex<T>> { public: - bool load(handle src, bool) { + bool load(handle src, bool convert) { if (!src) return false; + if (!convert && !PyComplex_Check(src.ptr())) + return false; Py_complex result = PyComplex_AsCComplex(src.ptr()); if (result.real == -1.0 && PyErr_Occurred()) { PyErr_Clear(); diff --git a/ext/pybind11/include/pybind11/eigen.h b/ext/pybind11/include/pybind11/eigen.h index 0a1208e16..6abe8c48f 100644 --- a/ext/pybind11/include/pybind11/eigen.h +++ b/ext/pybind11/include/pybind11/eigen.h @@ -17,158 +17,506 @@ # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wconversion" # pragma GCC diagnostic ignored "-Wdeprecated-declarations" +# if __GNUC__ >= 7 +# pragma GCC diagnostic ignored "-Wint-in-bool-context" +# endif #endif #include <Eigen/Core> #include <Eigen/SparseCore> -#if defined(__GNUG__) || defined(__clang__) -# pragma GCC diagnostic pop -#endif - #if defined(_MSC_VER) -#pragma warning(push) -#pragma warning(disable: 4127) // warning C4127: Conditional expression is constant +# pragma warning(push) +# pragma warning(disable: 4127) // warning C4127: Conditional expression is constant #endif +// Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit +// move constructors that break things. We could detect this an explicitly copy, but an extra copy +// of matrices seems highly undesirable. +static_assert(EIGEN_VERSION_AT_LEAST(3,2,7), "Eigen support in pybind11 requires Eigen >= 3.2.7"); + NAMESPACE_BEGIN(pybind11) + +// Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: +using EigenDStride = Eigen::Stride<Eigen::Dynamic, Eigen::Dynamic>; +template <typename MatrixType> using EigenDRef = Eigen::Ref<MatrixType, 0, EigenDStride>; +template <typename MatrixType> using EigenDMap = Eigen::Map<MatrixType, 0, EigenDStride>; + NAMESPACE_BEGIN(detail) -template <typename T> using is_eigen_dense = is_template_base_of<Eigen::DenseBase, T>; -template <typename T> using is_eigen_sparse = is_template_base_of<Eigen::SparseMatrixBase, T>; -template <typename T> using is_eigen_ref = is_template_base_of<Eigen::RefBase, T>; +#if EIGEN_VERSION_AT_LEAST(3,3,0) +using EigenIndex = Eigen::Index; +#else +using EigenIndex = EIGEN_DEFAULT_DENSE_INDEX_TYPE; +#endif +// Matches Eigen::Map, Eigen::Ref, blocks, etc: +template <typename T> using is_eigen_dense_map = all_of<is_template_base_of<Eigen::DenseBase, T>, std::is_base_of<Eigen::MapBase<T, Eigen::ReadOnlyAccessors>, T>>; +template <typename T> using is_eigen_mutable_map = std::is_base_of<Eigen::MapBase<T, Eigen::WriteAccessors>, T>; +template <typename T> using is_eigen_dense_plain = all_of<negation<is_eigen_dense_map<T>>, is_template_base_of<Eigen::PlainObjectBase, T>>; +template <typename T> using is_eigen_sparse = is_template_base_of<Eigen::SparseMatrixBase, T>; // Test for objects inheriting from EigenBase<Derived> that aren't captured by the above. This // basically covers anything that can be assigned to a dense matrix but that don't have a typical // matrix data layout that can be copied from their .data(). For example, DiagonalMatrix and // SelfAdjointView fall into this category. -template <typename T> using is_eigen_base = bool_constant< - is_template_base_of<Eigen::EigenBase, T>::value - && !is_eigen_dense<T>::value && !is_eigen_sparse<T>::value +template <typename T> using is_eigen_other = all_of< + is_template_base_of<Eigen::EigenBase, T>, + negation<any_of<is_eigen_dense_map<T>, is_eigen_dense_plain<T>, is_eigen_sparse<T>>> >; +// Captures numpy/eigen conformability status (returned by EigenProps::conformable()): +template <bool EigenRowMajor> struct EigenConformable { + bool conformable = false; + EigenIndex rows = 0, cols = 0; + EigenDStride stride{0, 0}; + + EigenConformable(bool fits = false) : conformable{fits} {} + // Matrix type: + EigenConformable(EigenIndex r, EigenIndex c, + EigenIndex rstride, EigenIndex cstride) : + conformable{true}, rows{r}, cols{c}, + stride(EigenRowMajor ? rstride : cstride /* outer stride */, + EigenRowMajor ? cstride : rstride /* inner stride */) + {} + // Vector type: + EigenConformable(EigenIndex r, EigenIndex c, EigenIndex stride) + : EigenConformable(r, c, r == 1 ? c*stride : stride, c == 1 ? r : r*stride) {} + + template <typename props> bool stride_compatible() const { + // To have compatible strides, we need (on both dimensions) one of fully dynamic strides, + // matching strides, or a dimension size of 1 (in which case the stride value is irrelevant) + return + (props::inner_stride == Eigen::Dynamic || props::inner_stride == stride.inner() || + (EigenRowMajor ? cols : rows) == 1) && + (props::outer_stride == Eigen::Dynamic || props::outer_stride == stride.outer() || + (EigenRowMajor ? rows : cols) == 1); + } + operator bool() const { return conformable; } +}; + +template <typename Type> struct eigen_extract_stride { using type = Type; }; +template <typename PlainObjectType, int MapOptions, typename StrideType> +struct eigen_extract_stride<Eigen::Map<PlainObjectType, MapOptions, StrideType>> { using type = StrideType; }; +template <typename PlainObjectType, int Options, typename StrideType> +struct eigen_extract_stride<Eigen::Ref<PlainObjectType, Options, StrideType>> { using type = StrideType; }; + +// Helper struct for extracting information from an Eigen type +template <typename Type_> struct EigenProps { + using Type = Type_; + using Scalar = typename Type::Scalar; + using StrideType = typename eigen_extract_stride<Type>::type; + static constexpr EigenIndex + rows = Type::RowsAtCompileTime, + cols = Type::ColsAtCompileTime, + size = Type::SizeAtCompileTime; + static constexpr bool + row_major = Type::IsRowMajor, + vector = Type::IsVectorAtCompileTime, // At least one dimension has fixed size 1 + fixed_rows = rows != Eigen::Dynamic, + fixed_cols = cols != Eigen::Dynamic, + fixed = size != Eigen::Dynamic, // Fully-fixed size + dynamic = !fixed_rows && !fixed_cols; // Fully-dynamic size + + template <EigenIndex i, EigenIndex ifzero> using if_zero = std::integral_constant<EigenIndex, i == 0 ? ifzero : i>; + static constexpr EigenIndex inner_stride = if_zero<StrideType::InnerStrideAtCompileTime, 1>::value, + outer_stride = if_zero<StrideType::OuterStrideAtCompileTime, + vector ? size : row_major ? cols : rows>::value; + static constexpr bool dynamic_stride = inner_stride == Eigen::Dynamic && outer_stride == Eigen::Dynamic; + static constexpr bool requires_row_major = !dynamic_stride && !vector && (row_major ? inner_stride : outer_stride) == 1; + static constexpr bool requires_col_major = !dynamic_stride && !vector && (row_major ? outer_stride : inner_stride) == 1; + + // Takes an input array and determines whether we can make it fit into the Eigen type. If + // the array is a vector, we attempt to fit it into either an Eigen 1xN or Nx1 vector + // (preferring the latter if it will fit in either, i.e. for a fully dynamic matrix type). + static EigenConformable<row_major> conformable(const array &a) { + const auto dims = a.ndim(); + if (dims < 1 || dims > 2) + return false; + + if (dims == 2) { // Matrix type: require exact match (or dynamic) + + EigenIndex + np_rows = a.shape(0), + np_cols = a.shape(1), + np_rstride = a.strides(0) / sizeof(Scalar), + np_cstride = a.strides(1) / sizeof(Scalar); + if ((fixed_rows && np_rows != rows) || (fixed_cols && np_cols != cols)) + return false; + + return {np_rows, np_cols, np_rstride, np_cstride}; + } + + // Otherwise we're storing an n-vector. Only one of the strides will be used, but whichever + // is used, we want the (single) numpy stride value. + const EigenIndex n = a.shape(0), + stride = a.strides(0) / sizeof(Scalar); + + if (vector) { // Eigen type is a compile-time vector + if (fixed && size != n) + return false; // Vector size mismatch + return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; + } + else if (fixed) { + // The type has a fixed size, but is not a vector: abort + return false; + } + else if (fixed_cols) { + // Since this isn't a vector, cols must be != 1. We allow this only if it exactly + // equals the number of elements (rows is Dynamic, and so 1 row is allowed). + if (cols != n) return false; + return {1, n, stride}; + } + else { + // Otherwise it's either fully dynamic, or column dynamic; both become a column vector + if (fixed_rows && rows != n) return false; + return {n, 1, stride}; + } + } + + static PYBIND11_DESCR descriptor() { + constexpr bool show_writeable = is_eigen_dense_map<Type>::value && is_eigen_mutable_map<Type>::value; + constexpr bool show_order = is_eigen_dense_map<Type>::value; + constexpr bool show_c_contiguous = show_order && requires_row_major; + constexpr bool show_f_contiguous = !show_c_contiguous && show_order && requires_col_major; + + return _("numpy.ndarray[") + npy_format_descriptor<Scalar>::name() + + _("[") + _<fixed_rows>(_<(size_t) rows>(), _("m")) + + _(", ") + _<fixed_cols>(_<(size_t) cols>(), _("n")) + + _("]") + + // For a reference type (e.g. Ref<MatrixXd>) we have other constraints that might need to be + // satisfied: writeable=True (for a mutable reference), and, depending on the map's stride + // options, possibly f_contiguous or c_contiguous. We include them in the descriptor output + // to provide some hint as to why a TypeError is occurring (otherwise it can be confusing to + // see that a function accepts a 'numpy.ndarray[float64[3,2]]' and an error message that you + // *gave* a numpy.ndarray of the right type and dimensions. + _<show_writeable>(", flags.writeable", "") + + _<show_c_contiguous>(", flags.c_contiguous", "") + + _<show_f_contiguous>(", flags.f_contiguous", "") + + _("]"); + } +}; + +// Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, +// otherwise it'll make a copy. writeable lets you turn off the writeable flag for the array. +template <typename props> handle eigen_array_cast(typename props::Type const &src, handle base = handle(), bool writeable = true) { + constexpr size_t elem_size = sizeof(typename props::Scalar); + std::vector<size_t> shape, strides; + if (props::vector) { + shape.push_back(src.size()); + strides.push_back(elem_size * src.innerStride()); + } + else { + shape.push_back(src.rows()); + shape.push_back(src.cols()); + strides.push_back(elem_size * src.rowStride()); + strides.push_back(elem_size * src.colStride()); + } + array a(std::move(shape), std::move(strides), src.data(), base); + if (!writeable) + array_proxy(a.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + + return a.release(); +} + +// Takes an lvalue ref to some Eigen type and a (python) base object, creating a numpy array that +// reference the Eigen object's data with `base` as the python-registered base class (if omitted, +// the base will be set to None, and lifetime management is up to the caller). The numpy array is +// non-writeable if the given type is const. +template <typename props, typename Type> +handle eigen_ref_array(Type &src, handle parent = none()) { + // none here is to get past array's should-we-copy detection, which currently always + // copies when there is no base. Setting the base to None should be harmless. + return eigen_array_cast<props>(src, parent, !std::is_const<Type>::value); +} + +// Takes a pointer to some dense, plain Eigen type, builds a capsule around it, then returns a numpy +// array that references the encapsulated data with a python-side reference to the capsule to tie +// its destruction to that of any dependent python objects. Const-ness is determined by whether or +// not the Type of the pointer given is const. +template <typename props, typename Type, typename = enable_if_t<is_eigen_dense_plain<Type>::value>> +handle eigen_encapsulate(Type *src) { + capsule base(src, [](void *o) { delete static_cast<Type *>(o); }); + return eigen_ref_array<props>(*src, base); +} + +// Type caster for regular, dense matrix types (e.g. MatrixXd), but not maps/refs/etc. of dense +// types. template<typename Type> -struct type_caster<Type, enable_if_t<is_eigen_dense<Type>::value && !is_eigen_ref<Type>::value>> { - typedef typename Type::Scalar Scalar; - static constexpr bool rowMajor = Type::Flags & Eigen::RowMajorBit; - static constexpr bool isVector = Type::IsVectorAtCompileTime; +struct type_caster<Type, enable_if_t<is_eigen_dense_plain<Type>::value>> { + using Scalar = typename Type::Scalar; + using props = EigenProps<Type>; bool load(handle src, bool) { auto buf = array_t<Scalar>::ensure(src); if (!buf) return false; - if (buf.ndim() == 1) { - typedef Eigen::InnerStride<> Strides; - if (!isVector && - !(Type::RowsAtCompileTime == Eigen::Dynamic && - Type::ColsAtCompileTime == Eigen::Dynamic)) - return false; + auto dims = buf.ndim(); + if (dims < 1 || dims > 2) + return false; - if (Type::SizeAtCompileTime != Eigen::Dynamic && - buf.shape(0) != (size_t) Type::SizeAtCompileTime) - return false; + auto fits = props::conformable(buf); + if (!fits) + return false; // Non-comformable vector/matrix types - Strides::Index n_elts = (Strides::Index) buf.shape(0); - Strides::Index unity = 1; + value = Eigen::Map<const Type, 0, EigenDStride>(buf.data(), fits.rows, fits.cols, fits.stride); - value = Eigen::Map<Type, 0, Strides>( - buf.mutable_data(), - rowMajor ? unity : n_elts, - rowMajor ? n_elts : unity, - Strides(buf.strides(0) / sizeof(Scalar)) - ); - } else if (buf.ndim() == 2) { - typedef Eigen::Stride<Eigen::Dynamic, Eigen::Dynamic> Strides; + return true; + } - if ((Type::RowsAtCompileTime != Eigen::Dynamic && buf.shape(0) != (size_t) Type::RowsAtCompileTime) || - (Type::ColsAtCompileTime != Eigen::Dynamic && buf.shape(1) != (size_t) Type::ColsAtCompileTime)) - return false; +private: + + // Cast implementation + template <typename CType> + static handle cast_impl(CType *src, return_value_policy policy, handle parent) { + switch (policy) { + case return_value_policy::take_ownership: + case return_value_policy::automatic: + return eigen_encapsulate<props>(src); + case return_value_policy::move: + return eigen_encapsulate<props>(new CType(std::move(*src))); + case return_value_policy::copy: + return eigen_array_cast<props>(*src); + case return_value_policy::reference: + case return_value_policy::automatic_reference: + return eigen_ref_array<props>(*src); + case return_value_policy::reference_internal: + return eigen_ref_array<props>(*src, parent); + default: + throw cast_error("unhandled return_value_policy: should not happen!"); + }; + } - value = Eigen::Map<Type, 0, Strides>( - buf.mutable_data(), - typename Strides::Index(buf.shape(0)), - typename Strides::Index(buf.shape(1)), - Strides(buf.strides(rowMajor ? 0 : 1) / sizeof(Scalar), - buf.strides(rowMajor ? 1 : 0) / sizeof(Scalar)) - ); - } else { - return false; - } - return true; +public: + + // Normal returned non-reference, non-const value: + static handle cast(Type &&src, return_value_policy /* policy */, handle parent) { + return cast_impl(&src, return_value_policy::move, parent); + } + // If you return a non-reference const, we mark the numpy array readonly: + static handle cast(const Type &&src, return_value_policy /* policy */, handle parent) { + return cast_impl(&src, return_value_policy::move, parent); + } + // lvalue reference return; default (automatic) becomes copy + static handle cast(Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) + policy = return_value_policy::copy; + return cast_impl(&src, policy, parent); + } + // const lvalue reference return; default (automatic) becomes copy + static handle cast(const Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) + policy = return_value_policy::copy; + return cast(&src, policy, parent); + } + // non-const pointer return + static handle cast(Type *src, return_value_policy policy, handle parent) { + return cast_impl(src, policy, parent); + } + // const pointer return + static handle cast(const Type *src, return_value_policy policy, handle parent) { + return cast_impl(src, policy, parent); } - static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { - if (isVector) { - return array( - { (size_t) src.size() }, // shape - { sizeof(Scalar) * static_cast<size_t>(src.innerStride()) }, // strides - src.data() // data - ).release(); - } else { - return array( - { (size_t) src.rows(), // shape - (size_t) src.cols() }, - { sizeof(Scalar) * static_cast<size_t>(src.rowStride()), // strides - sizeof(Scalar) * static_cast<size_t>(src.colStride()) }, - src.data() // data - ).release(); + static PYBIND11_DESCR name() { return type_descr(props::descriptor()); } + + operator Type*() { return &value; } + operator Type&() { return value; } + template <typename T> using cast_op_type = cast_op_type<T>; + +private: + Type value; +}; + +// Eigen Ref/Map classes have slightly different policy requirements, meaning we don't want to force +// `move` when a Ref/Map rvalue is returned; we treat Ref<> sort of like a pointer (we care about +// the underlying data, not the outer shell). +template <typename Return> +struct return_value_policy_override<Return, enable_if_t<is_eigen_dense_map<Return>::value>> { + static return_value_policy policy(return_value_policy p) { return p; } +}; + +// Base class for casting reference/map/block/etc. objects back to python. +template <typename MapType> struct eigen_map_caster { +private: + using props = EigenProps<MapType>; + +public: + + // Directly referencing a ref/map's data is a bit dangerous (whatever the map/ref points to has + // to stay around), but we'll allow it under the assumption that you know what you're doing (and + // have an appropriate keep_alive in place). We return a numpy array pointing directly at the + // ref's data (The numpy array ends up read-only if the ref was to a const matrix type.) Note + // that this means you need to ensure you don't destroy the object in some other way (e.g. with + // an appropriate keep_alive, or with a reference to a statically allocated matrix). + static handle cast(const MapType &src, return_value_policy policy, handle parent) { + switch (policy) { + case return_value_policy::copy: + return eigen_array_cast<props>(src); + case return_value_policy::reference_internal: + return eigen_array_cast<props>(src, parent, is_eigen_mutable_map<MapType>::value); + case return_value_policy::reference: + case return_value_policy::automatic: + case return_value_policy::automatic_reference: + return eigen_array_cast<props>(src, none(), is_eigen_mutable_map<MapType>::value); + default: + // move, take_ownership don't make any sense for a ref/map: + pybind11_fail("Invalid return_value_policy for Eigen Map/Ref/Block type"); } } - PYBIND11_TYPE_CASTER(Type, _("numpy.ndarray[") + npy_format_descriptor<Scalar>::name() + - _("[") + rows() + _(", ") + cols() + _("]]")); + static PYBIND11_DESCR name() { return props::descriptor(); } -protected: - template <typename T = Type, enable_if_t<T::RowsAtCompileTime == Eigen::Dynamic, int> = 0> - static PYBIND11_DESCR rows() { return _("m"); } - template <typename T = Type, enable_if_t<T::RowsAtCompileTime != Eigen::Dynamic, int> = 0> - static PYBIND11_DESCR rows() { return _<T::RowsAtCompileTime>(); } - template <typename T = Type, enable_if_t<T::ColsAtCompileTime == Eigen::Dynamic, int> = 0> - static PYBIND11_DESCR cols() { return _("n"); } - template <typename T = Type, enable_if_t<T::ColsAtCompileTime != Eigen::Dynamic, int> = 0> - static PYBIND11_DESCR cols() { return _<T::ColsAtCompileTime>(); } + // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return + // types but not bound arguments). We still provide them (with an explicitly delete) so that + // you end up here if you try anyway. + bool load(handle, bool) = delete; + operator MapType() = delete; + template <typename> using cast_op_type = MapType; }; -// Eigen::Ref<Derived> satisfies is_eigen_dense, but isn't constructable, so it needs a special -// type_caster to handle argument copying/forwarding. -template <typename CVDerived, int Options, typename StrideType> -struct type_caster<Eigen::Ref<CVDerived, Options, StrideType>> { -protected: - using Type = Eigen::Ref<CVDerived, Options, StrideType>; - using Derived = typename std::remove_const<CVDerived>::type; - using DerivedCaster = type_caster<Derived>; - DerivedCaster derived_caster; - std::unique_ptr<Type> value; +// We can return any map-like object (but can only load Refs, specialized next): +template <typename Type> struct type_caster<Type, enable_if_t<is_eigen_dense_map<Type>::value>> + : eigen_map_caster<Type> {}; + +// Loader for Ref<...> arguments. See the documentation for info on how to make this work without +// copying (it requires some extra effort in many cases). +template <typename PlainObjectType, typename StrideType> +struct type_caster< + Eigen::Ref<PlainObjectType, 0, StrideType>, + enable_if_t<is_eigen_dense_map<Eigen::Ref<PlainObjectType, 0, StrideType>>::value> +> : public eigen_map_caster<Eigen::Ref<PlainObjectType, 0, StrideType>> { +private: + using Type = Eigen::Ref<PlainObjectType, 0, StrideType>; + using props = EigenProps<Type>; + using Scalar = typename props::Scalar; + using MapType = Eigen::Map<PlainObjectType, 0, StrideType>; + using Array = array_t<Scalar, array::forcecast | + ((props::row_major ? props::inner_stride : props::outer_stride) == 1 ? array::c_style : + (props::row_major ? props::outer_stride : props::inner_stride) == 1 ? array::f_style : 0)>; + static constexpr bool need_writeable = is_eigen_mutable_map<Type>::value; + // Delay construction (these have no default constructor) + std::unique_ptr<MapType> map; + std::unique_ptr<Type> ref; + // Our array. When possible, this is just a numpy array pointing to the source data, but + // sometimes we can't avoid copying (e.g. input is not a numpy array at all, has an incompatible + // layout, or is an array of a type that needs to be converted). Using a numpy temporary + // (rather than an Eigen temporary) saves an extra copy when we need both type conversion and + // storage order conversion. (Note that we refuse to use this temporary copy when loading an + // argument for a Ref<M> with M non-const, i.e. a read-write reference). + Array copy_or_ref; public: - bool load(handle src, bool convert) { if (derived_caster.load(src, convert)) { value.reset(new Type(derived_caster.operator Derived&())); return true; } return false; } - static handle cast(const Type &src, return_value_policy policy, handle parent) { return DerivedCaster::cast(src, policy, parent); } - static handle cast(const Type *src, return_value_policy policy, handle parent) { return DerivedCaster::cast(*src, policy, parent); } + bool load(handle src, bool convert) { + // First check whether what we have is already an array of the right type. If not, we can't + // avoid a copy (because the copy is also going to do type conversion). + bool need_copy = !isinstance<Array>(src); + + EigenConformable<props::row_major> fits; + if (!need_copy) { + // We don't need a converting copy, but we also need to check whether the strides are + // compatible with the Ref's stride requirements + Array aref = reinterpret_borrow<Array>(src); + + if (aref && (!need_writeable || aref.writeable())) { + fits = props::conformable(aref); + if (!fits) return false; // Incompatible dimensions + if (!fits.template stride_compatible<props>()) + need_copy = true; + else + copy_or_ref = std::move(aref); + } + else { + need_copy = true; + } + } + + if (need_copy) { + // We need to copy: If we need a mutable reference, or we're not supposed to convert + // (either because we're in the no-convert overload pass, or because we're explicitly + // instructed not to copy (via `py::arg().noconvert()`) we have to fail loading. + if (!convert || need_writeable) return false; + + Array copy = Array::ensure(src); + if (!copy) return false; + fits = props::conformable(copy); + if (!fits || !fits.template stride_compatible<props>()) + return false; + copy_or_ref = std::move(copy); + } + + ref.reset(); + map.reset(new MapType(data(copy_or_ref), fits.rows, fits.cols, make_stride(fits.stride.outer(), fits.stride.inner()))); + ref.reset(new Type(*map)); - static PYBIND11_DESCR name() { return DerivedCaster::name(); } + return true; + } - operator Type*() { return value.get(); } - operator Type&() { if (!value) pybind11_fail("Eigen::Ref<...> value not loaded"); return *value; } + operator Type*() { return ref.get(); } + operator Type&() { return *ref; } template <typename _T> using cast_op_type = pybind11::detail::cast_op_type<_T>; + +private: + template <typename T = Type, enable_if_t<is_eigen_mutable_map<T>::value, int> = 0> + Scalar *data(Array &a) { return a.mutable_data(); } + + template <typename T = Type, enable_if_t<!is_eigen_mutable_map<T>::value, int> = 0> + const Scalar *data(Array &a) { return a.data(); } + + // Attempt to figure out a constructor of `Stride` that will work. + // If both strides are fixed, use a default constructor: + template <typename S> using stride_ctor_default = bool_constant< + S::InnerStrideAtCompileTime != Eigen::Dynamic && S::OuterStrideAtCompileTime != Eigen::Dynamic && + std::is_default_constructible<S>::value>; + // Otherwise, if there is a two-index constructor, assume it is (outer,inner) like + // Eigen::Stride, and use it: + template <typename S> using stride_ctor_dual = bool_constant< + !stride_ctor_default<S>::value && std::is_constructible<S, EigenIndex, EigenIndex>::value>; + // Otherwise, if there is a one-index constructor, and just one of the strides is dynamic, use + // it (passing whichever stride is dynamic). + template <typename S> using stride_ctor_outer = bool_constant< + !any_of<stride_ctor_default<S>, stride_ctor_dual<S>>::value && + S::OuterStrideAtCompileTime == Eigen::Dynamic && S::InnerStrideAtCompileTime != Eigen::Dynamic && + std::is_constructible<S, EigenIndex>::value>; + template <typename S> using stride_ctor_inner = bool_constant< + !any_of<stride_ctor_default<S>, stride_ctor_dual<S>>::value && + S::InnerStrideAtCompileTime == Eigen::Dynamic && S::OuterStrideAtCompileTime != Eigen::Dynamic && + std::is_constructible<S, EigenIndex>::value>; + + template <typename S = StrideType, enable_if_t<stride_ctor_default<S>::value, int> = 0> + static S make_stride(EigenIndex, EigenIndex) { return S(); } + template <typename S = StrideType, enable_if_t<stride_ctor_dual<S>::value, int> = 0> + static S make_stride(EigenIndex outer, EigenIndex inner) { return S(outer, inner); } + template <typename S = StrideType, enable_if_t<stride_ctor_outer<S>::value, int> = 0> + static S make_stride(EigenIndex outer, EigenIndex) { return S(outer); } + template <typename S = StrideType, enable_if_t<stride_ctor_inner<S>::value, int> = 0> + static S make_stride(EigenIndex, EigenIndex inner) { return S(inner); } + }; -// type_caster for special matrix types (e.g. DiagonalMatrix): load() is not supported, but we can -// cast them into the python domain by first copying to a regular Eigen::Matrix, then casting that. +// type_caster for special matrix types (e.g. DiagonalMatrix), which are EigenBase, but not +// EigenDense (i.e. they don't have a data(), at least not with the usual matrix layout). +// load() is not supported, but we can cast them into the python domain by first copying to a +// regular Eigen::Matrix, then casting that. template <typename Type> -struct type_caster<Type, enable_if_t<is_eigen_base<Type>::value && !is_eigen_ref<Type>::value>> { +struct type_caster<Type, enable_if_t<is_eigen_other<Type>::value>> { protected: - using Matrix = Eigen::Matrix<typename Type::Scalar, Eigen::Dynamic, Eigen::Dynamic>; - using MatrixCaster = type_caster<Matrix>; + using Matrix = Eigen::Matrix<typename Type::Scalar, Type::RowsAtCompileTime, Type::ColsAtCompileTime>; + using props = EigenProps<Matrix>; public: - [[noreturn]] bool load(handle, bool) { pybind11_fail("Unable to load() into specialized EigenBase object"); } - static handle cast(const Type &src, return_value_policy policy, handle parent) { return MatrixCaster::cast(Matrix(src), policy, parent); } - static handle cast(const Type *src, return_value_policy policy, handle parent) { return MatrixCaster::cast(Matrix(*src), policy, parent); } + static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { + handle h = eigen_encapsulate<props>(new Matrix(src)); + return h; + } + static handle cast(const Type *src, return_value_policy policy, handle parent) { return cast(*src, policy, parent); } - static PYBIND11_DESCR name() { return MatrixCaster::name(); } + static PYBIND11_DESCR name() { return props::descriptor(); } - [[noreturn]] operator Type*() { pybind11_fail("Loading not supported for specialized EigenBase object"); } - [[noreturn]] operator Type&() { pybind11_fail("Loading not supported for specialized EigenBase object"); } - template <typename _T> using cast_op_type = pybind11::detail::cast_op_type<_T>; + // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return + // types but not bound arguments). We still provide them (with an explicitly delete) so that + // you end up here if you try anyway. + bool load(handle, bool) = delete; + operator Type() = delete; + template <typename> using cast_op_type = Type; }; template<typename Type> @@ -176,7 +524,7 @@ struct type_caster<Type, enable_if_t<is_eigen_sparse<Type>::value>> { typedef typename Type::Scalar Scalar; typedef typename std::remove_reference<decltype(*std::declval<Type>().outerIndexPtr())>::type StorageIndex; typedef typename Type::Index Index; - static constexpr bool rowMajor = Type::Flags & Eigen::RowMajorBit; + static constexpr bool rowMajor = Type::IsRowMajor; bool load(handle src, bool) { if (!src) @@ -227,13 +575,15 @@ struct type_caster<Type, enable_if_t<is_eigen_sparse<Type>::value>> { ).release(); } - PYBIND11_TYPE_CASTER(Type, _<(Type::Flags & Eigen::RowMajorBit) != 0>("scipy.sparse.csr_matrix[", "scipy.sparse.csc_matrix[") + PYBIND11_TYPE_CASTER(Type, _<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", "scipy.sparse.csc_matrix[") + npy_format_descriptor<Scalar>::name() + _("]")); }; NAMESPACE_END(detail) NAMESPACE_END(pybind11) -#if defined(_MSC_VER) -#pragma warning(pop) +#if defined(__GNUG__) || defined(__clang__) +# pragma GCC diagnostic pop +#elif defined(_MSC_VER) +# pragma warning(pop) #endif diff --git a/ext/pybind11/include/pybind11/eval.h b/ext/pybind11/include/pybind11/eval.h index 204427d77..5b2b98272 100644 --- a/ext/pybind11/include/pybind11/eval.h +++ b/ext/pybind11/include/pybind11/eval.h @@ -95,8 +95,15 @@ object eval_file(str fname, object global = object(), object local = object()) { pybind11_fail("File \"" + fname_str + "\" could not be opened!"); } +#if PY_VERSION_HEX < 0x03000000 && defined(PYPY_VERSION) + PyObject *result = PyRun_File(f, fname_str.c_str(), start, global.ptr(), + local.ptr()); + (void) closeFile; +#else PyObject *result = PyRun_FileEx(f, fname_str.c_str(), start, global.ptr(), local.ptr(), closeFile); +#endif + if (!result) throw error_already_set(); return reinterpret_steal<object>(result); diff --git a/ext/pybind11/include/pybind11/functional.h b/ext/pybind11/include/pybind11/functional.h index f1b0ebbbf..a99ee737f 100644 --- a/ext/pybind11/include/pybind11/functional.h +++ b/ext/pybind11/include/pybind11/functional.h @@ -15,9 +15,12 @@ NAMESPACE_BEGIN(pybind11) NAMESPACE_BEGIN(detail) -template <typename Return, typename... Args> struct type_caster<std::function<Return(Args...)>> { - typedef std::function<Return(Args...)> type; - typedef typename std::conditional<std::is_same<Return, void>::value, void_type, Return>::type retval_type; +template <typename Return, typename... Args> +struct type_caster<std::function<Return(Args...)>> { + using type = std::function<Return(Args...)>; + using retval_type = conditional_t<std::is_same<Return, void>::value, void_type, Return>; + using function_type = Return (*) (Args...); + public: bool load(handle src_, bool) { if (src_.is_none()) @@ -36,12 +39,11 @@ public: captured variables), in which case the roundtrip can be avoided. */ if (PyCFunction_Check(src_.ptr())) { - auto c = reinterpret_borrow<capsule>(PyCFunction_GetSelf(src_.ptr())); + auto c = reinterpret_borrow<capsule>(PyCFunction_GET_SELF(src_.ptr())); auto rec = (function_record *) c; - using FunctionType = Return (*) (Args...); - if (rec && rec->is_stateless && rec->data[1] == &typeid(FunctionType)) { - struct capture { FunctionType f; }; + if (rec && rec->is_stateless && rec->data[1] == &typeid(function_type)) { + struct capture { function_type f; }; value = ((capture *) &rec->data)->f; return true; } @@ -50,7 +52,7 @@ public: auto src = reinterpret_borrow<object>(src_); value = [src](Args... args) -> Return { gil_scoped_acquire acq; - object retval(src(std::move(args)...)); + object retval(src(std::forward<Args>(args)...)); /* Visual studio 2015 parser issue: need parentheses around this expression */ return (retval.template cast<Return>()); }; @@ -62,7 +64,7 @@ public: if (!f_) return none().inc_ref(); - auto result = f_.template target<Return (*)(Args...)>(); + auto result = f_.template target<function_type>(); if (result) return cpp_function(*result, policy).release(); else @@ -71,7 +73,7 @@ public: PYBIND11_TYPE_CASTER(type, _("Callable[[") + argument_loader<Args...>::arg_names() + _("], ") + - type_caster<retval_type>::name() + + make_caster<retval_type>::name() + _("]")); }; diff --git a/ext/pybind11/include/pybind11/numpy.h b/ext/pybind11/include/pybind11/numpy.h index e6f4efdf9..3227a12eb 100644 --- a/ext/pybind11/include/pybind11/numpy.h +++ b/ext/pybind11/include/pybind11/numpy.h @@ -35,9 +35,11 @@ static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); NAMESPACE_BEGIN(pybind11) + +class array; // Forward declaration + NAMESPACE_BEGIN(detail) -template <typename type, typename SFINAE = void> struct npy_format_descriptor { }; -template <typename type> struct is_pod_struct; +template <typename type, typename SFINAE = void> struct npy_format_descriptor; struct PyArrayDescr_Proxy { PyObject_HEAD @@ -108,11 +110,11 @@ inline numpy_internals& get_numpy_internals() { struct npy_api { enum constants { - NPY_C_CONTIGUOUS_ = 0x0001, - NPY_F_CONTIGUOUS_ = 0x0002, + NPY_ARRAY_C_CONTIGUOUS_ = 0x0001, + NPY_ARRAY_F_CONTIGUOUS_ = 0x0002, NPY_ARRAY_OWNDATA_ = 0x0004, NPY_ARRAY_FORCECAST_ = 0x0010, - NPY_ENSURE_ARRAY_ = 0x0040, + NPY_ARRAY_ENSUREARRAY_ = 0x0040, NPY_ARRAY_ALIGNED_ = 0x0100, NPY_ARRAY_WRITEABLE_ = 0x0400, NPY_BOOL_ = 0, @@ -155,6 +157,7 @@ struct npy_api { int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, Py_ssize_t *, PyObject **, PyObject *); PyObject *(*PyArray_Squeeze_)(PyObject *); + int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); private: enum functions { API_PyArray_Type = 2, @@ -169,7 +172,8 @@ private: API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, API_PyArray_GetArrayParamsFromObject = 278, - API_PyArray_Squeeze = 136 + API_PyArray_Squeeze = 136, + API_PyArray_SetBaseObject = 282 }; static npy_api lookup() { @@ -195,6 +199,7 @@ private: DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_GetArrayParamsFromObject); DECL_NPY_API(PyArray_Squeeze); + DECL_NPY_API(PyArray_SetBaseObject); #undef DECL_NPY_API return api; } @@ -220,6 +225,128 @@ inline bool check_flags(const void* ptr, int flag) { return (flag == (array_proxy(ptr)->flags & flag)); } +template <typename T> struct is_std_array : std::false_type { }; +template <typename T, size_t N> struct is_std_array<std::array<T, N>> : std::true_type { }; +template <typename T> struct is_complex : std::false_type { }; +template <typename T> struct is_complex<std::complex<T>> : std::true_type { }; + +template <typename T> using is_pod_struct = all_of< + std::is_pod<T>, // since we're accessing directly in memory we need a POD type + satisfies_none_of<T, std::is_reference, std::is_array, is_std_array, std::is_arithmetic, is_complex, std::is_enum> +>; + +template <size_t Dim = 0, typename Strides> size_t byte_offset_unsafe(const Strides &) { return 0; } +template <size_t Dim = 0, typename Strides, typename... Ix> +size_t byte_offset_unsafe(const Strides &strides, size_t i, Ix... index) { + return i * strides[Dim] + byte_offset_unsafe<Dim + 1>(strides, index...); +} + +/** Proxy class providing unsafe, unchecked const access to array data. This is constructed through + * the `unchecked<T, N>()` method of `array` or the `unchecked<N>()` method of `array_t<T>`. `Dims` + * will be -1 for dimensions determined at runtime. + */ +template <typename T, ssize_t Dims> +class unchecked_reference { +protected: + static constexpr bool Dynamic = Dims < 0; + const unsigned char *data_; + // Storing the shape & strides in local variables (i.e. these arrays) allows the compiler to + // make large performance gains on big, nested loops, but requires compile-time dimensions + conditional_t<Dynamic, const size_t *, std::array<size_t, (size_t) Dims>> + shape_, strides_; + const size_t dims_; + + friend class pybind11::array; + // Constructor for compile-time dimensions: + template <bool Dyn = Dynamic> + unchecked_reference(const void *data, const size_t *shape, const size_t *strides, enable_if_t<!Dyn, size_t>) + : data_{reinterpret_cast<const unsigned char *>(data)}, dims_{Dims} { + for (size_t i = 0; i < dims_; i++) { + shape_[i] = shape[i]; + strides_[i] = strides[i]; + } + } + // Constructor for runtime dimensions: + template <bool Dyn = Dynamic> + unchecked_reference(const void *data, const size_t *shape, const size_t *strides, enable_if_t<Dyn, size_t> dims) + : data_{reinterpret_cast<const unsigned char *>(data)}, shape_{shape}, strides_{strides}, dims_{dims} {} + +public: + /** Unchecked const reference access to data at the given indices. For a compile-time known + * number of dimensions, this requires the correct number of arguments; for run-time + * dimensionality, this is not checked (and so is up to the caller to use safely). + */ + template <typename... Ix> const T &operator()(Ix... index) const { + static_assert(sizeof...(Ix) == Dims || Dynamic, + "Invalid number of indices for unchecked array reference"); + return *reinterpret_cast<const T *>(data_ + byte_offset_unsafe(strides_, size_t(index)...)); + } + /** Unchecked const reference access to data; this operator only participates if the reference + * is to a 1-dimensional array. When present, this is exactly equivalent to `obj(index)`. + */ + template <size_t D = Dims, typename = enable_if_t<D == 1 || Dynamic>> + const T &operator[](size_t index) const { return operator()(index); } + + /// Pointer access to the data at the given indices. + template <typename... Ix> const T *data(Ix... ix) const { return &operator()(size_t(ix)...); } + + /// Returns the item size, i.e. sizeof(T) + constexpr static size_t itemsize() { return sizeof(T); } + + /// Returns the shape (i.e. size) of dimension `dim` + size_t shape(size_t dim) const { return shape_[dim]; } + + /// Returns the number of dimensions of the array + size_t ndim() const { return dims_; } + + /// Returns the total number of elements in the referenced array, i.e. the product of the shapes + template <bool Dyn = Dynamic> + enable_if_t<!Dyn, size_t> size() const { + return std::accumulate(shape_.begin(), shape_.end(), (size_t) 1, std::multiplies<size_t>()); + } + template <bool Dyn = Dynamic> + enable_if_t<Dyn, size_t> size() const { + return std::accumulate(shape_, shape_ + ndim(), (size_t) 1, std::multiplies<size_t>()); + } + + /// Returns the total number of bytes used by the referenced data. Note that the actual span in + /// memory may be larger if the referenced array has non-contiguous strides (e.g. for a slice). + size_t nbytes() const { + return size() * itemsize(); + } +}; + +template <typename T, ssize_t Dims> +class unchecked_mutable_reference : public unchecked_reference<T, Dims> { + friend class pybind11::array; + using ConstBase = unchecked_reference<T, Dims>; + using ConstBase::ConstBase; + using ConstBase::Dynamic; +public: + /// Mutable, unchecked access to data at the given indices. + template <typename... Ix> T& operator()(Ix... index) { + static_assert(sizeof...(Ix) == Dims || Dynamic, + "Invalid number of indices for unchecked array reference"); + return const_cast<T &>(ConstBase::operator()(index...)); + } + /** Mutable, unchecked access data at the given index; this operator only participates if the + * reference is to a 1-dimensional array (or has runtime dimensions). When present, this is + * exactly equivalent to `obj(index)`. + */ + template <size_t D = Dims, typename = enable_if_t<D == 1 || Dynamic>> + T &operator[](size_t index) { return operator()(index); } + + /// Mutable pointer access to the data at the given indices. + template <typename... Ix> T *mutable_data(Ix... ix) { return &operator()(size_t(ix)...); } +}; + +template <typename T, size_t Dim> +struct type_caster<unchecked_reference<T, Dim>> { + static_assert(Dim == 0 && Dim > 0 /* always fail */, "unchecked array proxy object is not castable"); +}; +template <typename T, size_t Dim> +struct type_caster<unchecked_mutable_reference<T, Dim>> : type_caster<unchecked_reference<T, Dim>> {}; + NAMESPACE_END(detail) class dtype : public object { @@ -321,8 +448,8 @@ public: PYBIND11_OBJECT_CVT(array, buffer, detail::npy_api::get().PyArray_Check_, raw_array) enum { - c_style = detail::npy_api::NPY_C_CONTIGUOUS_, - f_style = detail::npy_api::NPY_F_CONTIGUOUS_, + c_style = detail::npy_api::NPY_ARRAY_C_CONTIGUOUS_, + f_style = detail::npy_api::NPY_ARRAY_F_CONTIGUOUS_, forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; @@ -340,7 +467,7 @@ public: int flags = 0; if (base && ptr) { if (isinstance<array>(base)) - /* Copy flags from base (except baseship bit) */ + /* Copy flags from base (except ownership bit) */ flags = reinterpret_borrow<array>(base).flags() & ~detail::npy_api::NPY_ARRAY_OWNDATA_; else /* Writable by default, easy to downgrade later on if needed */ @@ -348,13 +475,15 @@ public: } auto tmp = reinterpret_steal<object>(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(), - (Py_intptr_t *) strides.data(), const_cast<void *>(ptr), flags, nullptr)); + api.PyArray_Type_, descr.release().ptr(), (int) ndim, + reinterpret_cast<Py_intptr_t *>(const_cast<size_t*>(shape.data())), + reinterpret_cast<Py_intptr_t *>(const_cast<size_t*>(strides.data())), + const_cast<void *>(ptr), flags, nullptr)); if (!tmp) pybind11_fail("NumPy: unable to create array!"); if (ptr) { if (base) { - detail::array_proxy(tmp.ptr())->base = base.inc_ref().ptr(); + api.PyArray_SetBaseObject_(tmp.ptr(), base.inc_ref().ptr()); } else { tmp = reinterpret_steal<object>(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */)); } @@ -373,7 +502,7 @@ public: template<typename T> array(const std::vector<size_t>& shape, const std::vector<size_t>& strides, const T* ptr, handle base = handle()) - : array(pybind11::dtype::of<T>(), shape, strides, (void *) ptr, base) { } + : array(pybind11::dtype::of<T>(), shape, strides, (const void *) ptr, base) { } template <typename T> array(const std::vector<size_t> &shape, const T *ptr, @@ -486,6 +615,31 @@ public: return offset_at(index...) / itemsize(); } + /** Returns a proxy object that provides access to the array's data without bounds or + * dimensionality checking. Will throw if the array is missing the `writeable` flag. Use with + * care: the array must not be destroyed or reshaped for the duration of the returned object, + * and the caller must take care not to access invalid dimensions or dimension indices. + */ + template <typename T, ssize_t Dims = -1> detail::unchecked_mutable_reference<T, Dims> mutable_unchecked() { + if (Dims >= 0 && ndim() != (size_t) Dims) + throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + + "; expected " + std::to_string(Dims)); + return detail::unchecked_mutable_reference<T, Dims>(mutable_data(), shape(), strides(), ndim()); + } + + /** Returns a proxy object that provides const access to the array's data without bounds or + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed or + * reshaped for the duration of the returned object, and the caller must take care not to access + * invalid dimensions or dimension indices. + */ + template <typename T, ssize_t Dims = -1> detail::unchecked_reference<T, Dims> unchecked() const { + if (Dims >= 0 && ndim() != (size_t) Dims) + throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + + "; expected " + std::to_string(Dims)); + return detail::unchecked_reference<T, Dims>(data(), shape(), strides(), ndim()); + } + /// Return a new view with all of the dimensions of length 1 removed array squeeze() { auto& api = detail::npy_api::get(); @@ -511,18 +665,12 @@ protected: template<typename... Ix> size_t byte_offset(Ix... index) const { check_dimensions(index...); - return byte_offset_unsafe(index...); - } - - template<size_t dim = 0, typename... Ix> size_t byte_offset_unsafe(size_t i, Ix... index) const { - return i * strides()[dim] + byte_offset_unsafe<dim + 1>(index...); + return detail::byte_offset_unsafe(strides(), size_t(index)...); } - template<size_t dim = 0> size_t byte_offset_unsafe() const { return 0; } - void check_writeable() const { if (!writeable()) - throw std::runtime_error("array is not writeable"); + throw std::domain_error("array is not writeable"); } static std::vector<size_t> default_strides(const std::vector<size_t>& shape, size_t itemsize) { @@ -557,12 +705,14 @@ protected: if (ptr == nullptr) return nullptr; return detail::npy_api::get().PyArray_FromAny_( - ptr, nullptr, 0, 0, detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + ptr, nullptr, 0, 0, detail::npy_api::NPY_ARRAY_ENSUREARRAY_ | ExtraFlags, nullptr); } }; template <typename T, int ExtraFlags = array::forcecast> class array_t : public array { public: + using value_type = T; + array_t() : array(0, static_cast<const T *>(nullptr)) {} array_t(handle h, borrowed_t) : array(h, borrowed) { } array_t(handle h, stolen_t) : array(h, stolen) { } @@ -621,8 +771,27 @@ public: return *(static_cast<T*>(array::mutable_data()) + byte_offset(size_t(index)...) / itemsize()); } - /// Ensure that the argument is a NumPy array of the correct dtype. - /// In case of an error, nullptr is returned and the Python error is cleared. + /** Returns a proxy object that provides access to the array's data without bounds or + * dimensionality checking. Will throw if the array is missing the `writeable` flag. Use with + * care: the array must not be destroyed or reshaped for the duration of the returned object, + * and the caller must take care not to access invalid dimensions or dimension indices. + */ + template <ssize_t Dims = -1> detail::unchecked_mutable_reference<T, Dims> mutable_unchecked() { + return array::mutable_unchecked<T, Dims>(); + } + + /** Returns a proxy object that provides const access to the array's data without bounds or + * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying + * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped + * for the duration of the returned object, and the caller must take care not to access invalid + * dimensions or dimension indices. + */ + template <ssize_t Dims = -1> detail::unchecked_reference<T, Dims> unchecked() const { + return array::unchecked<T, Dims>(); + } + + /// Ensure that the argument is a NumPy array of the correct dtype (and if not, try to convert + /// it). In case of an error, nullptr is returned and the Python error is cleared. static array_t ensure(handle h) { auto result = reinterpret_steal<array_t>(raw_array_t(h.ptr())); if (!result) @@ -630,7 +799,7 @@ public: return result; } - static bool _check(handle h) { + static bool check_(handle h) { const auto &api = detail::npy_api::get(); return api.PyArray_Check_(h.ptr()) && api.PyArray_EquivTypes_(detail::array_proxy(h.ptr())->descr, dtype::of<T>().ptr()); @@ -643,7 +812,7 @@ protected: return nullptr; return detail::npy_api::get().PyArray_FromAny_( ptr, dtype::of<T>().release().ptr(), 0, 0, - detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + detail::npy_api::NPY_ARRAY_ENSUREARRAY_ | ExtraFlags, nullptr); } }; @@ -674,7 +843,9 @@ template <typename T, int ExtraFlags> struct pyobject_caster<array_t<T, ExtraFlags>> { using type = array_t<T, ExtraFlags>; - bool load(handle src, bool /* convert */) { + bool load(handle src, bool convert) { + if (!convert && !type::check_(src)) + return false; value = type::ensure(src); return static_cast<bool>(value); } @@ -685,65 +856,55 @@ struct pyobject_caster<array_t<T, ExtraFlags>> { PYBIND11_TYPE_CASTER(type, handle_type_name<type>::name()); }; -template <typename T> struct is_std_array : std::false_type { }; -template <typename T, size_t N> struct is_std_array<std::array<T, N>> : std::true_type { }; - template <typename T> -struct is_pod_struct { - enum { value = std::is_pod<T>::value && // offsetof only works correctly for POD types - !std::is_reference<T>::value && - !std::is_array<T>::value && - !is_std_array<T>::value && - !std::is_integral<T>::value && - !std::is_enum<T>::value && - !std::is_same<typename std::remove_cv<T>::type, float>::value && - !std::is_same<typename std::remove_cv<T>::type, double>::value && - !std::is_same<typename std::remove_cv<T>::type, bool>::value && - !std::is_same<typename std::remove_cv<T>::type, std::complex<float>>::value && - !std::is_same<typename std::remove_cv<T>::type, std::complex<double>>::value }; +struct compare_buffer_info<T, detail::enable_if_t<detail::is_pod_struct<T>::value>> { + static bool compare(const buffer_info& b) { + return npy_api::get().PyArray_EquivTypes_(dtype::of<T>().ptr(), dtype(b).ptr()); + } }; -template <typename T> struct npy_format_descriptor<T, enable_if_t<std::is_integral<T>::value>> { +template <typename T> struct npy_format_descriptor<T, enable_if_t<satisfies_any_of<T, std::is_arithmetic, is_complex>::value>> { private: - constexpr static const int values[8] = { - npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, - npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_ }; + // NB: the order here must match the one in common.h + constexpr static const int values[15] = { + npy_api::NPY_BOOL_, + npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, + npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_, + npy_api::NPY_FLOAT_, npy_api::NPY_DOUBLE_, npy_api::NPY_LONGDOUBLE_, + npy_api::NPY_CFLOAT_, npy_api::NPY_CDOUBLE_, npy_api::NPY_CLONGDOUBLE_ + }; + public: - enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned<T>::value ? 1 : 0)] }; + static constexpr int value = values[detail::is_fmt_numeric<T>::index]; + static pybind11::dtype dtype() { if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) return reinterpret_borrow<pybind11::dtype>(ptr); pybind11_fail("Unsupported buffer format!"); } - template <typename T2 = T, enable_if_t<std::is_signed<T2>::value, int> = 0> - static PYBIND11_DESCR name() { return _("int") + _<sizeof(T)*8>(); } - template <typename T2 = T, enable_if_t<!std::is_signed<T2>::value, int> = 0> - static PYBIND11_DESCR name() { return _("uint") + _<sizeof(T)*8>(); } + template <typename T2 = T, enable_if_t<std::is_integral<T2>::value, int> = 0> + static PYBIND11_DESCR name() { + return _<std::is_same<T, bool>::value>(_("bool"), + _<std::is_signed<T>::value>("int", "uint") + _<sizeof(T)*8>()); + } + template <typename T2 = T, enable_if_t<std::is_floating_point<T2>::value, int> = 0> + static PYBIND11_DESCR name() { + return _<std::is_same<T, float>::value || std::is_same<T, double>::value>( + _("float") + _<sizeof(T)*8>(), _("longdouble")); + } + template <typename T2 = T, enable_if_t<is_complex<T2>::value, int> = 0> + static PYBIND11_DESCR name() { + return _<std::is_same<typename T2::value_type, float>::value || std::is_same<typename T2::value_type, double>::value>( + _("complex") + _<sizeof(typename T2::value_type)*16>(), _("longcomplex")); + } }; -template <typename T> constexpr const int npy_format_descriptor< - T, enable_if_t<std::is_integral<T>::value>>::values[8]; - -#define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor<Type> { \ - enum { value = npy_api::NumPyName }; \ - static pybind11::dtype dtype() { \ - if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \ - return reinterpret_borrow<pybind11::dtype>(ptr); \ - pybind11_fail("Unsupported buffer format!"); \ - } \ - static PYBIND11_DESCR name() { return _(Name); } } -DECL_FMT(float, NPY_FLOAT_, "float32"); -DECL_FMT(double, NPY_DOUBLE_, "float64"); -DECL_FMT(bool, NPY_BOOL_, "bool"); -DECL_FMT(std::complex<float>, NPY_CFLOAT_, "complex64"); -DECL_FMT(std::complex<double>, NPY_CDOUBLE_, "complex128"); -#undef DECL_FMT - -#define DECL_CHAR_FMT \ + +#define PYBIND11_DECL_CHAR_FMT \ static PYBIND11_DESCR name() { return _("S") + _<N>(); } \ static pybind11::dtype dtype() { return pybind11::dtype(std::string("S") + std::to_string(N)); } -template <size_t N> struct npy_format_descriptor<char[N]> { DECL_CHAR_FMT }; -template <size_t N> struct npy_format_descriptor<std::array<char, N>> { DECL_CHAR_FMT }; -#undef DECL_CHAR_FMT +template <size_t N> struct npy_format_descriptor<char[N]> { PYBIND11_DECL_CHAR_FMT }; +template <size_t N> struct npy_format_descriptor<std::array<char, N>> { PYBIND11_DECL_CHAR_FMT }; +#undef PYBIND11_DECL_CHAR_FMT template<typename T> struct npy_format_descriptor<T, enable_if_t<std::is_enum<T>::value>> { private: @@ -798,9 +959,9 @@ inline PYBIND11_NOINLINE void register_structured_dtype( for (auto& field : ordered_fields) { if (field.offset > offset) oss << (field.offset - offset) << 'x'; - // mark unaligned fields with '=' + // mark unaligned fields with '^' (unaligned native type) if (field.offset % field.alignment) - oss << '='; + oss << '^'; oss << field.format << ':' << field.name << ':'; offset = field.offset + field.size; } @@ -820,9 +981,10 @@ inline PYBIND11_NOINLINE void register_structured_dtype( get_internals().direct_conversions[tindex].push_back(direct_converter); } -template <typename T> -struct npy_format_descriptor<T, enable_if_t<is_pod_struct<T>::value>> { - static PYBIND11_DESCR name() { return _("struct"); } +template <typename T, typename SFINAE> struct npy_format_descriptor { + static_assert(is_pod_struct<T>::value, "Attempt to use a non-POD or unimplemented POD type as a numpy dtype"); + + static PYBIND11_DESCR name() { return make_caster<T>::name(); } static pybind11::dtype dtype() { return reinterpret_borrow<pybind11::dtype>(dtype_ptr()); @@ -1043,87 +1205,146 @@ private: std::array<common_iter, N> m_common_iterator; }; +enum class broadcast_trivial { non_trivial, c_trivial, f_trivial }; + +// Populates the shape and number of dimensions for the set of buffers. Returns a broadcast_trivial +// enum value indicating whether the broadcast is "trivial"--that is, has each buffer being either a +// singleton or a full-size, C-contiguous (`c_trivial`) or Fortran-contiguous (`f_trivial`) storage +// buffer; returns `non_trivial` otherwise. template <size_t N> -bool broadcast(const std::array<buffer_info, N>& buffers, size_t& ndim, std::vector<size_t>& shape) { +broadcast_trivial broadcast(const std::array<buffer_info, N> &buffers, size_t &ndim, std::vector<size_t> &shape) { ndim = std::accumulate(buffers.begin(), buffers.end(), size_t(0), [](size_t res, const buffer_info& buf) { return std::max(res, buf.ndim); }); - shape = std::vector<size_t>(ndim, 1); - bool trivial_broadcast = true; + shape.clear(); + shape.resize(ndim, 1); + + // Figure out the output size, and make sure all input arrays conform (i.e. are either size 1 or + // the full size). for (size_t i = 0; i < N; ++i) { auto res_iter = shape.rbegin(); - bool i_trivial_broadcast = (buffers[i].size == 1) || (buffers[i].ndim == ndim); - for (auto shape_iter = buffers[i].shape.rbegin(); - shape_iter != buffers[i].shape.rend(); ++shape_iter, ++res_iter) { - - if (*res_iter == 1) - *res_iter = *shape_iter; - else if ((*shape_iter != 1) && (*res_iter != *shape_iter)) + auto end = buffers[i].shape.rend(); + for (auto shape_iter = buffers[i].shape.rbegin(); shape_iter != end; ++shape_iter, ++res_iter) { + const auto &dim_size_in = *shape_iter; + auto &dim_size_out = *res_iter; + + // Each input dimension can either be 1 or `n`, but `n` values must match across buffers + if (dim_size_out == 1) + dim_size_out = dim_size_in; + else if (dim_size_in != 1 && dim_size_in != dim_size_out) pybind11_fail("pybind11::vectorize: incompatible size/dimension of inputs!"); + } + } - i_trivial_broadcast = i_trivial_broadcast && (*res_iter == *shape_iter); + bool trivial_broadcast_c = true; + bool trivial_broadcast_f = true; + for (size_t i = 0; i < N && (trivial_broadcast_c || trivial_broadcast_f); ++i) { + if (buffers[i].size == 1) + continue; + + // Require the same number of dimensions: + if (buffers[i].ndim != ndim) + return broadcast_trivial::non_trivial; + + // Require all dimensions be full-size: + if (!std::equal(buffers[i].shape.cbegin(), buffers[i].shape.cend(), shape.cbegin())) + return broadcast_trivial::non_trivial; + + // Check for C contiguity (but only if previous inputs were also C contiguous) + if (trivial_broadcast_c) { + size_t expect_stride = buffers[i].itemsize; + auto end = buffers[i].shape.crend(); + for (auto shape_iter = buffers[i].shape.crbegin(), stride_iter = buffers[i].strides.crbegin(); + trivial_broadcast_c && shape_iter != end; ++shape_iter, ++stride_iter) { + if (expect_stride == *stride_iter) + expect_stride *= *shape_iter; + else + trivial_broadcast_c = false; + } + } + + // Check for Fortran contiguity (if previous inputs were also F contiguous) + if (trivial_broadcast_f) { + size_t expect_stride = buffers[i].itemsize; + auto end = buffers[i].shape.cend(); + for (auto shape_iter = buffers[i].shape.cbegin(), stride_iter = buffers[i].strides.cbegin(); + trivial_broadcast_f && shape_iter != end; ++shape_iter, ++stride_iter) { + if (expect_stride == *stride_iter) + expect_stride *= *shape_iter; + else + trivial_broadcast_f = false; + } } - trivial_broadcast = trivial_broadcast && i_trivial_broadcast; } - return trivial_broadcast; + + return + trivial_broadcast_c ? broadcast_trivial::c_trivial : + trivial_broadcast_f ? broadcast_trivial::f_trivial : + broadcast_trivial::non_trivial; } template <typename Func, typename Return, typename... Args> struct vectorize_helper { typename std::remove_reference<Func>::type f; + static constexpr size_t N = sizeof...(Args); template <typename T> explicit vectorize_helper(T&&f) : f(std::forward<T>(f)) { } - object operator()(array_t<Args, array::c_style | array::forcecast>... args) { - return run(args..., make_index_sequence<sizeof...(Args)>()); + object operator()(array_t<Args, array::forcecast>... args) { + return run(args..., make_index_sequence<N>()); } - template <size_t ... Index> object run(array_t<Args, array::c_style | array::forcecast>&... args, index_sequence<Index...> index) { + template <size_t ... Index> object run(array_t<Args, array::forcecast>&... args, index_sequence<Index...> index) { /* Request buffers from all parameters */ - const size_t N = sizeof...(Args); - std::array<buffer_info, N> buffers {{ args.request()... }}; /* Determine dimensions parameters of output array */ size_t ndim = 0; std::vector<size_t> shape(0); - bool trivial_broadcast = broadcast(buffers, ndim, shape); + auto trivial = broadcast(buffers, ndim, shape); size_t size = 1; std::vector<size_t> strides(ndim); if (ndim > 0) { - strides[ndim-1] = sizeof(Return); - for (size_t i = ndim - 1; i > 0; --i) { - strides[i - 1] = strides[i] * shape[i]; - size *= shape[i]; + if (trivial == broadcast_trivial::f_trivial) { + strides[0] = sizeof(Return); + for (size_t i = 1; i < ndim; ++i) { + strides[i] = strides[i - 1] * shape[i - 1]; + size *= shape[i - 1]; + } + size *= shape[ndim - 1]; + } + else { + strides[ndim-1] = sizeof(Return); + for (size_t i = ndim - 1; i > 0; --i) { + strides[i - 1] = strides[i] * shape[i]; + size *= shape[i]; + } + size *= shape[0]; } - size *= shape[0]; } if (size == 1) - return cast(f(*((Args *) buffers[Index].ptr)...)); + return cast(f(*reinterpret_cast<Args *>(buffers[Index].ptr)...)); array_t<Return> result(shape, strides); auto buf = result.request(); auto output = (Return *) buf.ptr; - if (trivial_broadcast) { - /* Call the function */ - for (size_t i = 0; i < size; ++i) { - output[i] = f((buffers[Index].size == 1 - ? *((Args *) buffers[Index].ptr) - : ((Args *) buffers[Index].ptr)[i])...); - } + /* Call the function */ + if (trivial == broadcast_trivial::non_trivial) { + apply_broadcast<Index...>(buffers, buf, index); } else { - apply_broadcast<N, Index...>(buffers, buf, index); + for (size_t i = 0; i < size; ++i) + output[i] = f((reinterpret_cast<Args *>(buffers[Index].ptr)[buffers[Index].size == 1 ? 0 : i])...); } return result; } - template <size_t N, size_t... Index> + template <size_t... Index> void apply_broadcast(const std::array<buffer_info, N> &buffers, buffer_info &output, index_sequence<Index...>) { using input_iterator = multi_array_iterator<N>; @@ -1140,26 +1361,29 @@ struct vectorize_helper { }; template <typename T, int Flags> struct handle_type_name<array_t<T, Flags>> { - static PYBIND11_DESCR name() { return _("numpy.ndarray[") + type_caster<T>::name() + _("]"); } + static PYBIND11_DESCR name() { + return _("numpy.ndarray[") + npy_format_descriptor<T>::name() + _("]"); + } }; NAMESPACE_END(detail) template <typename Func, typename Return, typename... Args> -detail::vectorize_helper<Func, Return, Args...> vectorize(const Func &f, Return (*) (Args ...)) { +detail::vectorize_helper<Func, Return, Args...> +vectorize(const Func &f, Return (*) (Args ...)) { return detail::vectorize_helper<Func, Return, Args...>(f); } template <typename Return, typename... Args> -detail::vectorize_helper<Return (*) (Args ...), Return, Args...> vectorize(Return (*f) (Args ...)) { +detail::vectorize_helper<Return (*) (Args ...), Return, Args...> +vectorize(Return (*f) (Args ...)) { return vectorize<Return (*) (Args ...), Return, Args...>(f, f); } -template <typename Func> +template <typename Func, typename FuncType = typename detail::remove_class<decltype(&std::remove_reference<Func>::type::operator())>::type> auto vectorize(Func &&f) -> decltype( - vectorize(std::forward<Func>(f), (typename detail::remove_class<decltype(&std::remove_reference<Func>::type::operator())>::type *) nullptr)) { - return vectorize(std::forward<Func>(f), (typename detail::remove_class<decltype( - &std::remove_reference<Func>::type::operator())>::type *) nullptr); + vectorize(std::forward<Func>(f), (FuncType *) nullptr)) { + return vectorize(std::forward<Func>(f), (FuncType *) nullptr); } NAMESPACE_END(pybind11) diff --git a/ext/pybind11/include/pybind11/pybind11.h b/ext/pybind11/include/pybind11/pybind11.h index 1db9efb8c..5976a36d8 100644 --- a/ext/pybind11/include/pybind11/pybind11.h +++ b/ext/pybind11/include/pybind11/pybind11.h @@ -35,6 +35,7 @@ #include "attr.h" #include "options.h" +#include "class_support.h" NAMESPACE_BEGIN(pybind11) @@ -50,10 +51,16 @@ public: } /// Construct a cpp_function from a lambda function (possibly with internal state) - template <typename Func, typename... Extra> cpp_function(Func &&f, const Extra&... extra) { + template <typename Func, typename... Extra, typename = detail::enable_if_t< + detail::satisfies_none_of< + typename std::remove_reference<Func>::type, + std::is_function, std::is_pointer, std::is_member_pointer + >::value> + > + cpp_function(Func &&f, const Extra&... extra) { + using FuncType = typename detail::remove_class<decltype(&std::remove_reference<Func>::type::operator())>::type; initialize(std::forward<Func>(f), - (typename detail::remove_class<decltype( - &std::remove_reference<Func>::type::operator())>::type *) nullptr, extra...); + (FuncType *) nullptr, extra...); } /// Construct a cpp_function from a class method (non-const) @@ -82,8 +89,6 @@ protected: /// Special internal constructor for functors, lambda functions, etc. template <typename Func, typename Return, typename... Args, typename... Extra> void initialize(Func &&f, Return (*)(Args...), const Extra&... extra) { - static_assert(detail::expected_num_args<Extra...>(sizeof...(Args)), - "The number of named arguments does not match the function signature"); struct capture { typename std::remove_reference<Func>::type f; }; @@ -116,32 +121,34 @@ protected: detail::conditional_t<std::is_void<Return>::value, detail::void_type, Return> >; + static_assert(detail::expected_num_args<Extra...>(sizeof...(Args), cast_in::has_args, cast_in::has_kwargs), + "The number of argument annotations does not match the number of function arguments"); + /* Dispatch code which converts function arguments and performs the actual function call */ - rec->impl = [](detail::function_record *rec, handle args, handle kwargs, handle parent) -> handle { + rec->impl = [](detail::function_call &call) -> handle { cast_in args_converter; /* Try to cast the function arguments into the C++ domain */ - if (!args_converter.load_args(args, kwargs, true)) + if (!args_converter.load_args(call)) return PYBIND11_TRY_NEXT_OVERLOAD; /* Invoke call policy pre-call hook */ - detail::process_attributes<Extra...>::precall(args); + detail::process_attributes<Extra...>::precall(call); /* Get a pointer to the capture object */ - capture *cap = (capture *) (sizeof(capture) <= sizeof(rec->data) - ? &rec->data : rec->data[0]); + auto data = (sizeof(capture) <= sizeof(call.func.data) + ? &call.func.data : call.func.data[0]); + capture *cap = const_cast<capture *>(reinterpret_cast<const capture *>(data)); - /* Override policy for rvalues -- always move */ - constexpr auto is_rvalue = !std::is_pointer<Return>::value - && !std::is_lvalue_reference<Return>::value; - const auto policy = is_rvalue ? return_value_policy::move : rec->policy; + /* Override policy for rvalues -- usually to enforce rvp::move on an rvalue */ + const auto policy = detail::return_value_policy_override<Return>::policy(call.func.policy); /* Perform the function call */ handle result = cast_out::cast(args_converter.template call<Return>(cap->f), - policy, parent); + policy, call.parent); /* Invoke call policy post-call hook */ - detail::process_attributes<Extra...>::postcall(args, result); + detail::process_attributes<Extra...>::postcall(call, result); return result; }; @@ -166,7 +173,7 @@ protected: sizeof(capture) == sizeof(void *); if (is_function_ptr) { rec->is_stateless = true; - rec->data[1] = (void *) &typeid(FunctionType); + rec->data[1] = const_cast<void *>(reinterpret_cast<const void *>(&typeid(FunctionType))); } } @@ -197,7 +204,7 @@ protected: if (c == '{') { // Write arg name for everything except *args, **kwargs and return type. if (type_depth == 0 && text[char_index] != '*' && arg_index < args) { - if (!rec->args.empty()) { + if (!rec->args.empty() && rec->args[arg_index].name) { signature += rec->args[arg_index].name; } else if (arg_index == 0 && rec->is_method) { signature += "self"; @@ -221,6 +228,11 @@ protected: if (!t) pybind11_fail("Internal error while parsing type signature (1)"); if (auto tinfo = detail::get_type_info(*t)) { +#if defined(PYPY_VERSION) + signature += handle((PyObject *) tinfo->type) + .attr("__module__") + .cast<std::string>() + "."; +#endif signature += tinfo->type->tp_name; } else { std::string tname(t->name()); @@ -251,7 +263,7 @@ protected: rec->signature = strdup(signature.c_str()); rec->args.shrink_to_fit(); rec->is_constructor = !strcmp(rec->name, "__init__") || !strcmp(rec->name, "__setstate__"); - rec->nargs = (uint16_t) args; + rec->nargs = (std::uint16_t) args; #if PY_MAJOR_VERSION < 3 if (rec->sibling && PyMethod_Check(rec->sibling.ptr())) @@ -261,7 +273,7 @@ protected: detail::function_record *chain = nullptr, *chain_start = rec; if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { - auto rec_capsule = reinterpret_borrow<capsule>(PyCFunction_GetSelf(rec->sibling.ptr())); + auto rec_capsule = reinterpret_borrow<capsule>(PyCFunction_GET_SELF(rec->sibling.ptr())); chain = (detail::function_record *) rec_capsule; /* Never append a method to an overload chain of a parent class; instead, hide the parent's overloads in this case */ @@ -282,8 +294,8 @@ protected: rec->def->ml_meth = reinterpret_cast<PyCFunction>(*dispatcher); rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; - capsule rec_capsule(rec, [](PyObject *o) { - destruct((detail::function_record *) PyCapsule_GetPointer(o, nullptr)); + capsule rec_capsule(rec, [](void *ptr) { + destruct((detail::function_record *) ptr); }); object scope_module; @@ -319,8 +331,10 @@ protected: signatures += "Overloaded function.\n\n"; } // Then specific overload signatures + bool first_user_def = true; for (auto it = chain_start; it != nullptr; it = it->next) { if (options::show_function_signatures()) { + if (index > 0) signatures += "\n"; if (chain) signatures += std::to_string(++index) + ". "; signatures += rec->name; @@ -328,18 +342,22 @@ protected: signatures += "\n"; } if (it->doc && strlen(it->doc) > 0 && options::show_user_defined_docstrings()) { + // If we're appending another docstring, and aren't printing function signatures, we + // need to append a newline first: + if (!options::show_function_signatures()) { + if (first_user_def) first_user_def = false; + else signatures += "\n"; + } if (options::show_function_signatures()) signatures += "\n"; signatures += it->doc; if (options::show_function_signatures()) signatures += "\n"; } - if (it->next) - signatures += "\n"; } /* Install docstring */ PyCFunctionObject *func = (PyCFunctionObject *) m_ptr; if (func->m_ml->ml_doc) - std::free((char *) func->m_ml->ml_doc); + std::free(const_cast<char *>(func->m_ml->ml_doc)); func->m_ml->ml_doc = strdup(signatures.c_str()); if (rec->is_method) { @@ -360,12 +378,12 @@ protected: std::free((char *) rec->doc); std::free((char *) rec->signature); for (auto &arg: rec->args) { - std::free((char *) arg.name); - std::free((char *) arg.descr); + std::free(const_cast<char *>(arg.name)); + std::free(const_cast<char *>(arg.descr)); arg.value.dec_ref(); } if (rec->def) { - std::free((char *) rec->def->ml_doc); + std::free(const_cast<char *>(rec->def->ml_doc)); delete rec->def; } delete rec; @@ -374,72 +392,205 @@ protected: } /// Main dispatch logic for calls to functions bound using pybind11 - static PyObject *dispatcher(PyObject *self, PyObject *args, PyObject *kwargs) { + static PyObject *dispatcher(PyObject *self, PyObject *args_in, PyObject *kwargs_in) { + using namespace detail; + /* Iterator over the list of potentially admissible overloads */ - detail::function_record *overloads = (detail::function_record *) PyCapsule_GetPointer(self, nullptr), - *it = overloads; + function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr), + *it = overloads; /* Need to know how many arguments + keyword arguments there are to pick the right overload */ - size_t nargs = (size_t) PyTuple_GET_SIZE(args), - nkwargs = kwargs ? (size_t) PyDict_Size(kwargs) : 0; + const size_t n_args_in = (size_t) PyTuple_GET_SIZE(args_in); - handle parent = nargs > 0 ? PyTuple_GET_ITEM(args, 0) : nullptr, + handle parent = n_args_in > 0 ? PyTuple_GET_ITEM(args_in, 0) : nullptr, result = PYBIND11_TRY_NEXT_OVERLOAD; + try { + // We do this in two passes: in the first pass, we load arguments with `convert=false`; + // in the second, we allow conversion (except for arguments with an explicit + // py::arg().noconvert()). This lets us prefer calls without conversion, with + // conversion as a fallback. + std::vector<function_call> second_pass; + + // However, if there are no overloads, we can just skip the no-convert pass entirely + const bool overloaded = it != nullptr && it->next != nullptr; + for (; it != nullptr; it = it->next) { - auto args_ = reinterpret_borrow<tuple>(args); - size_t kwargs_consumed = 0; /* For each overload: - 1. If the required list of arguments is longer than the - actually provided amount, create a copy of the argument - list and fill in any available keyword/default arguments. - 2. Ensure that all keyword arguments were "consumed" - 3. Call the function call dispatcher (function_record::impl) + 1. Copy all positional arguments we were given, also checking to make sure that + named positional arguments weren't *also* specified via kwarg. + 2. If we weren't given enough, try to make up the omitted ones by checking + whether they were provided by a kwarg matching the `py::arg("name")` name. If + so, use it (and remove it from kwargs; if not, see if the function binding + provided a default that we can use. + 3. Ensure that either all keyword arguments were "consumed", or that the function + takes a kwargs argument to accept unconsumed kwargs. + 4. Any positional arguments still left get put into a tuple (for args), and any + leftover kwargs get put into a dict. + 5. Pack everything into a vector; if we have py::args or py::kwargs, they are an + extra tuple or dict at the end of the positional arguments. + 6. Call the function call dispatcher (function_record::impl) + + If one of these fail, move on to the next overload and keep trying until we get a + result other than PYBIND11_TRY_NEXT_OVERLOAD. */ - size_t nargs_ = nargs; - if (nargs < it->args.size()) { - nargs_ = it->args.size(); - args_ = tuple(nargs_); - for (size_t i = 0; i < nargs; ++i) { - handle item = PyTuple_GET_ITEM(args, i); - PyTuple_SET_ITEM(args_.ptr(), i, item.inc_ref().ptr()); + + function_record &func = *it; + size_t pos_args = func.nargs; // Number of positional arguments that we need + if (func.has_args) --pos_args; // (but don't count py::args + if (func.has_kwargs) --pos_args; // or py::kwargs) + + if (!func.has_args && n_args_in > pos_args) + continue; // Too many arguments for this overload + + if (n_args_in < pos_args && func.args.size() < pos_args) + continue; // Not enough arguments given, and not enough defaults to fill in the blanks + + function_call call(func, parent); + + size_t args_to_copy = std::min(pos_args, n_args_in); + size_t args_copied = 0; + + // 1. Copy any position arguments given. + bool bad_kwarg = false; + for (; args_copied < args_to_copy; ++args_copied) { + if (kwargs_in && args_copied < func.args.size() && func.args[args_copied].name + && PyDict_GetItemString(kwargs_in, func.args[args_copied].name)) { + bad_kwarg = true; + break; } - int arg_ctr = 0; - for (auto const &it2 : it->args) { - int index = arg_ctr++; - if (PyTuple_GET_ITEM(args_.ptr(), index)) - continue; + call.args.push_back(PyTuple_GET_ITEM(args_in, args_copied)); + call.args_convert.push_back(args_copied < func.args.size() ? func.args[args_copied].convert : true); + } + if (bad_kwarg) + continue; // Maybe it was meant for another overload (issue #688) + + // We'll need to copy this if we steal some kwargs for defaults + dict kwargs = reinterpret_borrow<dict>(kwargs_in); + + // 2. Check kwargs and, failing that, defaults that may help complete the list + if (args_copied < pos_args) { + bool copied_kwargs = false; + + for (; args_copied < pos_args; ++args_copied) { + const auto &arg = func.args[args_copied]; handle value; - if (kwargs) - value = PyDict_GetItemString(kwargs, it2.name); + if (kwargs_in && arg.name) + value = PyDict_GetItemString(kwargs.ptr(), arg.name); - if (value) - kwargs_consumed++; - else if (it2.value) - value = it2.value; + if (value) { + // Consume a kwargs value + if (!copied_kwargs) { + kwargs = reinterpret_steal<dict>(PyDict_Copy(kwargs.ptr())); + copied_kwargs = true; + } + PyDict_DelItemString(kwargs.ptr(), arg.name); + } else if (arg.value) { + value = arg.value; + } if (value) { - PyTuple_SET_ITEM(args_.ptr(), index, value.inc_ref().ptr()); - } else { - kwargs_consumed = (size_t) -1; /* definite failure */ + call.args.push_back(value); + call.args_convert.push_back(arg.convert); + } + else break; + } + + if (args_copied < pos_args) + continue; // Not enough arguments, defaults, or kwargs to fill the positional arguments + } + + // 3. Check everything was consumed (unless we have a kwargs arg) + if (kwargs && kwargs.size() > 0 && !func.has_kwargs) + continue; // Unconsumed kwargs, but no py::kwargs argument to accept them + + // 4a. If we have a py::args argument, create a new tuple with leftovers + tuple extra_args; + if (func.has_args) { + if (args_to_copy == 0) { + // We didn't copy out any position arguments from the args_in tuple, so we + // can reuse it directly without copying: + extra_args = reinterpret_borrow<tuple>(args_in); + } else if (args_copied >= n_args_in) { + extra_args = tuple(0); + } else { + size_t args_size = n_args_in - args_copied; + extra_args = tuple(args_size); + for (size_t i = 0; i < args_size; ++i) { + handle item = PyTuple_GET_ITEM(args_in, args_copied + i); + extra_args[i] = item.inc_ref().ptr(); } } + call.args.push_back(extra_args); + call.args_convert.push_back(false); } + // 4b. If we have a py::kwargs, pass on any remaining kwargs + if (func.has_kwargs) { + if (!kwargs.ptr()) + kwargs = dict(); // If we didn't get one, send an empty one + call.args.push_back(kwargs); + call.args_convert.push_back(false); + } + + // 5. Put everything in a vector. Not technically step 5, we've been building it + // in `call.args` all along. + #if !defined(NDEBUG) + if (call.args.size() != func.nargs || call.args_convert.size() != func.nargs) + pybind11_fail("Internal error: function call dispatcher inserted wrong number of arguments!"); + #endif + + std::vector<bool> second_pass_convert; + if (overloaded) { + // We're in the first no-convert pass, so swap out the conversion flags for a + // set of all-false flags. If the call fails, we'll swap the flags back in for + // the conversion-allowed call below. + second_pass_convert.resize(func.nargs, false); + call.args_convert.swap(second_pass_convert); + } + + // 6. Call the function. try { - if ((kwargs_consumed == nkwargs || it->has_kwargs) && - (nargs_ == it->nargs || it->has_args)) - result = it->impl(it, args_, kwargs, parent); + result = func.impl(call); } catch (reference_cast_error &) { result = PYBIND11_TRY_NEXT_OVERLOAD; } if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) break; + + if (overloaded) { + // The (overloaded) call failed; if the call has at least one argument that + // permits conversion (i.e. it hasn't been explicitly specified `.noconvert()`) + // then add this call to the list of second pass overloads to try. + for (size_t i = func.is_method ? 1 : 0; i < pos_args; i++) { + if (second_pass_convert[i]) { + // Found one: swap the converting flags back in and store the call for + // the second pass. + call.args_convert.swap(second_pass_convert); + second_pass.push_back(std::move(call)); + break; + } + } + } + } + + if (overloaded && !second_pass.empty() && result.ptr() == PYBIND11_TRY_NEXT_OVERLOAD) { + // The no-conversion pass finished without success, try again with conversion allowed + for (auto &call : second_pass) { + try { + result = call.func.impl(call); + } catch (reference_cast_error &) { + result = PYBIND11_TRY_NEXT_OVERLOAD; + } + + if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) + break; + } } } catch (error_already_set &e) { e.restore(); @@ -457,7 +608,7 @@ protected: - delegate translation to the next translator by throwing a new type of exception. */ auto last_exception = std::current_exception(); - auto ®istered_exception_translators = pybind11::detail::get_internals().registered_exception_translators; + auto ®istered_exception_translators = get_internals().registered_exception_translators; for (auto& translator : registered_exception_translators) { try { translator(last_exception); @@ -480,7 +631,7 @@ protected: " arguments. The following argument types are supported:\n"; int ctr = 0; - for (detail::function_record *it2 = overloads; it2 != nullptr; it2 = it2->next) { + for (function_record *it2 = overloads; it2 != nullptr; it2 = it2->next) { msg += " "+ std::to_string(++ctr) + ". "; bool wrote_sig = false; @@ -507,12 +658,27 @@ protected: msg += "\n"; } msg += "\nInvoked with: "; - auto args_ = reinterpret_borrow<tuple>(args); + auto args_ = reinterpret_borrow<tuple>(args_in); + bool some_args = false; for (size_t ti = overloads->is_constructor ? 1 : 0; ti < args_.size(); ++ti) { + if (!some_args) some_args = true; + else msg += ", "; msg += pybind11::repr(args_[ti]); - if ((ti + 1) != args_.size() ) - msg += ", "; } + if (kwargs_in) { + auto kwargs = reinterpret_borrow<dict>(kwargs_in); + if (kwargs.size() > 0) { + if (some_args) msg += "; "; + msg += "kwargs: "; + bool first = true; + for (auto kwarg : kwargs) { + if (first) first = false; + else msg += ", "; + msg += pybind11::str("{}={!r}").format(kwarg.first, kwarg.second); + } + } + } + PyErr_SetString(PyExc_TypeError, msg.c_str()); return nullptr; } else if (!result) { @@ -525,9 +691,8 @@ protected: if (overloads->is_constructor) { /* When a constructor ran successfully, the corresponding holder type (e.g. std::unique_ptr) must still be initialized. */ - PyObject *inst = PyTuple_GET_ITEM(args, 0); - auto tinfo = detail::get_type_info(Py_TYPE(inst)); - tinfo->init_holder(inst, nullptr); + auto tinfo = get_type_info(Py_TYPE(parent.ptr())); + tinfo->init_holder(parent.ptr(), nullptr); } return result.ptr(); } @@ -539,6 +704,7 @@ class module : public object { public: PYBIND11_OBJECT_DEFAULT(module, object, PyModule_Check) + /// Create a new top-level Python module with the given name and docstring explicit module(const char *name, const char *doc = nullptr) { if (!options::show_user_defined_docstrings()) doc = nullptr; #if PY_MAJOR_VERSION >= 3 @@ -557,6 +723,11 @@ public: inc_ref(); } + /** \rst + Create Python binding for a new function within the module scope. ``Func`` + can be a plain C++ function, a function pointer, or a lambda function. For + details on the ``Extra&& ... extra`` argument, see section :ref:`extras`. + \endrst */ template <typename Func, typename... Extra> module &def(const char *name_, Func &&f, const Extra& ... extra) { cpp_function func(std::forward<Func>(f), name(name_), scope(*this), @@ -567,6 +738,16 @@ public: return *this; } + /** \rst + Create and return a new Python submodule with the given name and docstring. + This also works recursively, i.e. + + .. code-block:: cpp + + py::module m("example", "pybind11 example plugin"); + py::module m2 = m.def_submodule("sub", "A submodule of 'example'"); + py::module m3 = m2.def_submodule("subsub", "A submodule of 'example.sub'"); + \endrst */ module def_submodule(const char *name, const char *doc = nullptr) { std::string full_name = std::string(PyModule_GetName(m_ptr)) + std::string(".") + std::string(name); @@ -577,6 +758,7 @@ public: return result; } + /// Import and return a module or throws `error_already_set`. static module import(const char *name) { PyObject *obj = PyImport_ImportModule(name); if (!obj) @@ -600,175 +782,40 @@ public: }; NAMESPACE_BEGIN(detail) -extern "C" inline PyObject *get_dict(PyObject *op, void *) { - PyObject *&dict = *_PyObject_GetDictPtr(op); - if (!dict) { - dict = PyDict_New(); - } - Py_XINCREF(dict); - return dict; -} - -extern "C" inline int set_dict(PyObject *op, PyObject *new_dict, void *) { - if (!PyDict_Check(new_dict)) { - PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, not a '%.200s'", - Py_TYPE(new_dict)->tp_name); - return -1; - } - PyObject *&dict = *_PyObject_GetDictPtr(op); - Py_INCREF(new_dict); - Py_CLEAR(dict); - dict = new_dict; - return 0; -} - -static PyGetSetDef generic_getset[] = { - {const_cast<char*>("__dict__"), get_dict, set_dict, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr} -}; - /// Generic support for creating new Python heap types class generic_type : public object { template <typename...> friend class class_; public: PYBIND11_OBJECT_DEFAULT(generic_type, object, PyType_Check) protected: - void initialize(type_record *rec) { - auto &internals = get_internals(); - auto tindex = std::type_index(*(rec->type)); + void initialize(const type_record &rec) { + if (rec.scope && hasattr(rec.scope, rec.name)) + pybind11_fail("generic_type: cannot initialize type \"" + std::string(rec.name) + + "\": an object with that name is already defined"); - if (get_type_info(*(rec->type))) - pybind11_fail("generic_type: type \"" + std::string(rec->name) + + if (get_type_info(*rec.type)) + pybind11_fail("generic_type: type \"" + std::string(rec.name) + "\" is already registered!"); - auto name = reinterpret_steal<object>(PYBIND11_FROM_STRING(rec->name)); - object scope_module; - if (rec->scope) { - if (hasattr(rec->scope, rec->name)) - pybind11_fail("generic_type: cannot initialize type \"" + std::string(rec->name) + - "\": an object with that name is already defined"); - - if (hasattr(rec->scope, "__module__")) { - scope_module = rec->scope.attr("__module__"); - } else if (hasattr(rec->scope, "__name__")) { - scope_module = rec->scope.attr("__name__"); - } - } - -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - /* Qualified names for Python >= 3.3 */ - object scope_qualname; - if (rec->scope && hasattr(rec->scope, "__qualname__")) - scope_qualname = rec->scope.attr("__qualname__"); - object ht_qualname; - if (scope_qualname) { - ht_qualname = reinterpret_steal<object>(PyUnicode_FromFormat( - "%U.%U", scope_qualname.ptr(), name.ptr())); - } else { - ht_qualname = name; - } -#endif - - size_t num_bases = rec->bases.size(); - auto bases = tuple(rec->bases); - - std::string full_name = (scope_module ? ((std::string) pybind11::str(scope_module) + "." + rec->name) - : std::string(rec->name)); - - char *tp_doc = nullptr; - if (rec->doc && options::show_user_defined_docstrings()) { - /* Allocate memory for docstring (using PyObject_MALLOC, since - Python will free this later on) */ - size_t size = strlen(rec->doc) + 1; - tp_doc = (char *) PyObject_MALLOC(size); - memcpy((void *) tp_doc, rec->doc, size); - } - - /* Danger zone: from now (and until PyType_Ready), make sure to - issue no Python C API calls which could potentially invoke the - garbage collector (the GC will call type_traverse(), which will in - turn find the newly constructed type in an invalid state) */ - - auto type_holder = reinterpret_steal<object>(PyType_Type.tp_alloc(&PyType_Type, 0)); - auto type = (PyHeapTypeObject*) type_holder.ptr(); - - if (!type_holder || !name) - pybind11_fail(std::string(rec->name) + ": Unable to create type object!"); + m_ptr = make_new_python_type(rec); /* Register supplemental type information in C++ dict */ - detail::type_info *tinfo = new detail::type_info(); - tinfo->type = (PyTypeObject *) type; - tinfo->type_size = rec->type_size; - tinfo->init_holder = rec->init_holder; + auto *tinfo = new detail::type_info(); + tinfo->type = (PyTypeObject *) m_ptr; + tinfo->type_size = rec.type_size; + tinfo->operator_new = rec.operator_new; + tinfo->init_holder = rec.init_holder; + tinfo->dealloc = rec.dealloc; + + auto &internals = get_internals(); + auto tindex = std::type_index(*rec.type); tinfo->direct_conversions = &internals.direct_conversions[tindex]; + tinfo->default_holder = rec.default_holder; internals.registered_types_cpp[tindex] = tinfo; - internals.registered_types_py[type] = tinfo; - - /* Basic type attributes */ - type->ht_type.tp_name = strdup(full_name.c_str()); - type->ht_type.tp_basicsize = (ssize_t) rec->instance_size; - - if (num_bases > 0) { - type->ht_type.tp_base = (PyTypeObject *) ((object) bases[0]).inc_ref().ptr(); - type->ht_type.tp_bases = bases.release().ptr(); - rec->multiple_inheritance |= num_bases > 1; - } - - type->ht_name = name.release().ptr(); - -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - type->ht_qualname = ht_qualname.release().ptr(); -#endif + internals.registered_types_py[m_ptr] = tinfo; - /* Supported protocols */ - type->ht_type.tp_as_number = &type->as_number; - type->ht_type.tp_as_sequence = &type->as_sequence; - type->ht_type.tp_as_mapping = &type->as_mapping; - - /* Supported elementary operations */ - type->ht_type.tp_init = (initproc) init; - type->ht_type.tp_new = (newfunc) new_instance; - type->ht_type.tp_dealloc = rec->dealloc; - - /* Support weak references (needed for the keep_alive feature) */ - type->ht_type.tp_weaklistoffset = offsetof(instance_essentials<void>, weakrefs); - - /* Flags */ - type->ht_type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; -#if PY_MAJOR_VERSION < 3 - type->ht_type.tp_flags |= Py_TPFLAGS_CHECKTYPES; -#endif - type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC; - - /* Support dynamic attributes */ - if (rec->dynamic_attr) { - type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_GC; - type->ht_type.tp_dictoffset = type->ht_type.tp_basicsize; // place the dict at the end - type->ht_type.tp_basicsize += sizeof(PyObject *); // and allocate enough space for it - type->ht_type.tp_getset = generic_getset; - type->ht_type.tp_traverse = traverse; - type->ht_type.tp_clear = clear; - } - - type->ht_type.tp_doc = tp_doc; - - if (PyType_Ready(&type->ht_type) < 0) - pybind11_fail(std::string(rec->name) + ": PyType_Ready failed (" + - detail::error_string() + ")!"); - - m_ptr = type_holder.ptr(); - - if (scope_module) // Needed by pydoc - attr("__module__") = scope_module; - - /* Register type with the parent scope */ - if (rec->scope) - rec->scope.attr(handle(type->ht_name)) = *this; - - if (rec->multiple_inheritance) - mark_parents_nonsimple(&type->ht_type); - - type_holder.release(); + if (rec.bases.size() > 1 || rec.multiple_inheritance) + mark_parents_nonsimple(tinfo->type); } /// Helper function which tags all parents of a type using mult. inheritance @@ -782,141 +829,50 @@ protected: } } - /// Allocate a metaclass on demand (for static properties) - handle metaclass() { - auto &ht_type = ((PyHeapTypeObject *) m_ptr)->ht_type; - auto &ob_type = PYBIND11_OB_TYPE(ht_type); - - if (ob_type == &PyType_Type) { - std::string name_ = std::string(ht_type.tp_name) + "__Meta"; -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - auto ht_qualname = reinterpret_steal<object>(PyUnicode_FromFormat("%U__Meta", attr("__qualname__").ptr())); -#endif - auto name = reinterpret_steal<object>(PYBIND11_FROM_STRING(name_.c_str())); - auto type_holder = reinterpret_steal<object>(PyType_Type.tp_alloc(&PyType_Type, 0)); - if (!type_holder || !name) - pybind11_fail("generic_type::metaclass(): unable to create type object!"); - - auto type = (PyHeapTypeObject*) type_holder.ptr(); - type->ht_name = name.release().ptr(); - -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - /* Qualified names for Python >= 3.3 */ - type->ht_qualname = ht_qualname.release().ptr(); -#endif - type->ht_type.tp_name = strdup(name_.c_str()); - type->ht_type.tp_base = ob_type; - type->ht_type.tp_flags |= (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE) & - ~Py_TPFLAGS_HAVE_GC; - - if (PyType_Ready(&type->ht_type) < 0) - pybind11_fail("generic_type::metaclass(): PyType_Ready failed!"); - - ob_type = (PyTypeObject *) type_holder.release().ptr(); - } - return handle((PyObject *) ob_type); - } - - static int init(void *self, PyObject *, PyObject *) { - std::string msg = std::string(Py_TYPE(self)->tp_name) + ": No constructor defined!"; - PyErr_SetString(PyExc_TypeError, msg.c_str()); - return -1; - } - - static PyObject *new_instance(PyTypeObject *type, PyObject *, PyObject *) { - instance<void> *self = (instance<void> *) PyType_GenericAlloc((PyTypeObject *) type, 0); - auto tinfo = detail::get_type_info(type); - self->value = ::operator new(tinfo->type_size); - self->owned = true; - self->holder_constructed = false; - detail::get_internals().registered_instances.emplace(self->value, (PyObject *) self); - return (PyObject *) self; - } - - static void dealloc(instance<void> *self) { - if (self->value) { - auto instance_type = Py_TYPE(self); - auto ®istered_instances = detail::get_internals().registered_instances; - auto range = registered_instances.equal_range(self->value); - bool found = false; - for (auto it = range.first; it != range.second; ++it) { - if (instance_type == Py_TYPE(it->second)) { - registered_instances.erase(it); - found = true; - break; - } - } - if (!found) - pybind11_fail("generic_type::dealloc(): Tried to deallocate unregistered instance!"); - - if (self->weakrefs) - PyObject_ClearWeakRefs((PyObject *) self); - - PyObject **dict_ptr = _PyObject_GetDictPtr((PyObject *) self); - if (dict_ptr) { - Py_CLEAR(*dict_ptr); - } - } - Py_TYPE(self)->tp_free((PyObject*) self); - } - - static int traverse(PyObject *op, visitproc visit, void *arg) { - PyObject *&dict = *_PyObject_GetDictPtr(op); - Py_VISIT(dict); - return 0; - } - - static int clear(PyObject *op) { - PyObject *&dict = *_PyObject_GetDictPtr(op); - Py_CLEAR(dict); - return 0; - } - void install_buffer_funcs( buffer_info *(*get_buffer)(PyObject *, void *), void *get_buffer_data) { PyHeapTypeObject *type = (PyHeapTypeObject*) m_ptr; - type->ht_type.tp_as_buffer = &type->as_buffer; -#if PY_MAJOR_VERSION < 3 - type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; -#endif - type->as_buffer.bf_getbuffer = getbuffer; - type->as_buffer.bf_releasebuffer = releasebuffer; auto tinfo = detail::get_type_info(&type->ht_type); + + if (!type->ht_type.tp_as_buffer) + pybind11_fail( + "To be able to register buffer protocol support for the type '" + + std::string(tinfo->type->tp_name) + + "' the associated class<>(..) invocation must " + "include the pybind11::buffer_protocol() annotation!"); + tinfo->get_buffer = get_buffer; tinfo->get_buffer_data = get_buffer_data; } - static int getbuffer(PyObject *obj, Py_buffer *view, int flags) { - auto tinfo = detail::get_type_info(Py_TYPE(obj)); - if (view == nullptr || obj == nullptr || !tinfo || !tinfo->get_buffer) { - PyErr_SetString(PyExc_BufferError, "generic_type::getbuffer(): Internal error"); - return -1; - } - memset(view, 0, sizeof(Py_buffer)); - buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data); - view->obj = obj; - view->ndim = 1; - view->internal = info; - view->buf = info->ptr; - view->itemsize = (ssize_t) info->itemsize; - view->len = view->itemsize; - for (auto s : info->shape) - view->len *= s; - if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) - view->format = const_cast<char *>(info->format.c_str()); - if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { - view->ndim = (int) info->ndim; - view->strides = (ssize_t *) &info->strides[0]; - view->shape = (ssize_t *) &info->shape[0]; - } - Py_INCREF(view->obj); - return 0; - } + void def_property_static_impl(const char *name, + handle fget, handle fset, + detail::function_record *rec_fget) { + const auto is_static = !(rec_fget->is_method && rec_fget->scope); + const auto has_doc = rec_fget->doc && pybind11::options::show_user_defined_docstrings(); - static void releasebuffer(PyObject *, Py_buffer *view) { delete (buffer_info *) view->internal; } + auto property = handle((PyObject *) (is_static ? get_internals().static_property_type + : &PyProperty_Type)); + attr(name) = property(fget.ptr() ? fget : none(), + fset.ptr() ? fset : none(), + /*deleter*/none(), + pybind11::str(has_doc ? rec_fget->doc : "")); + } }; +/// Set the pointer to operator new if it exists. The cast is needed because it can be overloaded. +template <typename T, typename = void_t<decltype(static_cast<void *(*)(size_t)>(T::operator new))>> +void set_operator_new(type_record *r) { r->operator_new = &T::operator new; } + +template <typename> void set_operator_new(...) { } + +/// Call class-specific delete if it exists or global otherwise. Can also be an overload set. +template <typename T, typename = void_t<decltype(static_cast<void (*)(void *)>(T::operator delete))>> +void call_operator_delete(T *p) { T::operator delete(p); } + +inline void call_operator_delete(void *p) { ::operator delete(p); } + NAMESPACE_END(detail) template <typename type_, typename... options> @@ -924,12 +880,9 @@ class class_ : public detail::generic_type { template <typename T> using is_holder = detail::is_holder_type<type_, T>; template <typename T> using is_subtype = detail::bool_constant<std::is_base_of<type_, T>::value && !std::is_same<T, type_>::value>; template <typename T> using is_base = detail::bool_constant<std::is_base_of<T, type_>::value && !std::is_same<T, type_>::value>; - template <typename T> using is_valid_class_option = - detail::bool_constant< - is_holder<T>::value || - is_subtype<T>::value || - is_base<T>::value - >; + // struct instead of using here to help MSVC: + template <typename T> struct is_valid_class_option : + detail::any_of<is_holder<T>, is_subtype<T>, is_base<T>> {}; public: using type = type_; @@ -938,33 +891,46 @@ public: using holder_type = detail::first_of_t<is_holder, std::unique_ptr<type>, options...>; using instance_type = detail::instance<type, holder_type>; - static_assert(detail::all_of_t<is_valid_class_option, options...>::value, + static_assert(detail::all_of<is_valid_class_option<options>...>::value, "Unknown/invalid class_ template parameters provided"); PYBIND11_OBJECT(class_, generic_type, PyType_Check) template <typename... Extra> class_(handle scope, const char *name, const Extra &... extra) { - detail::type_record record; + using namespace detail; + + // MI can only be specified via class_ template options, not constructor parameters + static_assert( + none_of<is_pyobject<Extra>...>::value || // no base class arguments, or: + ( constexpr_sum(is_pyobject<Extra>::value...) == 1 && // Exactly one base + constexpr_sum(is_base<options>::value...) == 0 && // no template option bases + none_of<std::is_same<multiple_inheritance, Extra>...>::value), // no multiple_inheritance attr + "Error: multiple inheritance bases must be specified via class_ template options"); + + type_record record; record.scope = scope; record.name = name; record.type = &typeid(type); - record.type_size = sizeof(detail::conditional_t<has_alias, type_alias, type>); + record.type_size = sizeof(conditional_t<has_alias, type_alias, type>); record.instance_size = sizeof(instance_type); record.init_holder = init_holder; record.dealloc = dealloc; + record.default_holder = std::is_same<holder_type, std::unique_ptr<type>>::value; + + set_operator_new<type>(&record); /* Register base classes specified via template arguments to class_, if any */ bool unused[] = { (add_base<options>(record), false)..., false }; (void) unused; /* Process optional arguments, if any */ - detail::process_attributes<Extra...>::init(extra..., &record); + process_attributes<Extra...>::init(extra..., &record); - detail::generic_type::initialize(&record); + generic_type::initialize(record); if (has_alias) { - auto &instances = pybind11::detail::get_internals().registered_types_cpp; + auto &instances = get_internals().registered_types_cpp; instances[std::type_index(typeid(type_alias))] = instances[std::type_index(typeid(type))]; } } @@ -988,7 +954,9 @@ public: } template <typename Func, typename... Extra> class_ & - def_static(const char *name_, Func f, const Extra&... extra) { + def_static(const char *name_, Func &&f, const Extra&... extra) { + static_assert(!std::is_member_function_pointer<Func>::value, + "def_static(...) called with a non-static member function pointer"); cpp_function cf(std::forward<Func>(f), name(name_), scope(*this), sibling(getattr(*this, name_, none())), extra...); attr(cf.name()) = cf; @@ -1023,7 +991,7 @@ public: struct capture { Func func; }; capture *ptr = new capture { std::forward<Func>(func) }; install_buffer_funcs([](PyObject *obj, void *ptr) -> buffer_info* { - detail::type_caster<type> caster; + detail::make_caster<type> caster; if (!caster.load(obj, false)) return nullptr; return new buffer_info(((capture *) ptr)->func(caster)); @@ -1121,14 +1089,7 @@ public: rec_fset->doc = strdup(rec_fset->doc); } } - pybind11::str doc_obj = pybind11::str((rec_fget->doc && pybind11::options::show_user_defined_docstrings()) ? rec_fget->doc : ""); - const auto property = reinterpret_steal<object>( - PyObject_CallFunctionObjArgs((PyObject *) &PyProperty_Type, fget.ptr() ? fget.ptr() : Py_None, - fset.ptr() ? fset.ptr() : Py_None, Py_None, doc_obj.ptr(), nullptr)); - if (rec_fget->is_method && rec_fget->scope) - attr(name) = property; - else - metaclass().attr(name) = property; + def_property_static_impl(name, fget, fset, rec_fget); return *this; } @@ -1147,24 +1108,22 @@ private: } } + static void init_holder_from_existing(instance_type *inst, const holder_type *holder_ptr, + std::true_type /*is_copy_constructible*/) { + new (&inst->holder) holder_type(*holder_ptr); + } + + static void init_holder_from_existing(instance_type *inst, const holder_type *holder_ptr, + std::false_type /*is_copy_constructible*/) { + new (&inst->holder) holder_type(std::move(*const_cast<holder_type *>(holder_ptr))); + } + /// Initialize holder object, variant 2: try to construct from existing holder object, if possible - template <typename T = holder_type, - detail::enable_if_t<std::is_copy_constructible<T>::value, int> = 0> static void init_holder_helper(instance_type *inst, const holder_type *holder_ptr, const void * /* dummy */) { if (holder_ptr) { - new (&inst->holder) holder_type(*holder_ptr); + init_holder_from_existing(inst, holder_ptr, std::is_copy_constructible<holder_type>()); inst->holder_constructed = true; - } else if (inst->owned) { - new (&inst->holder) holder_type(inst->value); - inst->holder_constructed = true; - } - } - - /// Initialize holder object, variant 3: holder is not copy constructible (e.g. unique_ptr), always initialize from raw pointer - template <typename T = holder_type, - detail::enable_if_t<!std::is_copy_constructible<T>::value, int> = 0> - static void init_holder_helper(instance_type *inst, const holder_type * /* unused */, const void * /* dummy */) { - if (inst->owned) { + } else if (inst->owned || detail::always_construct_holder<holder_type>::value) { new (&inst->holder) holder_type(inst->value); inst->holder_constructed = true; } @@ -1181,14 +1140,12 @@ private: if (inst->holder_constructed) inst->holder.~holder_type(); else if (inst->owned) - ::operator delete(inst->value); - - generic_type::dealloc((detail::instance<void> *) inst); + detail::call_operator_delete(inst->value); } static detail::function_record *get_function_record(handle h) { h = detail::get_function(h); - return h ? (detail::function_record *) reinterpret_borrow<capsule>(PyCFunction_GetSelf(h.ptr())) + return h ? (detail::function_record *) reinterpret_borrow<capsule>(PyCFunction_GET_SELF(h.ptr())) : nullptr; } }; @@ -1197,26 +1154,33 @@ private: template <typename Type> class enum_ : public class_<Type> { public: using class_<Type>::def; + using class_<Type>::def_property_readonly_static; using Scalar = typename std::underlying_type<Type>::type; template <typename T> using arithmetic_tag = std::is_same<T, arithmetic>; template <typename... Extra> enum_(const handle &scope, const char *name, const Extra&... extra) - : class_<Type>(scope, name, extra...), m_parent(scope) { + : class_<Type>(scope, name, extra...), m_entries(), m_parent(scope) { constexpr bool is_arithmetic = !std::is_same<detail::first_of_t<arithmetic_tag, void, Extra...>, void>::value; - auto entries = new std::unordered_map<Scalar, const char *>(); - def("__repr__", [name, entries](Type value) -> std::string { - auto it = entries->find((Scalar) value); - return std::string(name) + "." + - ((it == entries->end()) ? std::string("???") - : std::string(it->second)); + auto m_entries_ptr = m_entries.inc_ref().ptr(); + def("__repr__", [name, m_entries_ptr](Type value) -> pybind11::str { + for (const auto &kv : reinterpret_borrow<dict>(m_entries_ptr)) { + if (pybind11::cast<Type>(kv.second) == value) + return pybind11::str("{}.{}").format(name, kv.first); + } + return pybind11::str("{}.???").format(name); }); + def_property_readonly_static("__members__", [m_entries_ptr](object /* self */) { + dict m; + for (const auto &kv : reinterpret_borrow<dict>(m_entries_ptr)) + m[kv.first] = kv.second; + return m; + }, return_value_policy::copy); def("__init__", [](Type& value, Scalar i) { value = (Type)i; }); - def("__init__", [](Type& value, Scalar i) { new (&value) Type((Type) i); }); def("__int__", [](Type value) { return (Scalar) value; }); def("__eq__", [](const Type &value, Type *value2) { return value2 && value == *value2; }); def("__ne__", [](const Type &value, Type *value2) { return !value2 || value != *value2; }); @@ -1253,28 +1217,25 @@ public: // Pickling and unpickling -- needed for use with the 'multiprocessing' module def("__getstate__", [](const Type &value) { return pybind11::make_tuple((Scalar) value); }); def("__setstate__", [](Type &p, tuple t) { new (&p) Type((Type) t[0].cast<Scalar>()); }); - m_entries = entries; } /// Export enumeration entries into the parent scope - enum_ &export_values() { - PyObject *dict = ((PyTypeObject *) this->m_ptr)->tp_dict; - PyObject *key, *value; - ssize_t pos = 0; - while (PyDict_Next(dict, &pos, &key, &value)) - if (PyObject_IsInstance(value, this->m_ptr)) - m_parent.attr(key) = value; + enum_& export_values() { + for (const auto &kv : m_entries) + m_parent.attr(kv.first) = kv.second; return *this; } /// Add an enumeration entry enum_& value(char const* name, Type value) { - this->attr(name) = pybind11::cast(value, return_value_policy::copy); - (*m_entries)[(Scalar) value] = name; + auto v = pybind11::cast(value, return_value_policy::copy); + this->attr(name) = v; + m_entries[pybind11::str(name)] = v; return *this; } + private: - std::unordered_map<Scalar, const char *> *m_entries; + dict m_entries; handle m_parent; }; @@ -1336,11 +1297,11 @@ inline void keep_alive_impl(handle nurse, handle patient) { (void) wr.release(); } -PYBIND11_NOINLINE inline void keep_alive_impl(int Nurse, int Patient, handle args, handle ret) { - handle nurse (Nurse > 0 ? PyTuple_GetItem(args.ptr(), Nurse - 1) : ret.ptr()); - handle patient(Patient > 0 ? PyTuple_GetItem(args.ptr(), Patient - 1) : ret.ptr()); - - keep_alive_impl(nurse, patient); +PYBIND11_NOINLINE inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret) { + keep_alive_impl( + Nurse == 0 ? ret : Nurse <= call.args.size() ? call.args[Nurse - 1] : handle(), + Patient == 0 ? ret : Patient <= call.args.size() ? call.args[Patient - 1] : handle() + ); } template <typename Iterator, typename Sentinel, bool KeyIterator, return_value_policy Policy> @@ -1355,6 +1316,7 @@ NAMESPACE_END(detail) template <typename... Args> detail::init<Args...> init() { return detail::init<Args...>(); } template <typename... Args> detail::init_alias<Args...> init_alias() { return detail::init_alias<Args...>(); } +/// Makes a python iterator from a first and past-the-end C++ InputIterator. template <return_value_policy Policy = return_value_policy::reference_internal, typename Iterator, typename Sentinel, @@ -1380,6 +1342,8 @@ iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) { return (iterator) cast(state { first, last, true }); } +/// Makes an python iterator over the keys (`.first`) of a iterator over pairs from a +/// first and past-the-end InputIterator. template <return_value_policy Policy = return_value_policy::reference_internal, typename Iterator, typename Sentinel, @@ -1405,11 +1369,15 @@ iterator make_key_iterator(Iterator first, Sentinel last, Extra &&... extra) { return (iterator) cast(state { first, last, true }); } +/// Makes an iterator over values of an stl container or other container supporting +/// `std::begin()`/`std::end()` template <return_value_policy Policy = return_value_policy::reference_internal, typename Type, typename... Extra> iterator make_iterator(Type &value, Extra&&... extra) { return make_iterator<Policy>(std::begin(value), std::end(value), extra...); } +/// Makes an iterator over the keys (`.first`) of a stl map-like container supporting +/// `std::begin()`/`std::end()` template <return_value_policy Policy = return_value_policy::reference_internal, typename Type, typename... Extra> iterator make_key_iterator(Type &value, Extra&&... extra) { return make_key_iterator<Policy>(std::begin(value), std::end(value), extra...); @@ -1417,7 +1385,7 @@ template <return_value_policy Policy = return_value_policy::reference_internal, template <typename InputType, typename OutputType> void implicitly_convertible() { auto implicit_caster = [](PyObject *obj, PyTypeObject *type) -> PyObject * { - if (!detail::type_caster<InputType>().load(obj, false)) + if (!detail::make_caster<InputType>().load(obj, false)) return nullptr; tuple args(1); args[0] = obj; @@ -1451,7 +1419,7 @@ public: exception(handle scope, const char *name, PyObject *base = PyExc_Exception) { std::string full_name = scope.attr("__name__").cast<std::string>() + std::string(".") + name; - m_ptr = PyErr_NewException((char *) full_name.c_str(), base, NULL); + m_ptr = PyErr_NewException(const_cast<char *>(full_name.c_str()), base, NULL); if (hasattr(scope, name)) pybind11_fail("Error during initialization: multiple incompatible " "definitions with name \"" + std::string(name) + "\""); @@ -1524,7 +1492,7 @@ void print(Args &&...args) { detail::print(c.args(), c.kwargs()); } -#if defined(WITH_THREAD) +#if defined(WITH_THREAD) && !defined(PYPY_VERSION) /* The functions below essentially reproduce the PyGILState_* API using a RAII * pattern, but there are a few important differences: @@ -1647,6 +1615,20 @@ private: PyThreadState *tstate; bool disassoc; }; +#elif defined(PYPY_VERSION) +class gil_scoped_acquire { + PyGILState_STATE state; +public: + gil_scoped_acquire() { state = PyGILState_Ensure(); } + ~gil_scoped_acquire() { PyGILState_Release(state); } +}; + +class gil_scoped_release { + PyThreadState *state; +public: + gil_scoped_release() { state = PyEval_SaveThread(); } + ~gil_scoped_release() { PyEval_RestoreThread(state); } +}; #else class gil_scoped_acquire { }; class gil_scoped_release { }; @@ -1655,16 +1637,15 @@ class gil_scoped_release { }; error_already_set::~error_already_set() { if (value) { gil_scoped_acquire gil; - PyErr_Restore(type, value, trace); - PyErr_Clear(); + clear(); } } inline function get_type_overload(const void *this_ptr, const detail::type_info *this_type, const char *name) { - handle py_object = detail::get_object_handle(this_ptr, this_type); - if (!py_object) + handle self = detail::get_object_handle(this_ptr, this_type); + if (!self) return function(); - handle type = py_object.get_type(); + handle type = self.get_type(); auto key = std::make_pair(type.ptr(), name); /* Cache functions that aren't overloaded in Python to avoid @@ -1673,22 +1654,47 @@ inline function get_type_overload(const void *this_ptr, const detail::type_info if (cache.find(key) != cache.end()) return function(); - function overload = getattr(py_object, name, function()); + function overload = getattr(self, name, function()); if (overload.is_cpp_function()) { cache.insert(key); return function(); } - /* Don't call dispatch code if invoked from overridden function */ + /* Don't call dispatch code if invoked from overridden function. + Unfortunately this doesn't work on PyPy. */ +#if !defined(PYPY_VERSION) PyFrameObject *frame = PyThreadState_Get()->frame; if (frame && (std::string) str(frame->f_code->co_name) == name && frame->f_code->co_argcount > 0) { PyFrame_FastToLocals(frame); PyObject *self_caller = PyDict_GetItem( frame->f_locals, PyTuple_GET_ITEM(frame->f_code->co_varnames, 0)); - if (self_caller == py_object.ptr()) + if (self_caller == self.ptr()) return function(); } +#else + /* PyPy currently doesn't provide a detailed cpyext emulation of + frame objects, so we have to emulate this using Python. This + is going to be slow..*/ + dict d; d["self"] = self; d["name"] = pybind11::str(name); + PyObject *result = PyRun_String( + "import inspect\n" + "frame = inspect.currentframe()\n" + "if frame is not None:\n" + " frame = frame.f_back\n" + " if frame is not None and str(frame.f_code.co_name) == name and " + "frame.f_code.co_argcount > 0:\n" + " self_caller = frame.f_locals[frame.f_code.co_varnames[0]]\n" + " if self_caller == self:\n" + " self = None\n", + Py_file_input, d.ptr(), d.ptr()); + if (result == nullptr) + throw error_already_set(); + if ((handle) d["self"] == Py_None) + return function(); + Py_DECREF(result); +#endif + return overload; } diff --git a/ext/pybind11/include/pybind11/pytypes.h b/ext/pybind11/include/pybind11/pytypes.h index 2b49ecfc9..900c57564 100644 --- a/ext/pybind11/include/pybind11/pytypes.h +++ b/ext/pybind11/include/pybind11/pytypes.h @@ -45,51 +45,130 @@ using tuple_accessor = accessor<accessor_policies::tuple_item>; class pyobject_tag { }; template <typename T> using is_pyobject = std::is_base_of<pyobject_tag, typename std::remove_reference<T>::type>; -/// Mixin which adds common functions to handle, object and various accessors. -/// The only requirement for `Derived` is to implement `PyObject *Derived::ptr() const`. +/** \rst + A mixin class which adds common functions to `handle`, `object` and various accessors. + The only requirement for `Derived` is to implement ``PyObject *Derived::ptr() const``. +\endrst */ template <typename Derived> class object_api : public pyobject_tag { const Derived &derived() const { return static_cast<const Derived &>(*this); } public: + /** \rst + Return an iterator equivalent to calling ``iter()`` in Python. The object + must be a collection which supports the iteration protocol. + \endrst */ iterator begin() const; + /// Return a sentinel which ends iteration. iterator end() const; + + /** \rst + Return an internal functor to invoke the object's sequence protocol. Casting + the returned ``detail::item_accessor`` instance to a `handle` or `object` + subclass causes a corresponding call to ``__getitem__``. Assigning a `handle` + or `object` subclass causes a call to ``__setitem__``. + \endrst */ item_accessor operator[](handle key) const; + /// See above (the only difference is that they key is provided as a string literal) item_accessor operator[](const char *key) const; + + /** \rst + Return an internal functor to access the object's attributes. Casting the + returned ``detail::obj_attr_accessor`` instance to a `handle` or `object` + subclass causes a corresponding call to ``getattr``. Assigning a `handle` + or `object` subclass causes a call to ``setattr``. + \endrst */ obj_attr_accessor attr(handle key) const; + /// See above (the only difference is that they key is provided as a string literal) str_attr_accessor attr(const char *key) const; + + /** \rst + Matches * unpacking in Python, e.g. to unpack arguments out of a ``tuple`` + or ``list`` for a function call. Applying another * to the result yields + ** unpacking, e.g. to unpack a dict as function keyword arguments. + See :ref:`calling_python_functions`. + \endrst */ args_proxy operator*() const; - template <typename T> bool contains(T &&key) const; + /// Check if the given item is contained within this object, i.e. ``item in obj``. + template <typename T> bool contains(T &&item) const; + + /** \rst + Assuming the Python object is a function or implements the ``__call__`` + protocol, ``operator()`` invokes the underlying function, passing an + arbitrary set of parameters. The result is returned as a `object` and + may need to be converted back into a Python object using `handle::cast()`. + + When some of the arguments cannot be converted to Python objects, the + function will throw a `cast_error` exception. When the Python function + call fails, a `error_already_set` exception is thrown. + \endrst */ template <return_value_policy policy = return_value_policy::automatic_reference, typename... Args> object operator()(Args &&...args) const; template <return_value_policy policy = return_value_policy::automatic_reference, typename... Args> PYBIND11_DEPRECATED("call(...) was deprecated in favor of operator()(...)") object call(Args&&... args) const; + /// Equivalent to ``obj is None`` in Python. bool is_none() const { return derived().ptr() == Py_None; } - PYBIND11_DEPRECATED("Instead of obj.str(), use py::str(obj)") + PYBIND11_DEPRECATED("Use py::str(obj) instead") pybind11::str str() const; + /// Return the object's current reference count int ref_count() const { return static_cast<int>(Py_REFCNT(derived().ptr())); } + /// Return a handle to the Python type object underlying the instance handle get_type() const; }; NAMESPACE_END(detail) -/// Holds a reference to a Python object (no reference counting) +/** \rst + Holds a reference to a Python object (no reference counting) + + The `handle` class is a thin wrapper around an arbitrary Python object (i.e. a + ``PyObject *`` in Python's C API). It does not perform any automatic reference + counting and merely provides a basic C++ interface to various Python API functions. + + .. seealso:: + The `object` class inherits from `handle` and adds automatic reference + counting features. +\endrst */ class handle : public detail::object_api<handle> { public: + /// The default constructor creates a handle with a ``nullptr``-valued pointer handle() = default; + /// Creates a ``handle`` from the given raw Python object pointer handle(PyObject *ptr) : m_ptr(ptr) { } // Allow implicit conversion from PyObject* + /// Return the underlying ``PyObject *`` pointer PyObject *ptr() const { return m_ptr; } PyObject *&ptr() { return m_ptr; } - const handle& inc_ref() const { Py_XINCREF(m_ptr); return *this; } - const handle& dec_ref() const { Py_XDECREF(m_ptr); return *this; } + /** \rst + Manually increase the reference count of the Python object. Usually, it is + preferable to use the `object` class which derives from `handle` and calls + this function automatically. Returns a reference to itself. + \endrst */ + const handle& inc_ref() const & { Py_XINCREF(m_ptr); return *this; } + + /** \rst + Manually decrease the reference count of the Python object. Usually, it is + preferable to use the `object` class which derives from `handle` and calls + this function automatically. Returns a reference to itself. + \endrst */ + const handle& dec_ref() const & { Py_XDECREF(m_ptr); return *this; } + + /** \rst + Attempt to cast the Python object into the given C++ type. A `cast_error` + will be throw upon failure. + \endrst */ template <typename T> T cast() const; + /// Return ``true`` when the `handle` wraps a valid Python object explicit operator bool() const { return m_ptr != nullptr; } + /** \rst + Check that the underlying pointers are the same. + Equivalent to ``obj1 is obj2`` in Python. + \endrst */ bool operator==(const handle &h) const { return m_ptr == h.m_ptr; } bool operator!=(const handle &h) const { return m_ptr != h.m_ptr; } PYBIND11_DEPRECATED("Use handle::operator bool() instead") @@ -98,16 +177,33 @@ protected: PyObject *m_ptr = nullptr; }; -/// Holds a reference to a Python object (with reference counting) +/** \rst + Holds a reference to a Python object (with reference counting) + + Like `handle`, the `object` class is a thin wrapper around an arbitrary Python + object (i.e. a ``PyObject *`` in Python's C API). In contrast to `handle`, it + optionally increases the object's reference count upon construction, and it + *always* decreases the reference count when the `object` instance goes out of + scope and is destructed. When using `object` instances consistently, it is much + easier to get reference counting right at the first attempt. +\endrst */ class object : public handle { public: object() = default; PYBIND11_DEPRECATED("Use reinterpret_borrow<object>() or reinterpret_steal<object>()") object(handle h, bool is_borrowed) : handle(h) { if (is_borrowed) inc_ref(); } + /// Copy constructor; always increases the reference count object(const object &o) : handle(o) { inc_ref(); } + /// Move constructor; steals the object from ``other`` and preserves its reference count object(object &&other) noexcept { m_ptr = other.m_ptr; other.m_ptr = nullptr; } + /// Destructor; automatically calls `handle::dec_ref()` ~object() { dec_ref(); } + /** \rst + Resets the internal pointer to ``nullptr`` without without decreasing the + object's reference count. The function returns a raw handle to the original + Python object. + \endrst */ handle release() { PyObject *tmp = m_ptr; m_ptr = nullptr; @@ -150,14 +246,43 @@ public: object(handle h, stolen_t) : handle(h) { } }; -/** The following functions don't do any kind of conversion, they simply declare - that a PyObject is a certain type and borrow or steal the reference. */ +/** \rst + Declare that a `handle` or ``PyObject *`` is a certain type and borrow the reference. + The target type ``T`` must be `object` or one of its derived classes. The function + doesn't do any conversions or checks. It's up to the user to make sure that the + target type is correct. + + .. code-block:: cpp + + PyObject *p = PyList_GetItem(obj, index); + py::object o = reinterpret_borrow<py::object>(p); + // or + py::tuple t = reinterpret_borrow<py::tuple>(p); // <-- `p` must be already be a `tuple` +\endrst */ template <typename T> T reinterpret_borrow(handle h) { return {h, object::borrowed}; } + +/** \rst + Like `reinterpret_borrow`, but steals the reference. + + .. code-block:: cpp + + PyObject *p = PyObject_Str(obj); + py::str s = reinterpret_steal<py::str>(p); // <-- `p` must be already be a `str` +\endrst */ template <typename T> T reinterpret_steal(handle h) { return {h, object::stolen}; } -/// Check if `obj` is an instance of type `T` +/** \defgroup python_builtins _ + Unless stated otherwise, the following C++ functions behave the same + as their Python counterparts. + */ + +/** \ingroup python_builtins + \rst + Return true if ``obj`` is an instance of ``T``. Type ``T`` must be a subclass of + `object` or a class which was exposed to Python as ``py::class_<T>``. +\endrst */ template <typename T, detail::enable_if_t<std::is_base_of<object, T>::value, int> = 0> -bool isinstance(handle obj) { return T::_check(obj); } +bool isinstance(handle obj) { return T::check_(obj); } template <typename T, detail::enable_if_t<!std::is_base_of<object, T>::value, int> = 0> bool isinstance(handle obj) { return detail::isinstance_generic(obj, typeid(T)); } @@ -165,6 +290,17 @@ bool isinstance(handle obj) { return detail::isinstance_generic(obj, typeid(T)); template <> inline bool isinstance<handle>(handle obj) = delete; template <> inline bool isinstance<object>(handle obj) { return obj.ptr() != nullptr; } +/// \ingroup python_builtins +/// Return true if ``obj`` is an instance of the ``type``. +inline bool isinstance(handle obj, handle type) { + const auto result = PyObject_IsInstance(obj.ptr(), type.ptr()); + if (result == -1) + throw error_already_set(); + return result != 0; +} + +/// \addtogroup python_builtins +/// @{ inline bool hasattr(handle obj, handle name) { return PyObject_HasAttr(obj.ptr(), name.ptr()) == 1; } @@ -210,6 +346,7 @@ inline void setattr(handle obj, handle name, handle value) { inline void setattr(handle obj, const char *name, handle value) { if (PyObject_SetAttrString(obj.ptr(), name, value.ptr()) != 0) { throw error_already_set(); } } +/// @} python_builtins NAMESPACE_BEGIN(detail) inline handle get_function(handle value) { @@ -316,7 +453,7 @@ struct sequence_item { static object get(handle obj, size_t index) { PyObject *result = PySequence_GetItem(obj.ptr(), static_cast<ssize_t>(index)); if (!result) { throw error_already_set(); } - return reinterpret_borrow<object>(result); + return reinterpret_steal<object>(result); } static void set(handle obj, size_t index, handle val) { @@ -362,24 +499,131 @@ struct tuple_item { }; NAMESPACE_END(accessor_policies) -struct dict_iterator { +/// STL iterator template used for tuple, list, sequence and dict +template <typename Policy> +class generic_iterator : public Policy { + using It = generic_iterator; + public: - explicit dict_iterator(handle dict = handle(), ssize_t pos = -1) : dict(dict), pos(pos) { } - dict_iterator& operator++() { - if (!PyDict_Next(dict.ptr(), &pos, &key.ptr(), &value.ptr())) - pos = -1; - return *this; - } - std::pair<handle, handle> operator*() const { - return std::make_pair(key, value); - } - bool operator==(const dict_iterator &it) const { return it.pos == pos; } - bool operator!=(const dict_iterator &it) const { return it.pos != pos; } + using difference_type = ssize_t; + using iterator_category = typename Policy::iterator_category; + using value_type = typename Policy::value_type; + using reference = typename Policy::reference; + using pointer = typename Policy::pointer; + + generic_iterator() = default; + generic_iterator(handle seq, ssize_t index) : Policy(seq, index) { } + + reference operator*() const { return Policy::dereference(); } + reference operator[](difference_type n) const { return *(*this + n); } + pointer operator->() const { return **this; } + + It &operator++() { Policy::increment(); return *this; } + It operator++(int) { auto copy = *this; Policy::increment(); return copy; } + It &operator--() { Policy::decrement(); return *this; } + It operator--(int) { auto copy = *this; Policy::decrement(); return copy; } + It &operator+=(difference_type n) { Policy::advance(n); return *this; } + It &operator-=(difference_type n) { Policy::advance(-n); return *this; } + + friend It operator+(const It &a, difference_type n) { auto copy = a; return copy += n; } + friend It operator+(difference_type n, const It &b) { return b + n; } + friend It operator-(const It &a, difference_type n) { auto copy = a; return copy -= n; } + friend difference_type operator-(const It &a, const It &b) { return a.distance_to(b); } + + friend bool operator==(const It &a, const It &b) { return a.equal(b); } + friend bool operator!=(const It &a, const It &b) { return !(a == b); } + friend bool operator< (const It &a, const It &b) { return b - a > 0; } + friend bool operator> (const It &a, const It &b) { return b < a; } + friend bool operator>=(const It &a, const It &b) { return !(a < b); } + friend bool operator<=(const It &a, const It &b) { return !(a > b); } +}; + +NAMESPACE_BEGIN(iterator_policies) +/// Quick proxy class needed to implement ``operator->`` for iterators which can't return pointers +template <typename T> +struct arrow_proxy { + T value; + + arrow_proxy(T &&value) : value(std::move(value)) { } + T *operator->() const { return &value; } +}; + +/// Lightweight iterator policy using just a simple pointer: see ``PySequence_Fast_ITEMS`` +class sequence_fast_readonly { +protected: + using iterator_category = std::random_access_iterator_tag; + using value_type = handle; + using reference = const handle; + using pointer = arrow_proxy<const handle>; + + sequence_fast_readonly(handle obj, ssize_t n) : ptr(PySequence_Fast_ITEMS(obj.ptr()) + n) { } + + reference dereference() const { return *ptr; } + void increment() { ++ptr; } + void decrement() { --ptr; } + void advance(ssize_t n) { ptr += n; } + bool equal(const sequence_fast_readonly &b) const { return ptr == b.ptr; } + ssize_t distance_to(const sequence_fast_readonly &b) const { return ptr - b.ptr; } + +private: + PyObject **ptr; +}; + +/// Full read and write access using the sequence protocol: see ``detail::sequence_accessor`` +class sequence_slow_readwrite { +protected: + using iterator_category = std::random_access_iterator_tag; + using value_type = object; + using reference = sequence_accessor; + using pointer = arrow_proxy<const sequence_accessor>; + + sequence_slow_readwrite(handle obj, ssize_t index) : obj(obj), index(index) { } + + reference dereference() const { return {obj, static_cast<size_t>(index)}; } + void increment() { ++index; } + void decrement() { --index; } + void advance(ssize_t n) { index += n; } + bool equal(const sequence_slow_readwrite &b) const { return index == b.index; } + ssize_t distance_to(const sequence_slow_readwrite &b) const { return index - b.index; } + private: - handle dict, key, value; - ssize_t pos = 0; + handle obj; + ssize_t index; }; +/// Python's dictionary protocol permits this to be a forward iterator +class dict_readonly { +protected: + using iterator_category = std::forward_iterator_tag; + using value_type = std::pair<handle, handle>; + using reference = const value_type; + using pointer = arrow_proxy<const value_type>; + + dict_readonly() = default; + dict_readonly(handle obj, ssize_t pos) : obj(obj), pos(pos) { increment(); } + + reference dereference() const { return {key, value}; } + void increment() { if (!PyDict_Next(obj.ptr(), &pos, &key, &value)) { pos = -1; } } + bool equal(const dict_readonly &b) const { return pos == b.pos; } + +private: + handle obj; + PyObject *key, *value; + ssize_t pos = -1; +}; +NAMESPACE_END(iterator_policies) + +#if !defined(PYPY_VERSION) +using tuple_iterator = generic_iterator<iterator_policies::sequence_fast_readonly>; +using list_iterator = generic_iterator<iterator_policies::sequence_fast_readonly>; +#else +using tuple_iterator = generic_iterator<iterator_policies::sequence_slow_readwrite>; +using list_iterator = generic_iterator<iterator_policies::sequence_slow_readwrite>; +#endif + +using sequence_iterator = generic_iterator<iterator_policies::sequence_slow_readwrite>; +using dict_iterator = generic_iterator<iterator_policies::dict_readonly>; + inline bool PyIterable_Check(PyObject *obj) { PyObject *iter = PyObject_GetIter(obj); if (iter) { @@ -410,12 +654,10 @@ public: template <typename T> using is_keyword = std::is_base_of<arg, T>; template <typename T> using is_s_unpacking = std::is_same<args_proxy, T>; // * unpacking template <typename T> using is_ds_unpacking = std::is_same<kwargs_proxy, T>; // ** unpacking -template <typename T> using is_positional = bool_constant< - !is_keyword<T>::value && !is_s_unpacking<T>::value && !is_ds_unpacking<T>::value ->; -template <typename T> using is_keyword_or_ds = bool_constant< - is_keyword<T>::value || is_ds_unpacking<T>::value +template <typename T> using is_positional = satisfies_none_of<T, + is_keyword, is_s_unpacking, is_ds_unpacking >; +template <typename T> using is_keyword_or_ds = satisfies_any_of<T, is_keyword, is_ds_unpacking>; // Call argument collector forward declarations template <return_value_policy policy = return_value_policy::automatic_reference> @@ -437,7 +679,7 @@ NAMESPACE_END(detail) Name(handle h, stolen_t) : Parent(h, stolen) { } \ PYBIND11_DEPRECATED("Use py::isinstance<py::python_type>(obj) instead") \ bool check() const { return m_ptr != nullptr && (bool) CheckFun(m_ptr); } \ - static bool _check(handle h) { return h.ptr() != nullptr && CheckFun(h.ptr()); } + static bool check_(handle h) { return h.ptr() != nullptr && CheckFun(h.ptr()); } #define PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun) \ PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ @@ -454,47 +696,74 @@ NAMESPACE_END(detail) PYBIND11_OBJECT(Name, Parent, CheckFun) \ Name() : Parent() { } +/// \addtogroup pytypes +/// @{ + +/** \rst + Wraps a Python iterator so that it can also be used as a C++ input iterator + + Caveat: copying an iterator does not (and cannot) clone the internal + state of the Python iterable. This also applies to the post-increment + operator. This iterator should only be used to retrieve the current + value using ``operator*()``. +\endrst */ class iterator : public object { public: - /** Caveat: copying an iterator does not (and cannot) clone the internal - state of the Python iterable */ + using iterator_category = std::input_iterator_tag; + using difference_type = ssize_t; + using value_type = handle; + using reference = const handle; + using pointer = const handle *; + PYBIND11_OBJECT_DEFAULT(iterator, object, PyIter_Check) iterator& operator++() { - if (m_ptr) - advance(); + advance(); return *this; } - /** Caveat: this postincrement operator does not (and cannot) clone the - internal state of the Python iterable. It should only be used to - retrieve the current iterate using <tt>operator*()</tt> */ iterator operator++(int) { - iterator rv(*this); - rv.value = value; - if (m_ptr) - advance(); + auto rv = *this; + advance(); return rv; } - bool operator==(const iterator &it) const { return *it == **this; } - bool operator!=(const iterator &it) const { return *it != **this; } - - handle operator*() const { - if (!ready && m_ptr) { + reference operator*() const { + if (m_ptr && !value.ptr()) { auto& self = const_cast<iterator &>(*this); self.advance(); - self.ready = true; } return value; } + pointer operator->() const { operator*(); return &value; } + + /** \rst + The value which marks the end of the iteration. ``it == iterator::sentinel()`` + is equivalent to catching ``StopIteration`` in Python. + + .. code-block:: cpp + + void foo(py::iterator it) { + while (it != py::iterator::sentinel()) { + // use `*it` + ++it; + } + } + \endrst */ + static iterator sentinel() { return {}; } + + friend bool operator==(const iterator &a, const iterator &b) { return a->ptr() == b->ptr(); } + friend bool operator!=(const iterator &a, const iterator &b) { return a->ptr() != b->ptr(); } + private: - void advance() { value = reinterpret_steal<object>(PyIter_Next(m_ptr)); } + void advance() { + value = reinterpret_steal<object>(PyIter_Next(m_ptr)); + if (PyErr_Occurred()) { throw error_already_set(); } + } private: object value = {}; - bool ready = false; }; class iterable : public object { @@ -523,6 +792,10 @@ public: explicit str(const bytes &b); + /** \rst + Return a string representation of the object. This is analogous to + the ``str()`` function in Python. + \endrst */ explicit str(handle h) : object(raw_str(h.ptr()), stolen) { } operator std::string() const { @@ -556,12 +829,17 @@ private: return str_value; } }; +/// @} pytypes inline namespace literals { -/// String literal version of str +/** \rst + String literal version of `str` + \endrst */ inline str operator"" _s(const char *s, size_t size) { return {s, size}; } } +/// \addtogroup pytypes +/// @{ class bytes : public object { public: PYBIND11_OBJECT(bytes, object, PYBIND11_BYTES_CHECK) @@ -726,10 +1004,44 @@ public: PYBIND11_OBJECT_DEFAULT(capsule, object, PyCapsule_CheckExact) PYBIND11_DEPRECATED("Use reinterpret_borrow<capsule>() or reinterpret_steal<capsule>()") capsule(PyObject *ptr, bool is_borrowed) : object(is_borrowed ? object(ptr, borrowed) : object(ptr, stolen)) { } - explicit capsule(const void *value, void (*destruct)(PyObject *) = nullptr) + + explicit capsule(const void *value) + : object(PyCapsule_New(const_cast<void *>(value), nullptr, nullptr), stolen) { + if (!m_ptr) + pybind11_fail("Could not allocate capsule object!"); + } + + PYBIND11_DEPRECATED("Please pass a destructor that takes a void pointer as input") + capsule(const void *value, void (*destruct)(PyObject *)) : object(PyCapsule_New(const_cast<void*>(value), nullptr, destruct), stolen) { - if (!m_ptr) pybind11_fail("Could not allocate capsule object!"); + if (!m_ptr) + pybind11_fail("Could not allocate capsule object!"); + } + + capsule(const void *value, void (*destructor)(void *)) { + m_ptr = PyCapsule_New(const_cast<void *>(value), nullptr, [](PyObject *o) { + auto destructor = reinterpret_cast<void (*)(void *)>(PyCapsule_GetContext(o)); + void *ptr = PyCapsule_GetPointer(o, nullptr); + destructor(ptr); + }); + + if (!m_ptr) + pybind11_fail("Could not allocate capsule object!"); + + if (PyCapsule_SetContext(m_ptr, (void *) destructor) != 0) + pybind11_fail("Could not set capsule context!"); } + + capsule(void (*destructor)()) { + m_ptr = PyCapsule_New(reinterpret_cast<void *>(destructor), nullptr, [](PyObject *o) { + auto destructor = reinterpret_cast<void (*)()>(PyCapsule_GetPointer(o, nullptr)); + destructor(); + }); + + if (!m_ptr) + pybind11_fail("Could not allocate capsule object!"); + } + template <typename T> operator T *() const { T * result = static_cast<T *>(PyCapsule_GetPointer(m_ptr, nullptr)); if (!result) pybind11_fail("Unable to extract capsule contents!"); @@ -745,6 +1057,8 @@ public: } size_t size() const { return (size_t) PyTuple_Size(m_ptr); } detail::tuple_accessor operator[](size_t index) const { return {*this, index}; } + detail::tuple_iterator begin() const { return {*this, 0}; } + detail::tuple_iterator end() const { return {*this, PyTuple_GET_SIZE(m_ptr)}; } }; class dict : public object { @@ -754,14 +1068,14 @@ public: if (!m_ptr) pybind11_fail("Could not allocate dict object!"); } template <typename... Args, - typename = detail::enable_if_t<detail::all_of_t<detail::is_keyword_or_ds, Args...>::value>, + typename = detail::enable_if_t<detail::all_of<detail::is_keyword_or_ds<Args>...>::value>, // MSVC workaround: it can't compile an out-of-line definition, so defer the collector typename collector = detail::deferred_t<detail::unpacking_collector<>, Args...>> explicit dict(Args &&...args) : dict(collector(std::forward<Args>(args)...).kwargs()) { } size_t size() const { return (size_t) PyDict_Size(m_ptr); } - detail::dict_iterator begin() const { return (++detail::dict_iterator(*this, 0)); } - detail::dict_iterator end() const { return detail::dict_iterator(); } + detail::dict_iterator begin() const { return {*this, 0}; } + detail::dict_iterator end() const { return {}; } void clear() const { PyDict_Clear(ptr()); } bool contains(handle key) const { return PyDict_Contains(ptr(), key.ptr()) == 1; } bool contains(const char *key) const { return PyDict_Contains(ptr(), pybind11::str(key).ptr()) == 1; } @@ -777,9 +1091,11 @@ private: class sequence : public object { public: - PYBIND11_OBJECT(sequence, object, PySequence_Check) + PYBIND11_OBJECT_DEFAULT(sequence, object, PySequence_Check) size_t size() const { return (size_t) PySequence_Size(m_ptr); } detail::sequence_accessor operator[](size_t index) const { return {*this, index}; } + detail::sequence_iterator begin() const { return {*this, 0}; } + detail::sequence_iterator end() const { return {*this, PySequence_Size(m_ptr)}; } }; class list : public object { @@ -790,6 +1106,8 @@ public: } size_t size() const { return (size_t) PyList_Size(m_ptr); } detail::list_accessor operator[](size_t index) const { return {*this, index}; } + detail::list_iterator begin() const { return {*this, 0}; } + detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; } template <typename T> void append(T &&val) const { PyList_Append(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()); } @@ -865,7 +1183,10 @@ public: PYBIND11_OBJECT_CVT(memoryview, object, PyMemoryView_Check, PyMemoryView_FromObject) }; +/// @} pytypes +/// \addtogroup python_builtins +/// @{ inline size_t len(handle h) { ssize_t result = PyObject_Length(h.ptr()); if (result < 0) @@ -884,13 +1205,16 @@ inline str repr(handle h) { return reinterpret_steal<str>(str_value); } -NAMESPACE_BEGIN(detail) -template <typename D> iterator object_api<D>::begin() const { - return reinterpret_steal<iterator>(PyObject_GetIter(derived().ptr())); -} -template <typename D> iterator object_api<D>::end() const { - return {}; +inline iterator iter(handle obj) { + PyObject *result = PyObject_GetIter(obj.ptr()); + if (!result) { throw error_already_set(); } + return reinterpret_steal<iterator>(result); } +/// @} python_builtins + +NAMESPACE_BEGIN(detail) +template <typename D> iterator object_api<D>::begin() const { return iter(derived()); } +template <typename D> iterator object_api<D>::end() const { return iterator::sentinel(); } template <typename D> item_accessor object_api<D>::operator[](handle key) const { return {derived(), reinterpret_borrow<object>(key)}; } @@ -906,8 +1230,8 @@ template <typename D> str_attr_accessor object_api<D>::attr(const char *key) con template <typename D> args_proxy object_api<D>::operator*() const { return args_proxy(derived().ptr()); } -template <typename D> template <typename T> bool object_api<D>::contains(T &&key) const { - return attr("__contains__")(std::forward<T>(key)).template cast<bool>(); +template <typename D> template <typename T> bool object_api<D>::contains(T &&item) const { + return attr("__contains__")(std::forward<T>(item)).template cast<bool>(); } template <typename D> diff --git a/ext/pybind11/include/pybind11/stl.h b/ext/pybind11/include/pybind11/stl.h index 7d6e8b102..37b6bcde3 100644 --- a/ext/pybind11/include/pybind11/stl.h +++ b/ext/pybind11/include/pybind11/stl.h @@ -139,7 +139,7 @@ public: auto value_ = reinterpret_steal<object>(value_conv::cast(value, policy, parent)); if (!value_) return handle(); - PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference + PyList_SET_ITEM(l.ptr(), (ssize_t) index++, value_.release().ptr()); // steals a reference } return l.release(); } @@ -192,7 +192,7 @@ public: auto value_ = reinterpret_steal<object>(value_conv::cast(value, policy, parent)); if (!value_) return handle(); - PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference + PyList_SET_ITEM(l.ptr(), (ssize_t) index++, value_.release().ptr()); // steals a reference } return l.release(); } diff --git a/ext/pybind11/include/pybind11/stl_bind.h b/ext/pybind11/include/pybind11/stl_bind.h index ef9950ebb..897188220 100644 --- a/ext/pybind11/include/pybind11/stl_bind.h +++ b/ext/pybind11/include/pybind11/stl_bind.h @@ -70,7 +70,7 @@ void vector_if_copy_constructible(enable_if_t< std::is_copy_constructible<Vector>::value && std::is_copy_constructible<typename Vector::value_type>::value, Class_> &cl) { - cl.def(pybind11::init<const Vector &>(), "Copy constructor"); + cl.def(init<const Vector &>(), "Copy constructor"); } template<typename Vector, typename Class_> @@ -93,7 +93,7 @@ void vector_if_equal_operator(enable_if_t<is_comparable<Vector>::value, Class_> if (p != v.end()) v.erase(p); else - throw pybind11::value_error(); + throw value_error(); }, arg("x"), "Remove the first item from the list whose value is x. " @@ -155,7 +155,7 @@ void vector_modifiers(enable_if_t<std::is_copy_constructible<typename Vector::va cl.def("pop", [](Vector &v) { if (v.empty()) - throw pybind11::index_error(); + throw index_error(); T t = v.back(); v.pop_back(); return t; @@ -166,7 +166,7 @@ void vector_modifiers(enable_if_t<std::is_copy_constructible<typename Vector::va cl.def("pop", [](Vector &v, SizeType i) { if (i >= v.size()) - throw pybind11::index_error(); + throw index_error(); T t = v[i]; v.erase(v.begin() + (DiffType) i); return t; @@ -178,7 +178,7 @@ void vector_modifiers(enable_if_t<std::is_copy_constructible<typename Vector::va cl.def("__setitem__", [](Vector &v, SizeType i, const T &t) { if (i >= v.size()) - throw pybind11::index_error(); + throw index_error(); v[i] = t; } ); @@ -189,7 +189,7 @@ void vector_modifiers(enable_if_t<std::is_copy_constructible<typename Vector::va size_t start, stop, step, slicelength; if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) - throw pybind11::error_already_set(); + throw error_already_set(); Vector *seq = new Vector(); seq->reserve((size_t) slicelength); @@ -208,7 +208,7 @@ void vector_modifiers(enable_if_t<std::is_copy_constructible<typename Vector::va [](Vector &v, slice slice, const Vector &value) { size_t start, stop, step, slicelength; if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) - throw pybind11::error_already_set(); + throw error_already_set(); if (slicelength != value.size()) throw std::runtime_error("Left and right hand size of slice assignment have different sizes!"); @@ -224,7 +224,7 @@ void vector_modifiers(enable_if_t<std::is_copy_constructible<typename Vector::va cl.def("__delitem__", [](Vector &v, SizeType i) { if (i >= v.size()) - throw pybind11::index_error(); + throw index_error(); v.erase(v.begin() + DiffType(i)); }, "Delete the list elements at index ``i``" @@ -235,7 +235,7 @@ void vector_modifiers(enable_if_t<std::is_copy_constructible<typename Vector::va size_t start, stop, step, slicelength; if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) - throw pybind11::error_already_set(); + throw error_already_set(); if (step == 1 && false) { v.erase(v.begin() + (DiffType) start, v.begin() + DiffType(start + slicelength)); @@ -253,8 +253,8 @@ void vector_modifiers(enable_if_t<std::is_copy_constructible<typename Vector::va // If the type has an operator[] that doesn't return a reference (most notably std::vector<bool>), // we have to access by copying; otherwise we return by reference. -template <typename Vector> using vector_needs_copy = bool_constant< - !std::is_same<decltype(std::declval<Vector>()[typename Vector::size_type()]), typename Vector::value_type &>::value>; +template <typename Vector> using vector_needs_copy = negation< + std::is_same<decltype(std::declval<Vector>()[typename Vector::size_type()]), typename Vector::value_type &>>; // The usual case: access and iterate by reference template <typename Vector, typename Class_> @@ -266,7 +266,7 @@ void vector_accessor(enable_if_t<!vector_needs_copy<Vector>::value, Class_> &cl) cl.def("__getitem__", [](Vector &v, SizeType i) -> T & { if (i >= v.size()) - throw pybind11::index_error(); + throw index_error(); return v[i]; }, return_value_policy::reference_internal // ref + keepalive @@ -274,7 +274,7 @@ void vector_accessor(enable_if_t<!vector_needs_copy<Vector>::value, Class_> &cl) cl.def("__iter__", [](Vector &v) { - return pybind11::make_iterator< + return make_iterator< return_value_policy::reference_internal, ItType, ItType, T&>( v.begin(), v.end()); }, @@ -291,14 +291,14 @@ void vector_accessor(enable_if_t<vector_needs_copy<Vector>::value, Class_> &cl) cl.def("__getitem__", [](const Vector &v, SizeType i) -> T { if (i >= v.size()) - throw pybind11::index_error(); + throw index_error(); return v[i]; } ); cl.def("__iter__", [](Vector &v) { - return pybind11::make_iterator< + return make_iterator< return_value_policy::copy, ItType, ItType, T>( v.begin(), v.end()); }, @@ -326,18 +326,64 @@ template <typename Vector, typename Class_> auto vector_if_insertion_operator(Cl ); } +// Provide the buffer interface for vectors if we have data() and we have a format for it +// GCC seems to have "void std::vector<bool>::data()" - doing SFINAE on the existence of data() is insufficient, we need to check it returns an appropriate pointer +template <typename Vector, typename = void> +struct vector_has_data_and_format : std::false_type {}; +template <typename Vector> +struct vector_has_data_and_format<Vector, enable_if_t<std::is_same<decltype(format_descriptor<typename Vector::value_type>::format(), std::declval<Vector>().data()), typename Vector::value_type*>::value>> : std::true_type {}; + +// Add the buffer interface to a vector +template <typename Vector, typename Class_, typename... Args> +enable_if_t<detail::any_of<std::is_same<Args, buffer_protocol>...>::value> +vector_buffer(Class_& cl) { + using T = typename Vector::value_type; + + static_assert(vector_has_data_and_format<Vector>::value, "There is not an appropriate format descriptor for this vector"); + + // numpy.h declares this for arbitrary types, but it may raise an exception and crash hard at runtime if PYBIND11_NUMPY_DTYPE hasn't been called, so check here + format_descriptor<T>::format(); + + cl.def_buffer([](Vector& v) -> buffer_info { + return buffer_info(v.data(), sizeof(T), format_descriptor<T>::format(), 1, {v.size()}, {sizeof(T)}); + }); + + cl.def("__init__", [](Vector& vec, buffer buf) { + auto info = buf.request(); + if (info.ndim != 1 || info.strides[0] <= 0 || info.strides[0] % sizeof(T)) + throw type_error("Only valid 1D buffers can be copied to a vector"); + if (!detail::compare_buffer_info<T>::compare(info) || sizeof(T) != info.itemsize) + throw type_error("Format mismatch (Python: " + info.format + " C++: " + format_descriptor<T>::format() + ")"); + new (&vec) Vector(); + vec.reserve(info.shape[0]); + T *p = static_cast<T*>(info.ptr); + auto step = info.strides[0] / sizeof(T); + T *end = p + info.shape[0] * step; + for (; p < end; p += step) + vec.push_back(*p); + }); + + return; +} + +template <typename Vector, typename Class_, typename... Args> +enable_if_t<!detail::any_of<std::is_same<Args, buffer_protocol>...>::value> vector_buffer(Class_&) {} + NAMESPACE_END(detail) // // std::vector // template <typename Vector, typename holder_type = std::unique_ptr<Vector>, typename... Args> -pybind11::class_<Vector, holder_type> bind_vector(pybind11::module &m, std::string const &name, Args&&... args) { - using Class_ = pybind11::class_<Vector, holder_type>; +class_<Vector, holder_type> bind_vector(module &m, std::string const &name, Args&&... args) { + using Class_ = class_<Vector, holder_type>; Class_ cl(m, name.c_str(), std::forward<Args>(args)...); - cl.def(pybind11::init<>()); + // Declare the buffer interface if a buffer_protocol() is passed in + detail::vector_buffer<Vector, Class_, Args...>(cl); + + cl.def(init<>()); // Register copy constructor (if possible) detail::vector_if_copy_constructible<Vector, Class_>(cl); @@ -368,7 +414,7 @@ pybind11::class_<Vector, holder_type> bind_vector(pybind11::module &m, std::stri #if 0 // C++ style functions deprecated, leaving it here as an example - cl.def(pybind11::init<size_type>()); + cl.def(init<size_type>()); cl.def("resize", (void (Vector::*) (size_type count)) & Vector::resize, @@ -377,7 +423,7 @@ pybind11::class_<Vector, holder_type> bind_vector(pybind11::module &m, std::stri cl.def("erase", [](Vector &v, SizeType i) { if (i >= v.size()) - throw pybind11::index_error(); + throw index_error(); v.erase(v.begin() + i); }, "erases element at index ``i``"); @@ -396,12 +442,12 @@ pybind11::class_<Vector, holder_type> bind_vector(pybind11::module &m, std::stri cl.def("front", [](Vector &v) { if (v.size()) return v.front(); - else throw pybind11::index_error(); + else throw index_error(); }, "access the first element"); cl.def("back", [](Vector &v) { if (v.size()) return v.back(); - else throw pybind11::index_error(); + else throw index_error(); }, "access the last element "); #endif @@ -484,14 +530,14 @@ template <typename Map, typename Class_> auto map_if_insertion_operator(Class_ & NAMESPACE_END(detail) template <typename Map, typename holder_type = std::unique_ptr<Map>, typename... Args> -pybind11::class_<Map, holder_type> bind_map(module &m, const std::string &name, Args&&... args) { +class_<Map, holder_type> bind_map(module &m, const std::string &name, Args&&... args) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; - using Class_ = pybind11::class_<Map, holder_type>; + using Class_ = class_<Map, holder_type>; Class_ cl(m, name.c_str(), std::forward<Args>(args)...); - cl.def(pybind11::init<>()); + cl.def(init<>()); // Register stream insertion operator (if possible) detail::map_if_insertion_operator<Map, Class_>(cl, name); @@ -502,20 +548,20 @@ pybind11::class_<Map, holder_type> bind_map(module &m, const std::string &name, ); cl.def("__iter__", - [](Map &m) { return pybind11::make_key_iterator(m.begin(), m.end()); }, - pybind11::keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ + [](Map &m) { return make_key_iterator(m.begin(), m.end()); }, + keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ ); cl.def("items", - [](Map &m) { return pybind11::make_iterator(m.begin(), m.end()); }, - pybind11::keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ + [](Map &m) { return make_iterator(m.begin(), m.end()); }, + keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ ); cl.def("__getitem__", [](Map &m, const KeyType &k) -> MappedType & { auto it = m.find(k); if (it == m.end()) - throw pybind11::key_error(); + throw key_error(); return it->second; }, return_value_policy::reference_internal // ref + keepalive @@ -528,7 +574,7 @@ pybind11::class_<Map, holder_type> bind_map(module &m, const std::string &name, [](Map &m, const KeyType &k) { auto it = m.find(k); if (it == m.end()) - throw pybind11::key_error(); + throw key_error(); return m.erase(it); } ); diff --git a/ext/pybind11/pybind11/_version.py b/ext/pybind11/pybind11/_version.py index be66f0732..2bb44f715 100644 --- a/ext/pybind11/pybind11/_version.py +++ b/ext/pybind11/pybind11/_version.py @@ -1,2 +1,2 @@ -version_info = (1, 9, 'dev0') +version_info = (2, 1, 1) __version__ = '.'.join(map(str, version_info)) diff --git a/ext/pybind11/setup.py b/ext/pybind11/setup.py index e9bc3c387..0cf4e47ce 100644 --- a/ext/pybind11/setup.py +++ b/ext/pybind11/setup.py @@ -4,21 +4,18 @@ from setuptools import setup from pybind11 import __version__ +import os -setup( - name='pybind11', - version=__version__, - description='Seamless operability between C++11 and Python', - author='Wenzel Jakob', - author_email='wenzel.jakob@epfl.ch', - url='https://github.com/wjakob/pybind11', - download_url='https://github.com/wjakob/pybind11/tarball/v' + __version__, - packages=['pybind11'], - license='BSD', - headers=[ +# Prevent installation of pybind11 headers by setting +# PYBIND11_USE_CMAKE. +if os.environ.get('PYBIND11_USE_CMAKE'): + headers = [] +else: + headers = [ 'include/pybind11/attr.h', 'include/pybind11/cast.h', 'include/pybind11/chrono.h', + 'include/pybind11/class_support.h', 'include/pybind11/common.h', 'include/pybind11/complex.h', 'include/pybind11/descr.h', @@ -32,8 +29,20 @@ setup( 'include/pybind11/pytypes.h', 'include/pybind11/stl.h', 'include/pybind11/stl_bind.h', - 'include/pybind11/typeid.h', - ], + 'include/pybind11/typeid.h' + ] + +setup( + name='pybind11', + version=__version__, + description='Seamless operability between C++11 and Python', + author='Wenzel Jakob', + author_email='wenzel.jakob@epfl.ch', + url='https://github.com/wjakob/pybind11', + download_url='https://github.com/wjakob/pybind11/tarball/v' + __version__, + packages=['pybind11'], + license='BSD', + headers=headers, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', @@ -46,14 +55,15 @@ setup( 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', - 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: BSD License' ], keywords='C++11, Python bindings', - long_description="""pybind11 is a lightweight header library that exposes -C++ types in Python and vice versa, mainly to create Python bindings of + long_description="""pybind11 is a lightweight header-only library that +exposes C++ types in Python and vice versa, mainly to create Python bindings of existing C++ code. Its goals and syntax are similar to the excellent -Boost.Python library by David Abrahams: to minimize boilerplate code in -traditional extension modules by inferring type information using compile-time +Boost.Python by David Abrahams: to minimize boilerplate code in traditional +extension modules by inferring type information using compile-time introspection. The main issue with Boost.Python-and the reason for creating such a similar @@ -66,9 +76,9 @@ become an excessively large and unnecessary dependency. Think of this library as a tiny self-contained version of Boost.Python with everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~2.5K lines of code and depend on -Python (2.7 or 3.x) and the C++ standard library. This compact implementation -was possible thanks to some of the new C++11 language features (specifically: -tuples, lambda functions and variadic templates). Since its creation, this -library has grown beyond Boost.Python in many ways, leading to dramatically -simpler binding code in many common situations.""") +comments, the core header files only require ~4K lines of code and depend on +Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This +compact implementation was possible thanks to some of the new C++11 language +features (specifically: tuples, lambda functions and variadic templates). Since +its creation, this library has grown beyond Boost.Python in many ways, leading +to dramatically simpler binding code in many common situations.""") diff --git a/ext/pybind11/tests/CMakeLists.txt b/ext/pybind11/tests/CMakeLists.txt index 27cb65291..11be49e53 100644 --- a/ext/pybind11/tests/CMakeLists.txt +++ b/ext/pybind11/tests/CMakeLists.txt @@ -1,3 +1,22 @@ +# CMakeLists.txt -- Build system for the pybind11 test suite +# +# Copyright (c) 2015 Wenzel Jakob <wenzel@inf.ethz.ch> +# +# All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +cmake_minimum_required(VERSION 2.8.12) + +option(PYBIND11_WERROR "Report all warnings as errors" OFF) + +if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + # We're being loaded directly, i.e. not via add_subdirectory, so make this + # work as its own project and load the pybind11Config to get the tools we need + project(pybind11_tests) + + find_package(pybind11 REQUIRED CONFIG) +endif() + if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) message(STATUS "Setting tests build type to MinSizeRel as none was specified") set(CMAKE_BUILD_TYPE MinSizeRel CACHE STRING "Choose the type of build." FORCE) @@ -54,9 +73,30 @@ string(REPLACE ".cpp" ".py" PYBIND11_PYTEST_FILES "${PYBIND11_TEST_FILES}") # skip message). list(FIND PYBIND11_TEST_FILES test_eigen.cpp PYBIND11_TEST_FILES_EIGEN_I) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) - find_package(Eigen3 QUIET) + # Try loading via newer Eigen's Eigen3Config first (bypassing tools/FindEigen3.cmake). + # Eigen 3.3.1+ exports a cmake 3.0+ target for handling dependency requirements, but also + # produces a fatal error if loaded from a pre-3.0 cmake. + if (NOT CMAKE_VERSION VERSION_LESS 3.0) + find_package(Eigen3 QUIET CONFIG) + if (EIGEN3_FOUND) + if (EIGEN3_VERSION_STRING AND NOT EIGEN3_VERSION_STRING VERSION_LESS 3.3.1) + set(PYBIND11_EIGEN_VIA_TARGET 1) + endif() + endif() + endif() + if (NOT EIGEN3_FOUND) + # Couldn't load via target, so fall back to allowing module mode finding, which will pick up + # tools/FindEigen3.cmake + find_package(Eigen3 QUIET) + endif() if(EIGEN3_FOUND) + # Eigen 3.3.1+ cmake sets EIGEN3_VERSION_STRING (and hard codes the version when installed + # rather than looking it up in the cmake script); older versions, and the + # tools/FindEigen3.cmake, set EIGEN3_VERSION instead. + if(NOT EIGEN3_VERSION AND EIGEN3_VERSION_STRING) + set(EIGEN3_VERSION ${EIGEN3_VERSION_STRING}) + endif() message(STATUS "Building tests with Eigen v${EIGEN3_VERSION}") else() list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) @@ -64,18 +104,40 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) endif() endif() +# Compile with compiler warnings turned on +function(pybind11_enable_warnings target_name) + if(MSVC) + target_compile_options(${target_name} PRIVATE /W4) + else() + target_compile_options(${target_name} PRIVATE -Wall -Wextra -Wconversion -Wcast-qual) + endif() + + if(PYBIND11_WERROR) + if(MSVC) + target_compile_options(${target_name} PRIVATE /WX) + else() + target_compile_options(${target_name} PRIVATE -Werror) + endif() + endif() +endfunction() + + # Create the binding library -pybind11_add_module(pybind11_tests pybind11_tests.cpp +pybind11_add_module(pybind11_tests THIN_LTO pybind11_tests.cpp ${PYBIND11_TEST_FILES} ${PYBIND11_HEADERS}) pybind11_enable_warnings(pybind11_tests) if(EIGEN3_FOUND) - target_include_directories(pybind11_tests PRIVATE ${EIGEN3_INCLUDE_DIR}) + if (PYBIND11_EIGEN_VIA_TARGET) + target_link_libraries(pybind11_tests PRIVATE Eigen3::Eigen) + else() + target_include_directories(pybind11_tests PRIVATE ${EIGEN3_INCLUDE_DIR}) + endif() target_compile_definitions(pybind11_tests PRIVATE -DPYBIND11_TEST_EIGEN) endif() -set(testdir ${PROJECT_SOURCE_DIR}/tests) +set(testdir ${CMAKE_CURRENT_SOURCE_DIR}) # Always write the output file directly into the 'tests' directory (even on MSVC) if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY) @@ -88,16 +150,20 @@ endif() # Make sure pytest is found or produce a fatal error if(NOT PYBIND11_PYTEST_FOUND) - execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pytest --version --noconftest OUTPUT_QUIET ERROR_QUIET - RESULT_VARIABLE PYBIND11_EXEC_PYTHON_ERR) - if(PYBIND11_EXEC_PYTHON_ERR) - message(FATAL_ERROR "Running the tests requires pytest. Please install it manually (try: ${PYTHON_EXECUTABLE} -m pip install pytest)") + execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import pytest; print(pytest.__version__)" + RESULT_VARIABLE pytest_not_found OUTPUT_VARIABLE pytest_version ERROR_QUIET) + if(pytest_not_found) + message(FATAL_ERROR "Running the tests requires pytest. Please install it manually" + " (try: ${PYTHON_EXECUTABLE} -m pip install pytest)") + elseif(pytest_version VERSION_LESS 3.0) + message(FATAL_ERROR "Running the tests requires pytest >= 3.0. Found: ${pytest_version}" + "Please update it (try: ${PYTHON_EXECUTABLE} -m pip install -U pytest)") endif() set(PYBIND11_PYTEST_FOUND TRUE CACHE INTERNAL "") endif() # A single command to compile and run the tests -add_custom_target(pytest COMMAND ${PYTHON_EXECUTABLE} -m pytest -rws ${PYBIND11_PYTEST_FILES} +add_custom_target(pytest COMMAND ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_PYTEST_FILES} DEPENDS pybind11_tests WORKING_DIRECTORY ${testdir}) if(PYBIND11_TEST_OVERRIDE) @@ -105,55 +171,68 @@ if(PYBIND11_TEST_OVERRIDE) COMMAND ${CMAKE_COMMAND} -E echo "Note: not all tests run: -DPYBIND11_TEST_OVERRIDE is in effect") endif() -# test use of installation -if(PYBIND11_INSTALL) - # 2.8.12 needed for test_installed_module - # 3.0 needed for interface library for test_installed_target +# Add a check target to run all the tests, starting with pytest (we add dependencies to this below) +add_custom_target(check DEPENDS pytest) + +# The remaining tests only apply when being built as part of the pybind11 project, but not if the +# tests are being built independently. +if (NOT PROJECT_NAME STREQUAL "pybind11") + return() +endif() + +# Add a post-build comment to show the .so size and, if a previous size, compare it: +add_custom_command(TARGET pybind11_tests POST_BUILD + COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/libsize.py + $<TARGET_FILE:pybind11_tests> ${CMAKE_CURRENT_BINARY_DIR}/sosize-$<TARGET_FILE_NAME:pybind11_tests>.txt) + +# Test CMake build using functions and targets from subdirectory or installed location +add_custom_target(test_cmake_build) +if(NOT CMAKE_VERSION VERSION_LESS 3.1) + # 3.0 needed for interface library for subdirectory_target/installed_target # 3.1 needed for cmake -E env for testing - if(NOT CMAKE_VERSION VERSION_LESS 3.1) - add_custom_target(test_installed_target - COMMAND ${CMAKE_COMMAND} - "-DCMAKE_INSTALL_PREFIX=${PROJECT_BINARY_DIR}/test_install" - -P "${PROJECT_BINARY_DIR}/cmake_install.cmake" - COMMAND ${CMAKE_CTEST_COMMAND} - --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/test_installed_target" - "${CMAKE_CURRENT_BINARY_DIR}/test_installed_target" - --build-noclean - --build-generator ${CMAKE_GENERATOR} - $<$<BOOL:${CMAKE_GENERATOR_PLATFORM}>:--build-generator-platform> ${CMAKE_GENERATOR_PLATFORM} - --build-makeprogram ${CMAKE_MAKE_PROGRAM} - --build-target check - --build-options "-DCMAKE_PREFIX_PATH=${PROJECT_BINARY_DIR}/test_install" - "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" - "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" - "-DPYBIND11_CPP_STANDARD=${PYBIND11_CPP_STANDARD}" + + include(CMakeParseArguments) + function(pybind11_add_build_test name) + cmake_parse_arguments(ARG "INSTALL" "" "" ${ARGN}) + + set(build_options "-DCMAKE_PREFIX_PATH=${PROJECT_BINARY_DIR}/mock_install" + "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" + "-DPYBIND11_CPP_STANDARD=${PYBIND11_CPP_STANDARD}") + if(NOT ARG_INSTALL) + list(APPEND build_options "-DPYBIND11_PROJECT_DIR=${PROJECT_SOURCE_DIR}") + endif() + + add_custom_target(test_${name} ${CMAKE_CTEST_COMMAND} + --quiet --output-log test_cmake_build/${name}.log + --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/test_cmake_build/${name}" + "${CMAKE_CURRENT_BINARY_DIR}/test_cmake_build/${name}" + --build-config Release + --build-noclean + --build-generator ${CMAKE_GENERATOR} + $<$<BOOL:${CMAKE_GENERATOR_PLATFORM}>:--build-generator-platform> ${CMAKE_GENERATOR_PLATFORM} + --build-makeprogram ${CMAKE_MAKE_PROGRAM} + --build-target check + --build-options ${build_options} ) - add_custom_target(test_installed_module - COMMAND ${CMAKE_COMMAND} - "-DCMAKE_INSTALL_PREFIX=${PROJECT_BINARY_DIR}/test_install" - -P "${PROJECT_BINARY_DIR}/cmake_install.cmake" - COMMAND ${CMAKE_CTEST_COMMAND} - --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/test_installed_module" - "${CMAKE_CURRENT_BINARY_DIR}/test_installed_module" - --build-noclean - --build-generator ${CMAKE_GENERATOR} - $<$<BOOL:${CMAKE_GENERATOR_PLATFORM}>:--build-generator-platform> ${CMAKE_GENERATOR_PLATFORM} - --build-makeprogram ${CMAKE_MAKE_PROGRAM} - --build-target check - --build-options "-DCMAKE_PREFIX_PATH=${PROJECT_BINARY_DIR}/test_install" - "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" - "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" - "-DPYBIND11_CPP_STANDARD=${PYBIND11_CPP_STANDARD}" + if(ARG_INSTALL) + add_dependencies(test_${name} mock_install) + endif() + add_dependencies(test_cmake_build test_${name}) + endfunction() + + pybind11_add_build_test(subdirectory_function) + pybind11_add_build_test(subdirectory_target) + + if(PYBIND11_INSTALL) + add_custom_target(mock_install ${CMAKE_COMMAND} + "-DCMAKE_INSTALL_PREFIX=${PROJECT_BINARY_DIR}/mock_install" + -P "${PROJECT_BINARY_DIR}/cmake_install.cmake" ) - else() - add_custom_target(test_installed_target) - add_custom_target(test_installed_module) + + pybind11_add_build_test(installed_function INSTALL) + pybind11_add_build_test(installed_target INSTALL) endif() - add_custom_target(test_install) - add_dependencies(test_install test_installed_target test_installed_module) endif() -# And another to show the .so size and, if a previous size, compare it: -add_custom_command(TARGET pybind11_tests POST_BUILD - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/libsize.py - $<TARGET_FILE:pybind11_tests> ${CMAKE_CURRENT_BINARY_DIR}/sosize-$<TARGET_FILE_NAME:pybind11_tests>.txt) +add_dependencies(check test_cmake_build) diff --git a/ext/pybind11/tests/conftest.py b/ext/pybind11/tests/conftest.py index d4335fc6d..5b08004e3 100644 --- a/ext/pybind11/tests/conftest.py +++ b/ext/pybind11/tests/conftest.py @@ -10,6 +10,8 @@ import difflib import re import sys import contextlib +import platform +import gc _unicode_marker = re.compile(r'u(\'[^\']*\')') _long_marker = re.compile(r'([0-9])L') @@ -101,9 +103,9 @@ class Capture(object): @pytest.fixture -def capture(capfd): - """Extended `capfd` with context manager and custom equality operators""" - return Capture(capfd) +def capture(capsys): + """Extended `capsys` with context manager and custom equality operators""" + return Capture(capsys) class SanitizedString(object): @@ -176,6 +178,13 @@ def suppress(exception): pass +def gc_collect(): + ''' Run the garbage collector twice (needed when running + reference counting tests with PyPy) ''' + gc.collect() + gc.collect() + + def pytest_namespace(): """Add import suppression and test requirements to `pytest` namespace""" try: @@ -190,6 +199,7 @@ def pytest_namespace(): from pybind11_tests import have_eigen except ImportError: have_eigen = False + pypy = platform.python_implementation() == "PyPy" skipif = pytest.mark.skipif return { @@ -200,6 +210,8 @@ def pytest_namespace(): reason="eigen and/or numpy are not installed"), 'requires_eigen_and_scipy': skipif(not have_eigen or not scipy, reason="eigen and/or scipy are not installed"), + 'unsupported_on_pypy': skipif(pypy, reason="unsupported on PyPy"), + 'gc_collect': gc_collect } diff --git a/ext/pybind11/tests/constructor_stats.h b/ext/pybind11/tests/constructor_stats.h index eb3e49cab..f66ff71df 100644 --- a/ext/pybind11/tests/constructor_stats.h +++ b/ext/pybind11/tests/constructor_stats.h @@ -24,7 +24,7 @@ function calls to constructors: ... } -You can find various examples of these in several of the existing example .cpp files. (Of course +You can find various examples of these in several of the existing testing .cpp files. (Of course you don't need to add any of the above constructors/operators that you don't actually have, except for the destructor). @@ -41,7 +41,7 @@ value constructor) for all of the above methods which will be included in the ou For testing, each of these also keeps track the created instances and allows you to check how many of the various constructors have been invoked from the Python side via code such as: - from example import ConstructorStats + from pybind11_tests import ConstructorStats cstats = ConstructorStats.get(MyClass) print(cstats.alive()) print(cstats.default_constructions) @@ -85,27 +85,51 @@ public: created(inst); copy_constructions++; } + void move_created(void *inst) { created(inst); move_constructions++; } + void default_created(void *inst) { created(inst); default_constructions++; } + void created(void *inst) { ++_instances[inst]; - }; + } + void destroyed(void *inst) { if (--_instances[inst] < 0) - throw std::runtime_error("cstats.destroyed() called with unknown instance; potential double-destruction or a missing cstats.created()"); + throw std::runtime_error("cstats.destroyed() called with unknown " + "instance; potential double-destruction " + "or a missing cstats.created()"); } - int alive() { + static void gc() { // Force garbage collection to ensure any pending destructors are invoked: +#if defined(PYPY_VERSION) + PyObject *globals = PyEval_GetGlobals(); + PyObject *result = PyRun_String( + "import gc\n" + "for i in range(2):" + " gc.collect()\n", + Py_file_input, globals, globals); + if (result == nullptr) + throw py::error_already_set(); + Py_DECREF(result); +#else py::module::import("gc").attr("collect")(); +#endif + } + + int alive() { + gc(); int total = 0; - for (const auto &p : _instances) if (p.second > 0) total += p.second; + for (const auto &p : _instances) + if (p.second > 0) + total += p.second; return total; } @@ -134,6 +158,9 @@ public: // Gets constructor stats from a C++ type template <typename T> static ConstructorStats& get() { +#if defined(PYPY_VERSION) + gc(); +#endif return get(typeid(T)); } diff --git a/ext/pybind11/tests/object.h b/ext/pybind11/tests/object.h index 753f654b2..9235f19c2 100644 --- a/ext/pybind11/tests/object.h +++ b/ext/pybind11/tests/object.h @@ -164,10 +164,10 @@ public: operator T* () { return m_ptr; } /// Return a const pointer to the referenced object - T* get() { return m_ptr; } + T* get_ptr() { return m_ptr; } /// Return a pointer to the referenced object - const T* get() const { return m_ptr; } + const T* get_ptr() const { return m_ptr; } private: T *m_ptr; }; diff --git a/ext/pybind11/tests/pybind11_tests.cpp b/ext/pybind11/tests/pybind11_tests.cpp index 9c593eee1..1d805d75b 100644 --- a/ext/pybind11/tests/pybind11_tests.cpp +++ b/ext/pybind11/tests/pybind11_tests.cpp @@ -10,6 +10,19 @@ #include "pybind11_tests.h" #include "constructor_stats.h" +/* +For testing purposes, we define a static global variable here in a function that each individual +test .cpp calls with its initialization lambda. It's convenient here because we can just not +compile some test files to disable/ignore some of the test code. + +It is NOT recommended as a way to use pybind11 in practice, however: the initialization order will +be essentially random, which is okay for our test scripts (there are no dependencies between the +individual pybind11 test .cpp files), but most likely not what you want when using pybind11 +productively. + +Instead, see the "How can I reduce the build time?" question in the "Frequently asked questions" +section of the documentation for good practice on splitting binding code over multiple files. +*/ std::list<std::function<void(py::module &)>> &initializers() { static std::list<std::function<void(py::module &)>> inits; return inits; @@ -32,7 +45,7 @@ void bind_ConstructorStats(py::module &m) { } PYBIND11_PLUGIN(pybind11_tests) { - py::module m("pybind11_tests", "pybind example plugin"); + py::module m("pybind11_tests", "pybind testing plugin"); bind_ConstructorStats(m); diff --git a/ext/pybind11/tests/pytest.ini b/ext/pybind11/tests/pytest.ini new file mode 100644 index 000000000..401cbe0ad --- /dev/null +++ b/ext/pybind11/tests/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +minversion = 3.0 +addopts = + # show summary of skipped tests + -rs + # capture only Python print and C++ py::print, but not C output (low-level Python errors) + --capture=sys diff --git a/ext/pybind11/tests/test_alias_initialization.py b/ext/pybind11/tests/test_alias_initialization.py index 0ed9d2f79..fb90cfc7b 100644 --- a/ext/pybind11/tests/test_alias_initialization.py +++ b/ext/pybind11/tests/test_alias_initialization.py @@ -1,10 +1,11 @@ -import gc +import pytest def test_alias_delay_initialization1(capture): - """A only initializes its trampoline class when we inherit from it; if we just - create and use an A instance directly, the trampoline initialization is bypassed - and we only initialize an A() instead (for performance reasons). + """ + A only initializes its trampoline class when we inherit from it; if we just + create and use an A instance directly, the trampoline initialization is + bypassed and we only initialize an A() instead (for performance reasons). """ from pybind11_tests import A, call_f @@ -20,7 +21,7 @@ def test_alias_delay_initialization1(capture): a = A() call_f(a) del a - gc.collect() + pytest.gc_collect() assert capture == "A.f()" # Python version @@ -28,7 +29,7 @@ def test_alias_delay_initialization1(capture): b = B() call_f(b) del b - gc.collect() + pytest.gc_collect() assert capture == """ PyA.PyA() PyA.f() @@ -57,7 +58,7 @@ def test_alias_delay_initialization2(capture): a2 = A2() call_f(a2) del a2 - gc.collect() + pytest.gc_collect() assert capture == """ PyA2.PyA2() PyA2.f() @@ -70,7 +71,7 @@ def test_alias_delay_initialization2(capture): b2 = B2() call_f(b2) del b2 - gc.collect() + pytest.gc_collect() assert capture == """ PyA2.PyA2() PyA2.f() diff --git a/ext/pybind11/tests/test_buffers.cpp b/ext/pybind11/tests/test_buffers.cpp index c3a7a9e02..057250d29 100644 --- a/ext/pybind11/tests/test_buffers.cpp +++ b/ext/pybind11/tests/test_buffers.cpp @@ -75,7 +75,7 @@ private: }; test_initializer buffers([](py::module &m) { - py::class_<Matrix> mtx(m, "Matrix"); + py::class_<Matrix> mtx(m, "Matrix", py::buffer_protocol()); mtx.def(py::init<size_t, size_t>()) /// Construct from a buffer diff --git a/ext/pybind11/tests/test_buffers.py b/ext/pybind11/tests/test_buffers.py index f0ea964d9..24843d95a 100644 --- a/ext/pybind11/tests/test_buffers.py +++ b/ext/pybind11/tests/test_buffers.py @@ -1,11 +1,38 @@ import pytest from pybind11_tests import Matrix, ConstructorStats +pytestmark = pytest.requires_numpy + with pytest.suppress(ImportError): import numpy as np -@pytest.requires_numpy +def test_from_python(): + with pytest.raises(RuntimeError) as excinfo: + Matrix(np.array([1, 2, 3])) # trying to assign a 1D array + assert str(excinfo.value) == "Incompatible buffer format!" + + m3 = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) + m4 = Matrix(m3) + + for i in range(m4.rows()): + for j in range(m4.cols()): + assert m3[i, j] == m4[i, j] + + cstats = ConstructorStats.get(Matrix) + assert cstats.alive() == 1 + del m3, m4 + assert cstats.alive() == 0 + assert cstats.values() == ["2x3 matrix"] + assert cstats.copy_constructions == 0 + # assert cstats.move_constructions >= 0 # Don't invoke any + assert cstats.copy_assignments == 0 + assert cstats.move_assignments == 0 + + +# PyPy: Memory leak in the "np.array(m, copy=False)" call +# https://bitbucket.org/pypy/pypy/issues/2444 +@pytest.unsupported_on_pypy def test_to_python(): m = Matrix(5, 5) @@ -23,35 +50,13 @@ def test_to_python(): cstats = ConstructorStats.get(Matrix) assert cstats.alive() == 1 del m + pytest.gc_collect() assert cstats.alive() == 1 del m2 # holds an m reference + pytest.gc_collect() assert cstats.alive() == 0 assert cstats.values() == ["5x5 matrix"] assert cstats.copy_constructions == 0 # assert cstats.move_constructions >= 0 # Don't invoke any assert cstats.copy_assignments == 0 assert cstats.move_assignments == 0 - - -@pytest.requires_numpy -def test_from_python(): - with pytest.raises(RuntimeError) as excinfo: - Matrix(np.array([1, 2, 3])) # trying to assign a 1D array - assert str(excinfo.value) == "Incompatible buffer format!" - - m3 = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - m4 = Matrix(m3) - - for i in range(m4.rows()): - for j in range(m4.cols()): - assert m3[i, j] == m4[i, j] - - cstats = ConstructorStats.get(Matrix) - assert cstats.alive() == 1 - del m3, m4 - assert cstats.alive() == 0 - assert cstats.values() == ["2x3 matrix"] - assert cstats.copy_constructions == 0 - # assert cstats.move_constructions >= 0 # Don't invoke any - assert cstats.copy_assignments == 0 - assert cstats.move_assignments == 0 diff --git a/ext/pybind11/tests/test_callbacks.cpp b/ext/pybind11/tests/test_callbacks.cpp index 31d4e39aa..f89cc1c79 100644 --- a/ext/pybind11/tests/test_callbacks.cpp +++ b/ext/pybind11/tests/test_callbacks.cpp @@ -74,6 +74,27 @@ struct Payload { /// Something to trigger a conversion error struct Unregistered {}; +class AbstractBase { +public: + virtual unsigned int func() = 0; +}; + +void func_accepting_func_accepting_base(std::function<double(AbstractBase&)>) { } + +struct MovableObject { + bool valid = true; + + MovableObject() = default; + MovableObject(const MovableObject &) = default; + MovableObject &operator=(const MovableObject &) = default; + MovableObject(MovableObject &&o) : valid(o.valid) { o.valid = false; } + MovableObject &operator=(MovableObject &&o) { + valid = o.valid; + o.valid = false; + return *this; + } +}; + test_initializer callbacks([](py::module &m) { m.def("test_callback1", &test_callback1); m.def("test_callback2", &test_callback2); @@ -136,6 +157,7 @@ test_initializer callbacks([](py::module &m) { return [p]() { /* p should be cleaned up when the returned function is garbage collected */ + (void) p; }; }); @@ -146,4 +168,15 @@ test_initializer callbacks([](py::module &m) { m.def("test_dummy_function", &test_dummy_function); // Export the payload constructor statistics for testing purposes: m.def("payload_cstats", &ConstructorStats::get<Payload>); + + m.def("func_accepting_func_accepting_base", + func_accepting_func_accepting_base); + + py::class_<MovableObject>(m, "MovableObject"); + + m.def("callback_with_movable", [](std::function<void(MovableObject &)> f) { + auto x = MovableObject(); + f(x); // lvalue reference shouldn't move out object + return x.valid; // must still return `true` + }); }); diff --git a/ext/pybind11/tests/test_callbacks.py b/ext/pybind11/tests/test_callbacks.py index c2668aa95..f94e7b64c 100644 --- a/ext/pybind11/tests/test_callbacks.py +++ b/ext/pybind11/tests/test_callbacks.py @@ -96,3 +96,9 @@ def test_function_signatures(doc): assert doc(test_callback3) == "test_callback3(arg0: Callable[[int], int]) -> str" assert doc(test_callback4) == "test_callback4() -> Callable[[int], int]" + + +def test_movable_object(): + from pybind11_tests import callback_with_movable + + assert callback_with_movable(lambda _: None) is True diff --git a/ext/pybind11/tests/test_chrono.cpp b/ext/pybind11/tests/test_chrono.cpp index b86f57adf..fcc1b6185 100644 --- a/ext/pybind11/tests/test_chrono.cpp +++ b/ext/pybind11/tests/test_chrono.cpp @@ -48,6 +48,11 @@ std::chrono::microseconds test_chrono7(std::chrono::microseconds t) { return t; } +// Float durations (issue #719) +std::chrono::duration<double> test_chrono_float_diff(std::chrono::duration<float> a, std::chrono::duration<float> b) { + return a - b; +} + test_initializer chrono([] (py::module &m) { m.def("test_chrono1", &test_chrono1); m.def("test_chrono2", &test_chrono2); @@ -56,4 +61,5 @@ test_initializer chrono([] (py::module &m) { m.def("test_chrono5", &test_chrono5); m.def("test_chrono6", &test_chrono6); m.def("test_chrono7", &test_chrono7); + m.def("test_chrono_float_diff", &test_chrono_float_diff); }); diff --git a/ext/pybind11/tests/test_chrono.py b/ext/pybind11/tests/test_chrono.py index 94ca55c76..55094edbf 100644 --- a/ext/pybind11/tests/test_chrono.py +++ b/ext/pybind11/tests/test_chrono.py @@ -104,7 +104,7 @@ def test_chrono_steady_clock_roundtrip(): def test_floating_point_duration(): - from pybind11_tests import test_chrono7 + from pybind11_tests import test_chrono7, test_chrono_float_diff import datetime # Test using 35.525123 seconds as an example floating point number in seconds @@ -114,3 +114,7 @@ def test_floating_point_duration(): assert time.seconds == 35 assert 525122 <= time.microseconds <= 525123 + + diff = test_chrono_float_diff(43.789012, 1.123456) + assert diff.seconds == 42 + assert 665556 <= diff.microseconds <= 665557 diff --git a/ext/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt b/ext/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt new file mode 100644 index 000000000..e0c20a8a3 --- /dev/null +++ b/ext/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 2.8.12) +project(test_installed_module CXX) + +set(CMAKE_MODULE_PATH "") + +find_package(pybind11 CONFIG REQUIRED) +message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") + +pybind11_add_module(test_cmake_build SHARED NO_EXTRAS ../main.cpp) + +add_custom_target(check ${CMAKE_COMMAND} -E env PYTHONPATH=$<TARGET_FILE_DIR:test_cmake_build> + ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME}) diff --git a/ext/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt b/ext/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt new file mode 100644 index 000000000..cd3ae6f7d --- /dev/null +++ b/ext/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.0) +project(test_installed_target CXX) + +set(CMAKE_MODULE_PATH "") + +find_package(pybind11 CONFIG REQUIRED) +message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") + +add_library(test_cmake_build MODULE ../main.cpp) + +target_link_libraries(test_cmake_build PRIVATE pybind11::module) + +# make sure result is, for example, test_installed_target.so, not libtest_installed_target.dylib +set_target_properties(test_cmake_build PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") + +# Do not treat includes from IMPORTED target as SYSTEM (Python headers in pybind11::module). +# This may be needed to resolve header conflicts, e.g. between Python release and debug headers. +set_target_properties(test_cmake_build PROPERTIES NO_SYSTEM_FROM_IMPORTED ON) + +add_custom_target(check ${CMAKE_COMMAND} -E env PYTHONPATH=$<TARGET_FILE_DIR:test_cmake_build> + ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME}) diff --git a/ext/pybind11/tests/test_installed_target/main.cpp b/ext/pybind11/tests/test_cmake_build/main.cpp index 2a84c11ce..e0f5b69c9 100644 --- a/ext/pybind11/tests/test_installed_target/main.cpp +++ b/ext/pybind11/tests/test_cmake_build/main.cpp @@ -1,8 +1,8 @@ #include <pybind11/pybind11.h> namespace py = pybind11; -PYBIND11_PLUGIN(test_installed_target) { - py::module m("test_installed_target"); +PYBIND11_PLUGIN(test_cmake_build) { + py::module m("test_cmake_build"); m.def("add", [](int i, int j) { return i + j; }); diff --git a/ext/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt b/ext/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt new file mode 100644 index 000000000..278007aeb --- /dev/null +++ b/ext/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 2.8.12) +project(test_subdirectory_module CXX) + +add_subdirectory(${PYBIND11_PROJECT_DIR} pybind11) +pybind11_add_module(test_cmake_build THIN_LTO ../main.cpp) + +add_custom_target(check ${CMAKE_COMMAND} -E env PYTHONPATH=$<TARGET_FILE_DIR:test_cmake_build> + ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME}) diff --git a/ext/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt b/ext/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt new file mode 100644 index 000000000..6b142d62a --- /dev/null +++ b/ext/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.0) +project(test_subdirectory_target CXX) + +add_subdirectory(${PYBIND11_PROJECT_DIR} pybind11) + +add_library(test_cmake_build MODULE ../main.cpp) + +target_link_libraries(test_cmake_build PRIVATE pybind11::module) + +# make sure result is, for example, test_installed_target.so, not libtest_installed_target.dylib +set_target_properties(test_cmake_build PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") + +add_custom_target(check ${CMAKE_COMMAND} -E env PYTHONPATH=$<TARGET_FILE_DIR:test_cmake_build> + ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME}) diff --git a/ext/pybind11/tests/test_cmake_build/test.py b/ext/pybind11/tests/test_cmake_build/test.py new file mode 100644 index 000000000..1467a61dc --- /dev/null +++ b/ext/pybind11/tests/test_cmake_build/test.py @@ -0,0 +1,5 @@ +import sys +import test_cmake_build + +assert test_cmake_build.add(1, 2) == 3 +print("{} imports, runs, and adds: 1 + 2 = 3".format(sys.argv[1])) diff --git a/ext/pybind11/tests/test_constants_and_functions.cpp b/ext/pybind11/tests/test_constants_and_functions.cpp index c8c0392c9..653bdf6b6 100644 --- a/ext/pybind11/tests/test_constants_and_functions.cpp +++ b/ext/pybind11/tests/test_constants_and_functions.cpp @@ -41,6 +41,26 @@ std::string print_bytes(py::bytes bytes) { return ret; } +// Test that we properly handle C++17 exception specifiers (which are part of the function signature +// in C++17). These should all still work before C++17, but don't affect the function signature. +namespace test_exc_sp { +int f1(int x) noexcept { return x+1; } +int f2(int x) noexcept(true) { return x+2; } +int f3(int x) noexcept(false) { return x+3; } +int f4(int x) throw() { return x+4; } // Deprecated equivalent to noexcept(true) +struct C { + int m1(int x) noexcept { return x-1; } + int m2(int x) const noexcept { return x-2; } + int m3(int x) noexcept(true) { return x-3; } + int m4(int x) const noexcept(true) { return x-4; } + int m5(int x) noexcept(false) { return x-5; } + int m6(int x) const noexcept(false) { return x-6; } + int m7(int x) throw() { return x-7; } + int m8(int x) const throw() { return x-8; } +}; +} + + test_initializer constants_and_functions([](py::module &m) { m.attr("some_constant") = py::int_(14); @@ -63,4 +83,22 @@ test_initializer constants_and_functions([](py::module &m) { m.def("return_bytes", &return_bytes); m.def("print_bytes", &print_bytes); + + using namespace test_exc_sp; + py::module m2 = m.def_submodule("exc_sp"); + py::class_<C>(m2, "C") + .def(py::init<>()) + .def("m1", &C::m1) + .def("m2", &C::m2) + .def("m3", &C::m3) + .def("m4", &C::m4) + .def("m5", &C::m5) + .def("m6", &C::m6) + .def("m7", &C::m7) + .def("m8", &C::m8) + ; + m2.def("f1", f1); + m2.def("f2", f2); + m2.def("f3", f3); + m2.def("f4", f4); }); diff --git a/ext/pybind11/tests/test_constants_and_functions.py b/ext/pybind11/tests/test_constants_and_functions.py index d13d3af1b..2a570d2e5 100644 --- a/ext/pybind11/tests/test_constants_and_functions.py +++ b/ext/pybind11/tests/test_constants_and_functions.py @@ -22,3 +22,22 @@ def test_bytes(): from pybind11_tests import return_bytes, print_bytes assert print_bytes(return_bytes()) == "bytes[1 0 2 0]" + + +def test_exception_specifiers(): + from pybind11_tests.exc_sp import C, f1, f2, f3, f4 + + c = C() + assert c.m1(2) == 1 + assert c.m2(3) == 1 + assert c.m3(5) == 2 + assert c.m4(7) == 3 + assert c.m5(10) == 5 + assert c.m6(14) == 8 + assert c.m7(20) == 13 + assert c.m8(29) == 21 + + assert f1(33) == 34 + assert f2(53) == 55 + assert f3(86) == 89 + assert f4(140) == 144 diff --git a/ext/pybind11/tests/test_docstring_options.cpp b/ext/pybind11/tests/test_docstring_options.cpp index 74178c272..9a9297cc3 100644 --- a/ext/pybind11/tests/test_docstring_options.cpp +++ b/ext/pybind11/tests/test_docstring_options.cpp @@ -24,6 +24,15 @@ test_initializer docstring_generation([](py::module &m) { m.def("test_function1", [](int, int) {}, py::arg("a"), py::arg("b")); m.def("test_function2", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); + m.def("test_overloaded1", [](int) {}, py::arg("i"), "Overload docstring"); + m.def("test_overloaded1", [](double) {}, py::arg("d")); + + m.def("test_overloaded2", [](int) {}, py::arg("i"), "overload docstring 1"); + m.def("test_overloaded2", [](double) {}, py::arg("d"), "overload docstring 2"); + + m.def("test_overloaded3", [](int) {}, py::arg("i")); + m.def("test_overloaded3", [](double) {}, py::arg("d"), "Overload docstr"); + options.enable_function_signatures(); m.def("test_function3", [](int, int) {}, py::arg("a"), py::arg("b")); diff --git a/ext/pybind11/tests/test_docstring_options.py b/ext/pybind11/tests/test_docstring_options.py index 66ad6b89f..5e40f6868 100644 --- a/ext/pybind11/tests/test_docstring_options.py +++ b/ext/pybind11/tests/test_docstring_options.py @@ -3,13 +3,23 @@ def test_docstring_options(): from pybind11_tests import (test_function1, test_function2, test_function3, test_function4, test_function5, test_function6, - test_function7, DocstringTestFoo) + test_function7, DocstringTestFoo, + test_overloaded1, test_overloaded2, test_overloaded3) # options.disable_function_signatures() assert not test_function1.__doc__ assert test_function2.__doc__ == "A custom docstring" + # docstring specified on just the first overload definition: + assert test_overloaded1.__doc__ == "Overload docstring" + + # docstring on both overloads: + assert test_overloaded2.__doc__ == "overload docstring 1\noverload docstring 2" + + # docstring on only second overload: + assert test_overloaded3.__doc__ == "Overload docstr" + # options.enable_function_signatures() assert test_function3.__doc__ .startswith("test_function3(a: int, b: int) -> None") diff --git a/ext/pybind11/tests/test_eigen.cpp b/ext/pybind11/tests/test_eigen.cpp index 588cdceb3..f2ec8fd2e 100644 --- a/ext/pybind11/tests/test_eigen.cpp +++ b/ext/pybind11/tests/test_eigen.cpp @@ -8,55 +8,160 @@ */ #include "pybind11_tests.h" +#include "constructor_stats.h" #include <pybind11/eigen.h> #include <Eigen/Cholesky> -Eigen::VectorXf double_col(const Eigen::VectorXf& x) -{ return 2.0f * x; } +using MatrixXdR = Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>; -Eigen::RowVectorXf double_row(const Eigen::RowVectorXf& x) -{ return 2.0f * x; } -Eigen::MatrixXf double_mat_cm(const Eigen::MatrixXf& x) -{ return 2.0f * x; } -// Different ways of passing via Eigen::Ref; the first and second are the Eigen-recommended -Eigen::MatrixXd cholesky1(Eigen::Ref<Eigen::MatrixXd> &x) { return x.llt().matrixL(); } -Eigen::MatrixXd cholesky2(const Eigen::Ref<const Eigen::MatrixXd> &x) { return x.llt().matrixL(); } -Eigen::MatrixXd cholesky3(const Eigen::Ref<Eigen::MatrixXd> &x) { return x.llt().matrixL(); } -Eigen::MatrixXd cholesky4(Eigen::Ref<const Eigen::MatrixXd> &x) { return x.llt().matrixL(); } -Eigen::MatrixXd cholesky5(Eigen::Ref<Eigen::MatrixXd> x) { return x.llt().matrixL(); } -Eigen::MatrixXd cholesky6(Eigen::Ref<const Eigen::MatrixXd> x) { return x.llt().matrixL(); } +// Sets/resets a testing reference matrix to have values of 10*r + c, where r and c are the +// (1-based) row/column number. +template <typename M> void reset_ref(M &x) { + for (int i = 0; i < x.rows(); i++) for (int j = 0; j < x.cols(); j++) + x(i, j) = 11 + 10*i + j; +} -typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> MatrixXfRowMajor; -MatrixXfRowMajor double_mat_rm(const MatrixXfRowMajor& x) -{ return 2.0f * x; } +// Returns a static, column-major matrix +Eigen::MatrixXd &get_cm() { + static Eigen::MatrixXd *x; + if (!x) { + x = new Eigen::MatrixXd(3, 3); + reset_ref(*x); + } + return *x; +} +// Likewise, but row-major +MatrixXdR &get_rm() { + static MatrixXdR *x; + if (!x) { + x = new MatrixXdR(3, 3); + reset_ref(*x); + } + return *x; +} +// Resets the values of the static matrices returned by get_cm()/get_rm() +void reset_refs() { + reset_ref(get_cm()); + reset_ref(get_rm()); +} + +// Returns element 2,1 from a matrix (used to test copy/nocopy) +double get_elem(Eigen::Ref<const Eigen::MatrixXd> m) { return m(2, 1); }; + + +// Returns a matrix with 10*r + 100*c added to each matrix element (to help test that the matrix +// reference is referencing rows/columns correctly). +template <typename MatrixArgType> Eigen::MatrixXd adjust_matrix(MatrixArgType m) { + Eigen::MatrixXd ret(m); + for (int c = 0; c < m.cols(); c++) for (int r = 0; r < m.rows(); r++) + ret(r, c) += 10*r + 100*c; + return ret; +} + +struct CustomOperatorNew { + CustomOperatorNew() = default; + + Eigen::Matrix4d a = Eigen::Matrix4d::Zero(); + Eigen::Matrix4d b = Eigen::Matrix4d::Identity(); + + EIGEN_MAKE_ALIGNED_OPERATOR_NEW; +}; test_initializer eigen([](py::module &m) { typedef Eigen::Matrix<float, 5, 6, Eigen::RowMajor> FixedMatrixR; typedef Eigen::Matrix<float, 5, 6> FixedMatrixC; typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> DenseMatrixR; typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic> DenseMatrixC; + typedef Eigen::Matrix<float, 4, Eigen::Dynamic> FourRowMatrixC; + typedef Eigen::Matrix<float, Eigen::Dynamic, 4> FourColMatrixC; + typedef Eigen::Matrix<float, 4, Eigen::Dynamic> FourRowMatrixR; + typedef Eigen::Matrix<float, Eigen::Dynamic, 4> FourColMatrixR; typedef Eigen::SparseMatrix<float, Eigen::RowMajor> SparseMatrixR; typedef Eigen::SparseMatrix<float> SparseMatrixC; m.attr("have_eigen") = true; - // Non-symmetric matrix with zero elements - Eigen::MatrixXf mat(5, 6); - mat << 0, 3, 0, 0, 0, 11, 22, 0, 0, 0, 17, 11, 7, 5, 0, 1, 0, 11, 0, - 0, 0, 0, 0, 11, 0, 0, 14, 0, 8, 11; - - m.def("double_col", &double_col); - m.def("double_row", &double_row); - m.def("double_mat_cm", &double_mat_cm); - m.def("double_mat_rm", &double_mat_rm); - m.def("cholesky1", &cholesky1); - m.def("cholesky2", &cholesky2); - m.def("cholesky3", &cholesky3); - m.def("cholesky4", &cholesky4); - m.def("cholesky5", &cholesky5); - m.def("cholesky6", &cholesky6); + m.def("double_col", [](const Eigen::VectorXf &x) -> Eigen::VectorXf { return 2.0f * x; }); + m.def("double_row", [](const Eigen::RowVectorXf &x) -> Eigen::RowVectorXf { return 2.0f * x; }); + m.def("double_complex", [](const Eigen::VectorXcf &x) -> Eigen::VectorXcf { return 2.0f * x; }); + m.def("double_threec", [](py::EigenDRef<Eigen::Vector3f> x) { x *= 2; }); + m.def("double_threer", [](py::EigenDRef<Eigen::RowVector3f> x) { x *= 2; }); + m.def("double_mat_cm", [](Eigen::MatrixXf x) -> Eigen::MatrixXf { return 2.0f * x; }); + m.def("double_mat_rm", [](DenseMatrixR x) -> DenseMatrixR { return 2.0f * x; }); + + // Different ways of passing via Eigen::Ref; the first and second are the Eigen-recommended + m.def("cholesky1", [](Eigen::Ref<MatrixXdR> x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); + m.def("cholesky2", [](const Eigen::Ref<const MatrixXdR> &x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); + m.def("cholesky3", [](const Eigen::Ref<MatrixXdR> &x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); + m.def("cholesky4", [](Eigen::Ref<const MatrixXdR> x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); + + // Mutators: these add some value to the given element using Eigen, but Eigen should be mapping into + // the numpy array data and so the result should show up there. There are three versions: one that + // works on a contiguous-row matrix (numpy's default), one for a contiguous-column matrix, and one + // for any matrix. + auto add_rm = [](Eigen::Ref<MatrixXdR> x, int r, int c, double v) { x(r,c) += v; }; + auto add_cm = [](Eigen::Ref<Eigen::MatrixXd> x, int r, int c, double v) { x(r,c) += v; }; + + // Mutators (Eigen maps into numpy variables): + m.def("add_rm", add_rm); // Only takes row-contiguous + m.def("add_cm", add_cm); // Only takes column-contiguous + // Overloaded versions that will accept either row or column contiguous: + m.def("add1", add_rm); + m.def("add1", add_cm); + m.def("add2", add_cm); + m.def("add2", add_rm); + // This one accepts a matrix of any stride: + m.def("add_any", [](py::EigenDRef<Eigen::MatrixXd> x, int r, int c, double v) { x(r,c) += v; }); + + // Return mutable references (numpy maps into eigen varibles) + m.def("get_cm_ref", []() { return Eigen::Ref<Eigen::MatrixXd>(get_cm()); }); + m.def("get_rm_ref", []() { return Eigen::Ref<MatrixXdR>(get_rm()); }); + // The same references, but non-mutable (numpy maps into eigen variables, but is !writeable) + m.def("get_cm_const_ref", []() { return Eigen::Ref<const Eigen::MatrixXd>(get_cm()); }); + m.def("get_rm_const_ref", []() { return Eigen::Ref<const MatrixXdR>(get_rm()); }); + // Just the corners (via a Map instead of a Ref): + m.def("get_cm_corners", []() { + auto &x = get_cm(); + return py::EigenDMap<Eigen::Matrix2d>( + x.data(), + py::EigenDStride(x.outerStride() * (x.rows() - 1), x.innerStride() * (x.cols() - 1))); + }); + m.def("get_cm_corners_const", []() { + const auto &x = get_cm(); + return py::EigenDMap<const Eigen::Matrix2d>( + x.data(), + py::EigenDStride(x.outerStride() * (x.rows() - 1), x.innerStride() * (x.cols() - 1))); + }); + + m.def("reset_refs", reset_refs); // Restores get_{cm,rm}_ref to original values + + // Increments and returns ref to (same) matrix + m.def("incr_matrix", [](Eigen::Ref<Eigen::MatrixXd> m, double v) { + m += Eigen::MatrixXd::Constant(m.rows(), m.cols(), v); + return m; + }, py::return_value_policy::reference); + + // Same, but accepts a matrix of any strides + m.def("incr_matrix_any", [](py::EigenDRef<Eigen::MatrixXd> m, double v) { + m += Eigen::MatrixXd::Constant(m.rows(), m.cols(), v); + return m; + }, py::return_value_policy::reference); + + // Returns an eigen slice of even rows + m.def("even_rows", [](py::EigenDRef<Eigen::MatrixXd> m) { + return py::EigenDMap<Eigen::MatrixXd>( + m.data(), (m.rows() + 1) / 2, m.cols(), + py::EigenDStride(m.outerStride(), 2 * m.innerStride())); + }, py::return_value_policy::reference); + + // Returns an eigen slice of even columns + m.def("even_cols", [](py::EigenDRef<Eigen::MatrixXd> m) { + return py::EigenDMap<Eigen::MatrixXd>( + m.data(), m.rows(), (m.cols() + 1) / 2, + py::EigenDStride(2 * m.outerStride(), m.innerStride())); + }, py::return_value_policy::reference); // Returns diagonals: a vector-like object with an inner stride != 1 m.def("diagonal", [](const Eigen::Ref<const Eigen::MatrixXd> &x) { return x.diagonal(); }); @@ -68,6 +173,52 @@ test_initializer eigen([](py::module &m) { return x.block(start_row, start_col, block_rows, block_cols); }); + // return value referencing/copying tests: + class ReturnTester { + Eigen::MatrixXd mat = create(); + public: + ReturnTester() { print_created(this); } + ~ReturnTester() { print_destroyed(this); } + static Eigen::MatrixXd create() { return Eigen::MatrixXd::Ones(10, 10); } + static const Eigen::MatrixXd createConst() { return Eigen::MatrixXd::Ones(10, 10); } + Eigen::MatrixXd &get() { return mat; } + Eigen::MatrixXd *getPtr() { return &mat; } + const Eigen::MatrixXd &view() { return mat; } + const Eigen::MatrixXd *viewPtr() { return &mat; } + Eigen::Ref<Eigen::MatrixXd> ref() { return mat; } + Eigen::Ref<const Eigen::MatrixXd> refConst() { return mat; } + Eigen::Block<Eigen::MatrixXd> block(int r, int c, int nrow, int ncol) { return mat.block(r, c, nrow, ncol); } + Eigen::Block<const Eigen::MatrixXd> blockConst(int r, int c, int nrow, int ncol) const { return mat.block(r, c, nrow, ncol); } + py::EigenDMap<Eigen::Matrix2d> corners() { return py::EigenDMap<Eigen::Matrix2d>(mat.data(), + py::EigenDStride(mat.outerStride() * (mat.outerSize()-1), mat.innerStride() * (mat.innerSize()-1))); } + py::EigenDMap<const Eigen::Matrix2d> cornersConst() const { return py::EigenDMap<const Eigen::Matrix2d>(mat.data(), + py::EigenDStride(mat.outerStride() * (mat.outerSize()-1), mat.innerStride() * (mat.innerSize()-1))); } + }; + using rvp = py::return_value_policy; + py::class_<ReturnTester>(m, "ReturnTester") + .def(py::init<>()) + .def_static("create", &ReturnTester::create) + .def_static("create_const", &ReturnTester::createConst) + .def("get", &ReturnTester::get, rvp::reference_internal) + .def("get_ptr", &ReturnTester::getPtr, rvp::reference_internal) + .def("view", &ReturnTester::view, rvp::reference_internal) + .def("view_ptr", &ReturnTester::view, rvp::reference_internal) + .def("copy_get", &ReturnTester::get) // Default rvp: copy + .def("copy_view", &ReturnTester::view) // " + .def("ref", &ReturnTester::ref) // Default for Ref is to reference + .def("ref_const", &ReturnTester::refConst) // Likewise, but const + .def("ref_safe", &ReturnTester::ref, rvp::reference_internal) + .def("ref_const_safe", &ReturnTester::refConst, rvp::reference_internal) + .def("copy_ref", &ReturnTester::ref, rvp::copy) + .def("copy_ref_const", &ReturnTester::refConst, rvp::copy) + .def("block", &ReturnTester::block) + .def("block_safe", &ReturnTester::block, rvp::reference_internal) + .def("block_const", &ReturnTester::blockConst, rvp::reference_internal) + .def("copy_block", &ReturnTester::block, rvp::copy) + .def("corners", &ReturnTester::corners, rvp::reference_internal) + .def("corners_const", &ReturnTester::cornersConst, rvp::reference_internal) + ; + // Returns a DiagonalMatrix with diagonal (1,2,3,...) m.def("incr_diag", [](int k) { Eigen::DiagonalMatrix<int, Eigen::Dynamic> m(k); @@ -84,51 +235,60 @@ test_initializer eigen([](py::module &m) { return m.selfadjointView<Eigen::Upper>(); }); - m.def("fixed_r", [mat]() -> FixedMatrixR { - return FixedMatrixR(mat); - }); - - m.def("fixed_c", [mat]() -> FixedMatrixC { - return FixedMatrixC(mat); - }); - - m.def("fixed_passthrough_r", [](const FixedMatrixR &m) -> FixedMatrixR { - return m; - }); - - m.def("fixed_passthrough_c", [](const FixedMatrixC &m) -> FixedMatrixC { - return m; - }); - - m.def("dense_r", [mat]() -> DenseMatrixR { - return DenseMatrixR(mat); - }); - - m.def("dense_c", [mat]() -> DenseMatrixC { - return DenseMatrixC(mat); - }); + // Test matrix for various functions below. + Eigen::MatrixXf mat(5, 6); + mat << 0, 3, 0, 0, 0, 11, + 22, 0, 0, 0, 17, 11, + 7, 5, 0, 1, 0, 11, + 0, 0, 0, 0, 0, 11, + 0, 0, 14, 0, 8, 11; - m.def("dense_passthrough_r", [](const DenseMatrixR &m) -> DenseMatrixR { - return m; - }); + m.def("fixed_r", [mat]() -> FixedMatrixR { return FixedMatrixR(mat); }); + m.def("fixed_r_const", [mat]() -> const FixedMatrixR { return FixedMatrixR(mat); }); + m.def("fixed_c", [mat]() -> FixedMatrixC { return FixedMatrixC(mat); }); + m.def("fixed_copy_r", [](const FixedMatrixR &m) -> FixedMatrixR { return m; }); + m.def("fixed_copy_c", [](const FixedMatrixC &m) -> FixedMatrixC { return m; }); + m.def("fixed_mutator_r", [](Eigen::Ref<FixedMatrixR>) {}); + m.def("fixed_mutator_c", [](Eigen::Ref<FixedMatrixC>) {}); + m.def("fixed_mutator_a", [](py::EigenDRef<FixedMatrixC>) {}); + m.def("dense_r", [mat]() -> DenseMatrixR { return DenseMatrixR(mat); }); + m.def("dense_c", [mat]() -> DenseMatrixC { return DenseMatrixC(mat); }); + m.def("dense_copy_r", [](const DenseMatrixR &m) -> DenseMatrixR { return m; }); + m.def("dense_copy_c", [](const DenseMatrixC &m) -> DenseMatrixC { return m; }); + m.def("sparse_r", [mat]() -> SparseMatrixR { return Eigen::SparseView<Eigen::MatrixXf>(mat); }); + m.def("sparse_c", [mat]() -> SparseMatrixC { return Eigen::SparseView<Eigen::MatrixXf>(mat); }); + m.def("sparse_copy_r", [](const SparseMatrixR &m) -> SparseMatrixR { return m; }); + m.def("sparse_copy_c", [](const SparseMatrixC &m) -> SparseMatrixC { return m; }); + m.def("partial_copy_four_rm_r", [](const FourRowMatrixR &m) -> FourRowMatrixR { return m; }); + m.def("partial_copy_four_rm_c", [](const FourColMatrixR &m) -> FourColMatrixR { return m; }); + m.def("partial_copy_four_cm_r", [](const FourRowMatrixC &m) -> FourRowMatrixC { return m; }); + m.def("partial_copy_four_cm_c", [](const FourColMatrixC &m) -> FourColMatrixC { return m; }); - m.def("dense_passthrough_c", [](const DenseMatrixC &m) -> DenseMatrixC { - return m; - }); + // Test that we can cast a numpy object to a Eigen::MatrixXd explicitly + m.def("cpp_copy", [](py::handle m) { return m.cast<Eigen::MatrixXd>()(1, 0); }); + m.def("cpp_ref_c", [](py::handle m) { return m.cast<Eigen::Ref<Eigen::MatrixXd>>()(1, 0); }); + m.def("cpp_ref_r", [](py::handle m) { return m.cast<Eigen::Ref<MatrixXdR>>()(1, 0); }); + m.def("cpp_ref_any", [](py::handle m) { return m.cast<py::EigenDRef<Eigen::MatrixXd>>()(1, 0); }); - m.def("sparse_r", [mat]() -> SparseMatrixR { - return Eigen::SparseView<Eigen::MatrixXf>(mat); - }); - m.def("sparse_c", [mat]() -> SparseMatrixC { - return Eigen::SparseView<Eigen::MatrixXf>(mat); - }); + // Test that we can prevent copying into an argument that would normally copy: First a version + // that would allow copying (if types or strides don't match) for comparison: + m.def("get_elem", &get_elem); + // Now this alternative that calls the tells pybind to fail rather than copy: + m.def("get_elem_nocopy", [](Eigen::Ref<const Eigen::MatrixXd> m) -> double { return get_elem(m); }, + py::arg().noconvert()); + // Also test a row-major-only no-copy const ref: + m.def("get_elem_rm_nocopy", [](Eigen::Ref<const Eigen::Matrix<long, -1, -1, Eigen::RowMajor>> &m) -> long { return m(2, 1); }, + py::arg().noconvert()); - m.def("sparse_passthrough_r", [](const SparseMatrixR &m) -> SparseMatrixR { - return m; - }); + // Issue #738: 1xN or Nx1 2D matrices were neither accepted nor properly copied with an + // incompatible stride value on the length-1 dimension--but that should be allowed (without + // requiring a copy!) because the stride value can be safely ignored on a size-1 dimension. + m.def("iss738_f1", &adjust_matrix<const Eigen::Ref<const Eigen::MatrixXd> &>, py::arg().noconvert()); + m.def("iss738_f2", &adjust_matrix<const Eigen::Ref<const Eigen::Matrix<double, -1, -1, Eigen::RowMajor>> &>, py::arg().noconvert()); - m.def("sparse_passthrough_c", [](const SparseMatrixC &m) -> SparseMatrixC { - return m; - }); + py::class_<CustomOperatorNew>(m, "CustomOperatorNew") + .def(py::init<>()) + .def_readonly("a", &CustomOperatorNew::a) + .def_readonly("b", &CustomOperatorNew::b); }); diff --git a/ext/pybind11/tests/test_eigen.py b/ext/pybind11/tests/test_eigen.py index b0092fc8b..df08edf3d 100644 --- a/ext/pybind11/tests/test_eigen.py +++ b/ext/pybind11/tests/test_eigen.py @@ -1,9 +1,11 @@ import pytest +pytestmark = pytest.requires_eigen_and_numpy + with pytest.suppress(ImportError): import numpy as np - ref = np.array([[ 0, 3, 0, 0, 0, 11], + ref = np.array([[ 0., 3, 0, 0, 0, 11], [22, 0, 0, 0, 17, 11], [ 7, 5, 0, 1, 0, 11], [ 0, 0, 0, 0, 0, 11], @@ -18,50 +20,140 @@ def assert_sparse_equal_ref(sparse_mat): assert_equal_ref(sparse_mat.todense()) -@pytest.requires_eigen_and_numpy def test_fixed(): - from pybind11_tests import fixed_r, fixed_c, fixed_passthrough_r, fixed_passthrough_c + from pybind11_tests import fixed_r, fixed_c, fixed_copy_r, fixed_copy_c assert_equal_ref(fixed_c()) assert_equal_ref(fixed_r()) - assert_equal_ref(fixed_passthrough_r(fixed_r())) - assert_equal_ref(fixed_passthrough_c(fixed_c())) - assert_equal_ref(fixed_passthrough_r(fixed_c())) - assert_equal_ref(fixed_passthrough_c(fixed_r())) + assert_equal_ref(fixed_copy_r(fixed_r())) + assert_equal_ref(fixed_copy_c(fixed_c())) + assert_equal_ref(fixed_copy_r(fixed_c())) + assert_equal_ref(fixed_copy_c(fixed_r())) -@pytest.requires_eigen_and_numpy def test_dense(): - from pybind11_tests import dense_r, dense_c, dense_passthrough_r, dense_passthrough_c + from pybind11_tests import dense_r, dense_c, dense_copy_r, dense_copy_c assert_equal_ref(dense_r()) assert_equal_ref(dense_c()) - assert_equal_ref(dense_passthrough_r(dense_r())) - assert_equal_ref(dense_passthrough_c(dense_c())) - assert_equal_ref(dense_passthrough_r(dense_c())) - assert_equal_ref(dense_passthrough_c(dense_r())) + assert_equal_ref(dense_copy_r(dense_r())) + assert_equal_ref(dense_copy_c(dense_c())) + assert_equal_ref(dense_copy_r(dense_c())) + assert_equal_ref(dense_copy_c(dense_r())) + + +def test_partially_fixed(): + from pybind11_tests import (partial_copy_four_rm_r, partial_copy_four_rm_c, + partial_copy_four_cm_r, partial_copy_four_cm_c) + + ref2 = np.array([[0., 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15]]) + np.testing.assert_array_equal(partial_copy_four_rm_r(ref2), ref2) + np.testing.assert_array_equal(partial_copy_four_rm_c(ref2), ref2) + np.testing.assert_array_equal(partial_copy_four_rm_r(ref2[:, 1]), ref2[:, [1]]) + np.testing.assert_array_equal(partial_copy_four_rm_c(ref2[0, :]), ref2[[0], :]) + np.testing.assert_array_equal(partial_copy_four_rm_r(ref2[:, (0, 2)]), ref2[:, (0, 2)]) + np.testing.assert_array_equal( + partial_copy_four_rm_c(ref2[(3, 1, 2), :]), ref2[(3, 1, 2), :]) + + np.testing.assert_array_equal(partial_copy_four_cm_r(ref2), ref2) + np.testing.assert_array_equal(partial_copy_four_cm_c(ref2), ref2) + np.testing.assert_array_equal(partial_copy_four_cm_r(ref2[:, 1]), ref2[:, [1]]) + np.testing.assert_array_equal(partial_copy_four_cm_c(ref2[0, :]), ref2[[0], :]) + np.testing.assert_array_equal(partial_copy_four_cm_r(ref2[:, (0, 2)]), ref2[:, (0, 2)]) + np.testing.assert_array_equal( + partial_copy_four_cm_c(ref2[(3, 1, 2), :]), ref2[(3, 1, 2), :]) + + +def test_mutator_descriptors(): + from pybind11_tests import fixed_mutator_r, fixed_mutator_c, fixed_mutator_a + zr = np.arange(30, dtype='float32').reshape(5, 6) # row-major + zc = zr.reshape(6, 5).transpose() # column-major + + fixed_mutator_r(zr) + fixed_mutator_c(zc) + fixed_mutator_a(zr) + fixed_mutator_a(zc) + with pytest.raises(TypeError) as excinfo: + fixed_mutator_r(zc) + assert ('(numpy.ndarray[float32[5, 6], flags.writeable, flags.c_contiguous]) -> arg0: None' + in str(excinfo.value)) + with pytest.raises(TypeError) as excinfo: + fixed_mutator_c(zr) + assert ('(numpy.ndarray[float32[5, 6], flags.writeable, flags.f_contiguous]) -> arg0: None' + in str(excinfo.value)) + with pytest.raises(TypeError) as excinfo: + fixed_mutator_a(np.array([[1, 2], [3, 4]], dtype='float32')) + assert ('(numpy.ndarray[float32[5, 6], flags.writeable]) -> arg0: None' + in str(excinfo.value)) + zr.flags.writeable = False + with pytest.raises(TypeError): + fixed_mutator_r(zr) + with pytest.raises(TypeError): + fixed_mutator_a(zr) + + +def test_cpp_casting(): + from pybind11_tests import (cpp_copy, cpp_ref_c, cpp_ref_r, cpp_ref_any, + fixed_r, fixed_c, get_cm_ref, get_rm_ref, ReturnTester) + assert cpp_copy(fixed_r()) == 22. + assert cpp_copy(fixed_c()) == 22. + z = np.array([[5., 6], [7, 8]]) + assert cpp_copy(z) == 7. + assert cpp_copy(get_cm_ref()) == 21. + assert cpp_copy(get_rm_ref()) == 21. + assert cpp_ref_c(get_cm_ref()) == 21. + assert cpp_ref_r(get_rm_ref()) == 21. + with pytest.raises(RuntimeError) as excinfo: + # Can't reference fixed_c: it contains floats, cpp_ref_any wants doubles + cpp_ref_any(fixed_c()) + assert 'Unable to cast Python instance' in str(excinfo.value) + with pytest.raises(RuntimeError) as excinfo: + # Can't reference fixed_r: it contains floats, cpp_ref_any wants doubles + cpp_ref_any(fixed_r()) + assert 'Unable to cast Python instance' in str(excinfo.value) + assert cpp_ref_any(ReturnTester.create()) == 1. + + assert cpp_ref_any(get_cm_ref()) == 21. + assert cpp_ref_any(get_cm_ref()) == 21. + + +def test_pass_readonly_array(): + from pybind11_tests import fixed_copy_r, fixed_r, fixed_r_const + z = np.full((5, 6), 42.0) + z.flags.writeable = False + np.testing.assert_array_equal(z, fixed_copy_r(z)) + np.testing.assert_array_equal(fixed_r_const(), fixed_r()) + assert not fixed_r_const().flags.writeable + np.testing.assert_array_equal(fixed_copy_r(fixed_r_const()), fixed_r_const()) -@pytest.requires_eigen_and_numpy def test_nonunit_stride_from_python(): - from pybind11_tests import double_row, double_col, double_mat_cm, double_mat_rm + from pybind11_tests import ( + double_row, double_col, double_complex, double_mat_cm, double_mat_rm, + double_threec, double_threer) counting_mat = np.arange(9.0, dtype=np.float32).reshape((3, 3)) - first_row = counting_mat[0, :] - first_col = counting_mat[:, 0] - assert np.array_equal(double_row(first_row), 2.0 * first_row) - assert np.array_equal(double_col(first_row), 2.0 * first_row) - assert np.array_equal(double_row(first_col), 2.0 * first_col) - assert np.array_equal(double_col(first_col), 2.0 * first_col) + second_row = counting_mat[1, :] + second_col = counting_mat[:, 1] + np.testing.assert_array_equal(double_row(second_row), 2.0 * second_row) + np.testing.assert_array_equal(double_col(second_row), 2.0 * second_row) + np.testing.assert_array_equal(double_complex(second_row), 2.0 * second_row) + np.testing.assert_array_equal(double_row(second_col), 2.0 * second_col) + np.testing.assert_array_equal(double_col(second_col), 2.0 * second_col) + np.testing.assert_array_equal(double_complex(second_col), 2.0 * second_col) counting_3d = np.arange(27.0, dtype=np.float32).reshape((3, 3, 3)) slices = [counting_3d[0, :, :], counting_3d[:, 0, :], counting_3d[:, :, 0]] for slice_idx, ref_mat in enumerate(slices): - assert np.array_equal(double_mat_cm(ref_mat), 2.0 * ref_mat) - assert np.array_equal(double_mat_rm(ref_mat), 2.0 * ref_mat) + np.testing.assert_array_equal(double_mat_cm(ref_mat), 2.0 * ref_mat) + np.testing.assert_array_equal(double_mat_rm(ref_mat), 2.0 * ref_mat) + + # Mutator: + double_threer(second_row) + double_threec(second_col) + np.testing.assert_array_equal(counting_mat, [[0., 2, 2], [6, 16, 10], [6, 14, 8]]) -@pytest.requires_eigen_and_numpy def test_nonunit_stride_to_python(): from pybind11_tests import diagonal, diagonal_1, diagonal_n, block @@ -75,23 +167,390 @@ def test_nonunit_stride_to_python(): assert np.all(block(ref, 1, 4, 3, 2) == ref[1:4, 4:]) -@pytest.requires_eigen_and_numpy def test_eigen_ref_to_python(): - from pybind11_tests import cholesky1, cholesky2, cholesky3, cholesky4, cholesky5, cholesky6 + from pybind11_tests import cholesky1, cholesky2, cholesky3, cholesky4 - chols = [cholesky1, cholesky2, cholesky3, cholesky4, cholesky5, cholesky6] + chols = [cholesky1, cholesky2, cholesky3, cholesky4] for i, chol in enumerate(chols, start=1): - mymat = chol(np.array([[1, 2, 4], [2, 13, 23], [4, 23, 77]])) + mymat = chol(np.array([[1., 2, 4], [2, 13, 23], [4, 23, 77]])) assert np.all(mymat == np.array([[1, 0, 0], [2, 3, 0], [4, 5, 6]])), "cholesky{}".format(i) -@pytest.requires_eigen_and_numpy +def assign_both(a1, a2, r, c, v): + a1[r, c] = v + a2[r, c] = v + + +def array_copy_but_one(a, r, c, v): + z = np.array(a, copy=True) + z[r, c] = v + return z + + +def test_eigen_return_references(): + """Tests various ways of returning references and non-referencing copies""" + from pybind11_tests import ReturnTester + master = np.ones((10, 10)) + a = ReturnTester() + a_get1 = a.get() + assert not a_get1.flags.owndata and a_get1.flags.writeable + assign_both(a_get1, master, 3, 3, 5) + a_get2 = a.get_ptr() + assert not a_get2.flags.owndata and a_get2.flags.writeable + assign_both(a_get1, master, 2, 3, 6) + + a_view1 = a.view() + assert not a_view1.flags.owndata and not a_view1.flags.writeable + with pytest.raises(ValueError): + a_view1[2, 3] = 4 + a_view2 = a.view_ptr() + assert not a_view2.flags.owndata and not a_view2.flags.writeable + with pytest.raises(ValueError): + a_view2[2, 3] = 4 + + a_copy1 = a.copy_get() + assert a_copy1.flags.owndata and a_copy1.flags.writeable + np.testing.assert_array_equal(a_copy1, master) + a_copy1[7, 7] = -44 # Shouldn't affect anything else + c1want = array_copy_but_one(master, 7, 7, -44) + a_copy2 = a.copy_view() + assert a_copy2.flags.owndata and a_copy2.flags.writeable + np.testing.assert_array_equal(a_copy2, master) + a_copy2[4, 4] = -22 # Shouldn't affect anything else + c2want = array_copy_but_one(master, 4, 4, -22) + + a_ref1 = a.ref() + assert not a_ref1.flags.owndata and a_ref1.flags.writeable + assign_both(a_ref1, master, 1, 1, 15) + a_ref2 = a.ref_const() + assert not a_ref2.flags.owndata and not a_ref2.flags.writeable + with pytest.raises(ValueError): + a_ref2[5, 5] = 33 + a_ref3 = a.ref_safe() + assert not a_ref3.flags.owndata and a_ref3.flags.writeable + assign_both(a_ref3, master, 0, 7, 99) + a_ref4 = a.ref_const_safe() + assert not a_ref4.flags.owndata and not a_ref4.flags.writeable + with pytest.raises(ValueError): + a_ref4[7, 0] = 987654321 + + a_copy3 = a.copy_ref() + assert a_copy3.flags.owndata and a_copy3.flags.writeable + np.testing.assert_array_equal(a_copy3, master) + a_copy3[8, 1] = 11 + c3want = array_copy_but_one(master, 8, 1, 11) + a_copy4 = a.copy_ref_const() + assert a_copy4.flags.owndata and a_copy4.flags.writeable + np.testing.assert_array_equal(a_copy4, master) + a_copy4[8, 4] = 88 + c4want = array_copy_but_one(master, 8, 4, 88) + + a_block1 = a.block(3, 3, 2, 2) + assert not a_block1.flags.owndata and a_block1.flags.writeable + a_block1[0, 0] = 55 + master[3, 3] = 55 + a_block2 = a.block_safe(2, 2, 3, 2) + assert not a_block2.flags.owndata and a_block2.flags.writeable + a_block2[2, 1] = -123 + master[4, 3] = -123 + a_block3 = a.block_const(6, 7, 4, 3) + assert not a_block3.flags.owndata and not a_block3.flags.writeable + with pytest.raises(ValueError): + a_block3[2, 2] = -44444 + + a_copy5 = a.copy_block(2, 2, 2, 3) + assert a_copy5.flags.owndata and a_copy5.flags.writeable + np.testing.assert_array_equal(a_copy5, master[2:4, 2:5]) + a_copy5[1, 1] = 777 + c5want = array_copy_but_one(master[2:4, 2:5], 1, 1, 777) + + a_corn1 = a.corners() + assert not a_corn1.flags.owndata and a_corn1.flags.writeable + a_corn1 *= 50 + a_corn1[1, 1] = 999 + master[0, 0] = 50 + master[0, 9] = 50 + master[9, 0] = 50 + master[9, 9] = 999 + a_corn2 = a.corners_const() + assert not a_corn2.flags.owndata and not a_corn2.flags.writeable + with pytest.raises(ValueError): + a_corn2[1, 0] = 51 + + # All of the changes made all the way along should be visible everywhere + # now (except for the copies, of course) + np.testing.assert_array_equal(a_get1, master) + np.testing.assert_array_equal(a_get2, master) + np.testing.assert_array_equal(a_view1, master) + np.testing.assert_array_equal(a_view2, master) + np.testing.assert_array_equal(a_ref1, master) + np.testing.assert_array_equal(a_ref2, master) + np.testing.assert_array_equal(a_ref3, master) + np.testing.assert_array_equal(a_ref4, master) + np.testing.assert_array_equal(a_block1, master[3:5, 3:5]) + np.testing.assert_array_equal(a_block2, master[2:5, 2:4]) + np.testing.assert_array_equal(a_block3, master[6:10, 7:10]) + np.testing.assert_array_equal(a_corn1, master[0::master.shape[0] - 1, 0::master.shape[1] - 1]) + np.testing.assert_array_equal(a_corn2, master[0::master.shape[0] - 1, 0::master.shape[1] - 1]) + + np.testing.assert_array_equal(a_copy1, c1want) + np.testing.assert_array_equal(a_copy2, c2want) + np.testing.assert_array_equal(a_copy3, c3want) + np.testing.assert_array_equal(a_copy4, c4want) + np.testing.assert_array_equal(a_copy5, c5want) + + +def assert_keeps_alive(cl, method, *args): + from pybind11_tests import ConstructorStats + cstats = ConstructorStats.get(cl) + start_with = cstats.alive() + a = cl() + assert cstats.alive() == start_with + 1 + z = method(a, *args) + assert cstats.alive() == start_with + 1 + del a + # Here's the keep alive in action: + assert cstats.alive() == start_with + 1 + del z + # Keep alive should have expired: + assert cstats.alive() == start_with + + +def test_eigen_keepalive(): + from pybind11_tests import ReturnTester, ConstructorStats + a = ReturnTester() + + cstats = ConstructorStats.get(ReturnTester) + assert cstats.alive() == 1 + unsafe = [a.ref(), a.ref_const(), a.block(1, 2, 3, 4)] + copies = [a.copy_get(), a.copy_view(), a.copy_ref(), a.copy_ref_const(), + a.copy_block(4, 3, 2, 1)] + del a + assert cstats.alive() == 0 + del unsafe + del copies + + for meth in [ReturnTester.get, ReturnTester.get_ptr, ReturnTester.view, + ReturnTester.view_ptr, ReturnTester.ref_safe, ReturnTester.ref_const_safe, + ReturnTester.corners, ReturnTester.corners_const]: + assert_keeps_alive(ReturnTester, meth) + + for meth in [ReturnTester.block_safe, ReturnTester.block_const]: + assert_keeps_alive(ReturnTester, meth, 4, 3, 2, 1) + + +def test_eigen_ref_mutators(): + """Tests whether Eigen can mutate numpy values""" + from pybind11_tests import add_rm, add_cm, add_any, add1, add2 + orig = np.array([[1., 2, 3], [4, 5, 6], [7, 8, 9]]) + zr = np.array(orig) + zc = np.array(orig, order='F') + add_rm(zr, 1, 0, 100) + assert np.all(zr == np.array([[1., 2, 3], [104, 5, 6], [7, 8, 9]])) + add_cm(zc, 1, 0, 200) + assert np.all(zc == np.array([[1., 2, 3], [204, 5, 6], [7, 8, 9]])) + + add_any(zr, 1, 0, 20) + assert np.all(zr == np.array([[1., 2, 3], [124, 5, 6], [7, 8, 9]])) + add_any(zc, 1, 0, 10) + assert np.all(zc == np.array([[1., 2, 3], [214, 5, 6], [7, 8, 9]])) + + # Can't reference a col-major array with a row-major Ref, and vice versa: + with pytest.raises(TypeError): + add_rm(zc, 1, 0, 1) + with pytest.raises(TypeError): + add_cm(zr, 1, 0, 1) + + # Overloads: + add1(zr, 1, 0, -100) + add2(zr, 1, 0, -20) + assert np.all(zr == orig) + add1(zc, 1, 0, -200) + add2(zc, 1, 0, -10) + assert np.all(zc == orig) + + # a non-contiguous slice (this won't work on either the row- or + # column-contiguous refs, but should work for the any) + cornersr = zr[0::2, 0::2] + cornersc = zc[0::2, 0::2] + + assert np.all(cornersr == np.array([[1., 3], [7, 9]])) + assert np.all(cornersc == np.array([[1., 3], [7, 9]])) + + with pytest.raises(TypeError): + add_rm(cornersr, 0, 1, 25) + with pytest.raises(TypeError): + add_cm(cornersr, 0, 1, 25) + with pytest.raises(TypeError): + add_rm(cornersc, 0, 1, 25) + with pytest.raises(TypeError): + add_cm(cornersc, 0, 1, 25) + add_any(cornersr, 0, 1, 25) + add_any(cornersc, 0, 1, 44) + assert np.all(zr == np.array([[1., 2, 28], [4, 5, 6], [7, 8, 9]])) + assert np.all(zc == np.array([[1., 2, 47], [4, 5, 6], [7, 8, 9]])) + + # You shouldn't be allowed to pass a non-writeable array to a mutating Eigen method: + zro = zr[0:4, 0:4] + zro.flags.writeable = False + with pytest.raises(TypeError): + add_rm(zro, 0, 0, 0) + with pytest.raises(TypeError): + add_any(zro, 0, 0, 0) + with pytest.raises(TypeError): + add1(zro, 0, 0, 0) + with pytest.raises(TypeError): + add2(zro, 0, 0, 0) + + # integer array shouldn't be passable to a double-matrix-accepting mutating func: + zi = np.array([[1, 2], [3, 4]]) + with pytest.raises(TypeError): + add_rm(zi) + + +def test_numpy_ref_mutators(): + """Tests numpy mutating Eigen matrices (for returned Eigen::Ref<...>s)""" + from pybind11_tests import ( + get_cm_ref, get_cm_const_ref, get_rm_ref, get_rm_const_ref, reset_refs) + reset_refs() # In case another test already changed it + + zc = get_cm_ref() + zcro = get_cm_const_ref() + zr = get_rm_ref() + zrro = get_rm_const_ref() + + assert [zc[1, 2], zcro[1, 2], zr[1, 2], zrro[1, 2]] == [23] * 4 + + assert not zc.flags.owndata and zc.flags.writeable + assert not zr.flags.owndata and zr.flags.writeable + assert not zcro.flags.owndata and not zcro.flags.writeable + assert not zrro.flags.owndata and not zrro.flags.writeable + + zc[1, 2] = 99 + expect = np.array([[11., 12, 13], [21, 22, 99], [31, 32, 33]]) + # We should have just changed zc, of course, but also zcro and the original eigen matrix + assert np.all(zc == expect) + assert np.all(zcro == expect) + assert np.all(get_cm_ref() == expect) + + zr[1, 2] = 99 + assert np.all(zr == expect) + assert np.all(zrro == expect) + assert np.all(get_rm_ref() == expect) + + # Make sure the readonly ones are numpy-readonly: + with pytest.raises(ValueError): + zcro[1, 2] = 6 + with pytest.raises(ValueError): + zrro[1, 2] = 6 + + # We should be able to explicitly copy like this (and since we're copying, + # the const should drop away) + y1 = np.array(get_cm_const_ref()) + + assert y1.flags.owndata and y1.flags.writeable + # We should get copies of the eigen data, which was modified above: + assert y1[1, 2] == 99 + y1[1, 2] += 12 + assert y1[1, 2] == 111 + assert zc[1, 2] == 99 # Make sure we aren't referencing the original + + +def test_both_ref_mutators(): + """Tests a complex chain of nested eigen/numpy references""" + from pybind11_tests import ( + incr_matrix, get_cm_ref, incr_matrix_any, even_cols, even_rows, reset_refs) + reset_refs() # In case another test already changed it + + z = get_cm_ref() # numpy -> eigen + z[0, 2] -= 3 + z2 = incr_matrix(z, 1) # numpy -> eigen -> numpy -> eigen + z2[1, 1] += 6 + z3 = incr_matrix(z, 2) # (numpy -> eigen)^3 + z3[2, 2] += -5 + z4 = incr_matrix(z, 3) # (numpy -> eigen)^4 + z4[1, 1] -= 1 + z5 = incr_matrix(z, 4) # (numpy -> eigen)^5 + z5[0, 0] = 0 + assert np.all(z == z2) + assert np.all(z == z3) + assert np.all(z == z4) + assert np.all(z == z5) + expect = np.array([[0., 22, 20], [31, 37, 33], [41, 42, 38]]) + assert np.all(z == expect) + + y = np.array(range(100), dtype='float64').reshape(10, 10) + y2 = incr_matrix_any(y, 10) # np -> eigen -> np + y3 = incr_matrix_any(y2[0::2, 0::2], -33) # np -> eigen -> np slice -> np -> eigen -> np + y4 = even_rows(y3) # numpy -> eigen slice -> (... y3) + y5 = even_cols(y4) # numpy -> eigen slice -> (... y4) + y6 = incr_matrix_any(y5, 1000) # numpy -> eigen -> (... y5) + + # Apply same mutations using just numpy: + yexpect = np.array(range(100), dtype='float64').reshape(10, 10) + yexpect += 10 + yexpect[0::2, 0::2] -= 33 + yexpect[0::4, 0::4] += 1000 + assert np.all(y6 == yexpect[0::4, 0::4]) + assert np.all(y5 == yexpect[0::4, 0::4]) + assert np.all(y4 == yexpect[0::4, 0::2]) + assert np.all(y3 == yexpect[0::2, 0::2]) + assert np.all(y2 == yexpect) + assert np.all(y == yexpect) + + +def test_nocopy_wrapper(): + from pybind11_tests import get_elem, get_elem_nocopy, get_elem_rm_nocopy + # get_elem requires a column-contiguous matrix reference, but should be + # callable with other types of matrix (via copying): + int_matrix_colmajor = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], order='F') + dbl_matrix_colmajor = np.array(int_matrix_colmajor, dtype='double', order='F', copy=True) + int_matrix_rowmajor = np.array(int_matrix_colmajor, order='C', copy=True) + dbl_matrix_rowmajor = np.array(int_matrix_rowmajor, dtype='double', order='C', copy=True) + + # All should be callable via get_elem: + assert get_elem(int_matrix_colmajor) == 8 + assert get_elem(dbl_matrix_colmajor) == 8 + assert get_elem(int_matrix_rowmajor) == 8 + assert get_elem(dbl_matrix_rowmajor) == 8 + + # All but the second should fail with get_elem_nocopy: + with pytest.raises(TypeError) as excinfo: + get_elem_nocopy(int_matrix_colmajor) + assert ('get_elem_nocopy(): incompatible function arguments.' in str(excinfo.value) and + ', flags.f_contiguous' in str(excinfo.value)) + assert get_elem_nocopy(dbl_matrix_colmajor) == 8 + with pytest.raises(TypeError) as excinfo: + get_elem_nocopy(int_matrix_rowmajor) + assert ('get_elem_nocopy(): incompatible function arguments.' in str(excinfo.value) and + ', flags.f_contiguous' in str(excinfo.value)) + with pytest.raises(TypeError) as excinfo: + get_elem_nocopy(dbl_matrix_rowmajor) + assert ('get_elem_nocopy(): incompatible function arguments.' in str(excinfo.value) and + ', flags.f_contiguous' in str(excinfo.value)) + + # For the row-major test, we take a long matrix in row-major, so only the third is allowed: + with pytest.raises(TypeError) as excinfo: + get_elem_rm_nocopy(int_matrix_colmajor) + assert ('get_elem_rm_nocopy(): incompatible function arguments.' in str(excinfo.value) and + ', flags.c_contiguous' in str(excinfo.value)) + with pytest.raises(TypeError) as excinfo: + get_elem_rm_nocopy(dbl_matrix_colmajor) + assert ('get_elem_rm_nocopy(): incompatible function arguments.' in str(excinfo.value) and + ', flags.c_contiguous' in str(excinfo.value)) + assert get_elem_rm_nocopy(int_matrix_rowmajor) == 8 + with pytest.raises(TypeError) as excinfo: + get_elem_rm_nocopy(dbl_matrix_rowmajor) + assert ('get_elem_rm_nocopy(): incompatible function arguments.' in str(excinfo.value) and + ', flags.c_contiguous' in str(excinfo.value)) + + def test_special_matrix_objects(): from pybind11_tests import incr_diag, symmetric_upper, symmetric_lower - assert np.all(incr_diag(7) == np.diag([1, 2, 3, 4, 5, 6, 7])) + assert np.all(incr_diag(7) == np.diag([1., 2, 3, 4, 5, 6, 7])) - asymm = np.array([[ 1, 2, 3, 4], + asymm = np.array([[ 1., 2, 3, 4], [ 5, 6, 7, 8], [ 9, 10, 11, 12], [13, 14, 15, 16]]) @@ -106,9 +565,8 @@ def test_special_matrix_objects(): assert np.all(symmetric_upper(asymm) == symm_upper) -@pytest.requires_eigen_and_numpy def test_dense_signature(doc): - from pybind11_tests import double_col, double_row, double_mat_rm + from pybind11_tests import double_col, double_row, double_complex, double_mat_rm assert doc(double_col) == """ double_col(arg0: numpy.ndarray[float32[m, 1]]) -> numpy.ndarray[float32[m, 1]] @@ -116,6 +574,9 @@ def test_dense_signature(doc): assert doc(double_row) == """ double_row(arg0: numpy.ndarray[float32[1, n]]) -> numpy.ndarray[float32[1, n]] """ + assert doc(double_complex) == """ + double_complex(arg0: numpy.ndarray[complex64[m, 1]]) -> numpy.ndarray[complex64[m, 1]] + """ assert doc(double_mat_rm) == """ double_mat_rm(arg0: numpy.ndarray[float32[m, n]]) -> numpy.ndarray[float32[m, n]] """ @@ -123,23 +584,43 @@ def test_dense_signature(doc): @pytest.requires_eigen_and_scipy def test_sparse(): - from pybind11_tests import sparse_r, sparse_c, sparse_passthrough_r, sparse_passthrough_c + from pybind11_tests import sparse_r, sparse_c, sparse_copy_r, sparse_copy_c assert_sparse_equal_ref(sparse_r()) assert_sparse_equal_ref(sparse_c()) - assert_sparse_equal_ref(sparse_passthrough_r(sparse_r())) - assert_sparse_equal_ref(sparse_passthrough_c(sparse_c())) - assert_sparse_equal_ref(sparse_passthrough_r(sparse_c())) - assert_sparse_equal_ref(sparse_passthrough_c(sparse_r())) + assert_sparse_equal_ref(sparse_copy_r(sparse_r())) + assert_sparse_equal_ref(sparse_copy_c(sparse_c())) + assert_sparse_equal_ref(sparse_copy_r(sparse_c())) + assert_sparse_equal_ref(sparse_copy_c(sparse_r())) @pytest.requires_eigen_and_scipy def test_sparse_signature(doc): - from pybind11_tests import sparse_passthrough_r, sparse_passthrough_c + from pybind11_tests import sparse_copy_r, sparse_copy_c - assert doc(sparse_passthrough_r) == """ - sparse_passthrough_r(arg0: scipy.sparse.csr_matrix[float32]) -> scipy.sparse.csr_matrix[float32] + assert doc(sparse_copy_r) == """ + sparse_copy_r(arg0: scipy.sparse.csr_matrix[float32]) -> scipy.sparse.csr_matrix[float32] """ # noqa: E501 line too long - assert doc(sparse_passthrough_c) == """ - sparse_passthrough_c(arg0: scipy.sparse.csc_matrix[float32]) -> scipy.sparse.csc_matrix[float32] + assert doc(sparse_copy_c) == """ + sparse_copy_c(arg0: scipy.sparse.csc_matrix[float32]) -> scipy.sparse.csc_matrix[float32] """ # noqa: E501 line too long + + +def test_issue738(): + from pybind11_tests import iss738_f1, iss738_f2 + + assert np.all(iss738_f1(np.array([[1., 2, 3]])) == np.array([[1., 102, 203]])) + assert np.all(iss738_f1(np.array([[1.], [2], [3]])) == np.array([[1.], [12], [23]])) + + assert np.all(iss738_f2(np.array([[1., 2, 3]])) == np.array([[1., 102, 203]])) + assert np.all(iss738_f2(np.array([[1.], [2], [3]])) == np.array([[1.], [12], [23]])) + + +def test_custom_operator_new(): + """Using Eigen types as member variables requires a class-specific + operator new with proper alignment""" + from pybind11_tests import CustomOperatorNew + + o = CustomOperatorNew() + np.testing.assert_allclose(o.a, 0.0) + np.testing.assert_allclose(o.b.diagonal(), 1.0) diff --git a/ext/pybind11/tests/test_enum.py b/ext/pybind11/tests/test_enum.py index de5f3c6f6..ba7e22ab0 100644 --- a/ext/pybind11/tests/test_enum.py +++ b/ext/pybind11/tests/test_enum.py @@ -7,6 +7,15 @@ def test_unscoped_enum(): assert str(UnscopedEnum.EOne) == "UnscopedEnum.EOne" assert str(UnscopedEnum.ETwo) == "UnscopedEnum.ETwo" assert str(EOne) == "UnscopedEnum.EOne" + # __members__ property + assert UnscopedEnum.__members__ == {"EOne": UnscopedEnum.EOne, "ETwo": UnscopedEnum.ETwo} + # __members__ readonly + with pytest.raises(AttributeError): + UnscopedEnum.__members__ = {} + # __members__ returns a copy + foo = UnscopedEnum.__members__ + foo["bar"] = "baz" + assert UnscopedEnum.__members__ == {"EOne": UnscopedEnum.EOne, "ETwo": UnscopedEnum.ETwo} # no TypeError exception for unscoped enum ==/!= int comparisons y = UnscopedEnum.ETwo diff --git a/ext/pybind11/tests/test_inheritance.cpp b/ext/pybind11/tests/test_inheritance.cpp index 2ec0b4a7a..c19f58dc2 100644 --- a/ext/pybind11/tests/test_inheritance.cpp +++ b/ext/pybind11/tests/test_inheritance.cpp @@ -36,6 +36,10 @@ public: Hamster(const std::string &name) : Pet(name, "rodent") {} }; +class Chimera : public Pet { + Chimera() : Pet("Kimmy", "chimera") {} +}; + std::string pet_name_species(const Pet &pet) { return pet.name() + " is a " + pet.species(); } @@ -49,6 +53,12 @@ struct BaseClass { virtual ~BaseClass() {} }; struct DerivedClass1 : BaseClass { }; struct DerivedClass2 : BaseClass { }; +struct MismatchBase1 { }; +struct MismatchDerived1 : MismatchBase1 { }; + +struct MismatchBase2 { }; +struct MismatchDerived2 : MismatchBase2 { }; + test_initializer inheritance([](py::module &m) { py::class_<Pet> pet_class(m, "Pet"); pet_class @@ -68,6 +78,8 @@ test_initializer inheritance([](py::module &m) { py::class_<Hamster, Pet>(m, "Hamster") .def(py::init<std::string>()); + py::class_<Chimera, Pet>(m, "Chimera"); + m.def("pet_name_species", pet_name_species); m.def("dog_bark", dog_bark); @@ -97,4 +109,15 @@ test_initializer inheritance([](py::module &m) { py::isinstance<Unregistered>(l[6]) ); }); + + m.def("test_mismatched_holder_type_1", []() { + auto m = py::module::import("__main__"); + py::class_<MismatchBase1, std::shared_ptr<MismatchBase1>>(m, "MismatchBase1"); + py::class_<MismatchDerived1, MismatchBase1>(m, "MismatchDerived1"); + }); + m.def("test_mismatched_holder_type_2", []() { + auto m = py::module::import("__main__"); + py::class_<MismatchBase2>(m, "MismatchBase2"); + py::class_<MismatchDerived2, std::shared_ptr<MismatchDerived2>, MismatchBase2>(m, "MismatchDerived2"); + }); }); diff --git a/ext/pybind11/tests/test_inheritance.py b/ext/pybind11/tests/test_inheritance.py index 7bb52be02..d1f537d1d 100644 --- a/ext/pybind11/tests/test_inheritance.py +++ b/ext/pybind11/tests/test_inheritance.py @@ -2,7 +2,7 @@ import pytest def test_inheritance(msg): - from pybind11_tests import Pet, Dog, Rabbit, Hamster, dog_bark, pet_name_species + from pybind11_tests import Pet, Dog, Rabbit, Hamster, Chimera, dog_bark, pet_name_species roger = Rabbit('Rabbit') assert roger.name() + " is a " + roger.species() == "Rabbit is a parrot" @@ -30,6 +30,10 @@ def test_inheritance(msg): Invoked with: <m.Pet object at 0> """ + with pytest.raises(TypeError) as excinfo: + Chimera("lion", "goat") + assert "No constructor defined!" in str(excinfo.value) + def test_automatic_upcasting(): from pybind11_tests import return_class_1, return_class_2, return_class_n, return_none @@ -37,7 +41,8 @@ def test_automatic_upcasting(): assert type(return_class_1()).__name__ == "DerivedClass1" assert type(return_class_2()).__name__ == "DerivedClass2" assert type(return_none()).__name__ == "NoneType" - # Repeat these a few times in a random order to ensure no invalid caching is applied + # Repeat these a few times in a random order to ensure no invalid caching + # is applied assert type(return_class_n(1)).__name__ == "DerivedClass1" assert type(return_class_n(2)).__name__ == "DerivedClass2" assert type(return_class_n(0)).__name__ == "BaseClass" @@ -53,3 +58,21 @@ def test_isinstance(): objects = [tuple(), dict(), Pet("Polly", "parrot")] + [Dog("Molly")] * 4 expected = (True, True, True, True, True, False, False) assert test_isinstance(objects) == expected + + +def test_holder(): + from pybind11_tests import test_mismatched_holder_type_1, test_mismatched_holder_type_2 + + with pytest.raises(RuntimeError) as excinfo: + test_mismatched_holder_type_1() + + assert str(excinfo.value) == ("generic_type: type \"MismatchDerived1\" does not have " + "a non-default holder type while its base " + "\"MismatchBase1\" does") + + with pytest.raises(RuntimeError) as excinfo: + test_mismatched_holder_type_2() + + assert str(excinfo.value) == ("generic_type: type \"MismatchDerived2\" has a " + "non-default holder type while its base " + "\"MismatchBase2\" does not") diff --git a/ext/pybind11/tests/test_installed_module/CMakeLists.txt b/ext/pybind11/tests/test_installed_module/CMakeLists.txt deleted file mode 100644 index 77fd49dc2..000000000 --- a/ext/pybind11/tests/test_installed_module/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -cmake_minimum_required(VERSION 2.8.12) -project(test_installed_module CXX) - -set(CMAKE_MODULE_PATH "") - -find_package(pybind11 CONFIG REQUIRED) - -message(STATUS "Found pybind11: ${pybind11_INCLUDE_DIRS} (found version ${pybind11_VERSION})") -message(STATUS "Found Python: ${PYTHON_INCLUDE_DIRS} (found version ${PYTHON_VERSION_STRING})") - -pybind11_add_module(test_installed_module SHARED main.cpp) - -add_custom_target(check ${CMAKE_COMMAND} -E env PYTHONPATH=$<TARGET_FILE_DIR:test_installed_module> - ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test.py) diff --git a/ext/pybind11/tests/test_installed_module/main.cpp b/ext/pybind11/tests/test_installed_module/main.cpp deleted file mode 100644 index a0bda4542..000000000 --- a/ext/pybind11/tests/test_installed_module/main.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include <pybind11/pybind11.h> -namespace py = pybind11; - -PYBIND11_PLUGIN(test_installed_module) { - py::module m("test_installed_module"); - - m.def("add", [](int i, int j) { return i + j; }); - - return m.ptr(); -} diff --git a/ext/pybind11/tests/test_installed_module/test.py b/ext/pybind11/tests/test_installed_module/test.py deleted file mode 100644 index 2f0632049..000000000 --- a/ext/pybind11/tests/test_installed_module/test.py +++ /dev/null @@ -1,3 +0,0 @@ -import test_installed_module -assert test_installed_module.add(11, 22) == 33 -print('test_installed_module imports, runs, and adds: 11 + 22 = 33') diff --git a/ext/pybind11/tests/test_installed_target/CMakeLists.txt b/ext/pybind11/tests/test_installed_target/CMakeLists.txt deleted file mode 100644 index 4333dc107..000000000 --- a/ext/pybind11/tests/test_installed_target/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -cmake_minimum_required(VERSION 3.0) -project(test_installed_target CXX) - -set(CMAKE_MODULE_PATH "") - -find_package(pybind11 CONFIG REQUIRED) - -message(STATUS "Found pybind11: ${pybind11_INCLUDE_DIRS} (found version ${pybind11_VERSION})") -message(STATUS "Found Python: ${PYTHON_INCLUDE_DIRS} (found version ${PYTHON_VERSION_STRING})") - -add_library(test_installed_target MODULE main.cpp) - -target_link_libraries(test_installed_target PRIVATE pybind11::pybind11) - -# make sure result is, for example, test_installed_target.so, not libtest_installed_target.dylib -set_target_properties(test_installed_target PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") - -add_custom_target(check ${CMAKE_COMMAND} -E env PYTHONPATH=$<TARGET_FILE_DIR:test_installed_target> - ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test.py) diff --git a/ext/pybind11/tests/test_installed_target/test.py b/ext/pybind11/tests/test_installed_target/test.py deleted file mode 100644 index b2888a72b..000000000 --- a/ext/pybind11/tests/test_installed_target/test.py +++ /dev/null @@ -1,3 +0,0 @@ -import test_installed_target -assert test_installed_target.add(1, 2) == 3 -print('test_installed_target imports, runs, and adds: 1 + 2 = 3') diff --git a/ext/pybind11/tests/test_issues.cpp b/ext/pybind11/tests/test_issues.cpp index 4c59a1b12..7da370045 100644 --- a/ext/pybind11/tests/test_issues.cpp +++ b/ext/pybind11/tests/test_issues.cpp @@ -74,7 +74,6 @@ namespace std { template <> struct hash<TplConstrClass> { size_t operator()(const TplConstrClass &t) const { return std::hash<std::string>()(t.str); } }; } - void init_issues(py::module &m) { py::module m2 = m.def_submodule("issues"); @@ -397,5 +396,5 @@ void init_issues(py::module &m) { #endif } -// MSVC workaround: trying to use a lambda here crashes MSCV +// MSVC workaround: trying to use a lambda here crashes MSVC test_initializer issues(&init_issues); diff --git a/ext/pybind11/tests/test_issues.py b/ext/pybind11/tests/test_issues.py index 2098ff8a3..e60b5ca90 100644 --- a/ext/pybind11/tests/test_issues.py +++ b/ext/pybind11/tests/test_issues.py @@ -1,5 +1,4 @@ import pytest -import gc from pybind11_tests import ConstructorStats @@ -55,7 +54,7 @@ def test_shared_ptr_gc(): el = ElementList() for i in range(10): el.add(ElementA(i)) - gc.collect() + pytest.gc_collect() for i, v in enumerate(el.get()): assert i == v.value() @@ -130,13 +129,13 @@ def test_nested(): assert c.b.a.as_base().value == 42 del c - gc.collect() + pytest.gc_collect() del a # Should't delete while abase is still alive - gc.collect() + pytest.gc_collect() assert abase.value == 42 del abase, b - gc.collect() + pytest.gc_collect() def test_move_fallback(): diff --git a/ext/pybind11/tests/test_keep_alive.py b/ext/pybind11/tests/test_keep_alive.py index 0cef34658..bfd7d40c3 100644 --- a/ext/pybind11/tests/test_keep_alive.py +++ b/ext/pybind11/tests/test_keep_alive.py @@ -1,4 +1,4 @@ -import gc +import pytest def test_keep_alive_argument(capture): @@ -9,14 +9,14 @@ def test_keep_alive_argument(capture): assert capture == "Allocating parent." with capture: p.addChild(Child()) - gc.collect() + pytest.gc_collect() assert capture == """ Allocating child. Releasing child. """ with capture: del p - gc.collect() + pytest.gc_collect() assert capture == "Releasing parent." with capture: @@ -24,11 +24,11 @@ def test_keep_alive_argument(capture): assert capture == "Allocating parent." with capture: p.addChildKeepAlive(Child()) - gc.collect() + pytest.gc_collect() assert capture == "Allocating child." with capture: del p - gc.collect() + pytest.gc_collect() assert capture == """ Releasing parent. Releasing child. @@ -43,14 +43,14 @@ def test_keep_alive_return_value(capture): assert capture == "Allocating parent." with capture: p.returnChild() - gc.collect() + pytest.gc_collect() assert capture == """ Allocating child. Releasing child. """ with capture: del p - gc.collect() + pytest.gc_collect() assert capture == "Releasing parent." with capture: @@ -58,11 +58,11 @@ def test_keep_alive_return_value(capture): assert capture == "Allocating parent." with capture: p.returnChildKeepAlive() - gc.collect() + pytest.gc_collect() assert capture == "Allocating child." with capture: del p - gc.collect() + pytest.gc_collect() assert capture == """ Releasing parent. Releasing child. @@ -77,11 +77,11 @@ def test_return_none(capture): assert capture == "Allocating parent." with capture: p.returnNullChildKeepAliveChild() - gc.collect() + pytest.gc_collect() assert capture == "" with capture: del p - gc.collect() + pytest.gc_collect() assert capture == "Releasing parent." with capture: @@ -89,9 +89,9 @@ def test_return_none(capture): assert capture == "Allocating parent." with capture: p.returnNullChildKeepAliveParent() - gc.collect() + pytest.gc_collect() assert capture == "" with capture: del p - gc.collect() + pytest.gc_collect() assert capture == "Releasing parent." diff --git a/ext/pybind11/tests/test_kwargs_and_defaults.cpp b/ext/pybind11/tests/test_kwargs_and_defaults.cpp index 24fc0cd5b..3180123df 100644 --- a/ext/pybind11/tests/test_kwargs_and_defaults.cpp +++ b/ext/pybind11/tests/test_kwargs_and_defaults.cpp @@ -28,6 +28,27 @@ py::tuple args_kwargs_function(py::args args, py::kwargs kwargs) { return py::make_tuple(args, kwargs); } +py::tuple mixed_plus_args(int i, double j, py::args args) { + return py::make_tuple(i, j, args); +} + +py::tuple mixed_plus_kwargs(int i, double j, py::kwargs kwargs) { + return py::make_tuple(i, j, kwargs); +} + +py::tuple mixed_plus_args_kwargs(int i, double j, py::args args, py::kwargs kwargs) { + return py::make_tuple(i, j, args, kwargs); +} + +// pybind11 won't allow these to be bound: args and kwargs, if present, must be at the end. +void bad_args1(py::args, int) {} +void bad_args2(py::kwargs, int) {} +void bad_args3(py::kwargs, py::args) {} +void bad_args4(py::args, int, py::kwargs) {} +void bad_args5(py::args, py::kwargs, int) {} +void bad_args6(py::args, py::args) {} +void bad_args7(py::kwargs, py::kwargs) {} + struct KWClass { void foo(int, float) {} }; @@ -53,4 +74,20 @@ test_initializer arg_keywords_and_defaults([](py::module &m) { py::class_<KWClass>(m, "KWClass") .def("foo0", &KWClass::foo) .def("foo1", &KWClass::foo, "x"_a, "y"_a); + + m.def("mixed_plus_args", &mixed_plus_args); + m.def("mixed_plus_kwargs", &mixed_plus_kwargs); + m.def("mixed_plus_args_kwargs", &mixed_plus_args_kwargs); + + m.def("mixed_plus_args_kwargs_defaults", &mixed_plus_args_kwargs, + py::arg("i") = 1, py::arg("j") = 3.14159); + + // Uncomment these to test that the static_assert is indeed working: +// m.def("bad_args1", &bad_args1); +// m.def("bad_args2", &bad_args2); +// m.def("bad_args3", &bad_args3); +// m.def("bad_args4", &bad_args4); +// m.def("bad_args5", &bad_args5); +// m.def("bad_args6", &bad_args6); +// m.def("bad_args7", &bad_args7); }); diff --git a/ext/pybind11/tests/test_kwargs_and_defaults.py b/ext/pybind11/tests/test_kwargs_and_defaults.py index 852d03c6e..90f8489ed 100644 --- a/ext/pybind11/tests/test_kwargs_and_defaults.py +++ b/ext/pybind11/tests/test_kwargs_and_defaults.py @@ -34,12 +34,8 @@ def test_named_arguments(msg): with pytest.raises(TypeError) as excinfo: # noinspection PyArgumentList kw_func2(x=5, y=10, z=12) - assert msg(excinfo.value) == """ - kw_func2(): incompatible function arguments. The following argument types are supported: - 1. (x: int=100, y: int=200) -> str - - Invoked with: - """ + assert excinfo.match( + r'(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$))' + '{3}$') assert kw_func4() == "{13 17}" assert kw_func4(myList=[1, 2, 3]) == "{1 2 3}" @@ -55,3 +51,58 @@ def test_arg_and_kwargs(): args = 'a1', 'a2' kwargs = dict(arg3='a3', arg4=4) assert args_kwargs_function(*args, **kwargs) == (args, kwargs) + + +def test_mixed_args_and_kwargs(msg): + from pybind11_tests import (mixed_plus_args, mixed_plus_kwargs, mixed_plus_args_kwargs, + mixed_plus_args_kwargs_defaults) + mpa = mixed_plus_args + mpk = mixed_plus_kwargs + mpak = mixed_plus_args_kwargs + mpakd = mixed_plus_args_kwargs_defaults + + assert mpa(1, 2.5, 4, 99.5, None) == (1, 2.5, (4, 99.5, None)) + assert mpa(1, 2.5) == (1, 2.5, ()) + with pytest.raises(TypeError) as excinfo: + assert mpa(1) + assert msg(excinfo.value) == """ + mixed_plus_args(): incompatible function arguments. The following argument types are supported: + 1. (arg0: int, arg1: float, *args) -> tuple + + Invoked with: 1 + """ # noqa: E501 line too long + with pytest.raises(TypeError) as excinfo: + assert mpa() + assert msg(excinfo.value) == """ + mixed_plus_args(): incompatible function arguments. The following argument types are supported: + 1. (arg0: int, arg1: float, *args) -> tuple + + Invoked with: + """ # noqa: E501 line too long + + assert mpk(-2, 3.5, pi=3.14159, e=2.71828) == (-2, 3.5, {'e': 2.71828, 'pi': 3.14159}) + assert mpak(7, 7.7, 7.77, 7.777, 7.7777, minusseven=-7) == ( + 7, 7.7, (7.77, 7.777, 7.7777), {'minusseven': -7}) + assert mpakd() == (1, 3.14159, (), {}) + assert mpakd(3) == (3, 3.14159, (), {}) + assert mpakd(j=2.71828) == (1, 2.71828, (), {}) + assert mpakd(k=42) == (1, 3.14159, (), {'k': 42}) + assert mpakd(1, 1, 2, 3, 5, 8, then=13, followedby=21) == ( + 1, 1, (2, 3, 5, 8), {'then': 13, 'followedby': 21}) + # Arguments specified both positionally and via kwargs should fail: + with pytest.raises(TypeError) as excinfo: + assert mpakd(1, i=1) + assert msg(excinfo.value) == """ + mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported: + 1. (i: int=1, j: float=3.14159, *args, **kwargs) -> tuple + + Invoked with: 1; kwargs: i=1 + """ # noqa: E501 line too long + with pytest.raises(TypeError) as excinfo: + assert mpakd(1, 2, j=1) + assert msg(excinfo.value) == """ + mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported: + 1. (i: int=1, j: float=3.14159, *args, **kwargs) -> tuple + + Invoked with: 1, 2; kwargs: j=1 + """ # noqa: E501 line too long diff --git a/ext/pybind11/tests/test_methods_and_attributes.cpp b/ext/pybind11/tests/test_methods_and_attributes.cpp index 11fc90091..b7b2edfd8 100644 --- a/ext/pybind11/tests/test_methods_and_attributes.cpp +++ b/ext/pybind11/tests/test_methods_and_attributes.cpp @@ -50,10 +50,14 @@ public: int *internal4() { return &value; } // return by pointer const int *internal5() { return &value; } // return by const pointer - py::str overloaded(int, float) { return "(int, float)"; } - py::str overloaded(float, int) { return "(float, int)"; } - py::str overloaded(int, float) const { return "(int, float) const"; } - py::str overloaded(float, int) const { return "(float, int) const"; } + py::str overloaded(int, float) { return "(int, float)"; } + py::str overloaded(float, int) { return "(float, int)"; } + py::str overloaded(int, int) { return "(int, int)"; } + py::str overloaded(float, float) { return "(float, float)"; } + py::str overloaded(int, float) const { return "(int, float) const"; } + py::str overloaded(float, int) const { return "(float, int) const"; } + py::str overloaded(int, int) const { return "(int, int) const"; } + py::str overloaded(float, float) const { return "(float, float) const"; } int value = 0; }; @@ -97,6 +101,58 @@ public: class CppDerivedDynamicClass : public DynamicClass { }; +// py::arg/py::arg_v testing: these arguments just record their argument when invoked +class ArgInspector1 { public: std::string arg = "(default arg inspector 1)"; }; +class ArgInspector2 { public: std::string arg = "(default arg inspector 2)"; }; +class ArgAlwaysConverts { }; +namespace pybind11 { namespace detail { +template <> struct type_caster<ArgInspector1> { +public: + PYBIND11_TYPE_CASTER(ArgInspector1, _("ArgInspector1")); + + bool load(handle src, bool convert) { + value.arg = "loading ArgInspector1 argument " + + std::string(convert ? "WITH" : "WITHOUT") + " conversion allowed. " + "Argument value = " + (std::string) str(src); + return true; + } + + static handle cast(const ArgInspector1 &src, return_value_policy, handle) { + return str(src.arg).release(); + } +}; +template <> struct type_caster<ArgInspector2> { +public: + PYBIND11_TYPE_CASTER(ArgInspector2, _("ArgInspector2")); + + bool load(handle src, bool convert) { + value.arg = "loading ArgInspector2 argument " + + std::string(convert ? "WITH" : "WITHOUT") + " conversion allowed. " + "Argument value = " + (std::string) str(src); + return true; + } + + static handle cast(const ArgInspector2 &src, return_value_policy, handle) { + return str(src.arg).release(); + } +}; +template <> struct type_caster<ArgAlwaysConverts> { +public: + PYBIND11_TYPE_CASTER(ArgAlwaysConverts, _("ArgAlwaysConverts")); + + bool load(handle, bool convert) { + return convert; + } + + static handle cast(const ArgAlwaysConverts &, return_value_policy, handle) { + return py::none(); + } +}; +}} + +/// Issue/PR #648: bad arg default debugging output +class NotRegistered {}; + test_initializer methods_and_attributes([](py::module &m) { py::class_<ExampleMandA>(m, "ExampleMandA") .def(py::init<>()) @@ -123,19 +179,28 @@ test_initializer methods_and_attributes([](py::module &m) { .def("internal4", &ExampleMandA::internal4) .def("internal5", &ExampleMandA::internal5) #if defined(PYBIND11_OVERLOAD_CAST) - .def("overloaded", py::overload_cast<int, float>(&ExampleMandA::overloaded)) - .def("overloaded", py::overload_cast<float, int>(&ExampleMandA::overloaded)) - .def("overloaded_const", py::overload_cast<int, float>(&ExampleMandA::overloaded, py::const_)) - .def("overloaded_const", py::overload_cast<float, int>(&ExampleMandA::overloaded, py::const_)) + .def("overloaded", py::overload_cast<int, float>(&ExampleMandA::overloaded)) + .def("overloaded", py::overload_cast<float, int>(&ExampleMandA::overloaded)) + .def("overloaded", py::overload_cast<int, int>(&ExampleMandA::overloaded)) + .def("overloaded", py::overload_cast<float, float>(&ExampleMandA::overloaded)) + .def("overloaded_float", py::overload_cast<float, float>(&ExampleMandA::overloaded)) + .def("overloaded_const", py::overload_cast<int, float>(&ExampleMandA::overloaded, py::const_)) + .def("overloaded_const", py::overload_cast<float, int>(&ExampleMandA::overloaded, py::const_)) + .def("overloaded_const", py::overload_cast<int, int>(&ExampleMandA::overloaded, py::const_)) + .def("overloaded_const", py::overload_cast<float, float>(&ExampleMandA::overloaded, py::const_)) #else - .def("overloaded", static_cast<py::str (ExampleMandA::*)(int, float)>(&ExampleMandA::overloaded)) - .def("overloaded", static_cast<py::str (ExampleMandA::*)(float, int)>(&ExampleMandA::overloaded)) - .def("overloaded_const", static_cast<py::str (ExampleMandA::*)(int, float) const>(&ExampleMandA::overloaded)) - .def("overloaded_const", static_cast<py::str (ExampleMandA::*)(float, int) const>(&ExampleMandA::overloaded)) + .def("overloaded", static_cast<py::str (ExampleMandA::*)(int, float)>(&ExampleMandA::overloaded)) + .def("overloaded", static_cast<py::str (ExampleMandA::*)(float, int)>(&ExampleMandA::overloaded)) + .def("overloaded", static_cast<py::str (ExampleMandA::*)(int, int)>(&ExampleMandA::overloaded)) + .def("overloaded", static_cast<py::str (ExampleMandA::*)(float, float)>(&ExampleMandA::overloaded)) + .def("overloaded_float", static_cast<py::str (ExampleMandA::*)(float, float)>(&ExampleMandA::overloaded)) + .def("overloaded_const", static_cast<py::str (ExampleMandA::*)(int, float) const>(&ExampleMandA::overloaded)) + .def("overloaded_const", static_cast<py::str (ExampleMandA::*)(float, int) const>(&ExampleMandA::overloaded)) + .def("overloaded_const", static_cast<py::str (ExampleMandA::*)(int, int) const>(&ExampleMandA::overloaded)) + .def("overloaded_const", static_cast<py::str (ExampleMandA::*)(float, float) const>(&ExampleMandA::overloaded)) #endif .def("__str__", &ExampleMandA::toString) - .def_readwrite("value", &ExampleMandA::value) - ; + .def_readwrite("value", &ExampleMandA::value); py::class_<TestProperties>(m, "TestProperties") .def(py::init<>()) @@ -149,7 +214,10 @@ test_initializer methods_and_attributes([](py::module &m) { [](py::object) { return TestProperties::static_get(); }) .def_property_static("def_property_static", [](py::object) { return TestProperties::static_get(); }, - [](py::object, int v) { return TestProperties::static_set(v); }); + [](py::object, int v) { TestProperties::static_set(v); }) + .def_property_static("static_cls", + [](py::object cls) { return cls; }, + [](py::object cls, py::function f) { f(cls); }); py::class_<SimpleValue>(m, "SimpleValue") .def_readwrite("value", &SimpleValue::value); @@ -177,9 +245,53 @@ test_initializer methods_and_attributes([](py::module &m) { .def_property_readonly("rvalue", &TestPropRVP::get_rvalue) .def_property_readonly_static("static_rvalue", [](py::object) { return SimpleValue(); }); + struct MetaclassOverride { }; + py::class_<MetaclassOverride>(m, "MetaclassOverride", py::metaclass((PyObject *) &PyType_Type)) + .def_property_readonly_static("readonly", [](py::object) { return 1; }); + +#if !defined(PYPY_VERSION) py::class_<DynamicClass>(m, "DynamicClass", py::dynamic_attr()) .def(py::init()); py::class_<CppDerivedDynamicClass, DynamicClass>(m, "CppDerivedDynamicClass") .def(py::init()); +#endif + + // Test converting. The ArgAlwaysConverts is just there to make the first no-conversion pass + // fail so that our call always ends up happening via the second dispatch (the one that allows + // some conversion). + class ArgInspector { + public: + ArgInspector1 f(ArgInspector1 a, ArgAlwaysConverts) { return a; } + std::string g(ArgInspector1 a, const ArgInspector1 &b, int c, ArgInspector2 *d, ArgAlwaysConverts) { + return a.arg + "\n" + b.arg + "\n" + std::to_string(c) + "\n" + d->arg; + } + static ArgInspector2 h(ArgInspector2 a, ArgAlwaysConverts) { return a; } + }; + py::class_<ArgInspector>(m, "ArgInspector") + .def(py::init<>()) + .def("f", &ArgInspector::f, py::arg(), py::arg() = ArgAlwaysConverts()) + .def("g", &ArgInspector::g, "a"_a.noconvert(), "b"_a, "c"_a.noconvert()=13, "d"_a=ArgInspector2(), py::arg() = ArgAlwaysConverts()) + .def_static("h", &ArgInspector::h, py::arg().noconvert(), py::arg() = ArgAlwaysConverts()) + ; + m.def("arg_inspect_func", [](ArgInspector2 a, ArgInspector1 b, ArgAlwaysConverts) { return a.arg + "\n" + b.arg; }, + py::arg().noconvert(false), py::arg_v(nullptr, ArgInspector1()).noconvert(true), py::arg() = ArgAlwaysConverts()); + + m.def("floats_preferred", [](double f) { return 0.5 * f; }, py::arg("f")); + m.def("floats_only", [](double f) { return 0.5 * f; }, py::arg("f").noconvert()); + + /// Issue/PR #648: bad arg default debugging output +#if !defined(NDEBUG) + m.attr("debug_enabled") = true; +#else + m.attr("debug_enabled") = false; +#endif + m.def("bad_arg_def_named", []{ + auto m = py::module::import("pybind11_tests.issues"); + m.def("should_fail", [](int, NotRegistered) {}, py::arg(), py::arg("a") = NotRegistered()); + }); + m.def("bad_arg_def_unnamed", []{ + auto m = py::module::import("pybind11_tests.issues"); + m.def("should_fail", [](int, NotRegistered) {}, py::arg(), py::arg() = NotRegistered()); + }); }); diff --git a/ext/pybind11/tests/test_methods_and_attributes.py b/ext/pybind11/tests/test_methods_and_attributes.py index 2b0f8d571..f185ac26d 100644 --- a/ext/pybind11/tests/test_methods_and_attributes.py +++ b/ext/pybind11/tests/test_methods_and_attributes.py @@ -33,8 +33,16 @@ def test_methods_and_attributes(): assert instance1.overloaded(1, 1.0) == "(int, float)" assert instance1.overloaded(2.0, 2) == "(float, int)" - assert instance1.overloaded_const(3, 3.0) == "(int, float) const" - assert instance1.overloaded_const(4.0, 4) == "(float, int) const" + assert instance1.overloaded(3, 3) == "(int, int)" + assert instance1.overloaded(4., 4.) == "(float, float)" + assert instance1.overloaded_const(5, 5.0) == "(int, float) const" + assert instance1.overloaded_const(6.0, 6) == "(float, int) const" + assert instance1.overloaded_const(7, 7) == "(int, int) const" + assert instance1.overloaded_const(8., 8.) == "(float, float) const" + assert instance1.overloaded_float(1, 1) == "(float, float)" + assert instance1.overloaded_float(1, 1.) == "(float, float)" + assert instance1.overloaded_float(1., 1) == "(float, float)" + assert instance1.overloaded_float(1., 1.) == "(float, float)" assert instance1.value == 320 instance1.value = 100 @@ -76,19 +84,63 @@ def test_static_properties(): from pybind11_tests import TestProperties as Type assert Type.def_readonly_static == 1 - with pytest.raises(AttributeError): + with pytest.raises(AttributeError) as excinfo: Type.def_readonly_static = 2 + assert "can't set attribute" in str(excinfo) Type.def_readwrite_static = 2 assert Type.def_readwrite_static == 2 assert Type.def_property_readonly_static == 2 - with pytest.raises(AttributeError): + with pytest.raises(AttributeError) as excinfo: Type.def_property_readonly_static = 3 + assert "can't set attribute" in str(excinfo) Type.def_property_static = 3 assert Type.def_property_static == 3 + # Static property read and write via instance + instance = Type() + + Type.def_readwrite_static = 0 + assert Type.def_readwrite_static == 0 + assert instance.def_readwrite_static == 0 + + instance.def_readwrite_static = 2 + assert Type.def_readwrite_static == 2 + assert instance.def_readwrite_static == 2 + + +def test_static_cls(): + """Static property getter and setters expect the type object as the their only argument""" + from pybind11_tests import TestProperties as Type + + instance = Type() + assert Type.static_cls is Type + assert instance.static_cls is Type + + def check_self(self): + assert self is Type + + Type.static_cls = check_self + instance.static_cls = check_self + + +def test_metaclass_override(): + """Overriding pybind11's default metaclass changes the behavior of `static_property`""" + from pybind11_tests import MetaclassOverride + + assert type(ExampleMandA).__name__ == "pybind11_type" + assert type(MetaclassOverride).__name__ == "type" + + assert MetaclassOverride.readonly == 1 + assert type(MetaclassOverride.__dict__["readonly"]).__name__ == "pybind11_static_property" + + # Regular `type` replaces the property instead of calling `__set__()` + MetaclassOverride.readonly = 2 + assert MetaclassOverride.readonly == 2 + assert isinstance(MetaclassOverride.__dict__["readonly"], int) + @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) def test_property_return_value_policies(access): @@ -125,10 +177,19 @@ def test_property_rvalue_policy(): instance = TestPropRVP() o = instance.rvalue assert o.value == 1 + + +def test_property_rvalue_policy_static(): + """When returning an rvalue, the return value policy is automatically changed from + `reference(_internal)` to `move`. The following would not work otherwise. + """ + from pybind11_tests import TestPropRVP o = TestPropRVP.static_rvalue assert o.value == 1 +# https://bitbucket.org/pypy/pypy/issues/2447 +@pytest.unsupported_on_pypy def test_dynamic_attributes(): from pybind11_tests import DynamicClass, CppDerivedDynamicClass @@ -171,6 +232,8 @@ def test_dynamic_attributes(): assert cstats.alive() == 0 +# https://bitbucket.org/pypy/pypy/issues/2447 +@pytest.unsupported_on_pypy def test_cyclic_gc(): from pybind11_tests import DynamicClass @@ -192,3 +255,71 @@ def test_cyclic_gc(): assert cstats.alive() == 2 del i1, i2 assert cstats.alive() == 0 + + +def test_noconvert_args(msg): + from pybind11_tests import ArgInspector, arg_inspect_func, floats_only, floats_preferred + + a = ArgInspector() + assert msg(a.f("hi")) == """ + loading ArgInspector1 argument WITH conversion allowed. Argument value = hi + """ + assert msg(a.g("this is a", "this is b")) == """ + loading ArgInspector1 argument WITHOUT conversion allowed. Argument value = this is a + loading ArgInspector1 argument WITH conversion allowed. Argument value = this is b + 13 + loading ArgInspector2 argument WITH conversion allowed. Argument value = (default arg inspector 2) + """ # noqa: E501 line too long + assert msg(a.g("this is a", "this is b", 42)) == """ + loading ArgInspector1 argument WITHOUT conversion allowed. Argument value = this is a + loading ArgInspector1 argument WITH conversion allowed. Argument value = this is b + 42 + loading ArgInspector2 argument WITH conversion allowed. Argument value = (default arg inspector 2) + """ # noqa: E501 line too long + assert msg(a.g("this is a", "this is b", 42, "this is d")) == """ + loading ArgInspector1 argument WITHOUT conversion allowed. Argument value = this is a + loading ArgInspector1 argument WITH conversion allowed. Argument value = this is b + 42 + loading ArgInspector2 argument WITH conversion allowed. Argument value = this is d + """ + assert (a.h("arg 1") == + "loading ArgInspector2 argument WITHOUT conversion allowed. Argument value = arg 1") + assert msg(arg_inspect_func("A1", "A2")) == """ + loading ArgInspector2 argument WITH conversion allowed. Argument value = A1 + loading ArgInspector1 argument WITHOUT conversion allowed. Argument value = A2 + """ + + assert floats_preferred(4) == 2.0 + assert floats_only(4.0) == 2.0 + with pytest.raises(TypeError) as excinfo: + floats_only(4) + assert msg(excinfo.value) == """ + floats_only(): incompatible function arguments. The following argument types are supported: + 1. (f: float) -> float + + Invoked with: 4 + """ + + +def test_bad_arg_default(msg): + from pybind11_tests import debug_enabled, bad_arg_def_named, bad_arg_def_unnamed + + with pytest.raises(RuntimeError) as excinfo: + bad_arg_def_named() + assert msg(excinfo.value) == ( + "arg(): could not convert default argument 'a: NotRegistered' in function 'should_fail' " + "into a Python object (type not registered yet?)" + if debug_enabled else + "arg(): could not convert default argument into a Python object (type not registered " + "yet?). Compile in debug mode for more information." + ) + + with pytest.raises(RuntimeError) as excinfo: + bad_arg_def_unnamed() + assert msg(excinfo.value) == ( + "arg(): could not convert default argument 'NotRegistered' in function 'should_fail' " + "into a Python object (type not registered yet?)" + if debug_enabled else + "arg(): could not convert default argument into a Python object (type not registered " + "yet?). Compile in debug mode for more information." + ) diff --git a/ext/pybind11/tests/test_modules.py b/ext/pybind11/tests/test_modules.py index fe72f190a..69620949b 100644 --- a/ext/pybind11/tests/test_modules.py +++ b/ext/pybind11/tests/test_modules.py @@ -52,3 +52,11 @@ def test_importing(): assert OD is OrderedDict assert str(OD([(1, 'a'), (2, 'b')])) == "OrderedDict([(1, 'a'), (2, 'b')])" + + +def test_pydoc(): + """Pydoc needs to be able to provide help() for everything inside a pybind11 module""" + import pybind11_tests + import pydoc + + assert pydoc.text.docmodule(pybind11_tests) diff --git a/ext/pybind11/tests/test_multiple_inheritance.cpp b/ext/pybind11/tests/test_multiple_inheritance.cpp index 3cb12b68d..3ebeb202b 100644 --- a/ext/pybind11/tests/test_multiple_inheritance.cpp +++ b/ext/pybind11/tests/test_multiple_inheritance.cpp @@ -10,7 +10,6 @@ #include "pybind11_tests.h" - struct Base1 { Base1(int i) : i(i) { } int foo() { return i; } @@ -32,18 +31,27 @@ struct MIType : Base12 { }; test_initializer multiple_inheritance([](py::module &m) { - py::class_<Base1>(m, "Base1") - .def(py::init<int>()) - .def("foo", &Base1::foo); + py::class_<Base1> b1(m, "Base1"); + b1.def(py::init<int>()) + .def("foo", &Base1::foo); - py::class_<Base2>(m, "Base2") - .def(py::init<int>()) - .def("bar", &Base2::bar); + py::class_<Base2> b2(m, "Base2"); + b2.def(py::init<int>()) + .def("bar", &Base2::bar); py::class_<Base12, Base1, Base2>(m, "Base12"); py::class_<MIType, Base12>(m, "MIType") .def(py::init<int, int>()); + + // Uncommenting this should result in a compile time failure (MI can only be specified via + // template parameters because pybind has to know the types involved; see discussion in #742 for + // details). +// struct Base12v2 : Base1, Base2 { +// Base12v2(int i, int j) : Base1(i), Base2(j) { } +// }; +// py::class_<Base12v2>(m, "Base12v2", b1, b2) +// .def(py::init<int, int>()); }); /* Test the case where not all base classes are specified, @@ -82,3 +90,73 @@ test_initializer multiple_inheritance_nonexplicit([](py::module &m) { m.def("bar_base2a", [](Base2a *b) { return b->bar(); }); m.def("bar_base2a_sharedptr", [](std::shared_ptr<Base2a> b) { return b->bar(); }); }); + +struct Vanilla { + std::string vanilla() { return "Vanilla"; }; +}; + +struct WithStatic1 { + static std::string static_func1() { return "WithStatic1"; }; + static int static_value1; +}; + +struct WithStatic2 { + static std::string static_func2() { return "WithStatic2"; }; + static int static_value2; +}; + +struct WithDict { }; + +struct VanillaStaticMix1 : Vanilla, WithStatic1, WithStatic2 { + static std::string static_func() { return "VanillaStaticMix1"; } + static int static_value; +}; + +struct VanillaStaticMix2 : WithStatic1, Vanilla, WithStatic2 { + static std::string static_func() { return "VanillaStaticMix2"; } + static int static_value; +}; + +struct VanillaDictMix1 : Vanilla, WithDict { }; +struct VanillaDictMix2 : WithDict, Vanilla { }; + +int WithStatic1::static_value1 = 1; +int WithStatic2::static_value2 = 2; +int VanillaStaticMix1::static_value = 12; +int VanillaStaticMix2::static_value = 12; + +test_initializer mi_static_properties([](py::module &pm) { + auto m = pm.def_submodule("mi"); + + py::class_<Vanilla>(m, "Vanilla") + .def(py::init<>()) + .def("vanilla", &Vanilla::vanilla); + + py::class_<WithStatic1>(m, "WithStatic1") + .def(py::init<>()) + .def_static("static_func1", &WithStatic1::static_func1) + .def_readwrite_static("static_value1", &WithStatic1::static_value1); + + py::class_<WithStatic2>(m, "WithStatic2") + .def(py::init<>()) + .def_static("static_func2", &WithStatic2::static_func2) + .def_readwrite_static("static_value2", &WithStatic2::static_value2); + + py::class_<VanillaStaticMix1, Vanilla, WithStatic1, WithStatic2>( + m, "VanillaStaticMix1") + .def(py::init<>()) + .def_static("static_func", &VanillaStaticMix1::static_func) + .def_readwrite_static("static_value", &VanillaStaticMix1::static_value); + + py::class_<VanillaStaticMix2, WithStatic1, Vanilla, WithStatic2>( + m, "VanillaStaticMix2") + .def(py::init<>()) + .def_static("static_func", &VanillaStaticMix2::static_func) + .def_readwrite_static("static_value", &VanillaStaticMix2::static_value); + +#if !defined(PYPY_VERSION) + py::class_<WithDict>(m, "WithDict", py::dynamic_attr()).def(py::init<>()); + py::class_<VanillaDictMix1, Vanilla, WithDict>(m, "VanillaDictMix1").def(py::init<>()); + py::class_<VanillaDictMix2, WithDict, Vanilla>(m, "VanillaDictMix2").def(py::init<>()); +#endif +}); diff --git a/ext/pybind11/tests/test_multiple_inheritance.py b/ext/pybind11/tests/test_multiple_inheritance.py index 581cf5687..7aaab7cd0 100644 --- a/ext/pybind11/tests/test_multiple_inheritance.py +++ b/ext/pybind11/tests/test_multiple_inheritance.py @@ -1,3 +1,4 @@ +import pytest def test_multiple_inheritance_cpp(): @@ -51,6 +52,17 @@ def test_multiple_inheritance_mix2(): assert mt.bar() == 4 +def test_multiple_inheritance_error(): + """Inheriting from multiple C++ bases in Python is not supported""" + from pybind11_tests import Base1, Base2 + + with pytest.raises(TypeError) as excinfo: + # noinspection PyUnusedLocal + class MI(Base1, Base2): + pass + assert "Can't inherit from multiple C++ classes in Python" in str(excinfo.value) + + def test_multiple_inheritance_virtbase(): from pybind11_tests import Base12a, bar_base2a, bar_base2a_sharedptr @@ -62,3 +74,38 @@ def test_multiple_inheritance_virtbase(): assert mt.bar() == 4 assert bar_base2a(mt) == 4 assert bar_base2a_sharedptr(mt) == 4 + + +def test_mi_static_properties(): + """Mixing bases with and without static properties should be possible + and the result should be independent of base definition order""" + from pybind11_tests import mi + + for d in (mi.VanillaStaticMix1(), mi.VanillaStaticMix2()): + assert d.vanilla() == "Vanilla" + assert d.static_func1() == "WithStatic1" + assert d.static_func2() == "WithStatic2" + assert d.static_func() == d.__class__.__name__ + + mi.WithStatic1.static_value1 = 1 + mi.WithStatic2.static_value2 = 2 + assert d.static_value1 == 1 + assert d.static_value2 == 2 + assert d.static_value == 12 + + d.static_value1 = 0 + assert d.static_value1 == 0 + d.static_value2 = 0 + assert d.static_value2 == 0 + d.static_value = 0 + assert d.static_value == 0 + + +@pytest.unsupported_on_pypy +def test_mi_dynamic_attributes(): + """Mixing bases with and without dynamic attribute support""" + from pybind11_tests import mi + + for d in (mi.VanillaDictMix1(), mi.VanillaDictMix2()): + d.dynamic = 1 + assert d.dynamic == 1 diff --git a/ext/pybind11/tests/test_numpy_array.cpp b/ext/pybind11/tests/test_numpy_array.cpp index 14c4c2999..cd6487249 100644 --- a/ext/pybind11/tests/test_numpy_array.cpp +++ b/ext/pybind11/tests/test_numpy_array.cpp @@ -17,6 +17,7 @@ using arr = py::array; using arr_t = py::array_t<uint16_t, 0>; +static_assert(std::is_same<arr_t::value_type, uint16_t>::value, ""); template<typename... Ix> arr data(const arr& a, Ix... index) { return arr(a.nbytes() - a.offset_at(index...), (const uint8_t *) a.data(index...)); @@ -67,6 +68,21 @@ template<typename... Ix> arr_t& mutate_at_t(arr_t& a, Ix... idx) { a.mutable_at( sm.def(#name, [](type a, int i, int j) { return name(a, i, j); }); \ sm.def(#name, [](type a, int i, int j, int k) { return name(a, i, j, k); }); +template <typename T, typename T2> py::handle auxiliaries(T &&r, T2 &&r2) { + if (r.ndim() != 2) throw std::domain_error("error: ndim != 2"); + py::list l; + l.append(*r.data(0, 0)); + l.append(*r2.mutable_data(0, 0)); + l.append(r.data(0, 1) == r2.mutable_data(0, 1)); + l.append(r.ndim()); + l.append(r.itemsize()); + l.append(r.shape(0)); + l.append(r.shape(1)); + l.append(r.size()); + l.append(r.nbytes()); + return l.release(); +} + test_initializer numpy_array([](py::module &m) { auto sm = m.def_submodule("array"); @@ -150,4 +166,102 @@ test_initializer numpy_array([](py::module &m) { "array_t<double>"_a=py::array_t<double>(o) ); }); + + // Overload resolution tests: + sm.def("overloaded", [](py::array_t<double>) { return "double"; }); + sm.def("overloaded", [](py::array_t<float>) { return "float"; }); + sm.def("overloaded", [](py::array_t<int>) { return "int"; }); + sm.def("overloaded", [](py::array_t<unsigned short>) { return "unsigned short"; }); + sm.def("overloaded", [](py::array_t<long long>) { return "long long"; }); + sm.def("overloaded", [](py::array_t<std::complex<double>>) { return "double complex"; }); + sm.def("overloaded", [](py::array_t<std::complex<float>>) { return "float complex"; }); + + sm.def("overloaded2", [](py::array_t<std::complex<double>>) { return "double complex"; }); + sm.def("overloaded2", [](py::array_t<double>) { return "double"; }); + sm.def("overloaded2", [](py::array_t<std::complex<float>>) { return "float complex"; }); + sm.def("overloaded2", [](py::array_t<float>) { return "float"; }); + + // Only accept the exact types: + sm.def("overloaded3", [](py::array_t<int>) { return "int"; }, py::arg().noconvert()); + sm.def("overloaded3", [](py::array_t<double>) { return "double"; }, py::arg().noconvert()); + + // Make sure we don't do unsafe coercion (e.g. float to int) when not using forcecast, but + // rather that float gets converted via the safe (conversion to double) overload: + sm.def("overloaded4", [](py::array_t<long long, 0>) { return "long long"; }); + sm.def("overloaded4", [](py::array_t<double, 0>) { return "double"; }); + + // But we do allow conversion to int if forcecast is enabled (but only if no overload matches + // without conversion) + sm.def("overloaded5", [](py::array_t<unsigned int>) { return "unsigned int"; }); + sm.def("overloaded5", [](py::array_t<double>) { return "double"; }); + + // Issue 685: ndarray shouldn't go to std::string overload + sm.def("issue685", [](std::string) { return "string"; }); + sm.def("issue685", [](py::array) { return "array"; }); + sm.def("issue685", [](py::object) { return "other"; }); + + sm.def("proxy_add2", [](py::array_t<double> a, double v) { + auto r = a.mutable_unchecked<2>(); + for (size_t i = 0; i < r.shape(0); i++) + for (size_t j = 0; j < r.shape(1); j++) + r(i, j) += v; + }, py::arg().noconvert(), py::arg()); + + sm.def("proxy_init3", [](double start) { + py::array_t<double, py::array::c_style> a({ 3, 3, 3 }); + auto r = a.mutable_unchecked<3>(); + for (size_t i = 0; i < r.shape(0); i++) + for (size_t j = 0; j < r.shape(1); j++) + for (size_t k = 0; k < r.shape(2); k++) + r(i, j, k) = start++; + return a; + }); + sm.def("proxy_init3F", [](double start) { + py::array_t<double, py::array::f_style> a({ 3, 3, 3 }); + auto r = a.mutable_unchecked<3>(); + for (size_t k = 0; k < r.shape(2); k++) + for (size_t j = 0; j < r.shape(1); j++) + for (size_t i = 0; i < r.shape(0); i++) + r(i, j, k) = start++; + return a; + }); + sm.def("proxy_squared_L2_norm", [](py::array_t<double> a) { + auto r = a.unchecked<1>(); + double sumsq = 0; + for (size_t i = 0; i < r.shape(0); i++) + sumsq += r[i] * r(i); // Either notation works for a 1D array + return sumsq; + }); + + sm.def("proxy_auxiliaries2", [](py::array_t<double> a) { + auto r = a.unchecked<2>(); + auto r2 = a.mutable_unchecked<2>(); + return auxiliaries(r, r2); + }); + + // Same as the above, but without a compile-time dimensions specification: + sm.def("proxy_add2_dyn", [](py::array_t<double> a, double v) { + auto r = a.mutable_unchecked(); + if (r.ndim() != 2) throw std::domain_error("error: ndim != 2"); + for (size_t i = 0; i < r.shape(0); i++) + for (size_t j = 0; j < r.shape(1); j++) + r(i, j) += v; + }, py::arg().noconvert(), py::arg()); + sm.def("proxy_init3_dyn", [](double start) { + py::array_t<double, py::array::c_style> a({ 3, 3, 3 }); + auto r = a.mutable_unchecked(); + if (r.ndim() != 3) throw std::domain_error("error: ndim != 3"); + for (size_t i = 0; i < r.shape(0); i++) + for (size_t j = 0; j < r.shape(1); j++) + for (size_t k = 0; k < r.shape(2); k++) + r(i, j, k) = start++; + return a; + }); + sm.def("proxy_auxiliaries2_dyn", [](py::array_t<double> a) { + return auxiliaries(a.unchecked(), a.mutable_unchecked()); + }); + + sm.def("array_auxiliaries2", [](py::array_t<double> a) { + return auxiliaries(a, a); + }); }); diff --git a/ext/pybind11/tests/test_numpy_array.py b/ext/pybind11/tests/test_numpy_array.py index 1c218a10b..14c25b371 100644 --- a/ext/pybind11/tests/test_numpy_array.py +++ b/ext/pybind11/tests/test_numpy_array.py @@ -1,5 +1,6 @@ import pytest -import gc + +pytestmark = pytest.requires_numpy with pytest.suppress(ImportError): import numpy as np @@ -7,10 +8,9 @@ with pytest.suppress(ImportError): @pytest.fixture(scope='function') def arr(): - return np.array([[1, 2, 3], [4, 5, 6]], '<u2') + return np.array([[1, 2, 3], [4, 5, 6]], '=u2') -@pytest.requires_numpy def test_array_attributes(): from pybind11_tests.array import ( ndim, shape, strides, writeable, size, itemsize, nbytes, owndata @@ -54,7 +54,6 @@ def test_array_attributes(): assert not owndata(a) -@pytest.requires_numpy @pytest.mark.parametrize('args, ret', [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)]) def test_index_offset(arr, args, ret): from pybind11_tests.array import index_at, index_at_t, offset_at, offset_at_t @@ -64,7 +63,6 @@ def test_index_offset(arr, args, ret): assert offset_at_t(arr, *args) == ret * arr.dtype.itemsize -@pytest.requires_numpy def test_dim_check_fail(arr): from pybind11_tests.array import (index_at, index_at_t, offset_at, offset_at_t, data, data_t, mutate_data, mutate_data_t) @@ -75,7 +73,6 @@ def test_dim_check_fail(arr): assert str(excinfo.value) == 'too many indices for an array: 3 (ndim = 2)' -@pytest.requires_numpy @pytest.mark.parametrize('args, ret', [([], [1, 2, 3, 4, 5, 6]), ([1], [4, 5, 6]), @@ -83,22 +80,21 @@ def test_dim_check_fail(arr): ([1, 2], [6])]) def test_data(arr, args, ret): from pybind11_tests.array import data, data_t + from sys import byteorder assert all(data_t(arr, *args) == ret) - assert all(data(arr, *args)[::2] == ret) - assert all(data(arr, *args)[1::2] == 0) + assert all(data(arr, *args)[(0 if byteorder == 'little' else 1)::2] == ret) + assert all(data(arr, *args)[(1 if byteorder == 'little' else 0)::2] == 0) -@pytest.requires_numpy def test_mutate_readonly(arr): from pybind11_tests.array import mutate_data, mutate_data_t, mutate_at_t arr.flags.writeable = False for func, args in (mutate_data, ()), (mutate_data_t, ()), (mutate_at_t, (0, 0)): - with pytest.raises(RuntimeError) as excinfo: + with pytest.raises(ValueError) as excinfo: func(arr, *args) assert str(excinfo.value) == 'array is not writeable' -@pytest.requires_numpy @pytest.mark.parametrize('dim', [0, 1, 3]) def test_at_fail(arr, dim): from pybind11_tests.array import at_t, mutate_at_t @@ -108,7 +104,6 @@ def test_at_fail(arr, dim): assert str(excinfo.value) == 'index dimension mismatch: {} (ndim = 2)'.format(dim) -@pytest.requires_numpy def test_at(arr): from pybind11_tests.array import at_t, mutate_at_t @@ -119,7 +114,6 @@ def test_at(arr): assert all(mutate_at_t(arr, 1, 0).ravel() == [1, 2, 4, 5, 5, 6]) -@pytest.requires_numpy def test_mutate_data(arr): from pybind11_tests.array import mutate_data, mutate_data_t @@ -136,7 +130,6 @@ def test_mutate_data(arr): assert all(mutate_data_t(arr, 1, 2).ravel() == [6, 19, 27, 68, 84, 197]) -@pytest.requires_numpy def test_bounds_check(arr): from pybind11_tests.array import (index_at, index_at_t, data, data_t, mutate_data, mutate_data_t, at_t, mutate_at_t) @@ -151,7 +144,6 @@ def test_bounds_check(arr): assert str(excinfo.value) == 'index 4 is out of bounds for axis 1 with size 3' -@pytest.requires_numpy def test_make_c_f_array(): from pybind11_tests.array import ( make_c_array, make_f_array @@ -162,11 +154,12 @@ def test_make_c_f_array(): assert not make_f_array().flags.c_contiguous -@pytest.requires_numpy def test_wrap(): from pybind11_tests.array import wrap - def assert_references(a, b): + def assert_references(a, b, base=None): + if base is None: + base = a assert a is not b assert a.__array_interface__['data'][0] == b.__array_interface__['data'][0] assert a.shape == b.shape @@ -178,7 +171,7 @@ def test_wrap(): assert a.flags.updateifcopy == b.flags.updateifcopy assert np.all(a == b) assert not b.flags.owndata - assert b.base is a + assert b.base is base if a.flags.writeable and a.ndim == 2: a[0, 0] = 1234 assert b[0, 0] == 1234 @@ -202,16 +195,15 @@ def test_wrap(): a2 = wrap(a1) assert_references(a1, a2) - a1 = a1.transpose() - a2 = wrap(a1) - assert_references(a1, a2) + a1t = a1.transpose() + a2 = wrap(a1t) + assert_references(a1t, a2, a1) - a1 = a1.diagonal() - a2 = wrap(a1) - assert_references(a1, a2) + a1d = a1.diagonal() + a2 = wrap(a1d) + assert_references(a1d, a2, a1) -@pytest.requires_numpy def test_numpy_view(capture): from pybind11_tests.array import ArrayClass with capture: @@ -220,7 +212,7 @@ def test_numpy_view(capture): ac_view_2 = ac.numpy_view() assert np.all(ac_view_1 == np.array([1, 2], dtype=np.int32)) del ac - gc.collect() + pytest.gc_collect() assert capture == """ ArrayClass() ArrayClass::numpy_view() @@ -233,20 +225,20 @@ def test_numpy_view(capture): with capture: del ac_view_1 del ac_view_2 - gc.collect() + pytest.gc_collect() + pytest.gc_collect() assert capture == """ ~ArrayClass() """ -@pytest.requires_numpy +@pytest.unsupported_on_pypy def test_cast_numpy_int64_to_uint64(): from pybind11_tests.array import function_taking_uint64 function_taking_uint64(123) function_taking_uint64(np.uint64(123)) -@pytest.requires_numpy def test_isinstance(): from pybind11_tests.array import isinstance_untyped, isinstance_typed @@ -254,7 +246,6 @@ def test_isinstance(): assert isinstance_typed(np.array([1.0, 2.0, 3.0])) -@pytest.requires_numpy def test_constructors(): from pybind11_tests.array import default_constructors, converting_constructors @@ -271,3 +262,118 @@ def test_constructors(): assert results["array"].dtype == np.int_ assert results["array_t<int32>"].dtype == np.int32 assert results["array_t<double>"].dtype == np.float64 + + +def test_overload_resolution(msg): + from pybind11_tests.array import overloaded, overloaded2, overloaded3, overloaded4, overloaded5 + + # Exact overload matches: + assert overloaded(np.array([1], dtype='float64')) == 'double' + assert overloaded(np.array([1], dtype='float32')) == 'float' + assert overloaded(np.array([1], dtype='ushort')) == 'unsigned short' + assert overloaded(np.array([1], dtype='intc')) == 'int' + assert overloaded(np.array([1], dtype='longlong')) == 'long long' + assert overloaded(np.array([1], dtype='complex')) == 'double complex' + assert overloaded(np.array([1], dtype='csingle')) == 'float complex' + + # No exact match, should call first convertible version: + assert overloaded(np.array([1], dtype='uint8')) == 'double' + + with pytest.raises(TypeError) as excinfo: + overloaded("not an array") + assert msg(excinfo.value) == """ + overloaded(): incompatible function arguments. The following argument types are supported: + 1. (arg0: numpy.ndarray[float64]) -> str + 2. (arg0: numpy.ndarray[float32]) -> str + 3. (arg0: numpy.ndarray[int32]) -> str + 4. (arg0: numpy.ndarray[uint16]) -> str + 5. (arg0: numpy.ndarray[int64]) -> str + 6. (arg0: numpy.ndarray[complex128]) -> str + 7. (arg0: numpy.ndarray[complex64]) -> str + + Invoked with: 'not an array' + """ + + assert overloaded2(np.array([1], dtype='float64')) == 'double' + assert overloaded2(np.array([1], dtype='float32')) == 'float' + assert overloaded2(np.array([1], dtype='complex64')) == 'float complex' + assert overloaded2(np.array([1], dtype='complex128')) == 'double complex' + assert overloaded2(np.array([1], dtype='float32')) == 'float' + + assert overloaded3(np.array([1], dtype='float64')) == 'double' + assert overloaded3(np.array([1], dtype='intc')) == 'int' + expected_exc = """ + overloaded3(): incompatible function arguments. The following argument types are supported: + 1. (arg0: numpy.ndarray[int32]) -> str + 2. (arg0: numpy.ndarray[float64]) -> str + + Invoked with:""" + + with pytest.raises(TypeError) as excinfo: + overloaded3(np.array([1], dtype='uintc')) + assert msg(excinfo.value) == expected_exc + " array([1], dtype=uint32)" + with pytest.raises(TypeError) as excinfo: + overloaded3(np.array([1], dtype='float32')) + assert msg(excinfo.value) == expected_exc + " array([ 1.], dtype=float32)" + with pytest.raises(TypeError) as excinfo: + overloaded3(np.array([1], dtype='complex')) + assert msg(excinfo.value) == expected_exc + " array([ 1.+0.j])" + + # Exact matches: + assert overloaded4(np.array([1], dtype='double')) == 'double' + assert overloaded4(np.array([1], dtype='longlong')) == 'long long' + # Non-exact matches requiring conversion. Since float to integer isn't a + # save conversion, it should go to the double overload, but short can go to + # either (and so should end up on the first-registered, the long long). + assert overloaded4(np.array([1], dtype='float32')) == 'double' + assert overloaded4(np.array([1], dtype='short')) == 'long long' + + assert overloaded5(np.array([1], dtype='double')) == 'double' + assert overloaded5(np.array([1], dtype='uintc')) == 'unsigned int' + assert overloaded5(np.array([1], dtype='float32')) == 'unsigned int' + + +def test_greedy_string_overload(): # issue 685 + from pybind11_tests.array import issue685 + + assert issue685("abc") == "string" + assert issue685(np.array([97, 98, 99], dtype='b')) == "array" + assert issue685(123) == "other" + + +def test_array_unchecked_fixed_dims(msg): + from pybind11_tests.array import (proxy_add2, proxy_init3F, proxy_init3, proxy_squared_L2_norm, + proxy_auxiliaries2, array_auxiliaries2) + + z1 = np.array([[1, 2], [3, 4]], dtype='float64') + proxy_add2(z1, 10) + assert np.all(z1 == [[11, 12], [13, 14]]) + + with pytest.raises(ValueError) as excinfo: + proxy_add2(np.array([1., 2, 3]), 5.0) + assert msg(excinfo.value) == "array has incorrect number of dimensions: 1; expected 2" + + expect_c = np.ndarray(shape=(3, 3, 3), buffer=np.array(range(3, 30)), dtype='int') + assert np.all(proxy_init3(3.0) == expect_c) + expect_f = np.transpose(expect_c) + assert np.all(proxy_init3F(3.0) == expect_f) + + assert proxy_squared_L2_norm(np.array(range(6))) == 55 + assert proxy_squared_L2_norm(np.array(range(6), dtype="float64")) == 55 + + assert proxy_auxiliaries2(z1) == [11, 11, True, 2, 8, 2, 2, 4, 32] + assert proxy_auxiliaries2(z1) == array_auxiliaries2(z1) + + +def test_array_unchecked_dyn_dims(msg): + from pybind11_tests.array import (proxy_add2_dyn, proxy_init3_dyn, proxy_auxiliaries2_dyn, + array_auxiliaries2) + z1 = np.array([[1, 2], [3, 4]], dtype='float64') + proxy_add2_dyn(z1, 10) + assert np.all(z1 == [[11, 12], [13, 14]]) + + expect_c = np.ndarray(shape=(3, 3, 3), buffer=np.array(range(3, 30)), dtype='int') + assert np.all(proxy_init3_dyn(3.0) == expect_c) + + assert proxy_auxiliaries2_dyn(z1) == [11, 11, True, 2, 8, 2, 2, 4, 32] + assert proxy_auxiliaries2_dyn(z1) == array_auxiliaries2(z1) diff --git a/ext/pybind11/tests/test_numpy_dtypes.cpp b/ext/pybind11/tests/test_numpy_dtypes.cpp index 3894f6a30..1f6c85704 100644 --- a/ext/pybind11/tests/test_numpy_dtypes.cpp +++ b/ext/pybind11/tests/test_numpy_dtypes.cpp @@ -19,23 +19,25 @@ namespace py = pybind11; struct SimpleStruct { - bool x; - uint32_t y; - float z; + bool bool_; + uint32_t uint_; + float float_; + long double ldbl_; }; std::ostream& operator<<(std::ostream& os, const SimpleStruct& v) { - return os << "s:" << v.x << "," << v.y << "," << v.z; + return os << "s:" << v.bool_ << "," << v.uint_ << "," << v.float_ << "," << v.ldbl_; } PYBIND11_PACKED(struct PackedStruct { - bool x; - uint32_t y; - float z; + bool bool_; + uint32_t uint_; + float float_; + long double ldbl_; }); std::ostream& operator<<(std::ostream& os, const PackedStruct& v) { - return os << "p:" << v.x << "," << v.y << "," << v.z; + return os << "p:" << v.bool_ << "," << v.uint_ << "," << v.float_ << "," << v.ldbl_; } PYBIND11_PACKED(struct NestedStruct { @@ -48,10 +50,11 @@ std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { } struct PartialStruct { - bool x; - uint32_t y; - float z; + bool bool_; + uint32_t uint_; + float float_; uint64_t dummy2; + long double ldbl_; }; struct PartialNestedStruct { @@ -99,13 +102,19 @@ py::array mkarray_via_buffer(size_t n) { 1, { n }, { sizeof(T) })); } +#define SET_TEST_VALS(s, i) do { \ + s.bool_ = (i) % 2 != 0; \ + s.uint_ = (uint32_t) (i); \ + s.float_ = (float) (i) * 1.5f; \ + s.ldbl_ = (long double) (i) * -2.5L; } while (0) + template <typename S> py::array_t<S, 0> create_recarray(size_t n) { auto arr = mkarray_via_buffer<S>(n); auto req = arr.request(); auto ptr = static_cast<S*>(req.ptr); for (size_t i = 0; i < n; i++) { - ptr[i].x = i % 2 != 0; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f; + SET_TEST_VALS(ptr[i], i); } return arr; } @@ -119,8 +128,8 @@ py::array_t<NestedStruct, 0> create_nested(size_t n) { auto req = arr.request(); auto ptr = static_cast<NestedStruct*>(req.ptr); for (size_t i = 0; i < n; i++) { - ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; - ptr[i].b.x = (i + 1) % 2 != 0; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f; + SET_TEST_VALS(ptr[i].a, i); + SET_TEST_VALS(ptr[i].b, i + 1); } return arr; } @@ -130,7 +139,7 @@ py::array_t<PartialNestedStruct, 0> create_partial_nested(size_t n) { auto req = arr.request(); auto ptr = static_cast<PartialNestedStruct*>(req.ptr); for (size_t i = 0; i < n; i++) { - ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; + SET_TEST_VALS(ptr[i].a, i); } return arr; } @@ -310,6 +319,22 @@ py::list test_dtype_methods() { return list; } +struct CompareStruct { + bool x; + uint32_t y; + float z; +}; + +py::list test_compare_buffer_info() { + py::list list; + list.append(py::bool_(py::detail::compare_buffer_info<float>::compare(py::buffer_info(nullptr, sizeof(float), "f", 1)))); + list.append(py::bool_(py::detail::compare_buffer_info<unsigned>::compare(py::buffer_info(nullptr, sizeof(int), "I", 1)))); + list.append(py::bool_(py::detail::compare_buffer_info<long>::compare(py::buffer_info(nullptr, sizeof(long), "l", 1)))); + list.append(py::bool_(py::detail::compare_buffer_info<long>::compare(py::buffer_info(nullptr, sizeof(long), sizeof(long) == sizeof(int) ? "i" : "q", 1)))); + list.append(py::bool_(py::detail::compare_buffer_info<CompareStruct>::compare(py::buffer_info(nullptr, sizeof(CompareStruct), "T{?:x:3xI:y:f:z:}", 1)))); + return list; +} + test_initializer numpy_dtypes([](py::module &m) { try { py::module::import("numpy"); @@ -320,20 +345,26 @@ test_initializer numpy_dtypes([](py::module &m) { // typeinfo may be registered before the dtype descriptor for scalar casts to work... py::class_<SimpleStruct>(m, "SimpleStruct"); - PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); - PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); + PYBIND11_NUMPY_DTYPE(PackedStruct, bool_, uint_, float_, ldbl_); PYBIND11_NUMPY_DTYPE(NestedStruct, a, b); - PYBIND11_NUMPY_DTYPE(PartialStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(PartialStruct, bool_, uint_, float_, ldbl_); PYBIND11_NUMPY_DTYPE(PartialNestedStruct, a); PYBIND11_NUMPY_DTYPE(StringStruct, a, b); PYBIND11_NUMPY_DTYPE(EnumStruct, e1, e2); PYBIND11_NUMPY_DTYPE(TrailingPaddingStruct, a, b); + PYBIND11_NUMPY_DTYPE(CompareStruct, x, y, z); // ... or after py::class_<PackedStruct>(m, "PackedStruct"); PYBIND11_NUMPY_DTYPE_EX(StructWithUglyNames, __x__, "x", __y__, "y"); + // If uncommented, this should produce a static_assert failure telling the user that the struct + // is not a POD type +// struct NotPOD { std::string v; NotPOD() : v("hi") {}; }; +// PYBIND11_NUMPY_DTYPE(NotPOD, v); + m.def("create_rec_simple", &create_recarray<SimpleStruct>); m.def("create_rec_packed", &create_recarray<PackedStruct>); m.def("create_rec_nested", &create_nested); @@ -352,12 +383,13 @@ test_initializer numpy_dtypes([](py::module &m) { m.def("test_array_ctors", &test_array_ctors); m.def("test_dtype_ctors", &test_dtype_ctors); m.def("test_dtype_methods", &test_dtype_methods); + m.def("compare_buffer_info", &test_compare_buffer_info); m.def("trailing_padding_dtype", &trailing_padding_dtype); m.def("buffer_to_dtype", &buffer_to_dtype); - m.def("f_simple", [](SimpleStruct s) { return s.y * 10; }); - m.def("f_packed", [](PackedStruct s) { return s.y * 10; }); - m.def("f_nested", [](NestedStruct s) { return s.a.y * 10; }); - m.def("register_dtype", []() { PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); }); + m.def("f_simple", [](SimpleStruct s) { return s.uint_ * 10; }); + m.def("f_packed", [](PackedStruct s) { return s.uint_ * 10; }); + m.def("f_nested", [](NestedStruct s) { return s.a.uint_ * 10; }); + m.def("register_dtype", []() { PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); }); }); #undef PYBIND11_PACKED diff --git a/ext/pybind11/tests/test_numpy_dtypes.py b/ext/pybind11/tests/test_numpy_dtypes.py index 52ebe0ede..f63814f9d 100644 --- a/ext/pybind11/tests/test_numpy_dtypes.py +++ b/ext/pybind11/tests/test_numpy_dtypes.py @@ -1,27 +1,69 @@ import re import pytest +pytestmark = pytest.requires_numpy + with pytest.suppress(ImportError): import numpy as np @pytest.fixture(scope='module') def simple_dtype(): - return np.dtype({'names': ['x', 'y', 'z'], - 'formats': ['?', 'u4', 'f4'], - 'offsets': [0, 4, 8]}) + ld = np.dtype('longdouble') + return np.dtype({'names': ['bool_', 'uint_', 'float_', 'ldbl_'], + 'formats': ['?', 'u4', 'f4', 'f{}'.format(ld.itemsize)], + 'offsets': [0, 4, 8, (16 if ld.alignment > 4 else 12)]}) @pytest.fixture(scope='module') def packed_dtype(): - return np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')]) + return np.dtype([('bool_', '?'), ('uint_', 'u4'), ('float_', 'f4'), ('ldbl_', 'g')]) + + +def dt_fmt(): + from sys import byteorder + e = '<' if byteorder == 'little' else '>' + return ("{{'names':['bool_','uint_','float_','ldbl_']," + " 'formats':['?','" + e + "u4','" + e + "f4','" + e + "f{}']," + " 'offsets':[0,4,8,{}], 'itemsize':{}}}") + + +def simple_dtype_fmt(): + ld = np.dtype('longdouble') + simple_ld_off = 12 + 4 * (ld.alignment > 4) + return dt_fmt().format(ld.itemsize, simple_ld_off, simple_ld_off + ld.itemsize) + + +def packed_dtype_fmt(): + from sys import byteorder + return "[('bool_', '?'), ('uint_', '{e}u4'), ('float_', '{e}f4'), ('ldbl_', '{e}f{}')]".format( + np.dtype('longdouble').itemsize, e='<' if byteorder == 'little' else '>') + + +def partial_ld_offset(): + return 12 + 4 * (np.dtype('uint64').alignment > 4) + 8 + 8 * ( + np.dtype('longdouble').alignment > 8) + + +def partial_dtype_fmt(): + ld = np.dtype('longdouble') + partial_ld_off = partial_ld_offset() + return dt_fmt().format(ld.itemsize, partial_ld_off, partial_ld_off + ld.itemsize) + + +def partial_nested_fmt(): + ld = np.dtype('longdouble') + partial_nested_off = 8 + 8 * (ld.alignment > 8) + partial_ld_off = partial_ld_offset() + partial_nested_size = partial_nested_off * 2 + partial_ld_off + ld.itemsize + return "{{'names':['a'], 'formats':[{}], 'offsets':[{}], 'itemsize':{}}}".format( + partial_dtype_fmt(), partial_nested_off, partial_nested_size) def assert_equal(actual, expected_data, expected_dtype): np.testing.assert_equal(actual, np.array(expected_data, dtype=expected_dtype)) -@pytest.requires_numpy def test_format_descriptors(): from pybind11_tests import get_format_unbound, print_format_descriptors @@ -29,33 +71,40 @@ def test_format_descriptors(): get_format_unbound() assert re.match('^NumPy type info missing for .*UnboundStruct.*$', str(excinfo.value)) + ld = np.dtype('longdouble') + ldbl_fmt = ('4x' if ld.alignment > 4 else '') + ld.char + ss_fmt = "T{?:bool_:3xI:uint_:f:float_:" + ldbl_fmt + ":ldbl_:}" + dbl = np.dtype('double') + partial_fmt = ("T{?:bool_:3xI:uint_:f:float_:" + + str(4 * (dbl.alignment > 4) + dbl.itemsize + 8 * (ld.alignment > 8)) + + "xg:ldbl_:}") + nested_extra = str(max(8, ld.alignment)) assert print_format_descriptors() == [ - "T{?:x:3xI:y:f:z:}", - "T{?:x:=I:y:=f:z:}", - "T{T{?:x:3xI:y:f:z:}:a:T{?:x:=I:y:=f:z:}:b:}", - "T{?:x:3xI:y:f:z:12x}", - "T{8xT{?:x:3xI:y:f:z:12x}:a:8x}", + ss_fmt, + "T{?:bool_:^I:uint_:^f:float_:^g:ldbl_:}", + "T{" + ss_fmt + ":a:T{?:bool_:^I:uint_:^f:float_:^g:ldbl_:}:b:}", + partial_fmt, + "T{" + nested_extra + "x" + partial_fmt + ":a:" + nested_extra + "x}", "T{3s:a:3s:b:}", 'T{q:e1:B:e2:}' ] -@pytest.requires_numpy def test_dtype(simple_dtype): from pybind11_tests import (print_dtypes, test_dtype_ctors, test_dtype_methods, trailing_padding_dtype, buffer_to_dtype) + from sys import byteorder + e = '<' if byteorder == 'little' else '>' assert print_dtypes() == [ - "{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':12}", - "[('x', '?'), ('y', '<u4'), ('z', '<f4')]", - "[('a', {'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8]," - " 'itemsize':12}), ('b', [('x', '?'), ('y', '<u4'), ('z', '<f4')])]", - "{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}", - "{'names':['a'], 'formats':[{'names':['x','y','z'], 'formats':['?','<u4','<f4']," - " 'offsets':[0,4,8], 'itemsize':24}], 'offsets':[8], 'itemsize':40}", + simple_dtype_fmt(), + packed_dtype_fmt(), + "[('a', {}), ('b', {})]".format(simple_dtype_fmt(), packed_dtype_fmt()), + partial_dtype_fmt(), + partial_nested_fmt(), "[('a', 'S3'), ('b', 'S3')]", - "[('e1', '<i8'), ('e2', 'u1')]", - "[('x', 'i1'), ('y', '<u8')]" + "[('e1', '" + e + "i8'), ('e2', 'u1')]", + "[('x', 'i1'), ('y', '" + e + "u8')]" ] d1 = np.dtype({'names': ['a', 'b'], 'formats': ['int32', 'float64'], @@ -70,13 +119,12 @@ def test_dtype(simple_dtype): assert trailing_padding_dtype() == buffer_to_dtype(np.zeros(1, trailing_padding_dtype())) -@pytest.requires_numpy def test_recarray(simple_dtype, packed_dtype): from pybind11_tests import (create_rec_simple, create_rec_packed, create_rec_nested, print_rec_simple, print_rec_packed, print_rec_nested, create_rec_partial, create_rec_partial_nested) - elements = [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)] + elements = [(False, 0, 0.0, -0.0), (True, 1, 1.5, -2.5), (False, 2, 3.0, -5.0)] for func, dtype in [(create_rec_simple, simple_dtype), (create_rec_packed, packed_dtype)]: arr = func(0) @@ -91,15 +139,15 @@ def test_recarray(simple_dtype, packed_dtype): if dtype == simple_dtype: assert print_rec_simple(arr) == [ - "s:0,0,0", - "s:1,1,1.5", - "s:0,2,3" + "s:0,0,0,-0", + "s:1,1,1.5,-2.5", + "s:0,2,3,-5" ] else: assert print_rec_packed(arr) == [ - "p:0,0,0", - "p:1,1,1.5", - "p:0,2,3" + "p:0,0,0,-0", + "p:1,1,1.5,-2.5", + "p:0,2,3,-5" ] nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)]) @@ -110,18 +158,17 @@ def test_recarray(simple_dtype, packed_dtype): arr = create_rec_nested(3) assert arr.dtype == nested_dtype - assert_equal(arr, [((False, 0, 0.0), (True, 1, 1.5)), - ((True, 1, 1.5), (False, 2, 3.0)), - ((False, 2, 3.0), (True, 3, 4.5))], nested_dtype) + assert_equal(arr, [((False, 0, 0.0, -0.0), (True, 1, 1.5, -2.5)), + ((True, 1, 1.5, -2.5), (False, 2, 3.0, -5.0)), + ((False, 2, 3.0, -5.0), (True, 3, 4.5, -7.5))], nested_dtype) assert print_rec_nested(arr) == [ - "n:a=s:0,0,0;b=p:1,1,1.5", - "n:a=s:1,1,1.5;b=p:0,2,3", - "n:a=s:0,2,3;b=p:1,3,4.5" + "n:a=s:0,0,0,-0;b=p:1,1,1.5,-2.5", + "n:a=s:1,1,1.5,-2.5;b=p:0,2,3,-5", + "n:a=s:0,2,3,-5;b=p:1,3,4.5,-7.5" ] arr = create_rec_partial(3) - assert str(arr.dtype) == \ - "{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}" + assert str(arr.dtype) == partial_dtype_fmt() partial_dtype = arr.dtype assert '' not in arr.dtype.fields assert partial_dtype.itemsize > simple_dtype.itemsize @@ -129,16 +176,13 @@ def test_recarray(simple_dtype, packed_dtype): assert_equal(arr, elements, packed_dtype) arr = create_rec_partial_nested(3) - assert str(arr.dtype) == \ - "{'names':['a'], 'formats':[{'names':['x','y','z'], 'formats':['?','<u4','<f4']," \ - " 'offsets':[0,4,8], 'itemsize':24}], 'offsets':[8], 'itemsize':40}" + assert str(arr.dtype) == partial_nested_fmt() assert '' not in arr.dtype.fields assert '' not in arr.dtype.fields['a'][0].fields assert arr.dtype.itemsize > partial_dtype.itemsize np.testing.assert_equal(arr['a'], create_rec_partial(3)) -@pytest.requires_numpy def test_array_constructors(): from pybind11_tests import test_array_ctors @@ -151,7 +195,6 @@ def test_array_constructors(): np.testing.assert_array_equal(test_array_ctors(40 + i), data) -@pytest.requires_numpy def test_string_array(): from pybind11_tests import create_string_array, print_string_array @@ -170,13 +213,14 @@ def test_string_array(): assert dtype == arr.dtype -@pytest.requires_numpy def test_enum_array(): from pybind11_tests import create_enum_array, print_enum_array + from sys import byteorder + e = '<' if byteorder == 'little' else '>' arr = create_enum_array(3) dtype = arr.dtype - assert dtype == np.dtype([('e1', '<i8'), ('e2', 'u1')]) + assert dtype == np.dtype([('e1', e + 'i8'), ('e2', 'u1')]) assert print_enum_array(arr) == [ "e1=A,e2=X", "e1=B,e2=Y", @@ -187,14 +231,12 @@ def test_enum_array(): assert create_enum_array(0).dtype == dtype -@pytest.requires_numpy def test_signature(doc): from pybind11_tests import create_rec_nested assert doc(create_rec_nested) == "create_rec_nested(arg0: int) -> numpy.ndarray[NestedStruct]" -@pytest.requires_numpy def test_scalar_conversion(): from pybind11_tests import (create_rec_simple, f_simple, create_rec_packed, f_packed, @@ -216,10 +258,15 @@ def test_scalar_conversion(): assert 'incompatible function arguments' in str(excinfo.value) -@pytest.requires_numpy def test_register_dtype(): from pybind11_tests import register_dtype with pytest.raises(RuntimeError) as excinfo: register_dtype() assert 'dtype is already registered' in str(excinfo.value) + + +@pytest.requires_numpy +def test_compare_buffer_info(): + from pybind11_tests import compare_buffer_info + assert all(compare_buffer_info()) diff --git a/ext/pybind11/tests/test_numpy_vectorize.cpp b/ext/pybind11/tests/test_numpy_vectorize.cpp index 6d94db2a1..8e951c6e1 100644 --- a/ext/pybind11/tests/test_numpy_vectorize.cpp +++ b/ext/pybind11/tests/test_numpy_vectorize.cpp @@ -38,4 +38,21 @@ test_initializer numpy_vectorize([](py::module &m) { m.def("selective_func", [](py::array_t<int, py::array::c_style>) { return "Int branch taken."; }); m.def("selective_func", [](py::array_t<float, py::array::c_style>) { return "Float branch taken."; }); m.def("selective_func", [](py::array_t<std::complex<float>, py::array::c_style>) { return "Complex float branch taken."; }); + + + // Internal optimization test for whether the input is trivially broadcastable: + py::enum_<py::detail::broadcast_trivial>(m, "trivial") + .value("f_trivial", py::detail::broadcast_trivial::f_trivial) + .value("c_trivial", py::detail::broadcast_trivial::c_trivial) + .value("non_trivial", py::detail::broadcast_trivial::non_trivial); + m.def("vectorized_is_trivial", []( + py::array_t<int, py::array::forcecast> arg1, + py::array_t<float, py::array::forcecast> arg2, + py::array_t<double, py::array::forcecast> arg3 + ) { + size_t ndim; + std::vector<size_t> shape; + std::array<py::buffer_info, 3> buffers {{ arg1.request(), arg2.request(), arg3.request() }}; + return py::detail::broadcast(buffers, ndim, shape); + }); }); diff --git a/ext/pybind11/tests/test_numpy_vectorize.py b/ext/pybind11/tests/test_numpy_vectorize.py index 718646efa..7ae777227 100644 --- a/ext/pybind11/tests/test_numpy_vectorize.py +++ b/ext/pybind11/tests/test_numpy_vectorize.py @@ -1,10 +1,11 @@ import pytest +pytestmark = pytest.requires_numpy + with pytest.suppress(ImportError): import numpy as np -@pytest.requires_numpy def test_vectorize(capture): from pybind11_tests import vectorized_func, vectorized_func2, vectorized_func3 @@ -24,6 +25,20 @@ def test_vectorize(capture): my_func(x:int=3, y:float=4, z:float=3) """ with capture: + a = np.array([[1, 2], [3, 4]], order='F') + b = np.array([[10, 20], [30, 40]], order='F') + c = 3 + result = f(a, b, c) + assert np.allclose(result, a * b * c) + assert result.flags.f_contiguous + # All inputs are F order and full or singletons, so we the result is in col-major order: + assert capture == """ + my_func(x:int=1, y:float=10, z:float=3) + my_func(x:int=3, y:float=30, z:float=3) + my_func(x:int=2, y:float=20, z:float=3) + my_func(x:int=4, y:float=40, z:float=3) + """ + with capture: a, b, c = np.array([[1, 3, 5], [7, 9, 11]]), np.array([[2, 4, 6], [8, 10, 12]]), 3 assert np.allclose(f(a, b, c), a * b * c) assert capture == """ @@ -56,9 +71,37 @@ def test_vectorize(capture): my_func(x:int=5, y:float=3, z:float=2) my_func(x:int=6, y:float=3, z:float=2) """ + with capture: + a, b, c = np.array([[1, 2, 3], [4, 5, 6]], order='F'), np.array([[2], [3]]), 2 + assert np.allclose(f(a, b, c), a * b * c) + assert capture == """ + my_func(x:int=1, y:float=2, z:float=2) + my_func(x:int=2, y:float=2, z:float=2) + my_func(x:int=3, y:float=2, z:float=2) + my_func(x:int=4, y:float=3, z:float=2) + my_func(x:int=5, y:float=3, z:float=2) + my_func(x:int=6, y:float=3, z:float=2) + """ + with capture: + a, b, c = np.array([[1, 2, 3], [4, 5, 6]])[::, ::2], np.array([[2], [3]]), 2 + assert np.allclose(f(a, b, c), a * b * c) + assert capture == """ + my_func(x:int=1, y:float=2, z:float=2) + my_func(x:int=3, y:float=2, z:float=2) + my_func(x:int=4, y:float=3, z:float=2) + my_func(x:int=6, y:float=3, z:float=2) + """ + with capture: + a, b, c = np.array([[1, 2, 3], [4, 5, 6]], order='F')[::, ::2], np.array([[2], [3]]), 2 + assert np.allclose(f(a, b, c), a * b * c) + assert capture == """ + my_func(x:int=1, y:float=2, z:float=2) + my_func(x:int=3, y:float=2, z:float=2) + my_func(x:int=4, y:float=3, z:float=2) + my_func(x:int=6, y:float=3, z:float=2) + """ -@pytest.requires_numpy def test_type_selection(): from pybind11_tests import selective_func @@ -67,10 +110,52 @@ def test_type_selection(): assert selective_func(np.array([1.0j], dtype=np.complex64)) == "Complex float branch taken." -@pytest.requires_numpy def test_docs(doc): from pybind11_tests import vectorized_func assert doc(vectorized_func) == """ - vectorized_func(arg0: numpy.ndarray[int], arg1: numpy.ndarray[float], arg2: numpy.ndarray[float]) -> object + vectorized_func(arg0: numpy.ndarray[int32], arg1: numpy.ndarray[float32], arg2: numpy.ndarray[float64]) -> object """ # noqa: E501 line too long + + +def test_trivial_broadcasting(): + from pybind11_tests import vectorized_is_trivial, trivial, vectorized_func + + assert vectorized_is_trivial(1, 2, 3) == trivial.c_trivial + assert vectorized_is_trivial(np.array(1), np.array(2), 3) == trivial.c_trivial + assert vectorized_is_trivial(np.array([1, 3]), np.array([2, 4]), 3) == trivial.c_trivial + assert trivial.c_trivial == vectorized_is_trivial( + np.array([[1, 3, 5], [7, 9, 11]]), np.array([[2, 4, 6], [8, 10, 12]]), 3) + assert vectorized_is_trivial( + np.array([[1, 2, 3], [4, 5, 6]]), np.array([2, 3, 4]), 2) == trivial.non_trivial + assert vectorized_is_trivial( + np.array([[1, 2, 3], [4, 5, 6]]), np.array([[2], [3]]), 2) == trivial.non_trivial + z1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]], dtype='int32') + z2 = np.array(z1, dtype='float32') + z3 = np.array(z1, dtype='float64') + assert vectorized_is_trivial(z1, z2, z3) == trivial.c_trivial + assert vectorized_is_trivial(1, z2, z3) == trivial.c_trivial + assert vectorized_is_trivial(z1, 1, z3) == trivial.c_trivial + assert vectorized_is_trivial(z1, z2, 1) == trivial.c_trivial + assert vectorized_is_trivial(z1[::2, ::2], 1, 1) == trivial.non_trivial + assert vectorized_is_trivial(1, 1, z1[::2, ::2]) == trivial.c_trivial + assert vectorized_is_trivial(1, 1, z3[::2, ::2]) == trivial.non_trivial + assert vectorized_is_trivial(z1, 1, z3[1::4, 1::4]) == trivial.c_trivial + + y1 = np.array(z1, order='F') + y2 = np.array(y1) + y3 = np.array(y1) + assert vectorized_is_trivial(y1, y2, y3) == trivial.f_trivial + assert vectorized_is_trivial(y1, 1, 1) == trivial.f_trivial + assert vectorized_is_trivial(1, y2, 1) == trivial.f_trivial + assert vectorized_is_trivial(1, 1, y3) == trivial.f_trivial + assert vectorized_is_trivial(y1, z2, 1) == trivial.non_trivial + assert vectorized_is_trivial(z1[1::4, 1::4], y2, 1) == trivial.f_trivial + assert vectorized_is_trivial(y1[1::4, 1::4], z2, 1) == trivial.c_trivial + + assert vectorized_func(z1, z2, z3).flags.c_contiguous + assert vectorized_func(y1, y2, y3).flags.f_contiguous + assert vectorized_func(z1, 1, 1).flags.c_contiguous + assert vectorized_func(1, y2, 1).flags.f_contiguous + assert vectorized_func(z1[1::4, 1::4], y2, 1).flags.f_contiguous + assert vectorized_func(y1[1::4, 1::4], z2, 1).flags.c_contiguous diff --git a/ext/pybind11/tests/test_opaque_types.py b/ext/pybind11/tests/test_opaque_types.py index 7781943b4..1cd410208 100644 --- a/ext/pybind11/tests/test_opaque_types.py +++ b/ext/pybind11/tests/test_opaque_types.py @@ -28,9 +28,10 @@ def test_pointers(msg): print_opaque_list, return_null_str, get_null_str_value, return_unique_ptr, ConstructorStats) + living_before = ConstructorStats.get(ExampleMandA).alive() assert get_void_ptr_value(return_void_ptr()) == 0x1234 assert get_void_ptr_value(ExampleMandA()) # Should also work for other C++ types - assert ConstructorStats.get(ExampleMandA).alive() == 0 + assert ConstructorStats.get(ExampleMandA).alive() == living_before with pytest.raises(TypeError) as excinfo: get_void_ptr_value([1, 2, 3]) # This should not work diff --git a/ext/pybind11/tests/test_operator_overloading.py b/ext/pybind11/tests/test_operator_overloading.py index e0d42391e..02ccb9633 100644 --- a/ext/pybind11/tests/test_operator_overloading.py +++ b/ext/pybind11/tests/test_operator_overloading.py @@ -1,4 +1,3 @@ - def test_operator_overloading(): from pybind11_tests import Vector2, Vector, ConstructorStats diff --git a/ext/pybind11/tests/test_pickling.cpp b/ext/pybind11/tests/test_pickling.cpp index 3941dc593..52b1dbc30 100644 --- a/ext/pybind11/tests/test_pickling.cpp +++ b/ext/pybind11/tests/test_pickling.cpp @@ -57,6 +57,7 @@ test_initializer pickling([](py::module &m) { p.setExtra2(t[2].cast<int>()); }); +#if !defined(PYPY_VERSION) py::class_<PickleableWithDict>(m, "PickleableWithDict", py::dynamic_attr()) .def(py::init<std::string>()) .def_readwrite("value", &PickleableWithDict::value) @@ -70,7 +71,7 @@ test_initializer pickling([](py::module &m) { throw std::runtime_error("Invalid state!"); /* Cast and construct */ auto& p = self.cast<PickleableWithDict&>(); - new (&p) Pickleable(t[0].cast<std::string>()); + new (&p) PickleableWithDict(t[0].cast<std::string>()); /* Assign C++ state */ p.extra = t[1].cast<int>(); @@ -78,4 +79,5 @@ test_initializer pickling([](py::module &m) { /* Assign Python state */ self.attr("__dict__") = t[2]; }); +#endif }); diff --git a/ext/pybind11/tests/test_pickling.py b/ext/pybind11/tests/test_pickling.py index 5e62e1fcc..548c618af 100644 --- a/ext/pybind11/tests/test_pickling.py +++ b/ext/pybind11/tests/test_pickling.py @@ -1,3 +1,5 @@ +import pytest + try: import cPickle as pickle # Use cPickle on Python 2.7 except ImportError: @@ -18,6 +20,7 @@ def test_roundtrip(): assert p2.extra2() == p.extra2() +@pytest.unsupported_on_pypy def test_roundtrip_with_dict(): from pybind11_tests import PickleableWithDict diff --git a/ext/pybind11/tests/test_python_types.cpp b/ext/pybind11/tests/test_python_types.cpp index 33c655b52..5696239b4 100644 --- a/ext/pybind11/tests/test_python_types.cpp +++ b/ext/pybind11/tests/test_python_types.cpp @@ -17,6 +17,11 @@ # include <fcntl.h> #endif +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable: 4127) // warning C4127: Conditional expression is constant +#endif + class ExamplePythonTypes { public: static ExamplePythonTypes *new_instance() { @@ -212,8 +217,7 @@ test_initializer python_types([](py::module &m) { .def("test_print", &ExamplePythonTypes::test_print, "test the print function") .def_static("new_instance", &ExamplePythonTypes::new_instance, "Return an instance") .def_readwrite_static("value", &ExamplePythonTypes::value, "Static value member") - .def_readonly_static("value2", &ExamplePythonTypes::value2, "Static value member (readonly)") - ; + .def_readonly_static("value2", &ExamplePythonTypes::value2, "Static value member (readonly)"); m.def("test_print_function", []() { py::print("Hello, World!"); @@ -327,14 +331,14 @@ test_initializer python_types([](py::module &m) { #ifdef PYBIND11_HAS_EXP_OPTIONAL has_exp_optional = true; - using opt_int = std::experimental::optional<int>; - m.def("double_or_zero_exp", [](const opt_int& x) -> int { + using exp_opt_int = std::experimental::optional<int>; + m.def("double_or_zero_exp", [](const exp_opt_int& x) -> int { return x.value_or(0) * 2; }); - m.def("half_or_none_exp", [](int x) -> opt_int { - return x ? opt_int(x / 2) : opt_int(); + m.def("half_or_none_exp", [](int x) -> exp_opt_int { + return x ? exp_opt_int(x / 2) : exp_opt_int(); }); - m.def("test_nullopt_exp", [](opt_int x) { + m.def("test_nullopt_exp", [](exp_opt_int x) { return x.value_or(42); }, py::arg_v("x", std::experimental::nullopt, "None")); #endif @@ -427,4 +431,65 @@ test_initializer python_types([](py::module &m) { "l"_a=l ); }); + + // Some test characters in utf16 and utf32 encodings. The last one (the π) contains a null byte + char32_t a32 = 0x61 /*a*/, z32 = 0x7a /*z*/, ib32 = 0x203d /*β½*/, cake32 = 0x1f382 /*π*/, mathbfA32 = 0x1d400 /*π*/; + char16_t b16 = 0x62 /*b*/, z16 = 0x7a, ib16 = 0x203d, cake16_1 = 0xd83c, cake16_2 = 0xdf82, mathbfA16_1 = 0xd835, mathbfA16_2 = 0xdc00; + std::wstring wstr; + wstr.push_back(0x61); // a + wstr.push_back(0x2e18); // βΈ + if (sizeof(wchar_t) == 2) { wstr.push_back(mathbfA16_1); wstr.push_back(mathbfA16_2); } // π, utf16 + else { wstr.push_back((wchar_t) mathbfA32); } // π, utf32 + wstr.push_back(0x7a); // z + + m.def("good_utf8_string", []() { return std::string(u8"Say utf8\u203d \U0001f382 \U0001d400"); }); // Say utf8β½ π π + m.def("good_utf16_string", [=]() { return std::u16string({ b16, ib16, cake16_1, cake16_2, mathbfA16_1, mathbfA16_2, z16 }); }); // bβ½ππz + m.def("good_utf32_string", [=]() { return std::u32string({ a32, mathbfA32, cake32, ib32, z32 }); }); // aππβ½z + m.def("good_wchar_string", [=]() { return wstr; }); // aβ½πz + m.def("bad_utf8_string", []() { return std::string("abc\xd0" "def"); }); + m.def("bad_utf16_string", [=]() { return std::u16string({ b16, char16_t(0xd800), z16 }); }); + // Under Python 2.7, invalid unicode UTF-32 characters don't appear to trigger UnicodeDecodeError + if (PY_MAJOR_VERSION >= 3) + m.def("bad_utf32_string", [=]() { return std::u32string({ a32, char32_t(0xd800), z32 }); }); + if (PY_MAJOR_VERSION >= 3 || sizeof(wchar_t) == 2) + m.def("bad_wchar_string", [=]() { return std::wstring({ wchar_t(0x61), wchar_t(0xd800) }); }); + m.def("u8_Z", []() -> char { return 'Z'; }); + m.def("u8_eacute", []() -> char { return '\xe9'; }); + m.def("u16_ibang", [=]() -> char16_t { return ib16; }); + m.def("u32_mathbfA", [=]() -> char32_t { return mathbfA32; }); + m.def("wchar_heart", []() -> wchar_t { return 0x2665; }); + + m.attr("wchar_size") = py::cast(sizeof(wchar_t)); + m.def("ord_char", [](char c) -> int { return static_cast<unsigned char>(c); }); + m.def("ord_char16", [](char16_t c) -> uint16_t { return c; }); + m.def("ord_char32", [](char32_t c) -> uint32_t { return c; }); + m.def("ord_wchar", [](wchar_t c) -> int { return c; }); + + m.def("return_none_string", []() -> std::string * { return nullptr; }); + m.def("return_none_char", []() -> const char * { return nullptr; }); + m.def("return_none_bool", []() -> bool * { return nullptr; }); + m.def("return_none_int", []() -> int * { return nullptr; }); + m.def("return_none_float", []() -> float * { return nullptr; }); + + m.def("return_capsule_with_destructor", + []() { + py::print("creating capsule"); + return py::capsule([]() { + py::print("destructing capsule"); + }); + } + ); + + m.def("return_capsule_with_destructor_2", + []() { + py::print("creating capsule"); + return py::capsule((void *) 1234, [](void *ptr) { + py::print("destructing capsule: {}"_s.format((size_t) ptr)); + }); + } + ); }); + +#if defined(_MSC_VER) +# pragma warning(pop) +#endif diff --git a/ext/pybind11/tests/test_python_types.py b/ext/pybind11/tests/test_python_types.py index 9fe1ef71e..7956c7cc6 100644 --- a/ext/pybind11/tests/test_python_types.py +++ b/ext/pybind11/tests/test_python_types.py @@ -1,8 +1,15 @@ +# Python < 3 needs this: coding=utf-8 import pytest from pybind11_tests import ExamplePythonTypes, ConstructorStats, has_optional, has_exp_optional +def test_repr(): + # In Python 3.3+, repr() accesses __qualname__ + assert "pybind11_type" in repr(type(ExamplePythonTypes)) + assert "ExamplePythonTypes" in repr(ExamplePythonTypes) + + def test_static(): ExamplePythonTypes.value = 15 assert ExamplePythonTypes.value == 15 @@ -132,8 +139,12 @@ def test_instance(capture): assert cstats.alive() == 0 -def test_docs(doc): +# PyPy does not seem to propagate the tp_docs field at the moment +def test_class_docs(doc): assert doc(ExamplePythonTypes) == "Example 2 documentation" + + +def test_method_docs(doc): assert doc(ExamplePythonTypes.get_dict) == """ get_dict(self: m.ExamplePythonTypes) -> dict @@ -400,3 +411,125 @@ def test_implicit_casting(): 'int_i1': 42, 'int_i2': 42, 'int_e': 43, 'int_p': 44 } assert z['l'] == [3, 6, 9, 12, 15] + + +def test_unicode_conversion(): + """Tests unicode conversion and error reporting.""" + import pybind11_tests + from pybind11_tests import (good_utf8_string, bad_utf8_string, + good_utf16_string, bad_utf16_string, + good_utf32_string, # bad_utf32_string, + good_wchar_string, # bad_wchar_string, + u8_Z, u8_eacute, u16_ibang, u32_mathbfA, wchar_heart) + + assert good_utf8_string() == u"Say utf8β½ π π" + assert good_utf16_string() == u"bβ½ππz" + assert good_utf32_string() == u"aππβ½z" + assert good_wchar_string() == u"aβΈπz" + + with pytest.raises(UnicodeDecodeError): + bad_utf8_string() + + with pytest.raises(UnicodeDecodeError): + bad_utf16_string() + + # These are provided only if they actually fail (they don't when 32-bit and under Python 2.7) + if hasattr(pybind11_tests, "bad_utf32_string"): + with pytest.raises(UnicodeDecodeError): + pybind11_tests.bad_utf32_string() + if hasattr(pybind11_tests, "bad_wchar_string"): + with pytest.raises(UnicodeDecodeError): + pybind11_tests.bad_wchar_string() + + assert u8_Z() == 'Z' + assert u8_eacute() == u'Γ©' + assert u16_ibang() == u'β½' + assert u32_mathbfA() == u'π' + assert wchar_heart() == u'β₯' + + +def test_single_char_arguments(): + """Tests failures for passing invalid inputs to char-accepting functions""" + from pybind11_tests import ord_char, ord_char16, ord_char32, ord_wchar, wchar_size + + def toobig_message(r): + return "Character code point not in range({0:#x})".format(r) + toolong_message = "Expected a character, but multi-character string found" + + assert ord_char(u'a') == 0x61 # simple ASCII + assert ord_char(u'Γ©') == 0xE9 # requires 2 bytes in utf-8, but can be stuffed in a char + with pytest.raises(ValueError) as excinfo: + assert ord_char(u'Δ') == 0x100 # requires 2 bytes, doesn't fit in a char + assert str(excinfo.value) == toobig_message(0x100) + with pytest.raises(ValueError) as excinfo: + assert ord_char(u'ab') + assert str(excinfo.value) == toolong_message + + assert ord_char16(u'a') == 0x61 + assert ord_char16(u'Γ©') == 0xE9 + assert ord_char16(u'Δ') == 0x100 + assert ord_char16(u'β½') == 0x203d + assert ord_char16(u'β₯') == 0x2665 + with pytest.raises(ValueError) as excinfo: + assert ord_char16(u'π') == 0x1F382 # requires surrogate pair + assert str(excinfo.value) == toobig_message(0x10000) + with pytest.raises(ValueError) as excinfo: + assert ord_char16(u'aa') + assert str(excinfo.value) == toolong_message + + assert ord_char32(u'a') == 0x61 + assert ord_char32(u'Γ©') == 0xE9 + assert ord_char32(u'Δ') == 0x100 + assert ord_char32(u'β½') == 0x203d + assert ord_char32(u'β₯') == 0x2665 + assert ord_char32(u'π') == 0x1F382 + with pytest.raises(ValueError) as excinfo: + assert ord_char32(u'aa') + assert str(excinfo.value) == toolong_message + + assert ord_wchar(u'a') == 0x61 + assert ord_wchar(u'Γ©') == 0xE9 + assert ord_wchar(u'Δ') == 0x100 + assert ord_wchar(u'β½') == 0x203d + assert ord_wchar(u'β₯') == 0x2665 + if wchar_size == 2: + with pytest.raises(ValueError) as excinfo: + assert ord_wchar(u'π') == 0x1F382 # requires surrogate pair + assert str(excinfo.value) == toobig_message(0x10000) + else: + assert ord_wchar(u'π') == 0x1F382 + with pytest.raises(ValueError) as excinfo: + assert ord_wchar(u'aa') + assert str(excinfo.value) == toolong_message + + +def test_builtins_cast_return_none(): + """Casters produced with PYBIND11_TYPE_CASTER() should convert nullptr to None""" + import pybind11_tests as m + + assert m.return_none_string() is None + assert m.return_none_char() is None + assert m.return_none_bool() is None + assert m.return_none_int() is None + assert m.return_none_float() is None + + +def test_capsule_with_destructor(capture): + import pybind11_tests as m + with capture: + a = m.return_capsule_with_destructor() + del a + pytest.gc_collect() + assert capture.unordered == """ + creating capsule + destructing capsule + """ + + with capture: + a = m.return_capsule_with_destructor_2() + del a + pytest.gc_collect() + assert capture.unordered == """ + creating capsule + destructing capsule: 1234 + """ diff --git a/ext/pybind11/tests/test_sequences_and_iterators.cpp b/ext/pybind11/tests/test_sequences_and_iterators.cpp index 323b4bf00..c2051fadb 100644 --- a/ext/pybind11/tests/test_sequences_and_iterators.cpp +++ b/ext/pybind11/tests/test_sequences_and_iterators.cpp @@ -169,7 +169,49 @@ bool operator==(const NonZeroIterator<std::pair<A, B>>& it, const NonZeroSentine return !(*it).first || !(*it).second; } -test_initializer sequences_and_iterators([](py::module &m) { +template <typename PythonType> +py::list test_random_access_iterator(PythonType x) { + if (x.size() < 5) + throw py::value_error("Please provide at least 5 elements for testing."); + + auto checks = py::list(); + auto assert_equal = [&checks](py::handle a, py::handle b) { + auto result = PyObject_RichCompareBool(a.ptr(), b.ptr(), Py_EQ); + if (result == -1) { throw py::error_already_set(); } + checks.append(result != 0); + }; + + auto it = x.begin(); + assert_equal(x[0], *it); + assert_equal(x[0], it[0]); + assert_equal(x[1], it[1]); + + assert_equal(x[1], *(++it)); + assert_equal(x[1], *(it++)); + assert_equal(x[2], *it); + assert_equal(x[3], *(it += 1)); + assert_equal(x[2], *(--it)); + assert_equal(x[2], *(it--)); + assert_equal(x[1], *it); + assert_equal(x[0], *(it -= 1)); + + assert_equal(it->attr("real"), x[0].attr("real")); + assert_equal((it + 1)->attr("real"), x[1].attr("real")); + + assert_equal(x[1], *(it + 1)); + assert_equal(x[1], *(1 + it)); + it += 3; + assert_equal(x[1], *(it - 2)); + + checks.append(static_cast<std::size_t>(x.end() - x.begin()) == x.size()); + checks.append((x.begin() + static_cast<std::ptrdiff_t>(x.size())) == x.end()); + checks.append(x.begin() < x.end()); + + return checks; +} + +test_initializer sequences_and_iterators([](py::module &pm) { + auto m = pm.def_submodule("sequences_and_iterators"); py::class_<Sequence> seq(m, "Sequence"); @@ -272,4 +314,41 @@ test_initializer sequences_and_iterators([](py::module &m) { On the actual Sequence object, the iterator would be constructed as follows: .def("__iter__", [](py::object s) { return PySequenceIterator(s.cast<const Sequence &>(), s); }) #endif + + m.def("object_to_list", [](py::object o) { + auto l = py::list(); + for (auto item : o) { + l.append(item); + } + return l; + }); + + m.def("iterator_to_list", [](py::iterator it) { + auto l = py::list(); + while (it != py::iterator::sentinel()) { + l.append(*it); + ++it; + } + return l; + }); + + // Make sure that py::iterator works with std algorithms + m.def("count_none", [](py::object o) { + return std::count_if(o.begin(), o.end(), [](py::handle h) { return h.is_none(); }); + }); + + m.def("find_none", [](py::object o) { + auto it = std::find_if(o.begin(), o.end(), [](py::handle h) { return h.is_none(); }); + return it->is_none(); + }); + + m.def("count_nonzeros", [](py::dict d) { + return std::count_if(d.begin(), d.end(), [](std::pair<py::handle, py::handle> p) { + return p.second.cast<int>() != 0; + }); + }); + + m.def("tuple_iterator", [](py::tuple x) { return test_random_access_iterator(x); }); + m.def("list_iterator", [](py::list x) { return test_random_access_iterator(x); }); + m.def("sequence_iterator", [](py::sequence x) { return test_random_access_iterator(x); }); }); diff --git a/ext/pybind11/tests/test_sequences_and_iterators.py b/ext/pybind11/tests/test_sequences_and_iterators.py index 76b9f43f6..30b6aaf4b 100644 --- a/ext/pybind11/tests/test_sequences_and_iterators.py +++ b/ext/pybind11/tests/test_sequences_and_iterators.py @@ -11,7 +11,7 @@ def allclose(a_list, b_list, rel_tol=1e-05, abs_tol=0.0): def test_generalized_iterators(): - from pybind11_tests import IntPairs + from pybind11_tests.sequences_and_iterators import IntPairs assert list(IntPairs([(1, 2), (3, 4), (0, 5)]).nonzero()) == [(1, 2), (3, 4)] assert list(IntPairs([(1, 2), (2, 0), (0, 3), (4, 5)]).nonzero()) == [(1, 2)] @@ -23,7 +23,8 @@ def test_generalized_iterators(): def test_sequence(): - from pybind11_tests import Sequence, ConstructorStats + from pybind11_tests import ConstructorStats + from pybind11_tests.sequences_and_iterators import Sequence cstats = ConstructorStats.get(Sequence) @@ -71,7 +72,7 @@ def test_sequence(): def test_map_iterator(): - from pybind11_tests import StringMap + from pybind11_tests.sequences_and_iterators import StringMap m = StringMap({'hi': 'bye', 'black': 'white'}) assert m['hi'] == 'bye' @@ -88,3 +89,37 @@ def test_map_iterator(): assert m[k] == expected[k] for k, v in m.items(): assert v == expected[k] + + +def test_python_iterator_in_cpp(): + import pybind11_tests.sequences_and_iterators as m + + t = (1, 2, 3) + assert m.object_to_list(t) == [1, 2, 3] + assert m.object_to_list(iter(t)) == [1, 2, 3] + assert m.iterator_to_list(iter(t)) == [1, 2, 3] + + with pytest.raises(TypeError) as excinfo: + m.object_to_list(1) + assert "object is not iterable" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + m.iterator_to_list(1) + assert "incompatible function arguments" in str(excinfo.value) + + def bad_next_call(): + raise RuntimeError("py::iterator::advance() should propagate errors") + + with pytest.raises(RuntimeError) as excinfo: + m.iterator_to_list(iter(bad_next_call, None)) + assert str(excinfo.value) == "py::iterator::advance() should propagate errors" + + l = [1, None, 0, None] + assert m.count_none(l) == 2 + assert m.find_none(l) is True + assert m.count_nonzeros({"a": 0, "b": 1, "c": 2}) == 2 + + r = range(5) + assert all(m.tuple_iterator(tuple(r))) + assert all(m.list_iterator(list(r))) + assert all(m.sequence_iterator(r)) diff --git a/ext/pybind11/tests/test_smart_ptr.cpp b/ext/pybind11/tests/test_smart_ptr.cpp index 07c3cb066..83c1c018a 100644 --- a/ext/pybind11/tests/test_smart_ptr.cpp +++ b/ext/pybind11/tests/test_smart_ptr.cpp @@ -82,10 +82,22 @@ private: }; /// Make pybind aware of the ref-counted wrapper type (s) -PYBIND11_DECLARE_HOLDER_TYPE(T, ref<T>); // Required for custom holder type + +// ref<T> is a wrapper for 'Object' which uses intrusive reference counting +// It is always possible to construct a ref<T> from an Object* pointer without +// possible incosistencies, hence the 'true' argument at the end. +PYBIND11_DECLARE_HOLDER_TYPE(T, ref<T>, true); PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>); // Not required any more for std::shared_ptr, // but it should compile without error +// Make pybind11 aware of the non-standard getter member function +namespace pybind11 { namespace detail { + template <typename T> + struct holder_helper<ref<T>> { + static const T *get(const ref<T> &p) { return p.get_ptr(); } + }; +}} + Object *make_object_1() { return new MyObject1(1); } ref<Object> make_object_2() { return new MyObject1(2); } @@ -125,6 +137,18 @@ test_initializer smart_ptr([](py::module &m) { py::class_<MyObject1, ref<MyObject1>>(m, "MyObject1", obj) .def(py::init<int>()); + m.def("test_object1_refcounting", + []() -> bool { + ref<MyObject1> o = new MyObject1(0); + bool good = o->getRefCount() == 1; + py::object o2 = py::cast(o, py::return_value_policy::reference); + // always request (partial) ownership for objects with intrusive + // reference counting even when using the 'reference' RVP + good &= o->getRefCount() == 2; + return good; + } + ); + m.def("make_object_1", &make_object_1); m.def("make_object_2", &make_object_2); m.def("make_myobject1_1", &make_myobject1_1); @@ -190,6 +214,18 @@ struct SharedFromThisRef { std::shared_ptr<B> shared = std::make_shared<B>(); }; +template <typename T> +class CustomUniquePtr { + std::unique_ptr<T> impl; + +public: + CustomUniquePtr(T* p) : impl(p) { } + T* get() const { return impl.get(); } + T* release_ptr() { return impl.release(); } +}; + +PYBIND11_DECLARE_HOLDER_TYPE(T, CustomUniquePtr<T>); + test_initializer smart_ptr_and_references([](py::module &pm) { auto m = pm.def_submodule("smart_ptr"); @@ -221,4 +257,18 @@ test_initializer smart_ptr_and_references([](py::module &pm) { py::return_value_policy::copy) .def("set_ref", [](SharedFromThisRef &, const B &) { return true; }) .def("set_holder", [](SharedFromThisRef &, std::shared_ptr<B>) { return true; }); + + struct C { + C() { print_created(this); } + ~C() { print_destroyed(this); } + }; + + py::class_<C, CustomUniquePtr<C>>(m, "TypeWithMoveOnlyHolder") + .def_static("make", []() { return CustomUniquePtr<C>(new C); }); + + struct HeldByDefaultHolder { }; + + py::class_<HeldByDefaultHolder>(m, "HeldByDefaultHolder") + .def(py::init<>()) + .def_static("load_shared_ptr", [](std::shared_ptr<HeldByDefaultHolder>) {}); }); diff --git a/ext/pybind11/tests/test_smart_ptr.py b/ext/pybind11/tests/test_smart_ptr.py index 3a33e1761..b5af3bd38 100644 --- a/ext/pybind11/tests/test_smart_ptr.py +++ b/ext/pybind11/tests/test_smart_ptr.py @@ -116,6 +116,11 @@ def test_smart_ptr(capture): assert cstats.move_assignments == 0 +def test_smart_ptr_refcounting(): + from pybind11_tests import test_object1_refcounting + assert test_object1_refcounting() + + def test_unique_nodelete(): from pybind11_tests import MyObject4 o = MyObject4(23) @@ -196,3 +201,22 @@ def test_shared_ptr_from_this_and_references(): del ref, bad_wp, copy, holder_ref, holder_copy, s assert stats.alive() == 0 + + +def test_move_only_holder(): + from pybind11_tests.smart_ptr import TypeWithMoveOnlyHolder + + a = TypeWithMoveOnlyHolder.make() + stats = ConstructorStats.get(TypeWithMoveOnlyHolder) + assert stats.alive() == 1 + del a + assert stats.alive() == 0 + + +def test_smart_ptr_from_default(): + from pybind11_tests.smart_ptr import HeldByDefaultHolder + + instance = HeldByDefaultHolder() + with pytest.raises(RuntimeError) as excinfo: + HeldByDefaultHolder.load_shared_ptr(instance) + assert "Unable to load a custom holder type from a default-holder instance" in str(excinfo) diff --git a/ext/pybind11/tests/test_stl_binders.cpp b/ext/pybind11/tests/test_stl_binders.cpp index b9b56c15d..f636c0b55 100644 --- a/ext/pybind11/tests/test_stl_binders.cpp +++ b/ext/pybind11/tests/test_stl_binders.cpp @@ -10,10 +10,16 @@ #include "pybind11_tests.h" #include <pybind11/stl_bind.h> +#include <pybind11/numpy.h> #include <map> #include <deque> #include <unordered_map> +#ifdef _MSC_VER +// We get some really long type names here which causes MSVC to emit warnings +# pragma warning(disable: 4503) // warning C4503: decorated name length exceeded, name was truncated +#endif + class El { public: El() = delete; @@ -53,17 +59,45 @@ template <class Map> Map *times_ten(int n) { return m; } +struct VStruct { + bool w; + uint32_t x; + double y; + bool z; +}; + +struct VUndeclStruct { //dtype not declared for this version + bool w; + uint32_t x; + double y; + bool z; +}; + test_initializer stl_binder_vector([](py::module &m) { py::class_<El>(m, "El") .def(py::init<int>()); - py::bind_vector<std::vector<unsigned int>>(m, "VectorInt"); + py::bind_vector<std::vector<unsigned char>>(m, "VectorUChar", py::buffer_protocol()); + py::bind_vector<std::vector<unsigned int>>(m, "VectorInt", py::buffer_protocol()); py::bind_vector<std::vector<bool>>(m, "VectorBool"); py::bind_vector<std::vector<El>>(m, "VectorEl"); py::bind_vector<std::vector<std::vector<El>>>(m, "VectorVectorEl"); + m.def("create_undeclstruct", [m] () mutable { + py::bind_vector<std::vector<VUndeclStruct>>(m, "VectorUndeclStruct", py::buffer_protocol()); + }); + + try { + py::module::import("numpy"); + } catch (...) { + return; + } + PYBIND11_NUMPY_DTYPE(VStruct, w, x, y, z); + py::class_<VStruct>(m, "VStruct").def_readwrite("x", &VStruct::x); + py::bind_vector<std::vector<VStruct>>(m, "VectorStruct", py::buffer_protocol()); + m.def("get_vectorstruct", [] {return std::vector<VStruct> {{0, 5, 3.0, 1}, {1, 30, -1e4, 0}};}); }); test_initializer stl_binder_map([](py::module &m) { @@ -92,4 +126,3 @@ test_initializer stl_binder_noncopyable([](py::module &m) { py::bind_map<std::unordered_map<int, E_nc>>(m, "UmapENC"); m.def("get_umnc", ×_ten<std::unordered_map<int, E_nc>>, py::return_value_policy::reference); }); - diff --git a/ext/pybind11/tests/test_stl_binders.py b/ext/pybind11/tests/test_stl_binders.py index c9bcc7935..0edf9e26e 100644 --- a/ext/pybind11/tests/test_stl_binders.py +++ b/ext/pybind11/tests/test_stl_binders.py @@ -1,3 +1,10 @@ +import pytest +import sys + +with pytest.suppress(ImportError): + import numpy as np + + def test_vector_int(): from pybind11_tests import VectorInt @@ -26,6 +33,57 @@ def test_vector_int(): assert v_int2 == VectorInt([0, 99, 2, 3]) +@pytest.unsupported_on_pypy +def test_vector_buffer(): + from pybind11_tests import VectorUChar, create_undeclstruct + b = bytearray([1, 2, 3, 4]) + v = VectorUChar(b) + assert v[1] == 2 + v[2] = 5 + m = memoryview(v) # We expose the buffer interface + if sys.version_info.major > 2: + assert m[2] == 5 + m[2] = 6 + else: + assert m[2] == '\x05' + m[2] = '\x06' + assert v[2] == 6 + + with pytest.raises(RuntimeError): + create_undeclstruct() # Undeclared struct contents, no buffer interface + + +@pytest.requires_numpy +def test_vector_buffer_numpy(): + from pybind11_tests import VectorInt, VectorStruct, get_vectorstruct + + a = np.array([1, 2, 3, 4], dtype=np.int32) + with pytest.raises(TypeError): + VectorInt(a) + + a = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], dtype=np.uintc) + v = VectorInt(a[0, :]) + assert len(v) == 4 + assert v[2] == 3 + m = np.asarray(v) + m[2] = 5 + assert v[2] == 5 + + v = VectorInt(a[:, 1]) + assert len(v) == 3 + assert v[2] == 10 + + v = get_vectorstruct() + assert v[0].x == 5 + m = np.asarray(v) + m[1]['x'] = 99 + assert v[1].x == 99 + + v = VectorStruct(np.zeros(3, dtype=np.dtype([('w', 'bool'), ('x', 'I'), + ('y', 'float64'), ('z', 'bool')], align=True))) + assert len(v) == 3 + + def test_vector_custom(): from pybind11_tests import El, VectorEl, VectorVectorEl diff --git a/ext/pybind11/tests/test_virtual_functions.py b/ext/pybind11/tests/test_virtual_functions.py index a9aecd67f..b11c699df 100644 --- a/ext/pybind11/tests/test_virtual_functions.py +++ b/ext/pybind11/tests/test_virtual_functions.py @@ -206,6 +206,9 @@ def test_inheriting_repeat(): assert obj.say_everything() == "BT -7" +# PyPy: Reference count > 1 causes call with noncopyable instance +# to fail in ncv1.print_nc() +@pytest.unsupported_on_pypy @pytest.mark.skipif(not hasattr(pybind11_tests, 'NCVirt'), reason="NCVirt test broken on ICPC") def test_move_support(): diff --git a/ext/pybind11/tools/pybind11Config.cmake.in b/ext/pybind11/tools/pybind11Config.cmake.in index 23588c300..ae6d2daee 100644 --- a/ext/pybind11/tools/pybind11Config.cmake.in +++ b/ext/pybind11/tools/pybind11Config.cmake.in @@ -26,12 +26,12 @@ # python detection and PYBIND11_CPP_STANDARD (-std=c++11 or -std=c++14) to # influence standard setting. :: # -# pybind11::pybind11 - the main pybind11 interface library (i.e., headers) +# pybind11::module - the main pybind11 interface library for extension modules (i.e., headers) # # find_package(pybind11 CONFIG REQUIRED) # message(STATUS "Found pybind11: ${pybind11_INCLUDE_DIR} (found version ${pybind11_VERSION} & Py${PYTHON_VERSION_STRING})") # add_library(mylib MODULE main.cpp) -# target_link_libraries(mylib pybind11::pybind11) +# target_link_libraries(mylib pybind11::module) # # Suggested usage:: # @@ -71,21 +71,21 @@ if(NOT (CMAKE_VERSION VERSION_LESS 3.0)) # Don't include targets if this file is being picked up by another # project which has already built this as a subproject #----------------------------------------------------------------------------- -if(NOT TARGET ${PN}::pybind11) +if(NOT TARGET ${PN}::module) include("${CMAKE_CURRENT_LIST_DIR}/${PN}Targets.cmake") find_package(PythonLibsNew ${PYBIND11_PYTHON_VERSION} MODULE REQUIRED) - set_property(TARGET ${PN}::pybind11 APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PYTHON_INCLUDE_DIRS}) + set_property(TARGET ${PN}::module APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PYTHON_INCLUDE_DIRS}) if(WIN32 OR CYGWIN) - set_property(TARGET ${PN}::pybind11 APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${PYTHON_LIBRARIES}) + set_property(TARGET ${PN}::module APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${PYTHON_LIBRARIES}) endif() select_cxx_standard() - set_property(TARGET ${PN}::pybind11 APPEND PROPERTY INTERFACE_COMPILE_OPTIONS "${PYBIND11_CPP_STANDARD}") + set_property(TARGET ${PN}::module APPEND PROPERTY INTERFACE_COMPILE_OPTIONS "${PYBIND11_CPP_STANDARD}") - get_property(_iid TARGET ${PN}::pybind11 PROPERTY INTERFACE_INCLUDE_DIRECTORIES) - get_property(_ill TARGET ${PN}::pybind11 PROPERTY INTERFACE_LINK_LIBRARIES) - get_property(_ico TARGET ${PN}::pybind11 PROPERTY INTERFACE_COMPILE_OPTIONS) + get_property(_iid TARGET ${PN}::module PROPERTY INTERFACE_INCLUDE_DIRECTORIES) + get_property(_ill TARGET ${PN}::module PROPERTY INTERFACE_LINK_LIBRARIES) + get_property(_ico TARGET ${PN}::module PROPERTY INTERFACE_COMPILE_OPTIONS) set(${PN}_INCLUDE_DIRS ${_iid}) set(${PN}_LIBRARIES ${_ico} ${_ill}) endif() diff --git a/ext/pybind11/tools/pybind11Tools.cmake b/ext/pybind11/tools/pybind11Tools.cmake index 4922982fa..3fbffeeff 100644 --- a/ext/pybind11/tools/pybind11Tools.cmake +++ b/ext/pybind11/tools/pybind11Tools.cmake @@ -8,12 +8,15 @@ cmake_minimum_required(VERSION 2.8.12) # Add a CMake parameter for choosing a desired Python version -set(PYBIND11_PYTHON_VERSION "" CACHE STRING "Python version to use for compiling modules") +if(NOT PYBIND11_PYTHON_VERSION) + set(PYBIND11_PYTHON_VERSION "" CACHE STRING "Python version to use for compiling modules") +endif() set(Python_ADDITIONAL_VERSIONS 3.7 3.6 3.5 3.4) find_package(PythonLibsNew ${PYBIND11_PYTHON_VERSION} REQUIRED) include(CheckCXXCompilerFlag) +include(CMakeParseArguments) function(select_cxx_standard) if(NOT MSVC AND NOT PYBIND11_CPP_STANDARD) @@ -33,29 +36,95 @@ function(select_cxx_standard) endif() endfunction() +# Checks whether the given CXX/linker flags can compile and link a cxx file. cxxflags and +# linkerflags are lists of flags to use. The result variable is a unique variable name for each set +# of flags: the compilation result will be cached base on the result variable. If the flags work, +# sets them in cxxflags_out/linkerflags_out internal cache variables (in addition to ${result}). +function(_pybind11_return_if_cxx_and_linker_flags_work result cxxflags linkerflags cxxflags_out linkerflags_out) + set(CMAKE_REQUIRED_LIBRARIES ${linkerflags}) + check_cxx_compiler_flag("${cxxflags}" ${result}) + if (${result}) + set(${cxxflags_out} "${cxxflags}" CACHE INTERNAL "" FORCE) + set(${linkerflags_out} "${linkerflags}" CACHE INTERNAL "" FORCE) + endif() +endfunction() + +# Internal: find the appropriate link time optimization flags for this compiler +function(_pybind11_add_lto_flags target_name prefer_thin_lto) + if (NOT DEFINED PYBIND11_LTO_CXX_FLAGS) + set(PYBIND11_LTO_CXX_FLAGS "" CACHE INTERNAL "") + set(PYBIND11_LTO_LINKER_FLAGS "" CACHE INTERNAL "") + + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(cxx_append "") + set(linker_append "") + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT APPLE) + # Clang Gold plugin does not support -Os; append -O3 to MinSizeRel builds to override it + set(linker_append ";$<$<CONFIG:MinSizeRel>:-O3>") + elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + set(cxx_append ";-fno-fat-lto-objects") + endif() + + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND prefer_thin_lto) + _pybind11_return_if_cxx_and_linker_flags_work(HAS_FLTO_THIN + "-flto=thin${cxx_append}" "-flto=thin${linker_append}" + PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() + + if (NOT HAS_FLTO_THIN) + _pybind11_return_if_cxx_and_linker_flags_work(HAS_FLTO + "-flto${cxx_append}" "-flto${linker_append}" + PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() + elseif (CMAKE_CXX_COMPILER_ID MATCHES "Intel") + # Intel equivalent to LTO is called IPO + _pybind11_return_if_cxx_and_linker_flags_work(HAS_INTEL_IPO + "-ipo" "-ipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + elseif(MSVC) + # cmake only interprets libraries as linker flags when they start with a - (otherwise it + # converts /LTCG to \LTCG as if it was a Windows path). Luckily MSVC supports passing flags + # with - instead of /, even if it is a bit non-standard: + _pybind11_return_if_cxx_and_linker_flags_work(HAS_MSVC_GL_LTCG + "/GL" "-LTCG" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() + + if (PYBIND11_LTO_CXX_FLAGS) + message(STATUS "LTO enabled") + else() + message(STATUS "LTO disabled (not supported by the compiler and/or linker)") + endif() + endif() + + # Enable LTO flags if found, except for Debug builds + if (PYBIND11_LTO_CXX_FLAGS) + target_compile_options(${target_name} PRIVATE "$<$<NOT:$<CONFIG:Debug>>:${PYBIND11_LTO_CXX_FLAGS}>") + endif() + if (PYBIND11_LTO_LINKER_FLAGS) + target_link_libraries(${target_name} PRIVATE "$<$<NOT:$<CONFIG:Debug>>:${PYBIND11_LTO_LINKER_FLAGS}>") + endif() +endfunction() + # Build a Python extension module: -# pybind11_add_module(<name> [MODULE | SHARED] [EXCLUDE_FROM_ALL] source1 [source2 ...]) +# pybind11_add_module(<name> [MODULE | SHARED] [EXCLUDE_FROM_ALL] +# [NO_EXTRAS] [THIN_LTO] source1 [source2 ...]) # function(pybind11_add_module target_name) - set(lib_type "MODULE") - set(do_lto True) - set(exclude_from_all "") - set(sources "") - - set(_args_to_try "${ARGN}") - foreach(_ex_arg IN LISTS _args_to_try) - if(${_ex_arg} STREQUAL "MODULE") - set(lib_type "MODULE") - elseif(${_ex_arg} STREQUAL "SHARED") - set(lib_type "SHARED") - elseif(${_ex_arg} STREQUAL "EXCLUDE_FROM_ALL") - set(exclude_from_all "EXCLUDE_FROM_ALL") - else() - list(APPEND sources "${_ex_arg}") - endif() - endforeach() + set(options MODULE SHARED EXCLUDE_FROM_ALL NO_EXTRAS THIN_LTO) + cmake_parse_arguments(ARG "${options}" "" "" ${ARGN}) + + if(ARG_MODULE AND ARG_SHARED) + message(FATAL_ERROR "Can't be both MODULE and SHARED") + elseif(ARG_SHARED) + set(lib_type SHARED) + else() + set(lib_type MODULE) + endif() - add_library(${target_name} ${lib_type} ${exclude_from_all} ${sources}) + if(ARG_EXCLUDE_FROM_ALL) + set(exclude_from_all EXCLUDE_FROM_ALL) + endif() + + add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS}) target_include_directories(${target_name} PRIVATE ${PYBIND11_INCLUDE_DIR} # from project CMakeLists.txt @@ -85,56 +154,44 @@ function(pybind11_add_module target_name) # import time. target_link_libraries(${target_name} PRIVATE "-undefined dynamic_lookup") + + if(ARG_SHARED) + # Suppress CMake >= 3.0 warning for shared libraries + set_target_properties(${target_name} PROPERTIES MACOSX_RPATH ON) + endif() endif() select_cxx_standard() if(NOT MSVC) # Make sure C++11/14 are enabled target_compile_options(${target_name} PUBLIC ${PYBIND11_CPP_STANDARD}) + endif() - # Enable link time optimization and set the default symbol - # visibility to hidden (very important to obtain small binaries) - string(TOUPPER "${CMAKE_BUILD_TYPE}" U_CMAKE_BUILD_TYPE) - if (NOT ${U_CMAKE_BUILD_TYPE} MATCHES DEBUG) - # Check for Link Time Optimization support (GCC/Clang) - check_cxx_compiler_flag("-flto" HAS_LTO_FLAG) - if(HAS_LTO_FLAG AND NOT CYGWIN) - target_compile_options(${target_name} PRIVATE -flto) - endif() + if(ARG_NO_EXTRAS) + return() + endif() - # Intel equivalent to LTO is called IPO - if(CMAKE_CXX_COMPILER_ID MATCHES "Intel") - check_cxx_compiler_flag("-ipo" HAS_IPO_FLAG) - if(HAS_IPO_FLAG) - target_compile_options(${target_name} PRIVATE -ipo) - endif() - endif() + _pybind11_add_lto_flags(${target_name} ${ARG_THIN_LTO}) + + if (NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug) + # Set the default symbol visibility to hidden (very important to obtain small binaries) + target_compile_options(${target_name} PRIVATE "-fvisibility=hidden") - # Default symbol visibility - target_compile_options(${target_name} PRIVATE "-fvisibility=hidden") - - # Strip unnecessary sections of the binary on Linux/Mac OS - if(CMAKE_STRIP) - if(APPLE) - add_custom_command(TARGET ${target_name} POST_BUILD - COMMAND ${CMAKE_STRIP} -u -r $<TARGET_FILE:${target_name}>) - else() - add_custom_command(TARGET ${target_name} POST_BUILD - COMMAND ${CMAKE_STRIP} $<TARGET_FILE:${target_name}>) - endif() + # Strip unnecessary sections of the binary on Linux/Mac OS + if(CMAKE_STRIP) + if(APPLE) + add_custom_command(TARGET ${target_name} POST_BUILD + COMMAND ${CMAKE_STRIP} -x $<TARGET_FILE:${target_name}>) + else() + add_custom_command(TARGET ${target_name} POST_BUILD + COMMAND ${CMAKE_STRIP} $<TARGET_FILE:${target_name}>) endif() endif() - elseif(MSVC) + endif() + + if(MSVC) # /MP enables multithreaded builds (relevant when there are many files), /bigobj is # needed for bigger binding projects due to the limit to 64k addressable sections target_compile_options(${target_name} PRIVATE /MP /bigobj) - - # Enforce link time code generation on MSVC, except in debug mode - target_compile_options(${target_name} PRIVATE $<$<NOT:$<CONFIG:Debug>>:/GL>) - - # Fancy generator expressions don't work with linker flags, for reasons unknown - set_property(TARGET ${target_name} APPEND_STRING PROPERTY LINK_FLAGS_RELEASE /LTCG) - set_property(TARGET ${target_name} APPEND_STRING PROPERTY LINK_FLAGS_MINSIZEREL /LTCG) - set_property(TARGET ${target_name} APPEND_STRING PROPERTY LINK_FLAGS_RELWITHDEBINFO /LTCG) endif() endfunction() |