Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NumPy scalars #2060

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions docs/advanced/pycpp/numpy.rst
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,42 @@ prevent many types of unsupported structures, it is still the user's
responsibility to use only "plain" structures that can be safely manipulated as
raw memory without violating invariants.

Scalar types
============

In some cases we may want to accept or return NumPy scalar values such as
``np.float32`` or ``np.uint16``. It is especially important in case of
C-side single-precision floats which by default will be bound to Python's
double-precision builtin floats, causing mismatch in float precision.

Luckily, there's a helper type for this occasion - ``py::numpy_scalar``:

.. code-block:: cpp

m.def("square_float", [](py::numpy_scalar<float> value) {
float v = value;
return py::make_scalar(v * v);
});

This type is trivially convertible to and from the type it wraps; currently
aldanor marked this conversation as resolved.
Show resolved Hide resolved
supported scalar types are NumPy arithmetic types: ``bool_``, ``int8``,
``int8``, ``int16``, ``int32``, ``int64``, ``uint8``, ``uint16``, ``uint32``,
``uint64``, ``float32``, ``float64``, ``complex64``, ``complex128``, all of
them mapping to respective C++ counterparts.

.. note::

This is a strict type, it will only allow input arguments of the specified
NumPy type and nothing else (e.g., ``py::numpy_scalar<int64_t>`` will not
accept built-in ``int`` or any other type for that matter).

.. note::

Native C types are mapped to NumPy types in a platform specific way: for
instance, ``char`` may be mapped to either ``np.int8`` or ``np.uint8``
depending on the platform. If you want to ensure specific NumPy types,
it is recommended to use fixed-width aliases from ``<cstdint>>``.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo:

Suggested change
it is recommended to use fixed-width aliases from ``<cstdint>>``.
it is recommended to use fixed-width aliases from ``<cstdint>``.


Vectorizing functions
=====================

Expand Down
186 changes: 147 additions & 39 deletions include/pybind11/numpy.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include <numeric>
#include <algorithm>
#include <array>
#include <complex>
#include <cstdint>
#include <cstdlib>
#include <cstring>
Expand All @@ -39,6 +40,8 @@ NAMESPACE_BEGIN(PYBIND11_NAMESPACE)

class array; // Forward declaration

template<typename> struct numpy_scalar; // Forward declaration

NAMESPACE_BEGIN(detail)
template <typename type, typename SFINAE = void> struct npy_format_descriptor;

Expand Down Expand Up @@ -109,16 +112,12 @@ inline numpy_internals& get_numpy_internals() {
return *ptr;
}

template <typename T> struct same_size {
template <typename U> using as = bool_constant<sizeof(T) == sizeof(U)>;
};

template <typename Concrete> constexpr int platform_lookup() { return -1; }
template <std::size_t> constexpr int platform_lookup() { return -1; }

// Lookup a type according to its size, and return a value corresponding to the NumPy typenum.
template <typename Concrete, typename T, typename... Ts, typename... Ints>
template <std::size_t size, typename T, typename... Ts, typename... Ints>
constexpr int platform_lookup(int I, Ints... Is) {
return sizeof(Concrete) == sizeof(T) ? I : platform_lookup<Concrete, Ts...>(Is...);
return size == sizeof(T) ? I : platform_lookup<size, Ts...>(Is...);
}

struct npy_api {
Expand Down Expand Up @@ -148,14 +147,25 @@ struct npy_api {
// `npy_common.h` defines the integer aliases. In order, it checks:
// NPY_BITSOF_LONG, NPY_BITSOF_LONGLONG, NPY_BITSOF_INT, NPY_BITSOF_SHORT, NPY_BITSOF_CHAR
// and assigns the alias to the first matching size, so we should check in this order.
NPY_INT32_ = platform_lookup<std::int32_t, long, int, short>(
NPY_INT32_ = platform_lookup<4, long, int, short>(
NPY_LONG_, NPY_INT_, NPY_SHORT_),
NPY_UINT32_ = platform_lookup<std::uint32_t, unsigned long, unsigned int, unsigned short>(
NPY_UINT32_ = platform_lookup<4, unsigned long, unsigned int, unsigned short>(
NPY_ULONG_, NPY_UINT_, NPY_USHORT_),
NPY_INT64_ = platform_lookup<std::int64_t, long, long long, int>(
NPY_INT64_ = platform_lookup<8, long, long long, int>(
NPY_LONG_, NPY_LONGLONG_, NPY_INT_),
NPY_UINT64_ = platform_lookup<std::uint64_t, unsigned long, unsigned long long, unsigned int>(
NPY_UINT64_ = platform_lookup<8, unsigned long, unsigned long long, unsigned int>(
NPY_ULONG_, NPY_ULONGLONG_, NPY_UINT_),
// The same type of lookups with floats; the order is: double, float, long double
NPY_FLOAT32_ = platform_lookup<4, double, float, long double>(
NPY_DOUBLE_, NPY_FLOAT_, NPY_LONGDOUBLE_),
NPY_FLOAT64_ = platform_lookup<8, double, float, long double>(
NPY_DOUBLE_, NPY_FLOAT_, NPY_LONGDOUBLE_),
NPY_COMPLEX64_ = platform_lookup<4, double, float, long double>(
aldanor marked this conversation as resolved.
Show resolved Hide resolved
NPY_CDOUBLE_, NPY_CFLOAT_, NPY_CLONGDOUBLE_),
NPY_COMPLEX128_ = platform_lookup<8, double, float, long double>(
NPY_CDOUBLE_, NPY_CFLOAT_, NPY_CLONGDOUBLE_),
// Native character type
NPY_CHAR_ = std::is_signed<char>::value ? NPY_BYTE_ : NPY_UBYTE_,
};

typedef struct {
Expand All @@ -177,6 +187,7 @@ struct npy_api {

unsigned int (*PyArray_GetNDArrayCFeatureVersion_)();
PyObject *(*PyArray_DescrFromType_)(int);
PyObject *(*PyArray_TypeObjectFromType_)(int);
PyObject *(*PyArray_NewFromDescr_)
(PyTypeObject *, PyObject *, int, Py_intptr_t *,
Py_intptr_t *, void *, int, PyObject *);
Expand All @@ -187,6 +198,8 @@ struct npy_api {
PyTypeObject *PyVoidArrType_Type_;
PyTypeObject *PyArrayDescr_Type_;
PyObject *(*PyArray_DescrFromScalar_)(PyObject *);
PyObject *(*PyArray_Scalar_)(void *, PyObject *, PyObject *);
void (*PyArray_ScalarAsCtype_)(PyObject *, void *);
PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *);
int (*PyArray_DescrConverter_) (PyObject *, PyObject **);
bool (*PyArray_EquivTypes_) (PyObject *, PyObject *);
Expand All @@ -202,13 +215,16 @@ struct npy_api {
API_PyArrayDescr_Type = 3,
API_PyVoidArrType_Type = 39,
API_PyArray_DescrFromType = 45,
API_PyArray_TypeObjectFromType = 46,
API_PyArray_DescrFromScalar = 57,
API_PyArray_Scalar = 60,
API_PyArray_ScalarAsCtype = 62,
API_PyArray_FromAny = 69,
API_PyArray_Resize = 80,
API_PyArray_CopyInto = 82,
API_PyArray_NewCopy = 85,
API_PyArray_NewFromDescr = 94,
API_PyArray_DescrNewFromType = 9,
API_PyArray_DescrNewFromType = 96,
API_PyArray_DescrConverter = 174,
API_PyArray_EquivTypes = 182,
API_PyArray_GetArrayParamsFromObject = 278,
Expand All @@ -233,7 +249,10 @@ struct npy_api {
DECL_NPY_API(PyVoidArrType_Type);
DECL_NPY_API(PyArrayDescr_Type);
DECL_NPY_API(PyArray_DescrFromType);
DECL_NPY_API(PyArray_TypeObjectFromType);
DECL_NPY_API(PyArray_DescrFromScalar);
DECL_NPY_API(PyArray_Scalar);
DECL_NPY_API(PyArray_ScalarAsCtype);
DECL_NPY_API(PyArray_FromAny);
DECL_NPY_API(PyArray_Resize);
DECL_NPY_API(PyArray_CopyInto);
Expand All @@ -250,6 +269,74 @@ struct npy_api {
}
};

template <typename T> struct is_complex : std::false_type { };
template <typename T> struct is_complex<std::complex<T>> : std::true_type { };

template <typename T, typename = void>
struct npy_format_descriptor_name;

template <typename T>
struct npy_format_descriptor_name<T, enable_if_t<std::is_integral<T>::value>> {
static constexpr auto name = _<std::is_same<T, bool>::value>(
_("bool"), _<std::is_signed<T>::value>("int", "uint") + _<sizeof(T)*8>()
);
};

template <typename T>
struct npy_format_descriptor_name<T, enable_if_t<std::is_floating_point<T>::value>> {
static constexpr auto name = _<std::is_same<T, float>::value || std::is_same<T, double>::value>(
_("float") + _<sizeof(T)*8>(), _("longdouble")
);
};

template <typename T>
struct npy_format_descriptor_name<T, enable_if_t<is_complex<T>::value>> {
static constexpr auto name = _<std::is_same<typename T::value_type, float>::value
|| std::is_same<typename T::value_type, double>::value>(
_("complex") + _<sizeof(typename T::value_type)*16>(), _("longcomplex")
);
};

template<typename T> struct numpy_scalar_info {};

#define DECL_NPY_SCALAR(ctype_, typenum_) \
template<> struct numpy_scalar_info<ctype_> { \
static constexpr auto name = npy_format_descriptor_name<ctype_>::name; \
static constexpr int typenum = npy_api::typenum_##_; \
}

// boolean type
DECL_NPY_SCALAR(bool, NPY_BOOL);

// character types
DECL_NPY_SCALAR(char, NPY_CHAR);
DECL_NPY_SCALAR(signed char, NPY_BYTE);
DECL_NPY_SCALAR(unsigned char, NPY_UBYTE);

// signed integer types
DECL_NPY_SCALAR(short, NPY_SHORT);
DECL_NPY_SCALAR(int, NPY_INT);
DECL_NPY_SCALAR(long, NPY_LONG);
DECL_NPY_SCALAR(long long, NPY_LONGLONG);

// unsigned integer types
DECL_NPY_SCALAR(unsigned short, NPY_USHORT);
DECL_NPY_SCALAR(unsigned int, NPY_UINT);
DECL_NPY_SCALAR(unsigned long, NPY_ULONG);
DECL_NPY_SCALAR(unsigned long long, NPY_ULONGLONG);

// floating point types
DECL_NPY_SCALAR(float, NPY_FLOAT);
DECL_NPY_SCALAR(double, NPY_DOUBLE);
DECL_NPY_SCALAR(long double, NPY_LONGDOUBLE);

// complex types
DECL_NPY_SCALAR(std::complex<float>, NPY_CFLOAT);
DECL_NPY_SCALAR(std::complex<double>, NPY_CDOUBLE);
DECL_NPY_SCALAR(std::complex<long double>, NPY_CLONGDOUBLE);

#undef DECL_NPY_SCALAR

inline PyArray_Proxy* array_proxy(void* ptr) {
return reinterpret_cast<PyArray_Proxy*>(ptr);
}
Expand All @@ -272,8 +359,6 @@ inline bool check_flags(const void* ptr, int flag) {

template <typename T> struct is_std_array : std::false_type { };
template <typename T, size_t N> struct is_std_array<std::array<T, N>> : std::true_type { };
template <typename T> struct is_complex : std::false_type { };
template <typename T> struct is_complex<std::complex<T>> : std::true_type { };

template <typename T> struct array_info_scalar {
typedef T type;
Expand Down Expand Up @@ -439,8 +524,56 @@ struct type_caster<unchecked_reference<T, Dim>> {
template <typename T, ssize_t Dim>
struct type_caster<unchecked_mutable_reference<T, Dim>> : type_caster<unchecked_reference<T, Dim>> {};

template<typename T>
struct type_caster<numpy_scalar<T>> {
using value_type = T;
using type_info = numpy_scalar_info<T>;

PYBIND11_TYPE_CASTER(numpy_scalar<T>, type_info::name);

static handle& target_type() {
static handle tp = npy_api::get().PyArray_TypeObjectFromType_(type_info::typenum);
return tp;
}

static handle& target_dtype() {
static handle tp = npy_api::get().PyArray_DescrFromType_(type_info::typenum);
return tp;
}

bool load(handle src, bool) {
if (isinstance(src, target_type())) {
npy_api::get().PyArray_ScalarAsCtype_(src.ptr(), &value.value);
return true;
}
return false;
}

static handle cast(numpy_scalar<T> src, return_value_policy, handle) {
return npy_api::get().PyArray_Scalar_(&src.value, target_dtype().ptr(), nullptr);
}
};

NAMESPACE_END(detail)

template<typename T>
struct numpy_scalar {
using value_type = T;

value_type value;

numpy_scalar() = default;
numpy_scalar(value_type value) : value(value) {}

operator value_type() { return value; }
numpy_scalar& operator=(value_type value) { this->value = value; return *this; }
};

template<typename T>
numpy_scalar<T> make_scalar(T value) {
return numpy_scalar<T>(value);
}

class dtype : public object {
public:
PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_);
Expand Down Expand Up @@ -1001,31 +1134,6 @@ struct compare_buffer_info<T, detail::enable_if_t<detail::is_pod_struct<T>::valu
}
};

template <typename T, typename = void>
struct npy_format_descriptor_name;

template <typename T>
struct npy_format_descriptor_name<T, enable_if_t<std::is_integral<T>::value>> {
static constexpr auto name = _<std::is_same<T, bool>::value>(
_("bool"), _<std::is_signed<T>::value>("int", "uint") + _<sizeof(T)*8>()
);
};

template <typename T>
struct npy_format_descriptor_name<T, enable_if_t<std::is_floating_point<T>::value>> {
static constexpr auto name = _<std::is_same<T, float>::value || std::is_same<T, double>::value>(
_("float") + _<sizeof(T)*8>(), _("longdouble")
);
};

template <typename T>
struct npy_format_descriptor_name<T, enable_if_t<is_complex<T>::value>> {
static constexpr auto name = _<std::is_same<typename T::value_type, float>::value
|| std::is_same<typename T::value_type, double>::value>(
_("complex") + _<sizeof(typename T::value_type)*16>(), _("longcomplex")
);
};

template <typename T>
struct npy_format_descriptor<T, enable_if_t<satisfies_any_of<T, std::is_arithmetic, is_complex>::value>>
: npy_format_descriptor_name<T> {
Expand Down
1 change: 1 addition & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ set(PYBIND11_TEST_FILES
test_multiple_inheritance.cpp
test_numpy_array.cpp
test_numpy_dtypes.cpp
test_numpy_scalars.cpp
test_numpy_vectorize.cpp
test_opaque_types.cpp
test_operator_overloading.cpp
Expand Down
60 changes: 60 additions & 0 deletions tests/test_numpy_scalars.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
tests/test_numpy_scalars.cpp -- strict NumPy scalars

Copyright (c) 2020 Ivan Smirnov

All rights reserved. Use of this source code is governed by a
BSD-style license that can be found in the LICENSE file.
*/

#include <complex>
#include <cstdint>
#include <string>
#include <utility>

#include "pybind11_tests.h"
#include <pybind11/numpy.h>

namespace py = pybind11;

template<typename T, typename F>
void register_test(py::module& m, const char *name, F&& func) {
m.def("test_numpy_scalars", [=](py::numpy_scalar<T> v) {
return std::make_tuple(name, py::make_scalar(static_cast<T>(func(v.value))));
}, py::arg("x"));
m.def((std::string("test_") + name).c_str(), [=](py::numpy_scalar<T> v) {
return std::make_tuple(name, py::make_scalar(static_cast<T>(func(v.value))));
}, py::arg("x"));
}

template<typename T>
struct add {
T x;
add(T x) : x(x) {}
T operator()(T y) const { return static_cast<T>(x + y); }
};

TEST_SUBMODULE(numpy_scalars, m) {
try { py::module::import("numpy"); }
catch (...) { return; }

using cfloat = std::complex<float>;
using cdouble = std::complex<double>;
using clongdouble = std::complex<long double>;

register_test<bool>(m, "bool", [](bool x) { return !x; });
register_test<int8_t>(m, "int8", add<int8_t>(-8));
register_test<int16_t>(m, "int16", add<int16_t>(-16));
register_test<int32_t>(m, "int32", add<int32_t>(-32));
register_test<int64_t>(m, "int64", add<int64_t>(-64));
register_test<uint8_t>(m, "uint8", add<uint8_t>(8));
register_test<uint16_t>(m, "uint16", add<uint16_t>(16));
register_test<uint32_t>(m, "uint32", add<uint32_t>(32));
register_test<uint64_t>(m, "uint64", add<uint64_t>(64));
register_test<float>(m, "float32", add<float>(0.125f));
register_test<double>(m, "float64", add<double>(0.25f));
register_test<long double>(m, "longdouble", add<long double>(0.5L));
register_test<cfloat>(m, "complex64", add<cfloat>({0, -0.125f}));
register_test<cdouble>(m, "complex128", add<cdouble>({0, -0.25}));
register_test<clongdouble>(m, "longcomplex", add<clongdouble>({0, -0.5L}));
}
Loading