From 3dd325b77280be9ef83cfd986309d2489a709686 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 18 Jun 2016 00:46:12 +0100 Subject: [PATCH 01/87] Change npy_format_descriptor typenum to static fn --- include/pybind11/numpy.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 5d355e2fcc..e8182acd1f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -84,7 +84,7 @@ class array : public buffer { template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::value); + PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::typenum()); if (descr == nullptr) pybind11_fail("NumPy: unsupported buffer format!"); Py_intptr_t shape = (Py_intptr_t) size; @@ -135,7 +135,7 @@ template class array_t : public if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::value); + PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::typenum()); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -152,7 +152,7 @@ template struct npy_format_descriptor::value ? 1 : 0)] }; + static int typenum() { return values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)]; } template ::value, int>::type = 0> static PYBIND11_DESCR name() { return _("int") + _(); } template ::value, int>::type = 0> @@ -162,10 +162,13 @@ template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - enum { value = array::API::NumPyName }; \ + static int typenum() { return array::API::NumPyName; } \ static PYBIND11_DESCR name() { return _(Name); } } -DECL_FMT(float, NPY_FLOAT_, "float32"); DECL_FMT(double, NPY_DOUBLE_, "float64"); DECL_FMT(bool, NPY_BOOL_, "bool"); -DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); +DECL_FMT(float, NPY_FLOAT_, "float32"); +DECL_FMT(double, NPY_DOUBLE_, "float64"); +DECL_FMT(bool, NPY_BOOL_, "bool"); +DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); +DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #undef DECL_FMT template From a7e62e1ca621c7a7c1b8956873a16c255f2939b6 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:37:55 +0100 Subject: [PATCH 02/87] Add buffer_info::as_pybuffer() method --- include/pybind11/common.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/include/pybind11/common.h b/include/pybind11/common.h index 0b20929204..d105350787 100644 --- a/include/pybind11/common.h +++ b/include/pybind11/common.h @@ -231,6 +231,31 @@ struct buffer_info { ~buffer_info() { if (view) { PyBuffer_Release(view); delete view; } } + + Py_buffer& as_pybuffer() const { + static Py_buffer buf { }; + // Py_buffer uses signed sizes, strides and shape!.. + static std::vector py_strides { }; + static std::vector py_shape { }; + buf.buf = ptr; + buf.itemsize = (Py_ssize_t) itemsize; + buf.format = const_cast(format.c_str()); + buf.ndim = (int) ndim; + buf.len = (Py_ssize_t) size; + py_strides.clear(); + py_shape.clear(); + for (size_t i = 0; i < ndim; ++i) { + py_strides.push_back((Py_ssize_t) strides[i]); + py_shape.push_back((Py_ssize_t) shape[i]); + } + buf.strides = py_strides.data(); + buf.shape = py_shape.data(); + buf.suboffsets = nullptr; + buf.readonly = false; + buf.internal = nullptr; + return buf; + } + private: Py_buffer *view = nullptr; }; From 42ad3284815b06cc74fa9d19d767a73d86e73546 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:39:41 +0100 Subject: [PATCH 03/87] Change format_descriptor::value to a static func --- docs/advanced.rst | 16 ++++++++-------- example/example-buffers.cpp | 14 +++++++------- include/pybind11/common.h | 18 +++++++++++++----- include/pybind11/eigen.h | 12 ++++++------ include/pybind11/numpy.h | 2 +- 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index c68c33affc..aca1325946 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -1224,12 +1224,12 @@ completely avoid copy operations with Python expressions like py::class_(m, "Matrix") .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( - m.data(), /* Pointer to buffer */ - sizeof(float), /* Size of one scalar */ - py::format_descriptor::value, /* Python struct-style format descriptor */ - 2, /* Number of dimensions */ - { m.rows(), m.cols() }, /* Buffer dimensions */ - { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ + m.data(), /* Pointer to buffer */ + sizeof(float), /* Size of one scalar */ + py::format_descriptor::value(), /* Python struct-style format descriptor */ + 2, /* Number of dimensions */ + { m.rows(), m.cols() }, /* Buffer dimensions */ + { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ sizeof(float) } ); }); @@ -1273,7 +1273,7 @@ buffer objects (e.g. a NumPy matrix). py::buffer_info info = b.request(); /* Some sanity checks ... */ - if (info.format != py::format_descriptor::value) + if (info.format != py::format_descriptor::value()) throw std::runtime_error("Incompatible format: expected a double array!"); if (info.ndim != 2) @@ -1299,7 +1299,7 @@ as follows: m.data(), /* Pointer to buffer */ sizeof(Scalar), /* Size of one scalar */ /* Python struct-style format descriptor */ - py::format_descriptor::value, + py::format_descriptor::value(), /* Number of dimensions */ 2, /* Buffer dimensions */ diff --git a/example/example-buffers.cpp b/example/example-buffers.cpp index 17c8d271fc..2deee6f836 100644 --- a/example/example-buffers.cpp +++ b/example/example-buffers.cpp @@ -81,7 +81,7 @@ void init_ex_buffers(py::module &m) { /// Construct from a buffer .def("__init__", [](Matrix &v, py::buffer b) { py::buffer_info info = b.request(); - if (info.format != py::format_descriptor::value || info.ndim != 2) + if (info.format != py::format_descriptor::value() || info.ndim != 2) throw std::runtime_error("Incompatible buffer format!"); new (&v) Matrix(info.shape[0], info.shape[1]); memcpy(v.data(), info.ptr, sizeof(float) * v.rows() * v.cols()); @@ -104,12 +104,12 @@ void init_ex_buffers(py::module &m) { /// Provide buffer access .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( - m.data(), /* Pointer to buffer */ - sizeof(float), /* Size of one scalar */ - py::format_descriptor::value, /* Python struct-style format descriptor */ - 2, /* Number of dimensions */ - { m.rows(), m.cols() }, /* Buffer dimensions */ - { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ + m.data(), /* Pointer to buffer */ + sizeof(float), /* Size of one scalar */ + py::format_descriptor::value(), /* Python struct-style format descriptor */ + 2, /* Number of dimensions */ + { m.rows(), m.cols() }, /* Buffer dimensions */ + { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ sizeof(float) } ); }) diff --git a/include/pybind11/common.h b/include/pybind11/common.h index d105350787..ca5765ab15 100644 --- a/include/pybind11/common.h +++ b/include/pybind11/common.h @@ -204,7 +204,7 @@ struct buffer_info { void *ptr; // Pointer to the underlying storage size_t itemsize; // Size of individual items in bytes size_t size; // Total number of entries - std::string format; // For homogeneous buffers, this should be set to format_descriptor::value + std::string format; // For homogeneous buffers, this should be set to format_descriptor::value() size_t ndim; // Number of dimensions std::vector shape; // Shape of the tensor (1 entry per dimension) std::vector strides; // Number of entries between adjacent entries (for each per dimension) @@ -348,14 +348,22 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error) /// Used internally [[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const std::string &reason) { throw std::runtime_error(reason); } /// Format strings for basic number types -#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor { static constexpr const char *value = v; } +#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor \ + { static constexpr const char* value() { return v; } }; + template struct format_descriptor { }; + template struct format_descriptor::value>::type> { - static constexpr const char value[2] = + static constexpr const char* value() { return format; } + static constexpr const char format[2] = { "bBhHiIqQ"[detail::log2(sizeof(T))*2 + (std::is_unsigned::value ? 1 : 0)], '\0' }; }; + template constexpr const char format_descriptor< - T, typename std::enable_if::value>::type>::value[2]; -PYBIND11_DECL_FMT(float, "f"); PYBIND11_DECL_FMT(double, "d"); PYBIND11_DECL_FMT(bool, "?"); + T, typename std::enable_if::value>::type>::format[2]; + +PYBIND11_DECL_FMT(float, "f"); +PYBIND11_DECL_FMT(double, "d"); +PYBIND11_DECL_FMT(bool, "?"); NAMESPACE_END(pybind11) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index bfb934060f..db8f9eb172 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -133,7 +133,7 @@ struct type_caster::value && /* Size of one scalar */ sizeof(Scalar), /* Python struct-style format descriptor */ - format_descriptor::value, + format_descriptor::value(), /* Number of dimensions */ 1, /* Buffer dimensions */ @@ -148,7 +148,7 @@ struct type_caster::value && /* Size of one scalar */ sizeof(Scalar), /* Python struct-style format descriptor */ - format_descriptor::value, + format_descriptor::value(), /* Number of dimensions */ isVector ? 1 : 2, /* Buffer dimensions */ @@ -233,7 +233,7 @@ struct type_caster::value>:: try { obj = matrix_type(obj); } catch (const error_already_set &) { - PyErr_Clear(); + PyErr_Clear(); return false; } } @@ -276,7 +276,7 @@ struct type_caster::value>:: // Size of one scalar sizeof(Scalar), // Python struct-style format descriptor - format_descriptor::value, + format_descriptor::value(), // Number of dimensions 1, // Buffer dimensions @@ -291,7 +291,7 @@ struct type_caster::value>:: // Size of one scalar sizeof(StorageIndex), // Python struct-style format descriptor - format_descriptor::value, + format_descriptor::value(), // Number of dimensions 1, // Buffer dimensions @@ -306,7 +306,7 @@ struct type_caster::value>:: // Size of one scalar sizeof(StorageIndex), // Python struct-style format descriptor - format_descriptor::value, + format_descriptor::value(), // Number of dimensions 1, // Buffer dimensions diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index e8182acd1f..7dff28daf4 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -351,7 +351,7 @@ struct vectorize_helper { return cast(f(*((Args *) buffers[Index].ptr)...)); array result(buffer_info(nullptr, sizeof(Return), - format_descriptor::value, + format_descriptor::value(), ndim, shape, strides)); buffer_info buf = result.request(); From 8b5fc8b5e1c54d20bd4b6d692fb6315ca4007caa Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:40:04 +0100 Subject: [PATCH 04/87] Dump test output if the test runner fails --- example/run_test.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/example/run_test.py b/example/run_test.py index a11c3fc684..2785c8eaae 100755 --- a/example/run_test.py +++ b/example/run_test.py @@ -46,12 +46,11 @@ def sanitize(lines): try: output_bytes = subprocess.check_output([sys.executable, "-u", name + ".py"], stderr=subprocess.STDOUT) -except subprocess.CalledProcessError as e: - if e.returncode == 99: - print('Test "%s" could not be run.' % name) - exit(0) - else: - raise +except subprocess.CalledProcessError as exc: + print('Test `{}` failed:\n{}\n'.format(name, '-' * 50)) + print(exc.output.decode()) + print('-' * 50) + sys.exit(1) output = sanitize(output_bytes.decode('utf-8')) reference = sanitize(open(name + '.ref', 'r').read()) From 7709d6b77dceb925b8f41819bc3e3b1b67c7cfca Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:40:30 +0100 Subject: [PATCH 05/87] Add memoryview type --- include/pybind11/pytypes.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 0a0f508781..7a6cf6fbd3 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -570,6 +570,19 @@ class buffer : public object { } }; +class memoryview : public object { +public: + memoryview(const buffer_info& info) : memoryview(&info.as_pybuffer()) { } + + memoryview(Py_buffer* view) + : object(PyMemoryView_FromBuffer(view), false) { + if (!m_ptr) + pybind11_fail("Unable to create memoryview from buffer descriptor"); + } + + PYBIND11_OBJECT_DEFAULT(memoryview, object, PyMemoryView_Check) +}; + inline size_t len(handle h) { ssize_t result = PyObject_Length(h.ptr()); if (result < 0) From f7143dc589f2c105b95e3cab5402d3a6ea8c177d Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:41:21 +0100 Subject: [PATCH 06/87] Update gitignore to ignore debug test builds --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 16701940f1..4b9df18aa1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ CMakeFiles Makefile cmake_install.cmake .DS_Store -/example/example.so +/example/example*.so /example/example.cpython*.so /example/example.pyd /example/example*.dll From ea2755ccdc25c6f5ec8c4061d2e69d2e56521442 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:44:20 +0100 Subject: [PATCH 07/87] Use a macro for numpy API definitions --- include/pybind11/numpy.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 7dff28daf4..8e8abe3d89 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -55,11 +55,13 @@ class array : public buffer { void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); #endif API api; - api.PyArray_Type_ = (decltype(api.PyArray_Type_)) api_ptr[API_PyArray_Type]; - api.PyArray_DescrFromType_ = (decltype(api.PyArray_DescrFromType_)) api_ptr[API_PyArray_DescrFromType]; - api.PyArray_FromAny_ = (decltype(api.PyArray_FromAny_)) api_ptr[API_PyArray_FromAny]; - api.PyArray_NewCopy_ = (decltype(api.PyArray_NewCopy_)) api_ptr[API_PyArray_NewCopy]; - api.PyArray_NewFromDescr_ = (decltype(api.PyArray_NewFromDescr_)) api_ptr[API_PyArray_NewFromDescr]; +#define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; + DECL_NPY_API(PyArray_Type); + DECL_NPY_API(PyArray_DescrFromType); + DECL_NPY_API(PyArray_FromAny); + DECL_NPY_API(PyArray_NewCopy); + DECL_NPY_API(PyArray_NewFromDescr); +#undef DECL_NPY_API return api; } From a67c2b52e470613620e610e6210687ed084b4949 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:50:06 +0100 Subject: [PATCH 08/87] Use memoryview for constructing array from buffer --- include/pybind11/numpy.h | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8e8abe3d89..19965ba850 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -13,6 +13,7 @@ #include "complex.h" #include #include +#include #if defined(_MSC_VER) #pragma warning(push) @@ -31,6 +32,7 @@ class array : public buffer { API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, + API_PyArray_GetArrayParamsFromObject = 278, NPY_C_CONTIGUOUS_ = 0x0001, NPY_F_CONTIGUOUS_ = 0x0002, @@ -61,6 +63,7 @@ class array : public buffer { DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); + DECL_NPY_API(PyArray_GetArrayParamsFromObject); #undef DECL_NPY_API return api; } @@ -74,6 +77,8 @@ class array : public buffer { PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); + int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, + Py_ssize_t *, PyObject **, PyObject *); }; PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check_) @@ -100,24 +105,22 @@ class array : public buffer { } array(const buffer_info &info) { - API& api = lookup_api(); - if ((info.format.size() < 1) || (info.format.size() > 2)) - pybind11_fail("Unsupported buffer format!"); - int fmt = (int) info.format[0]; - if (info.format == "Zd") fmt = API::NPY_CDOUBLE_; - else if (info.format == "Zf") fmt = API::NPY_CFLOAT_; + PyObject *arr = nullptr, *descr = nullptr; + int ndim = 0; + Py_ssize_t dims[32]; - PyObject *descr = api.PyArray_DescrFromType_(fmt); - if (descr == nullptr) - pybind11_fail("NumPy: unsupported buffer format '" + info.format + "'!"); - object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0], - (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); - if (info.ptr && tmp) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); - if (!tmp) - pybind11_fail("NumPy: unable to create array!"); - m_ptr = tmp.release().ptr(); + // allocate zeroed memory if it hasn't been provided + auto buf_info = info; + if (!buf_info.ptr) + buf_info.ptr = std::calloc(info.size, info.itemsize); + auto view = py::memoryview(buf_info); + + API& api = lookup_api(); + auto res = api.PyArray_GetArrayParamsFromObject_(view.ptr(), nullptr, 1, &descr, + &ndim, dims, &arr, nullptr); + if (res < 0 || !arr || descr) + pybind11_fail("NumPy: unable to convert buffer to an array"); + m_ptr = arr; } protected: From fab02efb10d5d3a057eba85752b91d1d77bfcd4b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:53:20 +0100 Subject: [PATCH 09/87] Switch away from typenums for numpy descriptors --- include/pybind11/numpy.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 19965ba850..49489ceb10 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -91,9 +91,7 @@ class array : public buffer { template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::typenum()); - if (descr == nullptr) - pybind11_fail("NumPy: unsupported buffer format!"); + PyObject *descr = detail::npy_format_descriptor::descr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); @@ -128,6 +126,8 @@ class array : public buffer { static API api = API::lookup(); return api; } + + template friend struct detail::npy_format_descriptor; }; template class array_t : public array { @@ -140,7 +140,7 @@ template class array_t : public if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::typenum()); + PyObject *descr = detail::npy_format_descriptor::descr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -158,6 +158,10 @@ template struct npy_format_descriptor::value ? 1 : 0)]; } + static PyObject* descr() { + if (auto obj = array::lookup_api().PyArray_DescrFromType_(typenum())) return obj; + else pybind11_fail("Unsupported buffer format!"); + } template ::value, int>::type = 0> static PYBIND11_DESCR name() { return _("int") + _(); } template ::value, int>::type = 0> @@ -167,7 +171,11 @@ template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - static int typenum() { return array::API::NumPyName; } \ + static int typenum() { return array::API::NumPyName; } \ + static PyObject* descr() { \ + if (auto obj = array::lookup_api().PyArray_DescrFromType_(typenum())) return obj; \ + else pybind11_fail("Unsupported buffer format!"); \ + } \ static PYBIND11_DESCR name() { return _(Name); } } DECL_FMT(float, NPY_FLOAT_, "float32"); DECL_FMT(double, NPY_DOUBLE_, "float64"); From 2488b32066e6a7199719bd1f056037bbffc39b52 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 15:48:55 +0100 Subject: [PATCH 10/87] Add PYBIND11_DTYPE macro for registering dtypes --- include/pybind11/numpy.h | 107 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 49489ceb10..186b82f950 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #if defined(_MSC_VER) #pragma warning(push) @@ -32,6 +34,7 @@ class array : public buffer { API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, + API_PyArray_DescrConverter = 174, API_PyArray_GetArrayParamsFromObject = 278, NPY_C_CONTIGUOUS_ = 0x0001, @@ -63,6 +66,7 @@ class array : public buffer { DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); + DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_GetArrayParamsFromObject); #undef DECL_NPY_API return api; @@ -77,6 +81,7 @@ class array : public buffer { PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); + int (*PyArray_DescrConverter_) (PyObject *, PyObject **); int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, Py_ssize_t *, PyObject **, PyObject *); }; @@ -149,6 +154,19 @@ template class array_t : public } }; +template struct format_descriptor +::value && + !std::is_integral::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same>::value && + !std::is_same>::value>::type> +{ + static const char *value() { + return detail::npy_format_descriptor::format_str(); + } +}; + NAMESPACE_BEGIN(detail) template struct npy_format_descriptor::value>::type> { @@ -184,6 +202,95 @@ DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #undef DECL_FMT +struct field_descriptor { + const char *name; + int offset; + PyObject *descr; +}; + +template struct npy_format_descriptor +::value && // offsetof only works correctly for POD types + !std::is_integral::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same>::value && + !std::is_same>::value>::type> +{ + static PYBIND11_DESCR name() { return _("user-defined"); } + + static PyObject* descr() { + if (!descr_()) + pybind11_fail("NumPy: unsupported buffer format!"); + return descr_(); + } + + static const char* format_str() { + return format_str_(); + } + + static void register_dtype(std::initializer_list fields) { + array::API& api = array::lookup_api(); + auto args = py::dict(); + py::list names { }, offsets { }, formats { }; + std::vector dtypes; + for (auto field : fields) { + names.append(py::str(field.name)); + offsets.append(py::int_(field.offset)); + if (!field.descr) + pybind11_fail("NumPy: unsupported field dtype"); + dtypes.emplace_back(field.descr, false); + formats.append(dtypes.back()); + } + args["names"] = names; + args["offsets"] = offsets; + args["formats"] = formats; + if (!api.PyArray_DescrConverter_(args.ptr(), &descr_()) || !descr_()) + pybind11_fail("NumPy: failed to create structured dtype"); + auto np = module::import("numpy"); + auto empty = (object) np.attr("empty"); + if (auto arr = (object) empty(py::int_(0), object(descr(), true))) + if (auto view = PyMemoryView_FromObject(arr.ptr())) + if (auto info = PyMemoryView_GET_BUFFER(view)) { + std::strncpy(format_str_(), info->format, 4096); + return; + } + pybind11_fail("NumPy: failed to extract buffer format"); + } + +private: + static inline PyObject*& descr_() { static PyObject *ptr = nullptr; return ptr; } + static inline char* format_str_() { static char s[4096]; return s; } +}; + +#define FIELD_DESCRIPTOR(Type, Field) \ + ::pybind11::detail::field_descriptor { \ + #Field, offsetof(Type, Field), \ + ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() } + +// The main idea of this macro is borrowed from https://github.com/swansontec/map-macro +// (C) William Swanson, Paul Fultz +#define EVAL0(...) __VA_ARGS__ +#define EVAL1(...) EVAL0 (EVAL0 (EVAL0 (__VA_ARGS__))) +#define EVAL2(...) EVAL1 (EVAL1 (EVAL1 (__VA_ARGS__))) +#define EVAL3(...) EVAL2 (EVAL2 (EVAL2 (__VA_ARGS__))) +#define EVAL4(...) EVAL3 (EVAL3 (EVAL3 (__VA_ARGS__))) +#define EVAL(...) EVAL4 (EVAL4 (EVAL4 (__VA_ARGS__))) +#define MAP_END(...) +#define MAP_OUT +#define MAP_COMMA , +#define MAP_GET_END() 0, MAP_END +#define MAP_NEXT0(test, next, ...) next MAP_OUT +#define MAP_NEXT1(test, next) MAP_NEXT0 (test, next, 0) +#define MAP_NEXT(test, next) MAP_NEXT1 (MAP_GET_END test, next) +#define MAP_LIST_NEXT1(test, next) MAP_NEXT0 (test, MAP_COMMA next, 0) +#define MAP_LIST_NEXT(test, next) MAP_LIST_NEXT1 (MAP_GET_END test, next) +#define MAP_LIST0(f, t, x, peek, ...) f(t, x) MAP_LIST_NEXT (peek, MAP_LIST1) (f, t, peek, __VA_ARGS__) +#define MAP_LIST1(f, t, x, peek, ...) f(t, x) MAP_LIST_NEXT (peek, MAP_LIST0) (f, t, peek, __VA_ARGS__) +#define MAP_LIST(f, t, ...) EVAL (MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) + +#define PYBIND11_DTYPE(Type, ...) \ + ::pybind11::detail::npy_format_descriptor::register_dtype({MAP_LIST(FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + template using array_iterator = typename std::add_pointer::type; From bb4015ded39592a593f4af437d7e62b1ae4366a4 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 15:50:31 +0100 Subject: [PATCH 11/87] Add a basic test for recarrays and complex dtypes --- example/CMakeLists.txt | 2 +- example/example.cpp | 2 ++ example/example20.cpp | 57 ++++++++++++++++++++++++++++++++++++++++++ example/example20.py | 19 ++++++++++++++ example/example20.ref | 0 5 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 example/example20.cpp create mode 100644 example/example20.py create mode 100644 example/example20.ref diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 22d44c808e..b6d3804973 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -26,6 +26,7 @@ set(PYBIND11_EXAMPLES example-stl-binder-vector.cpp example-eval.cpp example-custom-exceptions.cpp + example20.cpp issues.cpp ) @@ -65,4 +66,3 @@ foreach(VALUE ${PYBIND11_EXAMPLES}) string(REGEX REPLACE "^(.+).cpp$" "\\1" EXAMPLE_NAME "${VALUE}") add_test(NAME ${EXAMPLE_NAME} COMMAND ${RUN_TEST} ${EXAMPLE_NAME}) endforeach() - diff --git a/example/example.cpp b/example/example.cpp index b831e3e84c..819f69f449 100644 --- a/example/example.cpp +++ b/example/example.cpp @@ -29,6 +29,7 @@ void init_ex_inheritance(py::module &); void init_ex_stl_binder_vector(py::module &); void init_ex_eval(py::module &); void init_ex_custom_exceptions(py::module &); +void init_ex20(py::module &); void init_issues(py::module &); #if defined(PYBIND11_TEST_EIGEN) @@ -72,6 +73,7 @@ PYBIND11_PLUGIN(example) { init_ex_stl_binder_vector(m); init_ex_eval(m); init_ex_custom_exceptions(m); + init_ex20(m); init_issues(m); #if defined(PYBIND11_TEST_EIGEN) diff --git a/example/example20.cpp b/example/example20.cpp new file mode 100644 index 0000000000..c22eb213ee --- /dev/null +++ b/example/example20.cpp @@ -0,0 +1,57 @@ +/* + example/example20.cpp -- Usage of structured numpy dtypes + + Copyright (c) 2016 Ivan Smirnov + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include "example.h" + +#include +#include +#include + +namespace py = pybind11; + +struct Struct { + bool x; + uint32_t y; + float z; +}; + +struct PackedStruct { + bool x; + uint32_t y; + float z; +} __attribute__((packed)); + +struct NestedStruct { + Struct a; + PackedStruct b; +}; + +template +py::array_t create_recarray(size_t n) { + auto arr = py::array(py::buffer_info(nullptr, sizeof(S), + py::format_descriptor::value(), + 1, { n }, { sizeof(S) })); + auto buf = arr.request(); + auto ptr = static_cast(buf.ptr); + for (size_t i = 0; i < n; i++) { + ptr[i].x = i % 2; + ptr[i].y = i; + ptr[i].z = i * 1.5; + } + return arr; +} + +void init_ex20(py::module &m) { + PYBIND11_DTYPE(Struct, x, y, z); + PYBIND11_DTYPE(PackedStruct, x, y, z); + PYBIND11_DTYPE(NestedStruct, a, b); + + m.def("create_rec_simple", &create_recarray); + m.def("create_rec_packed", &create_recarray); +} diff --git a/example/example20.py b/example/example20.py new file mode 100644 index 0000000000..ed3454860f --- /dev/null +++ b/example/example20.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +from __future__ import print_function + +import numpy as np +from example import create_rec_simple + + +def check_eq(arr, data, dtype): + np.testing.assert_equal(arr, np.array(data, dtype=dtype)) + +dtype = np.dtype({'names': ['x', 'y', 'z'], + 'formats': ['?', 'u4', 'f4'], + 'offsets': [0, 4, 8]}) +base_dtype = np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')]) + +arr = create_rec_simple(3) +assert arr.dtype == dtype +check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], dtype) +check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], base_dtype) diff --git a/example/example20.ref b/example/example20.ref new file mode 100644 index 0000000000..e69de29bb2 From f10c84eb9b91e62a545eb326da90194c502f023b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 16:04:01 +0100 Subject: [PATCH 12/87] Release format descriptor args before converting --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 186b82f950..92ab116702 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -244,7 +244,7 @@ template struct npy_format_descriptor args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; - if (!api.PyArray_DescrConverter_(args.ptr(), &descr_()) || !descr_()) + if (!api.PyArray_DescrConverter_(args.release().ptr(), &descr_()) || !descr_()) pybind11_fail("NumPy: failed to create structured dtype"); auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); From 2e1565e414bbc34e72a67bfdbf44c2706546d355 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 16:05:23 +0100 Subject: [PATCH 13/87] Add empty recarray test, check for calloc fail --- example/example20.py | 5 +++++ include/pybind11/numpy.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/example/example20.py b/example/example20.py index ed3454860f..d1bc942386 100644 --- a/example/example20.py +++ b/example/example20.py @@ -17,3 +17,8 @@ def check_eq(arr, data, dtype): assert arr.dtype == dtype check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], dtype) check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], base_dtype) + +arr = create_rec_simple(0) +assert arr.dtype == dtype +check_eq(arr, [], dtype) +check_eq(arr, [], base_dtype) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 92ab116702..6e9a629dbb 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -116,6 +116,8 @@ class array : public buffer { auto buf_info = info; if (!buf_info.ptr) buf_info.ptr = std::calloc(info.size, info.itemsize); + if (!buf_info.ptr) + pybind11_fail("NumPy: failed to allocate memory for buffer"); auto view = py::memoryview(buf_info); API& api = lookup_api(); From 8502f542b36d0cbb1f3af532300df149f9ca773a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 16:09:44 +0100 Subject: [PATCH 14/87] Add packed recarray tests --- example/example20.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/example/example20.py b/example/example20.py index d1bc942386..0e42c68d6c 100644 --- a/example/example20.py +++ b/example/example20.py @@ -2,23 +2,24 @@ from __future__ import print_function import numpy as np -from example import create_rec_simple +from example import create_rec_simple, create_rec_packed def check_eq(arr, data, dtype): np.testing.assert_equal(arr, np.array(data, dtype=dtype)) -dtype = np.dtype({'names': ['x', 'y', 'z'], - 'formats': ['?', 'u4', 'f4'], - 'offsets': [0, 4, 8]}) -base_dtype = np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')]) +simple_dtype = np.dtype({'names': ['x', 'y', 'z'], + 'formats': ['?', 'u4', 'f4'], + 'offsets': [0, 4, 8]}) +packed_dtype = np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')]) -arr = create_rec_simple(3) -assert arr.dtype == dtype -check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], dtype) -check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], base_dtype) +for func, dtype in [(create_rec_simple, simple_dtype), (create_rec_packed, packed_dtype)]: + arr = func(3) + assert arr.dtype == dtype + check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], simple_dtype) + check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], packed_dtype) -arr = create_rec_simple(0) -assert arr.dtype == dtype -check_eq(arr, [], dtype) -check_eq(arr, [], base_dtype) + arr = func(0) + assert arr.dtype == dtype + check_eq(arr, [], simple_dtype) + check_eq(arr, [], packed_dtype) From 80a3785a66a61d4aaa3543a450ef249fbec33d5a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 16:40:52 +0100 Subject: [PATCH 15/87] Borrow field descriptors for recarray dtype --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6e9a629dbb..3f538e56b1 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -240,7 +240,7 @@ template struct npy_format_descriptor offsets.append(py::int_(field.offset)); if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); - dtypes.emplace_back(field.descr, false); + dtypes.emplace_back(field.descr, true); formats.append(dtypes.back()); } args["names"] = names; From 7f913aecabed087bc368708cc7e2b23d0edb58aa Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 16:41:15 +0100 Subject: [PATCH 16/87] Add tests for nested recarrays --- example/example20.cpp | 32 +++++++++++++++++++++++--------- example/example20.py | 23 ++++++++++++++++++----- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/example/example20.cpp b/example/example20.cpp index c22eb213ee..2535b258bb 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -30,21 +30,34 @@ struct PackedStruct { struct NestedStruct { Struct a; PackedStruct b; -}; +} __attribute__((packed)); + +template +py::array mkarray_via_buffer(size_t n) { + return py::array(py::buffer_info(nullptr, sizeof(T), + py::format_descriptor::value(), + 1, { n }, { sizeof(T) })); +} template py::array_t create_recarray(size_t n) { - auto arr = py::array(py::buffer_info(nullptr, sizeof(S), - py::format_descriptor::value(), - 1, { n }, { sizeof(S) })); - auto buf = arr.request(); - auto ptr = static_cast(buf.ptr); + auto arr = mkarray_via_buffer(n); + auto ptr = static_cast(arr.request().ptr); + for (size_t i = 0; i < n; i++) { + ptr[i].x = i % 2; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f; + } + return arr; +} + +py::array_t create_nested(size_t n) { + auto arr = mkarray_via_buffer(n); + auto ptr = static_cast(arr.request().ptr); for (size_t i = 0; i < n; i++) { - ptr[i].x = i % 2; - ptr[i].y = i; - ptr[i].z = i * 1.5; + ptr[i].a.x = i % 2; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; + ptr[i].b.x = (i + 1) % 2; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f; } return arr; + } void init_ex20(py::module &m) { @@ -54,4 +67,5 @@ void init_ex20(py::module &m) { m.def("create_rec_simple", &create_recarray); m.def("create_rec_packed", &create_recarray); + m.def("create_rec_nested", &create_nested); } diff --git a/example/example20.py b/example/example20.py index 0e42c68d6c..83725b5284 100644 --- a/example/example20.py +++ b/example/example20.py @@ -2,7 +2,7 @@ from __future__ import print_function import numpy as np -from example import create_rec_simple, create_rec_packed +from example import create_rec_simple, create_rec_packed, create_rec_nested def check_eq(arr, data, dtype): @@ -14,12 +14,25 @@ def check_eq(arr, data, dtype): packed_dtype = np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')]) for func, dtype in [(create_rec_simple, simple_dtype), (create_rec_packed, packed_dtype)]: + arr = func(0) + assert arr.dtype == dtype + check_eq(arr, [], simple_dtype) + check_eq(arr, [], packed_dtype) + arr = func(3) assert arr.dtype == dtype check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], simple_dtype) check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], packed_dtype) - arr = func(0) - assert arr.dtype == dtype - check_eq(arr, [], simple_dtype) - check_eq(arr, [], packed_dtype) + +nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)]) + +arr = create_rec_nested(0) +assert arr.dtype == nested_dtype +check_eq(arr, [], nested_dtype) + +arr = create_rec_nested(3) +assert arr.dtype == nested_dtype +check_eq(arr, [((False, 0, 0.0), (True, 1, 1.5)), + ((True, 1, 1.5), (False, 2, 3.0)), + ((False, 2, 3.0), (True, 3, 4.5))], nested_dtype) From bdc9902041300a158eb3230f5af441310e90b162 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 16:54:07 +0100 Subject: [PATCH 17/87] Add explicit test for recarray format descriptors --- example/example20.cpp | 7 +++++++ example/example20.py | 6 +++++- example/example20.ref | 3 +++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/example/example20.cpp b/example/example20.cpp index 2535b258bb..7066e321b6 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -57,7 +57,13 @@ py::array_t create_nested(size_t n) { ptr[i].b.x = (i + 1) % 2; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f; } return arr; +} + +void print_format_descriptors() { + std::cout << py::format_descriptor::value() << std::endl; + std::cout << py::format_descriptor::value() << std::endl; + std::cout << py::format_descriptor::value() << std::endl; } void init_ex20(py::module &m) { @@ -68,4 +74,5 @@ void init_ex20(py::module &m) { m.def("create_rec_simple", &create_recarray); m.def("create_rec_packed", &create_recarray); m.def("create_rec_nested", &create_nested); + m.def("print_format_descriptors", &print_format_descriptors); } diff --git a/example/example20.py b/example/example20.py index 83725b5284..83a914c248 100644 --- a/example/example20.py +++ b/example/example20.py @@ -2,12 +2,16 @@ from __future__ import print_function import numpy as np -from example import create_rec_simple, create_rec_packed, create_rec_nested +from example import ( + create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors +) def check_eq(arr, data, dtype): np.testing.assert_equal(arr, np.array(data, dtype=dtype)) +print_format_descriptors() + simple_dtype = np.dtype({'names': ['x', 'y', 'z'], 'formats': ['?', 'u4', 'f4'], 'offsets': [0, 4, 8]}) diff --git a/example/example20.ref b/example/example20.ref index e69de29bb2..6023a87f24 100644 --- a/example/example20.ref +++ b/example/example20.ref @@ -0,0 +1,3 @@ +T{?:x:xxxI:y:f:z:} +T{?:x:=I:y:f:z:} +T{T{?:x:xxxI:y:f:z:}:a:T{?:x:=I:y:f:z:}:b:} From f5b166d042b8c1edd9bff6e2f8604e54843d7924 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Tue, 21 Jun 2016 21:03:58 +0100 Subject: [PATCH 18/87] Simplify npy_format_descriptor slightly --- include/pybind11/numpy.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3f538e56b1..1d957d1a33 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -234,14 +234,12 @@ template struct npy_format_descriptor array::API& api = array::lookup_api(); auto args = py::dict(); py::list names { }, offsets { }, formats { }; - std::vector dtypes; for (auto field : fields) { - names.append(py::str(field.name)); - offsets.append(py::int_(field.offset)); if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); - dtypes.emplace_back(field.descr, true); - formats.append(dtypes.back()); + names.append(py::str(field.name)); + offsets.append(py::int_(field.offset)); + formats.append(object(field.descr, true)); } args["names"] = names; args["offsets"] = offsets; From 669e14269d58620a75ec85bd8cdb93776769a8ae Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Tue, 21 Jun 2016 21:05:29 +0100 Subject: [PATCH 19/87] Add test for a function accepting recarray (WIP) --- example/example20.cpp | 22 ++++++++++++++++++++++ example/example20.py | 10 +++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/example/example20.cpp b/example/example20.cpp index 7066e321b6..1d4088fd37 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -21,17 +21,29 @@ struct Struct { float z; }; +std::ostream& operator<<(std::ostream& os, const Struct& v) { + return os << v.x << "," << v.y << "," << v.z; +} + struct PackedStruct { bool x; uint32_t y; float z; } __attribute__((packed)); +std::ostream& operator<<(std::ostream& os, const PackedStruct& v) { + return os << v.x << "," << v.y << "," << v.z; +} + struct NestedStruct { Struct a; PackedStruct b; } __attribute__((packed)); +std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { + return os << v.a << "|" << v.b; +} + template py::array mkarray_via_buffer(size_t n) { return py::array(py::buffer_info(nullptr, sizeof(T), @@ -59,6 +71,13 @@ py::array_t create_nested(size_t n) { return arr; } +template +void print_recarray(py::array_t arr) { + auto buf = arr.request(); + auto ptr = static_cast(buf.ptr); + for (size_t i = 0; i < buf.size; i++) + std::cout << ptr[i] << std::endl; +} void print_format_descriptors() { std::cout << py::format_descriptor::value() << std::endl; @@ -75,4 +94,7 @@ void init_ex20(py::module &m) { m.def("create_rec_packed", &create_recarray); m.def("create_rec_nested", &create_nested); m.def("print_format_descriptors", &print_format_descriptors); + m.def("print_rec_simple", &print_recarray); + m.def("print_rec_packed", &print_recarray); + m.def("print_rec_nested", &print_recarray); } diff --git a/example/example20.py b/example/example20.py index 83a914c248..78e717d913 100644 --- a/example/example20.py +++ b/example/example20.py @@ -3,7 +3,8 @@ import numpy as np from example import ( - create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors + create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, + print_rec_simple, print_rec_packed, print_rec_nested ) @@ -28,6 +29,12 @@ def check_eq(arr, data, dtype): check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], simple_dtype) check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], packed_dtype) + # uncomment lines below to cause a segfault upon exit in Py_Finalize :( + + # if dtype == simple_dtype: + # print_rec_simple(arr) + # else: + # print_rec_packed(arr) nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)]) @@ -40,3 +47,4 @@ def check_eq(arr, data, dtype): check_eq(arr, [((False, 0, 0.0), (True, 1, 1.5)), ((True, 1, 1.5), (False, 2, 3.0)), ((False, 2, 3.0), (True, 3, 4.5))], nested_dtype) +# print_rec_nested(arr) From 2a7acb6d55c0f1e59df1ddb518de7fe6c2fa6456 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 00:33:56 +0100 Subject: [PATCH 20/87] Incref descriptors properly when creating arrays --- example/example20.py | 12 +++++------- example/example20.ref | 9 +++++++++ include/pybind11/numpy.h | 4 ++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/example/example20.py b/example/example20.py index 78e717d913..7dd60e3463 100644 --- a/example/example20.py +++ b/example/example20.py @@ -29,12 +29,10 @@ def check_eq(arr, data, dtype): check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], simple_dtype) check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], packed_dtype) - # uncomment lines below to cause a segfault upon exit in Py_Finalize :( - - # if dtype == simple_dtype: - # print_rec_simple(arr) - # else: - # print_rec_packed(arr) + if dtype == simple_dtype: + print_rec_simple(arr) + else: + print_rec_packed(arr) nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)]) @@ -47,4 +45,4 @@ def check_eq(arr, data, dtype): check_eq(arr, [((False, 0, 0.0), (True, 1, 1.5)), ((True, 1, 1.5), (False, 2, 3.0)), ((False, 2, 3.0), (True, 3, 4.5))], nested_dtype) -# print_rec_nested(arr) +print_rec_nested(arr) diff --git a/example/example20.ref b/example/example20.ref index 6023a87f24..315a82a00a 100644 --- a/example/example20.ref +++ b/example/example20.ref @@ -1,3 +1,12 @@ T{?:x:xxxI:y:f:z:} T{?:x:=I:y:f:z:} T{T{?:x:xxxI:y:f:z:}:a:T{?:x:=I:y:f:z:}:b:} +0,0,0 +1,1,1.5 +0,2,3 +0,0,0 +1,1,1.5 +0,2,3 +0,0,0|1,1,1.5 +1,1,1.5|0,2,3 +0,2,3|1,3,4.5 diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1d957d1a33..41d0fea903 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -96,7 +96,7 @@ class array : public buffer { template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = detail::npy_format_descriptor::descr(); + PyObject *descr = object(detail::npy_format_descriptor::descr(), true).release().ptr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); @@ -147,7 +147,7 @@ template class array_t : public if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = detail::npy_format_descriptor::descr(); + PyObject *descr = object(detail::npy_format_descriptor::descr(), true).release().ptr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); From 1f54cd92096b163185fa15669f359e0dc70c1a73 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 00:42:10 +0100 Subject: [PATCH 21/87] Use object instead of ptrs in numpy descriptors --- include/pybind11/numpy.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 41d0fea903..288909987a 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -96,7 +96,7 @@ class array : public buffer { template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = object(detail::npy_format_descriptor::descr(), true).release().ptr(); + PyObject *descr = detail::npy_format_descriptor::descr().release().ptr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); @@ -147,7 +147,7 @@ template class array_t : public if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = object(detail::npy_format_descriptor::descr(), true).release().ptr(); + PyObject *descr = detail::npy_format_descriptor::descr().release().ptr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -178,8 +178,8 @@ template struct npy_format_descriptor::value ? 1 : 0)]; } - static PyObject* descr() { - if (auto obj = array::lookup_api().PyArray_DescrFromType_(typenum())) return obj; + static py::object descr() { + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return py::object(ptr, true); else pybind11_fail("Unsupported buffer format!"); } template ::value, int>::type = 0> @@ -192,8 +192,8 @@ template constexpr const int npy_format_descriptor< #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ static int typenum() { return array::API::NumPyName; } \ - static PyObject* descr() { \ - if (auto obj = array::lookup_api().PyArray_DescrFromType_(typenum())) return obj; \ + static py::object descr() { \ + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return py::object(ptr, true); \ else pybind11_fail("Unsupported buffer format!"); \ } \ static PYBIND11_DESCR name() { return _(Name); } } @@ -207,7 +207,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); struct field_descriptor { const char *name; int offset; - PyObject *descr; + py::object descr; }; template struct npy_format_descriptor @@ -220,10 +220,10 @@ template struct npy_format_descriptor { static PYBIND11_DESCR name() { return _("user-defined"); } - static PyObject* descr() { + static py::object descr() { if (!descr_()) pybind11_fail("NumPy: unsupported buffer format!"); - return descr_(); + return py::object(descr_(), true); } static const char* format_str() { @@ -239,7 +239,7 @@ template struct npy_format_descriptor pybind11_fail("NumPy: unsupported field dtype"); names.append(py::str(field.name)); offsets.append(py::int_(field.offset)); - formats.append(object(field.descr, true)); + formats.append(field.descr); } args["names"] = names; args["offsets"] = offsets; @@ -265,7 +265,8 @@ template struct npy_format_descriptor #define FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ - ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() } + ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() \ + } // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz From 873d2674716fdfbdf5b7ccf4d6cd951693d9a555 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 00:48:36 +0100 Subject: [PATCH 22/87] Prefix all macros in numpy.h to avoid name clashes --- include/pybind11/numpy.h | 41 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 288909987a..c41b8da312 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -270,27 +270,30 @@ template struct npy_format_descriptor // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz -#define EVAL0(...) __VA_ARGS__ -#define EVAL1(...) EVAL0 (EVAL0 (EVAL0 (__VA_ARGS__))) -#define EVAL2(...) EVAL1 (EVAL1 (EVAL1 (__VA_ARGS__))) -#define EVAL3(...) EVAL2 (EVAL2 (EVAL2 (__VA_ARGS__))) -#define EVAL4(...) EVAL3 (EVAL3 (EVAL3 (__VA_ARGS__))) -#define EVAL(...) EVAL4 (EVAL4 (EVAL4 (__VA_ARGS__))) -#define MAP_END(...) -#define MAP_OUT -#define MAP_COMMA , -#define MAP_GET_END() 0, MAP_END -#define MAP_NEXT0(test, next, ...) next MAP_OUT -#define MAP_NEXT1(test, next) MAP_NEXT0 (test, next, 0) -#define MAP_NEXT(test, next) MAP_NEXT1 (MAP_GET_END test, next) -#define MAP_LIST_NEXT1(test, next) MAP_NEXT0 (test, MAP_COMMA next, 0) -#define MAP_LIST_NEXT(test, next) MAP_LIST_NEXT1 (MAP_GET_END test, next) -#define MAP_LIST0(f, t, x, peek, ...) f(t, x) MAP_LIST_NEXT (peek, MAP_LIST1) (f, t, peek, __VA_ARGS__) -#define MAP_LIST1(f, t, x, peek, ...) f(t, x) MAP_LIST_NEXT (peek, MAP_LIST0) (f, t, peek, __VA_ARGS__) -#define MAP_LIST(f, t, ...) EVAL (MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) +#define PB11_IMPL_EVAL0(...) __VA_ARGS__ +#define PB11_IMPL_EVAL1(...) PB11_IMPL_EVAL0 (PB11_IMPL_EVAL0 (PB11_IMPL_EVAL0 (__VA_ARGS__))) +#define PB11_IMPL_EVAL2(...) PB11_IMPL_EVAL1 (PB11_IMPL_EVAL1 (PB11_IMPL_EVAL1 (__VA_ARGS__))) +#define PB11_IMPL_EVAL3(...) PB11_IMPL_EVAL2 (PB11_IMPL_EVAL2 (PB11_IMPL_EVAL2 (__VA_ARGS__))) +#define PB11_IMPL_EVAL4(...) PB11_IMPL_EVAL3 (PB11_IMPL_EVAL3 (PB11_IMPL_EVAL3 (__VA_ARGS__))) +#define PB11_IMPL_EVAL(...) PB11_IMPL_EVAL4 (PB11_IMPL_EVAL4 (PB11_IMPL_EVAL4 (__VA_ARGS__))) +#define PB11_IMPL_MAP_END(...) +#define PB11_IMPL_MAP_OUT +#define PB11_IMPL_MAP_COMMA , +#define PB11_IMPL_MAP_GET_END() 0, PB11_IMPL_MAP_END +#define PB11_IMPL_MAP_NEXT0(test, next, ...) next PB11_IMPL_MAP_OUT +#define PB11_IMPL_MAP_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, next, 0) +#define PB11_IMPL_MAP_NEXT(test, next) PB11_IMPL_MAP_NEXT1 (PB11_IMPL_MAP_GET_END test, next) +#define PB11_IMPL_MAP_LIST_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0) +#define PB11_IMPL_MAP_LIST_NEXT(test, next) PB11_IMPL_MAP_LIST_NEXT1 (PB11_IMPL_MAP_GET_END test, next) +#define PB11_IMPL_MAP_LIST0(f, t, x, peek, ...) \ + f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST1) (f, t, peek, __VA_ARGS__) +#define PB11_IMPL_MAP_LIST1(f, t, x, peek, ...) \ + f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST0) (f, t, peek, __VA_ARGS__) +#define PB11_IMPL_MAP_LIST(f, t, ...) PB11_IMPL_EVAL (PB11_IMPL_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) #define PYBIND11_DTYPE(Type, ...) \ - ::pybind11::detail::npy_format_descriptor::register_dtype({MAP_LIST(FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + ::pybind11::detail::npy_format_descriptor::register_dtype \ + ({PB11_IMPL_MAP_LIST(FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) template using array_iterator = typename std::add_pointer::type; From 036e8cd32f067953d7236a4612d9f3e4674650ce Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 00:52:16 +0100 Subject: [PATCH 23/87] Remove erroneous py:: prefix in numpy.h --- include/pybind11/numpy.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index c41b8da312..1fc6752008 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -118,7 +118,7 @@ class array : public buffer { buf_info.ptr = std::calloc(info.size, info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); - auto view = py::memoryview(buf_info); + auto view = memoryview(buf_info); API& api = lookup_api(); auto res = api.PyArray_GetArrayParamsFromObject_(view.ptr(), nullptr, 1, &descr, @@ -178,8 +178,8 @@ template struct npy_format_descriptor::value ? 1 : 0)]; } - static py::object descr() { - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return py::object(ptr, true); + static object descr() { + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return object(ptr, true); else pybind11_fail("Unsupported buffer format!"); } template ::value, int>::type = 0> @@ -192,8 +192,8 @@ template constexpr const int npy_format_descriptor< #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ static int typenum() { return array::API::NumPyName; } \ - static py::object descr() { \ - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return py::object(ptr, true); \ + static object descr() { \ + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return object(ptr, true); \ else pybind11_fail("Unsupported buffer format!"); \ } \ static PYBIND11_DESCR name() { return _(Name); } } @@ -207,7 +207,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); struct field_descriptor { const char *name; int offset; - py::object descr; + object descr; }; template struct npy_format_descriptor @@ -220,10 +220,10 @@ template struct npy_format_descriptor { static PYBIND11_DESCR name() { return _("user-defined"); } - static py::object descr() { + static object descr() { if (!descr_()) pybind11_fail("NumPy: unsupported buffer format!"); - return py::object(descr_(), true); + return object(descr_(), true); } static const char* format_str() { @@ -232,13 +232,13 @@ template struct npy_format_descriptor static void register_dtype(std::initializer_list fields) { array::API& api = array::lookup_api(); - auto args = py::dict(); - py::list names { }, offsets { }, formats { }; + auto args = dict(); + list names { }, offsets { }, formats { }; for (auto field : fields) { if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); - names.append(py::str(field.name)); - offsets.append(py::int_(field.offset)); + names.append(str(field.name)); + offsets.append(int_(field.offset)); formats.append(field.descr); } args["names"] = names; @@ -248,7 +248,7 @@ template struct npy_format_descriptor pybind11_fail("NumPy: failed to create structured dtype"); auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); - if (auto arr = (object) empty(py::int_(0), object(descr(), true))) + if (auto arr = (object) empty(int_(0), object(descr(), true))) if (auto view = PyMemoryView_FromObject(arr.ptr())) if (auto info = PyMemoryView_GET_BUFFER(view)) { std::strncpy(format_str_(), info->format, 4096); From 4f164217e48ae1dbce5cfb896195c7b56521fc2a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 01:07:20 +0100 Subject: [PATCH 24/87] Add dtype_of() function, update the tests --- example/example20.cpp | 30 ++++++++++++++++++++---------- example/example20.py | 3 ++- example/example20.ref | 21 ++++++++++++--------- include/pybind11/numpy.h | 5 +++++ 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/example/example20.cpp b/example/example20.cpp index 1d4088fd37..201c470e0a 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -15,14 +15,14 @@ namespace py = pybind11; -struct Struct { +struct SimpleStruct { bool x; uint32_t y; float z; }; -std::ostream& operator<<(std::ostream& os, const Struct& v) { - return os << v.x << "," << v.y << "," << v.z; +std::ostream& operator<<(std::ostream& os, const SimpleStruct& v) { + return os << "s:" << v.x << "," << v.y << "," << v.z; } struct PackedStruct { @@ -32,16 +32,16 @@ struct PackedStruct { } __attribute__((packed)); std::ostream& operator<<(std::ostream& os, const PackedStruct& v) { - return os << v.x << "," << v.y << "," << v.z; + return os << "p:" << v.x << "," << v.y << "," << v.z; } struct NestedStruct { - Struct a; + SimpleStruct a; PackedStruct b; } __attribute__((packed)); std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { - return os << v.a << "|" << v.b; + return os << "n:a=" << v.a << ";b=" << v.b; } template @@ -80,21 +80,31 @@ void print_recarray(py::array_t arr) { } void print_format_descriptors() { - std::cout << py::format_descriptor::value() << std::endl; + std::cout << py::format_descriptor::value() << std::endl; std::cout << py::format_descriptor::value() << std::endl; std::cout << py::format_descriptor::value() << std::endl; } +void print_dtypes() { + auto to_str = [](py::object obj) { + return (std::string) (py::str) ((py::object) obj.attr("__str__"))(); + }; + std::cout << to_str(py::dtype_of()) << std::endl; + std::cout << to_str(py::dtype_of()) << std::endl; + std::cout << to_str(py::dtype_of()) << std::endl; +} + void init_ex20(py::module &m) { - PYBIND11_DTYPE(Struct, x, y, z); + PYBIND11_DTYPE(SimpleStruct, x, y, z); PYBIND11_DTYPE(PackedStruct, x, y, z); PYBIND11_DTYPE(NestedStruct, a, b); - m.def("create_rec_simple", &create_recarray); + m.def("create_rec_simple", &create_recarray); m.def("create_rec_packed", &create_recarray); m.def("create_rec_nested", &create_nested); m.def("print_format_descriptors", &print_format_descriptors); - m.def("print_rec_simple", &print_recarray); + m.def("print_rec_simple", &print_recarray); m.def("print_rec_packed", &print_recarray); m.def("print_rec_nested", &print_recarray); + m.def("print_dtypes", &print_dtypes); } diff --git a/example/example20.py b/example/example20.py index 7dd60e3463..c581cf3f72 100644 --- a/example/example20.py +++ b/example/example20.py @@ -4,7 +4,7 @@ import numpy as np from example import ( create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, - print_rec_simple, print_rec_packed, print_rec_nested + print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes ) @@ -12,6 +12,7 @@ def check_eq(arr, data, dtype): np.testing.assert_equal(arr, np.array(data, dtype=dtype)) print_format_descriptors() +print_dtypes() simple_dtype = np.dtype({'names': ['x', 'y', 'z'], 'formats': ['?', 'u4', 'f4'], diff --git a/example/example20.ref b/example/example20.ref index 315a82a00a..32e2b4b7af 100644 --- a/example/example20.ref +++ b/example/example20.ref @@ -1,12 +1,15 @@ T{?:x:xxxI:y:f:z:} T{?:x:=I:y:f:z:} T{T{?:x:xxxI:y:f:z:}:a:T{?:x:=I:y:f:z:}:b:} -0,0,0 -1,1,1.5 -0,2,3 -0,0,0 -1,1,1.5 -0,2,3 -0,0,0|1,1,1.5 -1,1,1.5|0,2,3 -0,2,3|1,3,4.5 +{'names':['x','y','z'], 'formats':['?',' struct format_descriptor } }; +template +object dtype_of() { + return detail::npy_format_descriptor::descr(); +} + NAMESPACE_BEGIN(detail) template struct npy_format_descriptor::value>::type> { From 5e71e17bdfa13c15e4ac752fec1fa1d7684d15e7 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 12:42:34 +0100 Subject: [PATCH 25/87] Make changes to format_descriptor backwards-compat The format strings that are known at compile time are now accessible via both ::value and ::format(), and format strings for everything else is accessible via ::format(). This makes it backwards compatible. --- docs/advanced.rst | 19 +++++++++---------- example/example-buffers.cpp | 14 +++++++------- example/example20.cpp | 8 ++++---- include/pybind11/common.h | 11 ++++++----- include/pybind11/eigen.h | 10 +++++----- include/pybind11/numpy.h | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index aca1325946..1620db7172 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -1224,12 +1224,12 @@ completely avoid copy operations with Python expressions like py::class_(m, "Matrix") .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( - m.data(), /* Pointer to buffer */ - sizeof(float), /* Size of one scalar */ - py::format_descriptor::value(), /* Python struct-style format descriptor */ - 2, /* Number of dimensions */ - { m.rows(), m.cols() }, /* Buffer dimensions */ - { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ + m.data(), /* Pointer to buffer */ + sizeof(float), /* Size of one scalar */ + py::format_descriptor::format(), /* Python struct-style format descriptor */ + 2, /* Number of dimensions */ + { m.rows(), m.cols() }, /* Buffer dimensions */ + { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ sizeof(float) } ); }); @@ -1273,7 +1273,7 @@ buffer objects (e.g. a NumPy matrix). py::buffer_info info = b.request(); /* Some sanity checks ... */ - if (info.format != py::format_descriptor::value()) + if (info.format != py::format_descriptor::format()) throw std::runtime_error("Incompatible format: expected a double array!"); if (info.ndim != 2) @@ -1299,7 +1299,7 @@ as follows: m.data(), /* Pointer to buffer */ sizeof(Scalar), /* Size of one scalar */ /* Python struct-style format descriptor */ - py::format_descriptor::value(), + py::format_descriptor::format(), /* Number of dimensions */ 2, /* Buffer dimensions */ @@ -1439,7 +1439,7 @@ simply using ``vectorize``). auto result = py::array(py::buffer_info( nullptr, /* Pointer to data (nullptr -> ask NumPy to allocate!) */ sizeof(double), /* Size of one item */ - py::format_descriptor::value, /* Buffer format */ + py::format_descriptor::format(), /* Buffer format */ buf1.ndim, /* How many dimensions? */ { buf1.shape[0] }, /* Number of elements for each dimension */ { sizeof(double) } /* Strides for each dimension */ @@ -1830,4 +1830,3 @@ is always ``none``). // Evaluate the statements in an separate Python file on disk py::eval_file("script.py", scope); - diff --git a/example/example-buffers.cpp b/example/example-buffers.cpp index 2deee6f836..fa3178b51d 100644 --- a/example/example-buffers.cpp +++ b/example/example-buffers.cpp @@ -81,7 +81,7 @@ void init_ex_buffers(py::module &m) { /// Construct from a buffer .def("__init__", [](Matrix &v, py::buffer b) { py::buffer_info info = b.request(); - if (info.format != py::format_descriptor::value() || info.ndim != 2) + if (info.format != py::format_descriptor::format() || info.ndim != 2) throw std::runtime_error("Incompatible buffer format!"); new (&v) Matrix(info.shape[0], info.shape[1]); memcpy(v.data(), info.ptr, sizeof(float) * v.rows() * v.cols()); @@ -104,12 +104,12 @@ void init_ex_buffers(py::module &m) { /// Provide buffer access .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( - m.data(), /* Pointer to buffer */ - sizeof(float), /* Size of one scalar */ - py::format_descriptor::value(), /* Python struct-style format descriptor */ - 2, /* Number of dimensions */ - { m.rows(), m.cols() }, /* Buffer dimensions */ - { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ + m.data(), /* Pointer to buffer */ + sizeof(float), /* Size of one scalar */ + py::format_descriptor::format(), /* Python struct-style format descriptor */ + 2, /* Number of dimensions */ + { m.rows(), m.cols() }, /* Buffer dimensions */ + { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ sizeof(float) } ); }) diff --git a/example/example20.cpp b/example/example20.cpp index 201c470e0a..77cda6b5c7 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -47,7 +47,7 @@ std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { template py::array mkarray_via_buffer(size_t n) { return py::array(py::buffer_info(nullptr, sizeof(T), - py::format_descriptor::value(), + py::format_descriptor::format(), 1, { n }, { sizeof(T) })); } @@ -80,9 +80,9 @@ void print_recarray(py::array_t arr) { } void print_format_descriptors() { - std::cout << py::format_descriptor::value() << std::endl; - std::cout << py::format_descriptor::value() << std::endl; - std::cout << py::format_descriptor::value() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; } void print_dtypes() { diff --git a/include/pybind11/common.h b/include/pybind11/common.h index ca5765ab15..32e8abf53c 100644 --- a/include/pybind11/common.h +++ b/include/pybind11/common.h @@ -204,7 +204,7 @@ struct buffer_info { void *ptr; // Pointer to the underlying storage size_t itemsize; // Size of individual items in bytes size_t size; // Total number of entries - std::string format; // For homogeneous buffers, this should be set to format_descriptor::value() + std::string format; // For homogeneous buffers, this should be set to format_descriptor::format() size_t ndim; // Number of dimensions std::vector shape; // Shape of the tensor (1 entry per dimension) std::vector strides; // Number of entries between adjacent entries (for each per dimension) @@ -349,18 +349,19 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error) /// Used internally /// Format strings for basic number types #define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor \ - { static constexpr const char* value() { return v; } }; + { static constexpr const char* value = v; /* for backwards compatibility */ \ + static constexpr const char* format() { return value; } } template struct format_descriptor { }; template struct format_descriptor::value>::type> { - static constexpr const char* value() { return format; } - static constexpr const char format[2] = + static constexpr const char value[2] = { "bBhHiIqQ"[detail::log2(sizeof(T))*2 + (std::is_unsigned::value ? 1 : 0)], '\0' }; + static constexpr const char* format() { return value; } }; template constexpr const char format_descriptor< - T, typename std::enable_if::value>::type>::format[2]; + T, typename std::enable_if::value>::type>::value[2]; PYBIND11_DECL_FMT(float, "f"); PYBIND11_DECL_FMT(double, "d"); diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index db8f9eb172..2ea7d486b8 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -133,7 +133,7 @@ struct type_caster::value && /* Size of one scalar */ sizeof(Scalar), /* Python struct-style format descriptor */ - format_descriptor::value(), + format_descriptor::format(), /* Number of dimensions */ 1, /* Buffer dimensions */ @@ -148,7 +148,7 @@ struct type_caster::value && /* Size of one scalar */ sizeof(Scalar), /* Python struct-style format descriptor */ - format_descriptor::value(), + format_descriptor::format(), /* Number of dimensions */ isVector ? 1 : 2, /* Buffer dimensions */ @@ -276,7 +276,7 @@ struct type_caster::value>:: // Size of one scalar sizeof(Scalar), // Python struct-style format descriptor - format_descriptor::value(), + format_descriptor::format(), // Number of dimensions 1, // Buffer dimensions @@ -291,7 +291,7 @@ struct type_caster::value>:: // Size of one scalar sizeof(StorageIndex), // Python struct-style format descriptor - format_descriptor::value(), + format_descriptor::format(), // Number of dimensions 1, // Buffer dimensions @@ -306,7 +306,7 @@ struct type_caster::value>:: // Size of one scalar sizeof(StorageIndex), // Python struct-style format descriptor - format_descriptor::value(), + format_descriptor::format(), // Number of dimensions 1, // Buffer dimensions diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 19ba4de728..cac19ee723 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -164,8 +164,8 @@ template struct format_descriptor !std::is_same>::value && !std::is_same>::value>::type> { - static const char *value() { - return detail::npy_format_descriptor::format_str(); + static const char *format() { + return detail::npy_format_descriptor::format(); } }; @@ -231,8 +231,8 @@ template struct npy_format_descriptor return object(descr_(), true); } - static const char* format_str() { - return format_str_(); + static const char* format() { + return format_(); } static void register_dtype(std::initializer_list fields) { @@ -256,7 +256,7 @@ template struct npy_format_descriptor if (auto arr = (object) empty(int_(0), object(descr(), true))) if (auto view = PyMemoryView_FromObject(arr.ptr())) if (auto info = PyMemoryView_GET_BUFFER(view)) { - std::strncpy(format_str_(), info->format, 4096); + std::strncpy(format_(), info->format, 4096); return; } pybind11_fail("NumPy: failed to extract buffer format"); @@ -264,7 +264,7 @@ template struct npy_format_descriptor private: static inline PyObject*& descr_() { static PyObject *ptr = nullptr; return ptr; } - static inline char* format_str_() { static char s[4096]; return s; } + static inline char* format_() { static char s[4096]; return s; } }; #define FIELD_DESCRIPTOR(Type, Field) \ @@ -480,7 +480,7 @@ struct vectorize_helper { return cast(f(*((Args *) buffers[Index].ptr)...)); array result(buffer_info(nullptr, sizeof(Return), - format_descriptor::value(), + format_descriptor::format(), ndim, shape, strides)); buffer_info buf = result.request(); From 95e9b1232262000fe798fe63a5d0fd8b794faf2e Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:18:46 +0100 Subject: [PATCH 26/87] Prefix the FIELD_DESCRIPTOR macro --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index cac19ee723..aacc993dbe 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -267,7 +267,7 @@ template struct npy_format_descriptor static inline char* format_() { static char s[4096]; return s; } }; -#define FIELD_DESCRIPTOR(Type, Field) \ +#define PB11_IMPL_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() \ @@ -298,7 +298,7 @@ template struct npy_format_descriptor #define PYBIND11_DTYPE(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ - ({PB11_IMPL_MAP_LIST(FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + ({PB11_IMPL_MAP_LIST(PB11_IMPL_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) template using array_iterator = typename std::add_pointer::type; From 40eadfeb736fa46aa1f5d1932a4bda07712071f4 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:19:18 +0100 Subject: [PATCH 27/87] Make npy_format_descriptor backwards-compat The typenum for non-structured types is still accessible at ::value, and the dtype object for all types is accessible at ::dtype(). --- include/pybind11/numpy.h | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index aacc993dbe..e14a58ed13 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -96,7 +96,7 @@ class array : public buffer { template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = detail::npy_format_descriptor::descr().release().ptr(); + PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); @@ -147,7 +147,7 @@ template class array_t : public if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = detail::npy_format_descriptor::descr().release().ptr(); + PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -171,7 +171,7 @@ template struct format_descriptor template object dtype_of() { - return detail::npy_format_descriptor::descr(); + return detail::npy_format_descriptor::dtype(); } NAMESPACE_BEGIN(detail) @@ -182,10 +182,11 @@ template struct npy_format_descriptor::value ? 1 : 0)]; } - static object descr() { - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return object(ptr, true); - else pybind11_fail("Unsupported buffer format!"); + enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; + static object dtype() { + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(value)) + return object(ptr, true); + pybind11_fail("Unsupported buffer format!"); } template ::value, int>::type = 0> static PYBIND11_DESCR name() { return _("int") + _(); } @@ -196,10 +197,11 @@ template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - static int typenum() { return array::API::NumPyName; } \ - static object descr() { \ - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return object(ptr, true); \ - else pybind11_fail("Unsupported buffer format!"); \ + enum { value = array::API::NumPyName }; \ + static object dtype() { \ + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(value)) \ + return object(ptr, true); \ + pybind11_fail("Unsupported buffer format!"); \ } \ static PYBIND11_DESCR name() { return _(Name); } } DECL_FMT(float, NPY_FLOAT_, "float32"); @@ -225,10 +227,10 @@ template struct npy_format_descriptor { static PYBIND11_DESCR name() { return _("user-defined"); } - static object descr() { - if (!descr_()) + static object dtype() { + if (!dtype_()) pybind11_fail("NumPy: unsupported buffer format!"); - return object(descr_(), true); + return object(dtype_(), true); } static const char* format() { @@ -249,11 +251,11 @@ template struct npy_format_descriptor args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; - if (!api.PyArray_DescrConverter_(args.release().ptr(), &descr_()) || !descr_()) + if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) pybind11_fail("NumPy: failed to create structured dtype"); auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); - if (auto arr = (object) empty(int_(0), object(descr(), true))) + if (auto arr = (object) empty(int_(0), dtype())) if (auto view = PyMemoryView_FromObject(arr.ptr())) if (auto info = PyMemoryView_GET_BUFFER(view)) { std::strncpy(format_(), info->format, 4096); @@ -263,14 +265,14 @@ template struct npy_format_descriptor } private: - static inline PyObject*& descr_() { static PyObject *ptr = nullptr; return ptr; } + static inline PyObject*& dtype_() { static PyObject *ptr = nullptr; return ptr; } static inline char* format_() { static char s[4096]; return s; } }; #define PB11_IMPL_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ - ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() \ + ::pybind11::detail::npy_format_descriptor(0)->Field)>::dtype() \ } // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro From 5a47a16e47cdbb690b2c1e9337e15d3db6f90fb5 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:21:34 +0100 Subject: [PATCH 28/87] Revert accidental whitespace change --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index e14a58ed13..f9408526f7 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -482,7 +482,7 @@ struct vectorize_helper { return cast(f(*((Args *) buffers[Index].ptr)...)); array result(buffer_info(nullptr, sizeof(Return), - format_descriptor::format(), + format_descriptor::format(), ndim, shape, strides)); buffer_info buf = result.request(); From a0e37f250edf86b68ff0e26380883e9a3bf88434 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:34:39 +0100 Subject: [PATCH 29/87] npy_format_descriptor::format() - fail if unbound --- include/pybind11/numpy.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index f9408526f7..73cdce0e3e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -234,6 +234,8 @@ template struct npy_format_descriptor } static const char* format() { + if (!dtype_()) + pybind11_fail("NumPy: unsupported buffer format!"); return format_(); } From d0bafd90e0a8472e8b4de0ba17de636d34b195a0 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:35:28 +0100 Subject: [PATCH 30/87] Add a test for buffer format of unbound struct --- example/example20.cpp | 7 +++++++ example/example20.py | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/example/example20.cpp b/example/example20.cpp index 77cda6b5c7..662dc473a5 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -44,6 +44,8 @@ std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { return os << "n:a=" << v.a << ";b=" << v.b; } +struct UnboundStruct { }; + template py::array mkarray_via_buffer(size_t n) { return py::array(py::buffer_info(nullptr, sizeof(T), @@ -61,6 +63,10 @@ py::array_t create_recarray(size_t n) { return arr; } +std::string get_format_unbound() { + return py::format_descriptor::format(); +} + py::array_t create_nested(size_t n) { auto arr = mkarray_via_buffer(n); auto ptr = static_cast(arr.request().ptr); @@ -107,4 +113,5 @@ void init_ex20(py::module &m) { m.def("print_rec_packed", &print_recarray); m.def("print_rec_nested", &print_recarray); m.def("print_dtypes", &print_dtypes); + m.def("get_format_unbound", &get_format_unbound); } diff --git a/example/example20.py b/example/example20.py index c581cf3f72..e0a0018040 100644 --- a/example/example20.py +++ b/example/example20.py @@ -1,16 +1,20 @@ #!/usr/bin/env python from __future__ import print_function +import unittest import numpy as np from example import ( create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, - print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes + print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound ) def check_eq(arr, data, dtype): np.testing.assert_equal(arr, np.array(data, dtype=dtype)) +unittest.TestCase().assertRaisesRegex( + RuntimeError, 'unsupported buffer format', get_format_unbound) + print_format_descriptors() print_dtypes() From 73f56830f8e4b024f24717e323d48c5e98fa1275 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:46:40 +0100 Subject: [PATCH 31/87] Add detail::is_pod_struct helper --- include/pybind11/numpy.h | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 73cdce0e3e..bbd68cfb84 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -23,7 +23,19 @@ #endif NAMESPACE_BEGIN(pybind11) -namespace detail { template struct npy_format_descriptor { }; } +namespace detail { +template struct npy_format_descriptor { }; + +template +struct is_pod_struct { + enum { value = std::is_pod::value && // offsetof only works correctly for POD types + !std::is_integral::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same>::value && + !std::is_same>::value }; +}; +} class array : public buffer { public: @@ -156,14 +168,8 @@ template class array_t : public } }; -template struct format_descriptor -::value && - !std::is_integral::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same>::value && - !std::is_same>::value>::type> -{ +template +struct format_descriptor::value>::type> { static const char *format() { return detail::npy_format_descriptor::format(); } @@ -217,13 +223,8 @@ struct field_descriptor { object descr; }; -template struct npy_format_descriptor -::value && // offsetof only works correctly for POD types - !std::is_integral::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same>::value && - !std::is_same>::value>::type> +template +struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("user-defined"); } From 5dc6c5445d74bf82f9183fffcb92aafd62ec4a8c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 27 Jun 2016 15:47:51 +0100 Subject: [PATCH 32/87] Cosmetic: fix indentation --- include/pybind11/numpy.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index bbd68cfb84..64fa1dd0df 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -224,8 +224,7 @@ struct field_descriptor { }; template -struct npy_format_descriptor::value>::type> -{ +struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("user-defined"); } static object dtype() { From 7bdd74a9fbed302a8aa408574b667af3c0fd5bf0 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 27 Jun 2016 17:01:22 +0100 Subject: [PATCH 33/87] Fix PYBIND11_DTYPE to work with MSVC compiler --- include/pybind11/numpy.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 64fa1dd0df..abad9ff17e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -292,8 +292,15 @@ struct npy_format_descriptor::value> #define PB11_IMPL_MAP_NEXT0(test, next, ...) next PB11_IMPL_MAP_OUT #define PB11_IMPL_MAP_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, next, 0) #define PB11_IMPL_MAP_NEXT(test, next) PB11_IMPL_MAP_NEXT1 (PB11_IMPL_MAP_GET_END test, next) -#define PB11_IMPL_MAP_LIST_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0) -#define PB11_IMPL_MAP_LIST_NEXT(test, next) PB11_IMPL_MAP_LIST_NEXT1 (PB11_IMPL_MAP_GET_END test, next) +#ifdef _MSC_VER +#define PB11_IMPL_MAP_LIST_NEXT1(test, next) \ + PB11_IMPL_EVAL0 (PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0)) +#else +#define PB11_IMPL_MAP_LIST_NEXT1(test, next) \ + PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0) +#endif +#define PB11_IMPL_MAP_LIST_NEXT(test, next) \ + PB11_IMPL_MAP_LIST_NEXT1 (PB11_IMPL_MAP_GET_END test, next) #define PB11_IMPL_MAP_LIST0(f, t, x, peek, ...) \ f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST1) (f, t, peek, __VA_ARGS__) #define PB11_IMPL_MAP_LIST1(f, t, x, peek, ...) \ From 95545e6256911ea371957f88efdf731628bc402b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 27 Jun 2016 23:02:21 +0100 Subject: [PATCH 34/87] Change PB11_IMPL prefix to PYBIND11, add comment --- include/pybind11/numpy.h | 55 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index abad9ff17e..dc4000ba9e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -271,7 +271,7 @@ struct npy_format_descriptor::value> static inline char* format_() { static char s[4096]; return s; } }; -#define PB11_IMPL_FIELD_DESCRIPTOR(Type, Field) \ +#define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ ::pybind11::detail::npy_format_descriptor(0)->Field)>::dtype() \ @@ -279,37 +279,38 @@ struct npy_format_descriptor::value> // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz -#define PB11_IMPL_EVAL0(...) __VA_ARGS__ -#define PB11_IMPL_EVAL1(...) PB11_IMPL_EVAL0 (PB11_IMPL_EVAL0 (PB11_IMPL_EVAL0 (__VA_ARGS__))) -#define PB11_IMPL_EVAL2(...) PB11_IMPL_EVAL1 (PB11_IMPL_EVAL1 (PB11_IMPL_EVAL1 (__VA_ARGS__))) -#define PB11_IMPL_EVAL3(...) PB11_IMPL_EVAL2 (PB11_IMPL_EVAL2 (PB11_IMPL_EVAL2 (__VA_ARGS__))) -#define PB11_IMPL_EVAL4(...) PB11_IMPL_EVAL3 (PB11_IMPL_EVAL3 (PB11_IMPL_EVAL3 (__VA_ARGS__))) -#define PB11_IMPL_EVAL(...) PB11_IMPL_EVAL4 (PB11_IMPL_EVAL4 (PB11_IMPL_EVAL4 (__VA_ARGS__))) -#define PB11_IMPL_MAP_END(...) -#define PB11_IMPL_MAP_OUT -#define PB11_IMPL_MAP_COMMA , -#define PB11_IMPL_MAP_GET_END() 0, PB11_IMPL_MAP_END -#define PB11_IMPL_MAP_NEXT0(test, next, ...) next PB11_IMPL_MAP_OUT -#define PB11_IMPL_MAP_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, next, 0) -#define PB11_IMPL_MAP_NEXT(test, next) PB11_IMPL_MAP_NEXT1 (PB11_IMPL_MAP_GET_END test, next) -#ifdef _MSC_VER -#define PB11_IMPL_MAP_LIST_NEXT1(test, next) \ - PB11_IMPL_EVAL0 (PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0)) +#define PYBIND11_EVAL0(...) __VA_ARGS__ +#define PYBIND11_EVAL1(...) PYBIND11_EVAL0 (PYBIND11_EVAL0 (PYBIND11_EVAL0 (__VA_ARGS__))) +#define PYBIND11_EVAL2(...) PYBIND11_EVAL1 (PYBIND11_EVAL1 (PYBIND11_EVAL1 (__VA_ARGS__))) +#define PYBIND11_EVAL3(...) PYBIND11_EVAL2 (PYBIND11_EVAL2 (PYBIND11_EVAL2 (__VA_ARGS__))) +#define PYBIND11_EVAL4(...) PYBIND11_EVAL3 (PYBIND11_EVAL3 (PYBIND11_EVAL3 (__VA_ARGS__))) +#define PYBIND11_EVAL(...) PYBIND11_EVAL4 (PYBIND11_EVAL4 (PYBIND11_EVAL4 (__VA_ARGS__))) +#define PYBIND11_MAP_END(...) +#define PYBIND11_MAP_OUT +#define PYBIND11_MAP_COMMA , +#define PYBIND11_MAP_GET_END() 0, PYBIND11_MAP_END +#define PYBIND11_MAP_NEXT0(test, next, ...) next PYBIND11_MAP_OUT +#define PYBIND11_MAP_NEXT1(test, next) PYBIND11_MAP_NEXT0 (test, next, 0) +#define PYBIND11_MAP_NEXT(test, next) PYBIND11_MAP_NEXT1 (PYBIND11_MAP_GET_END test, next) +#ifdef _MSC_VER // MSVC is not as eager to expand macros +#define PYBIND11_MAP_LIST_NEXT1(test, next) \ + PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)) #else -#define PB11_IMPL_MAP_LIST_NEXT1(test, next) \ - PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0) +#define PYBIND11_MAP_LIST_NEXT1(test, next) \ + PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0) #endif -#define PB11_IMPL_MAP_LIST_NEXT(test, next) \ - PB11_IMPL_MAP_LIST_NEXT1 (PB11_IMPL_MAP_GET_END test, next) -#define PB11_IMPL_MAP_LIST0(f, t, x, peek, ...) \ - f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST1) (f, t, peek, __VA_ARGS__) -#define PB11_IMPL_MAP_LIST1(f, t, x, peek, ...) \ - f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST0) (f, t, peek, __VA_ARGS__) -#define PB11_IMPL_MAP_LIST(f, t, ...) PB11_IMPL_EVAL (PB11_IMPL_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) +#define PYBIND11_MAP_LIST_NEXT(test, next) \ + PYBIND11_MAP_LIST_NEXT1 (PYBIND11_MAP_GET_END test, next) +#define PYBIND11_MAP_LIST0(f, t, x, peek, ...) \ + f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST1) (f, t, peek, __VA_ARGS__) +#define PYBIND11_MAP_LIST1(f, t, x, peek, ...) \ + f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST0) (f, t, peek, __VA_ARGS__) +#define PYBIND11_MAP_LIST(f, t, ...) \ + PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) #define PYBIND11_DTYPE(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ - ({PB11_IMPL_MAP_LIST(PB11_IMPL_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + ({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) template using array_iterator = typename std::add_pointer::type; From b38ca22e947f7cc7fcb58297e1f4ce2d36f2d52a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 29 Jun 2016 15:16:49 +0100 Subject: [PATCH 35/87] Add a few braces for clarity --- include/pybind11/numpy.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index dc4000ba9e..5fdbe6df9d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -257,12 +257,14 @@ struct npy_format_descriptor::value> pybind11_fail("NumPy: failed to create structured dtype"); auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); - if (auto arr = (object) empty(int_(0), dtype())) - if (auto view = PyMemoryView_FromObject(arr.ptr())) + if (auto arr = (object) empty(int_(0), dtype())) { + if (auto view = PyMemoryView_FromObject(arr.ptr())) { if (auto info = PyMemoryView_GET_BUFFER(view)) { std::strncpy(format_(), info->format, 4096); return; } + } + } pybind11_fail("NumPy: failed to extract buffer format"); } From 2f01f01866a1efe137e0dbd7160165bca44d087e Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 29 Jun 2016 15:21:10 +0100 Subject: [PATCH 36/87] Always allocate at least one element --- include/pybind11/numpy.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 5fdbe6df9d..43f0d57ff0 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -127,7 +127,8 @@ class array : public buffer { // allocate zeroed memory if it hasn't been provided auto buf_info = info; if (!buf_info.ptr) - buf_info.ptr = std::calloc(info.size, info.itemsize); + // always allocate at least 1 element, same way as NumPy does it + buf_info.ptr = std::calloc(std::max(info.size, 1ul), info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); auto view = memoryview(buf_info); From 3b803846d50783f4cb4fe2e41d76b0f9c9f520ea Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 29 Jun 2016 15:21:51 +0100 Subject: [PATCH 37/87] Add a few comments throughout numpy.h --- include/pybind11/numpy.h | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 43f0d57ff0..a3bab986fe 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -123,20 +123,29 @@ class array : public buffer { PyObject *arr = nullptr, *descr = nullptr; int ndim = 0; Py_ssize_t dims[32]; + API& api = lookup_api(); - // allocate zeroed memory if it hasn't been provided + // Allocate zeroed memory if it hasn't been provided by the caller. + // Normally, we could leave this null for NumPy to allocate memory for us, but + // since we need a memoryview, the data pointer has to be non-null. NumPy uses + // malloc if NPY_NEEDS_INIT is not set (in which case it uses calloc); however, + // we don't have a descriptor yet (only a buffer format string), so we can't + // access the flags. The safest thing to do is thus to use calloc. auto buf_info = info; if (!buf_info.ptr) // always allocate at least 1 element, same way as NumPy does it buf_info.ptr = std::calloc(std::max(info.size, 1ul), info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); - auto view = memoryview(buf_info); - API& api = lookup_api(); + // PyArray_GetArrayParamsFromObject seems to be the only low-level API function + // that will accept arbitrary buffers (including structured types) + auto view = memoryview(buf_info); auto res = api.PyArray_GetArrayParamsFromObject_(view.ptr(), nullptr, 1, &descr, &ndim, dims, &arr, nullptr); if (res < 0 || !arr || descr) + // We expect arr to have a pointer to a newly created array, in which case all + // other parameters like descr would be set to null, according to the API. pybind11_fail("NumPy: unable to convert buffer to an array"); m_ptr = arr; } @@ -161,7 +170,8 @@ template class array_t : public return nullptr; API &api = lookup_api(); PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); - PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, + API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); Py_DECREF(ptr); @@ -254,8 +264,11 @@ struct npy_format_descriptor::value> args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; + // This is essentially the same as calling np.dtype() constructor in Python and passing + // it a dict of the form {'names': ..., 'formats': ..., 'offsets': ...}. if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) pybind11_fail("NumPy: failed to create structured dtype"); + // Let NumPy figure the buffer format string for us: memoryview(np.empty(0, dtype)).format auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); if (auto arr = (object) empty(int_(0), dtype())) { @@ -274,6 +287,7 @@ struct npy_format_descriptor::value> static inline char* format_() { static char s[4096]; return s; } }; +// Extract name, offset and format descriptor for a struct field #define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ @@ -295,7 +309,7 @@ struct npy_format_descriptor::value> #define PYBIND11_MAP_NEXT0(test, next, ...) next PYBIND11_MAP_OUT #define PYBIND11_MAP_NEXT1(test, next) PYBIND11_MAP_NEXT0 (test, next, 0) #define PYBIND11_MAP_NEXT(test, next) PYBIND11_MAP_NEXT1 (PYBIND11_MAP_GET_END test, next) -#ifdef _MSC_VER // MSVC is not as eager to expand macros +#ifdef _MSC_VER // MSVC is not as eager to expand macros, hence this workaround #define PYBIND11_MAP_LIST_NEXT1(test, next) \ PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)) #else @@ -308,6 +322,7 @@ struct npy_format_descriptor::value> f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST1) (f, t, peek, __VA_ARGS__) #define PYBIND11_MAP_LIST1(f, t, x, peek, ...) \ f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST0) (f, t, peek, __VA_ARGS__) +// PYBIND11_MAP_LIST(f, t, a1, a2, ...) expands to f(t, a1), f(t, a2), ... #define PYBIND11_MAP_LIST(f, t, ...) \ PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) From 4c9a160a1d85be98c7111f50daa2f41e5fb5cd70 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 29 Jun 2016 15:27:21 +0100 Subject: [PATCH 38/87] Exclude double type from is_pod_struct --- include/pybind11/numpy.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index a3bab986fe..393c887e8b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -31,6 +31,7 @@ struct is_pod_struct { enum { value = std::is_pod::value && // offsetof only works correctly for POD types !std::is_integral::value && !std::is_same::value && + !std::is_same::value && !std::is_same::value && !std::is_same>::value && !std::is_same>::value }; From 223afe37fad4b89418633f8f91a522ae6cefab85 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 2 Jul 2016 15:33:04 +0100 Subject: [PATCH 39/87] Add documentation re: PYBIND11_DTYPE macro --- docs/advanced.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/advanced.rst b/docs/advanced.rst index 1620db7172..649454c91e 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -1358,6 +1358,30 @@ template paramenter, and it ensures that non-conforming arguments are converted into an array satisfying the specified requirements instead of trying the next function overload. +NumPy structured types +====================== + +In order for ``py::array_t`` to work with structured (record) types, we first need +to register the memory layout of the type. This could be done via ``PYBIND11_DTYPE`` +macro which expects the type followed by field names: + +.. code-block:: cpp + + struct A { + int x; + double y; + }; + + struct B { + int z; + A a; + }; + + PYBIND11_DTYPE(A, x, y); + PYBIND11_DTYPE(B, z, a); + + /* now both A and B can be used as template arguments to py::array_t */ + Vectorizing functions ===================== From 872bd92575299cb701823dfe098479915d4e854a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 2 Jul 2016 16:14:57 +0100 Subject: [PATCH 40/87] Use proper type for an int literal --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 393c887e8b..6375e160df 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -135,7 +135,7 @@ class array : public buffer { auto buf_info = info; if (!buf_info.ptr) // always allocate at least 1 element, same way as NumPy does it - buf_info.ptr = std::calloc(std::max(info.size, 1ul), info.itemsize); + buf_info.ptr = std::calloc(std::max(info.size, (size_t) 1), info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); From 5afe9df30a07d65fd4706d5ded8d5a13902e38ba Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 2 Jul 2016 16:17:21 +0100 Subject: [PATCH 41/87] Minor fix in the docs --- docs/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index 649454c91e..dca4a29d20 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -1362,7 +1362,7 @@ NumPy structured types ====================== In order for ``py::array_t`` to work with structured (record) types, we first need -to register the memory layout of the type. This could be done via ``PYBIND11_DTYPE`` +to register the memory layout of the type. This can be done via ``PYBIND11_DTYPE`` macro which expects the type followed by field names: .. code-block:: cpp From 5412a05cf03dc6472456c6dbc2497d764c952d90 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 2 Jul 2016 16:18:42 +0100 Subject: [PATCH 42/87] Rename PYBIND11_DTYPE to PYBIND11_NUMPY_DTYPE --- docs/advanced.rst | 6 +++--- example/example20.cpp | 6 +++--- include/pybind11/numpy.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index dca4a29d20..5190e0b773 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -1362,7 +1362,7 @@ NumPy structured types ====================== In order for ``py::array_t`` to work with structured (record) types, we first need -to register the memory layout of the type. This can be done via ``PYBIND11_DTYPE`` +to register the memory layout of the type. This can be done via ``PYBIND11_NUMPY_DTYPE`` macro which expects the type followed by field names: .. code-block:: cpp @@ -1377,8 +1377,8 @@ macro which expects the type followed by field names: A a; }; - PYBIND11_DTYPE(A, x, y); - PYBIND11_DTYPE(B, z, a); + PYBIND11_NUMPY_DTYPE(A, x, y); + PYBIND11_NUMPY_DTYPE(B, z, a); /* now both A and B can be used as template arguments to py::array_t */ diff --git a/example/example20.cpp b/example/example20.cpp index 662dc473a5..2c24e6dd3e 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -101,9 +101,9 @@ void print_dtypes() { } void init_ex20(py::module &m) { - PYBIND11_DTYPE(SimpleStruct, x, y, z); - PYBIND11_DTYPE(PackedStruct, x, y, z); - PYBIND11_DTYPE(NestedStruct, a, b); + PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(NestedStruct, a, b); m.def("create_rec_simple", &create_recarray); m.def("create_rec_packed", &create_recarray); diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6375e160df..c8854ea7d9 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -327,7 +327,7 @@ struct npy_format_descriptor::value> #define PYBIND11_MAP_LIST(f, t, ...) \ PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) -#define PYBIND11_DTYPE(Type, ...) \ +#define PYBIND11_NUMPY_DTYPE(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ ({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) From 511401599c4e655675efa75b0b0bf92e219bdda2 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 2 Jul 2016 16:43:21 +0100 Subject: [PATCH 43/87] Use malloc insterad of calloc for numpy arrays --- include/pybind11/numpy.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index c8854ea7d9..2d1082f8d4 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -126,16 +126,17 @@ class array : public buffer { Py_ssize_t dims[32]; API& api = lookup_api(); - // Allocate zeroed memory if it hasn't been provided by the caller. + // Allocate non-zeroed memory if it hasn't been provided by the caller. // Normally, we could leave this null for NumPy to allocate memory for us, but // since we need a memoryview, the data pointer has to be non-null. NumPy uses // malloc if NPY_NEEDS_INIT is not set (in which case it uses calloc); however, - // we don't have a descriptor yet (only a buffer format string), so we can't - // access the flags. The safest thing to do is thus to use calloc. + // we don't have a desriptor yet (only a buffer format string), so we can't + // access the flags. As long as we're not dealing with object dtypes/fields + // though, the memory doesn't have to be zeroed so we use malloc. auto buf_info = info; if (!buf_info.ptr) // always allocate at least 1 element, same way as NumPy does it - buf_info.ptr = std::calloc(std::max(info.size, (size_t) 1), info.itemsize); + buf_info.ptr = std::malloc(std::max(info.size, (size_t) 1) * info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); From eeb4c043f9b9fbc9a270910101ed2f45d823533b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 3 Jul 2016 10:22:10 +0100 Subject: [PATCH 44/87] Change field descriptor offset type to size_t --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 2d1082f8d4..1046cd43ee 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -232,7 +232,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); struct field_descriptor { const char *name; - int offset; + size_t offset; object descr; }; From 13022f1b8cf533711994b505858025fcd3ea345c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 3 Jul 2016 10:22:40 +0100 Subject: [PATCH 45/87] Bugfix: pass struct size as itemsize to descriptor Without this, partially bound structs will have incorrect itemsize. --- include/pybind11/numpy.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1046cd43ee..423403dd8e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -266,6 +266,7 @@ struct npy_format_descriptor::value> args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; + args["itemsize"] = int_(sizeof(T)); // This is essentially the same as calling np.dtype() constructor in Python and passing // it a dict of the form {'names': ..., 'formats': ..., 'offsets': ...}. if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) From 8fa09cb871d2241355bb2a794dc75db502b20997 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 6 Jul 2016 00:28:12 +0100 Subject: [PATCH 46/87] Strip padding fields in dtypes, update the tests --- example/example20.cpp | 36 ++++++++- example/example20.py | 25 +++++- example/example20.ref | 12 ++- include/pybind11/numpy.h | 165 +++++++++++++++++++++++++++------------ 4 files changed, 180 insertions(+), 58 deletions(-) diff --git a/example/example20.cpp b/example/example20.cpp index 2c24e6dd3e..32b50e3d0e 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -44,6 +44,19 @@ std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { return os << "n:a=" << v.a << ";b=" << v.b; } +struct PartialStruct { + bool x; + uint32_t y; + float z; + long dummy2; +}; + +struct PartialNestedStruct { + long dummy1; + PartialStruct a; + long dummy2; +}; + struct UnboundStruct { }; template @@ -54,7 +67,7 @@ py::array mkarray_via_buffer(size_t n) { } template -py::array_t create_recarray(size_t n) { +py::array_t create_recarray(size_t n) { auto arr = mkarray_via_buffer(n); auto ptr = static_cast(arr.request().ptr); for (size_t i = 0; i < n; i++) { @@ -67,7 +80,7 @@ std::string get_format_unbound() { return py::format_descriptor::format(); } -py::array_t create_nested(size_t n) { +py::array_t create_nested(size_t n) { auto arr = mkarray_via_buffer(n); auto ptr = static_cast(arr.request().ptr); for (size_t i = 0; i < n; i++) { @@ -77,8 +90,17 @@ py::array_t create_nested(size_t n) { return arr; } +py::array_t create_partial_nested(size_t n) { + auto arr = mkarray_via_buffer(n); + auto ptr = static_cast(arr.request().ptr); + for (size_t i = 0; i < n; i++) { + ptr[i].a.x = i % 2; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; + } + return arr; +} + template -void print_recarray(py::array_t arr) { +void print_recarray(py::array_t arr) { auto buf = arr.request(); auto ptr = static_cast(buf.ptr); for (size_t i = 0; i < buf.size; i++) @@ -89,6 +111,8 @@ void print_format_descriptors() { std::cout << py::format_descriptor::format() << std::endl; std::cout << py::format_descriptor::format() << std::endl; std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; } void print_dtypes() { @@ -98,16 +122,22 @@ void print_dtypes() { std::cout << to_str(py::dtype_of()) << std::endl; std::cout << to_str(py::dtype_of()) << std::endl; std::cout << to_str(py::dtype_of()) << std::endl; + std::cout << to_str(py::dtype_of()) << std::endl; + std::cout << to_str(py::dtype_of()) << std::endl; } void init_ex20(py::module &m) { PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z); PYBIND11_NUMPY_DTYPE(NestedStruct, a, b); + PYBIND11_NUMPY_DTYPE(PartialStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(PartialNestedStruct, a); m.def("create_rec_simple", &create_recarray); m.def("create_rec_packed", &create_recarray); m.def("create_rec_nested", &create_nested); + m.def("create_rec_partial", &create_recarray); + m.def("create_rec_partial_nested", &create_partial_nested); m.def("print_format_descriptors", &print_format_descriptors); m.def("print_rec_simple", &print_recarray); m.def("print_rec_packed", &print_recarray); diff --git a/example/example20.py b/example/example20.py index e0a0018040..85ea9ae5fa 100644 --- a/example/example20.py +++ b/example/example20.py @@ -5,7 +5,8 @@ import numpy as np from example import ( create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, - print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound + print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound, + create_rec_partial, create_rec_partial_nested ) @@ -23,6 +24,8 @@ def check_eq(arr, data, dtype): 'offsets': [0, 4, 8]}) packed_dtype = np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')]) +elements = [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)] + for func, dtype in [(create_rec_simple, simple_dtype), (create_rec_packed, packed_dtype)]: arr = func(0) assert arr.dtype == dtype @@ -31,14 +34,30 @@ def check_eq(arr, data, dtype): arr = func(3) assert arr.dtype == dtype - check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], simple_dtype) - check_eq(arr, [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)], packed_dtype) + check_eq(arr, elements, simple_dtype) + check_eq(arr, elements, packed_dtype) if dtype == simple_dtype: print_rec_simple(arr) else: print_rec_packed(arr) + +arr = create_rec_partial(3) +print(arr.dtype) +partial_dtype = arr.dtype +assert '' not in arr.dtype.fields +assert partial_dtype.itemsize > simple_dtype.itemsize +check_eq(arr, elements, simple_dtype) +check_eq(arr, elements, packed_dtype) + +arr = create_rec_partial_nested(3) +print(arr.dtype) +assert '' not in arr.dtype.fields +assert '' not in arr.dtype.fields['a'][0].fields +assert arr.dtype.itemsize > partial_dtype.itemsize +np.testing.assert_equal(arr['a'], create_rec_partial(3)) + nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)]) arr = create_rec_nested(0) diff --git a/example/example20.ref b/example/example20.ref index 32e2b4b7af..72a6c18586 100644 --- a/example/example20.ref +++ b/example/example20.ref @@ -1,15 +1,21 @@ -T{?:x:xxxI:y:f:z:} -T{?:x:=I:y:f:z:} -T{T{?:x:xxxI:y:f:z:}:a:T{?:x:=I:y:f:z:}:b:} +T{=?:x:3x=I:y:=f:z:} +T{=?:x:=I:y:=f:z:} +T{=T{=?:x:3x=I:y:=f:z:}:a:=T{=?:x:=I:y:=f:z:}:b:} +T{=?:x:3x=I:y:=f:z:12x} +T{8x=T{=?:x:3x=I:y:=f:z:12x}:a:8x} {'names':['x','y','z'], 'formats':['?',' #include #include +#include #include #if defined(_MSC_VER) @@ -26,6 +27,8 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; +object fix_dtype(object); + template struct is_pod_struct { enum { value = std::is_pod::value && // offsetof only works correctly for POD types @@ -47,7 +50,9 @@ class array : public buffer { API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, + API_PyArray_DescrNewFromType = 9, API_PyArray_DescrConverter = 174, + API_PyArray_EquivTypes = 182, API_PyArray_GetArrayParamsFromObject = 278, NPY_C_CONTIGUOUS_ = 0x0001, @@ -61,7 +66,9 @@ class array : public buffer { NPY_LONG_, NPY_ULONG_, NPY_LONGLONG_, NPY_ULONGLONG_, NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, - NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_ + NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, + NPY_OBJECT_ = 17, + NPY_STRING_, NPY_UNICODE_, NPY_VOID_ }; static API lookup() { @@ -79,7 +86,9 @@ class array : public buffer { DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); + DECL_NPY_API(PyArray_DescrNewFromType); DECL_NPY_API(PyArray_DescrConverter); + DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_GetArrayParamsFromObject); #undef DECL_NPY_API return api; @@ -91,10 +100,12 @@ class array : public buffer { PyObject *(*PyArray_NewFromDescr_) (PyTypeObject *, PyObject *, int, Py_intptr_t *, Py_intptr_t *, void *, int, PyObject *); + PyObject *(*PyArray_DescrNewFromType_)(int); PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); int (*PyArray_DescrConverter_) (PyObject *, PyObject **); + bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, Py_ssize_t *, PyObject **, PyObject *); }; @@ -113,52 +124,83 @@ class array : public buffer { Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); - if (ptr && tmp) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); + if (ptr) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); m_ptr = tmp.release().ptr(); } array(const buffer_info &info) { - PyObject *arr = nullptr, *descr = nullptr; - int ndim = 0; - Py_ssize_t dims[32]; - API& api = lookup_api(); - - // Allocate non-zeroed memory if it hasn't been provided by the caller. - // Normally, we could leave this null for NumPy to allocate memory for us, but - // since we need a memoryview, the data pointer has to be non-null. NumPy uses - // malloc if NPY_NEEDS_INIT is not set (in which case it uses calloc); however, - // we don't have a desriptor yet (only a buffer format string), so we can't - // access the flags. As long as we're not dealing with object dtypes/fields - // though, the memory doesn't have to be zeroed so we use malloc. - auto buf_info = info; - if (!buf_info.ptr) - // always allocate at least 1 element, same way as NumPy does it - buf_info.ptr = std::malloc(std::max(info.size, (size_t) 1) * info.itemsize); - if (!buf_info.ptr) - pybind11_fail("NumPy: failed to allocate memory for buffer"); - - // PyArray_GetArrayParamsFromObject seems to be the only low-level API function - // that will accept arbitrary buffers (including structured types) - auto view = memoryview(buf_info); - auto res = api.PyArray_GetArrayParamsFromObject_(view.ptr(), nullptr, 1, &descr, - &ndim, dims, &arr, nullptr); - if (res < 0 || !arr || descr) - // We expect arr to have a pointer to a newly created array, in which case all - // other parameters like descr would be set to null, according to the API. - pybind11_fail("NumPy: unable to convert buffer to an array"); - m_ptr = arr; + auto& api = lookup_api(); + + // _dtype_from_pep3118 returns dtypes with padding fields in, however the array + // constructor seems to then consume them, so we don't need to strip them ourselves + auto numpy_internal = module::import("numpy.core._internal"); + auto dtype_from_fmt = (object) numpy_internal.attr("_dtype_from_pep3118"); + auto dtype = dtype_from_fmt(pybind11::str(info.format)); + auto dtype2 = strip_padding_fields(dtype); + + object tmp(api.PyArray_NewFromDescr_( + api.PyArray_Type_, dtype2.release().ptr(), (int) info.ndim, (Py_intptr_t *) &info.shape[0], + (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); + if (!tmp) + pybind11_fail("NumPy: unable to create array!"); + if (info.ptr) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + m_ptr = tmp.release().ptr(); + auto d = (object) this->attr("dtype"); } -protected: +// protected: static API &lookup_api() { static API api = API::lookup(); return api; } template friend struct detail::npy_format_descriptor; + + static object strip_padding_fields(object dtype) { + // Recursively strip all void fields with empty names that are generated for + // padding fields (as of NumPy v1.11). + auto fields = dtype.attr("fields").cast(); + if (fields.ptr() == Py_None) + return dtype; + + struct field_descr { pybind11::str name; object format; int_ offset; }; + std::vector field_descriptors; + + auto items = fields.attr("items").cast(); + for (auto field : items()) { + auto spec = object(field, true).cast(); + auto name = spec[0].cast(); + auto format = spec[1].cast()[0].cast(); + auto offset = spec[1].cast()[1].cast(); + if (!len(name) && (std::string) dtype.attr("kind").cast() == "V") + continue; + field_descriptors.push_back({name, strip_padding_fields(format), offset}); + } + + std::sort(field_descriptors.begin(), field_descriptors.end(), + [](const field_descr& a, const field_descr& b) { + return (int) a.offset < (int) b.offset; + }); + + list names, formats, offsets; + for (auto& descr : field_descriptors) { + names.append(descr.name); + formats.append(descr.format); + offsets.append(descr.offset); + } + auto args = dict(); + args["names"] = names; args["formats"] = formats; args["offsets"] = offsets; + args["itemsize"] = dtype.attr("itemsize").cast(); + + PyObject *descr = nullptr; + if (!lookup_api().PyArray_DescrConverter_(args.release().ptr(), &descr) || !descr) + pybind11_fail("NumPy: failed to create structured dtype"); + return object(descr, false); + } }; template class array_t : public array { @@ -233,9 +275,12 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); struct field_descriptor { const char *name; size_t offset; + size_t size; + const char *format; object descr; }; + template struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("user-defined"); } @@ -253,7 +298,7 @@ struct npy_format_descriptor::value> } static void register_dtype(std::initializer_list fields) { - array::API& api = array::lookup_api(); + auto& api = array::lookup_api(); auto args = dict(); list names { }, offsets { }, formats { }; for (auto field : fields) { @@ -263,26 +308,47 @@ struct npy_format_descriptor::value> offsets.append(int_(field.offset)); formats.append(field.descr); } - args["names"] = names; - args["offsets"] = offsets; - args["formats"] = formats; + args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; args["itemsize"] = int_(sizeof(T)); // This is essentially the same as calling np.dtype() constructor in Python and passing // it a dict of the form {'names': ..., 'formats': ..., 'offsets': ...}. if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) pybind11_fail("NumPy: failed to create structured dtype"); - // Let NumPy figure the buffer format string for us: memoryview(np.empty(0, dtype)).format - auto np = module::import("numpy"); - auto empty = (object) np.attr("empty"); - if (auto arr = (object) empty(int_(0), dtype())) { - if (auto view = PyMemoryView_FromObject(arr.ptr())) { - if (auto info = PyMemoryView_GET_BUFFER(view)) { - std::strncpy(format_(), info->format, 4096); - return; - } - } + + // There is an existing bug in NumPy (as of v1.11): trailing bytes are + // not encoded explicitly into the format string. This will supposedly + // get fixed in v1.12; for further details, see these: + // - https://github.com/numpy/numpy/issues/7797 + // - https://github.com/numpy/numpy/pull/7798 + // Because of this, we won't use numpy's logic to generate buffer format + // strings and will just do it ourselves. + std::vector ordered_fields(fields); + std::sort(ordered_fields.begin(), ordered_fields.end(), + [](const field_descriptor& a, const field_descriptor &b) { + return a.offset < b.offset; + }); + size_t offset = 0; + std::ostringstream oss; + oss << "T{"; + for (auto& field : ordered_fields) { + if (field.offset > offset) + oss << (field.offset - offset) << 'x'; + // note that '=' is required to cover the case of unaligned fields + oss << '=' << field.format << ':' << field.name << ':'; + offset = field.offset + field.size; } - pybind11_fail("NumPy: failed to extract buffer format"); + if (sizeof(T) > offset) + oss << (sizeof(T) - offset) << 'x'; + oss << '}'; + std::strncpy(format_(), oss.str().c_str(), 4096); + + // Sanity check: verify that NumPy properly parses our buffer format string + auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); + auto dtype = (object) arr.attr("dtype"); + auto fixed_dtype = dtype; + // auto fixed_dtype = array::strip_padding_fields(object(dtype_(), true)); + // if (!api.PyArray_EquivTypes_(dtype_(), fixed_dtype.ptr())) + // pybind11_fail("NumPy: invalid buffer descriptor!"); } private: @@ -293,7 +359,8 @@ struct npy_format_descriptor::value> // Extract name, offset and format descriptor for a struct field #define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ - #Field, offsetof(Type, Field), \ + #Field, offsetof(Type, Field), sizeof(decltype(static_cast(0)->Field)), \ + ::pybind11::format_descriptor(0)->Field)>::format(), \ ::pybind11::detail::npy_format_descriptor(0)->Field)>::dtype() \ } From 8f2f7cd61cae57680baa3cef2c00c307f51b5146 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 17 Jul 2016 11:07:49 +0100 Subject: [PATCH 47/87] Various cleanup --- include/pybind11/numpy.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b8827c34bb..e756db8e23 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -138,21 +138,19 @@ class array : public buffer { // constructor seems to then consume them, so we don't need to strip them ourselves auto numpy_internal = module::import("numpy.core._internal"); auto dtype_from_fmt = (object) numpy_internal.attr("_dtype_from_pep3118"); - auto dtype = dtype_from_fmt(pybind11::str(info.format)); - auto dtype2 = strip_padding_fields(dtype); + auto dtype = strip_padding_fields(dtype_from_fmt(pybind11::str(info.format))); object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, dtype2.release().ptr(), (int) info.ndim, (Py_intptr_t *) &info.shape[0], + api.PyArray_Type_, dtype.release().ptr(), (int) info.ndim, (Py_intptr_t *) &info.shape[0], (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); if (info.ptr) tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); m_ptr = tmp.release().ptr(); - auto d = (object) this->attr("dtype"); } -// protected: +protected: static API &lookup_api() { static API api = API::lookup(); return api; From 076b953ccd6a0849af652755feb7e1d1d19296dc Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 17 Jul 2016 11:10:19 +0100 Subject: [PATCH 48/87] Restore dtype equivalence sanity check --- include/pybind11/numpy.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index e756db8e23..58d9695fe9 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -342,11 +342,9 @@ struct npy_format_descriptor::value> // Sanity check: verify that NumPy properly parses our buffer format string auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); - auto dtype = (object) arr.attr("dtype"); - auto fixed_dtype = dtype; - // auto fixed_dtype = array::strip_padding_fields(object(dtype_(), true)); - // if (!api.PyArray_EquivTypes_(dtype_(), fixed_dtype.ptr())) - // pybind11_fail("NumPy: invalid buffer descriptor!"); + auto fixed_dtype = array::strip_padding_fields(object(dtype_(), true)); + if (!api.PyArray_EquivTypes_(dtype_(), fixed_dtype.ptr())) + pybind11_fail("NumPy: invalid buffer descriptor!"); } private: From 41c339902123b96ce3bf23f2522758f5d4954be8 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 19:58:20 +0100 Subject: [PATCH 49/87] Update npy_format_descriptor::name() --- example/example20.py | 2 ++ include/pybind11/numpy.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/example/example20.py b/example/example20.py index 85ea9ae5fa..bb57590c35 100644 --- a/example/example20.py +++ b/example/example20.py @@ -70,3 +70,5 @@ def check_eq(arr, data, dtype): ((True, 1, 1.5), (False, 2, 3.0)), ((False, 2, 3.0), (True, 3, 4.5))], nested_dtype) print_rec_nested(arr) + +assert create_rec_nested.__doc__.strip().endswith('numpy.ndarray[dtype=NestedStruct]') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 58d9695fe9..030787b7db 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -281,7 +281,7 @@ struct field_descriptor { template struct npy_format_descriptor::value>::type> { - static PYBIND11_DESCR name() { return _("user-defined"); } + static PYBIND11_DESCR name() { return _("struct"); } static object dtype() { if (!dtype_()) From bf2510ee8650fb5b340c47c3a46a5ae42b732b27 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 22:24:53 +0100 Subject: [PATCH 50/87] Make buffer_info::as_pybuffer a memoryview ctor --- include/pybind11/common.h | 24 ------------------------ include/pybind11/pytypes.h | 25 ++++++++++++++++++++++--- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/include/pybind11/common.h b/include/pybind11/common.h index 32e8abf53c..302d75a600 100644 --- a/include/pybind11/common.h +++ b/include/pybind11/common.h @@ -232,30 +232,6 @@ struct buffer_info { if (view) { PyBuffer_Release(view); delete view; } } - Py_buffer& as_pybuffer() const { - static Py_buffer buf { }; - // Py_buffer uses signed sizes, strides and shape!.. - static std::vector py_strides { }; - static std::vector py_shape { }; - buf.buf = ptr; - buf.itemsize = (Py_ssize_t) itemsize; - buf.format = const_cast(format.c_str()); - buf.ndim = (int) ndim; - buf.len = (Py_ssize_t) size; - py_strides.clear(); - py_shape.clear(); - for (size_t i = 0; i < ndim; ++i) { - py_strides.push_back((Py_ssize_t) strides[i]); - py_shape.push_back((Py_ssize_t) shape[i]); - } - buf.strides = py_strides.data(); - buf.shape = py_shape.data(); - buf.suboffsets = nullptr; - buf.readonly = false; - buf.internal = nullptr; - return buf; - } - private: Py_buffer *view = nullptr; }; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 7a6cf6fbd3..db87b088a3 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -572,10 +572,29 @@ class buffer : public object { class memoryview : public object { public: - memoryview(const buffer_info& info) : memoryview(&info.as_pybuffer()) { } + memoryview(const buffer_info& info) { + static Py_buffer buf { }; + // Py_buffer uses signed sizes, strides and shape!.. + static std::vector py_strides { }; + static std::vector py_shape { }; + buf.buf = info.ptr; + buf.itemsize = (Py_ssize_t) info.itemsize; + buf.format = const_cast(info.format.c_str()); + buf.ndim = (int) info.ndim; + buf.len = (Py_ssize_t) info.size; + py_strides.clear(); + py_shape.clear(); + for (size_t i = 0; i < info.ndim; ++i) { + py_strides.push_back((Py_ssize_t) info.strides[i]); + py_shape.push_back((Py_ssize_t) info.shape[i]); + } + buf.strides = py_strides.data(); + buf.shape = py_shape.data(); + buf.suboffsets = nullptr; + buf.readonly = false; + buf.internal = nullptr; - memoryview(Py_buffer* view) - : object(PyMemoryView_FromBuffer(view), false) { + m_ptr = PyMemoryView_FromBuffer(&buf); if (!m_ptr) pybind11_fail("Unable to create memoryview from buffer descriptor"); } From b51fa02cc383ce2dca2e678374d78763fc7fbd27 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 22:35:50 +0100 Subject: [PATCH 51/87] Store array requests in local variables in tests --- example/example20.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/example/example20.cpp b/example/example20.cpp index 32b50e3d0e..6557021cf9 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -69,7 +69,8 @@ py::array mkarray_via_buffer(size_t n) { template py::array_t create_recarray(size_t n) { auto arr = mkarray_via_buffer(n); - auto ptr = static_cast(arr.request().ptr); + auto req = arr.request(); + auto ptr = static_cast(req.ptr); for (size_t i = 0; i < n; i++) { ptr[i].x = i % 2; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f; } @@ -82,7 +83,8 @@ std::string get_format_unbound() { py::array_t create_nested(size_t n) { auto arr = mkarray_via_buffer(n); - auto ptr = static_cast(arr.request().ptr); + auto req = arr.request(); + auto ptr = static_cast(req.ptr); for (size_t i = 0; i < n; i++) { ptr[i].a.x = i % 2; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; ptr[i].b.x = (i + 1) % 2; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f; @@ -92,7 +94,8 @@ py::array_t create_nested(size_t n) { py::array_t create_partial_nested(size_t n) { auto arr = mkarray_via_buffer(n); - auto ptr = static_cast(arr.request().ptr); + auto req = arr.request(); + auto ptr = static_cast(req.ptr); for (size_t i = 0; i < n; i++) { ptr[i].a.x = i % 2; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; } @@ -101,9 +104,9 @@ py::array_t create_partial_nested(size_t n) { template void print_recarray(py::array_t arr) { - auto buf = arr.request(); - auto ptr = static_cast(buf.ptr); - for (size_t i = 0; i < buf.size; i++) + auto req = arr.request(); + auto ptr = static_cast(req.ptr); + for (size_t i = 0; i < req.size; i++) std::cout << ptr[i] << std::endl; } From b37985ee0c2c46e689e43542649760872f1a996c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 22:37:42 +0100 Subject: [PATCH 52/87] Fix a comment and wrong indentation --- include/pybind11/numpy.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 030787b7db..12073ba74d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -134,8 +134,7 @@ class array : public buffer { array(const buffer_info &info) { auto& api = lookup_api(); - // _dtype_from_pep3118 returns dtypes with padding fields in, however the array - // constructor seems to then consume them, so we don't need to strip them ourselves + // _dtype_from_pep3118 returns dtypes with padding fields in, so we need to strip them auto numpy_internal = module::import("numpy.core._internal"); auto dtype_from_fmt = (object) numpy_internal.attr("_dtype_from_pep3118"); auto dtype = strip_padding_fields(dtype_from_fmt(pybind11::str(info.format))); @@ -175,7 +174,7 @@ class array : public buffer { auto format = spec[1].cast()[0].cast(); auto offset = spec[1].cast()[1].cast(); if (!len(name) && (std::string) dtype.attr("kind").cast() == "V") - continue; + continue; field_descriptors.push_back({name, strip_padding_fields(format), offset}); } From f5f75c65449ab6956b58626ad13d6530357c2cf9 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 22:47:40 +0100 Subject: [PATCH 53/87] Make struct packing in example20 MSVC-compliant --- example/example20.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/example/example20.cpp b/example/example20.cpp index 6557021cf9..8b18c05d47 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -13,6 +13,12 @@ #include #include +#ifdef __GNUC__ +#define PYBIND11_PACKED(cls) cls __attribute__((__packed__)) +#else +#define PYBIND11_PACKED(cls) __pragma(pack(push, 1)) cls __pragma(pack(pop)) +#endif + namespace py = pybind11; struct SimpleStruct { @@ -25,20 +31,20 @@ std::ostream& operator<<(std::ostream& os, const SimpleStruct& v) { return os << "s:" << v.x << "," << v.y << "," << v.z; } -struct PackedStruct { +PYBIND11_PACKED(struct PackedStruct { bool x; uint32_t y; float z; -} __attribute__((packed)); +}); std::ostream& operator<<(std::ostream& os, const PackedStruct& v) { return os << "p:" << v.x << "," << v.y << "," << v.z; } -struct NestedStruct { +PYBIND11_PACKED(struct NestedStruct { SimpleStruct a; PackedStruct b; -} __attribute__((packed)); +}); std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { return os << "n:a=" << v.a << ";b=" << v.b; @@ -148,3 +154,5 @@ void init_ex20(py::module &m) { m.def("print_dtypes", &print_dtypes); m.def("get_format_unbound", &get_format_unbound); } + +#undef PYBIND11_PACKED From 098f9aef7382f483128919c97a74c81359a7c911 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 22:52:08 +0100 Subject: [PATCH 54/87] Replace 4096B format buffer with std::string --- include/pybind11/numpy.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 12073ba74d..10eb895328 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -291,7 +291,7 @@ struct npy_format_descriptor::value> static const char* format() { if (!dtype_()) pybind11_fail("NumPy: unsupported buffer format!"); - return format_(); + return format_().c_str(); } static void register_dtype(std::initializer_list fields) { @@ -337,7 +337,7 @@ struct npy_format_descriptor::value> if (sizeof(T) > offset) oss << (sizeof(T) - offset) << 'x'; oss << '}'; - std::strncpy(format_(), oss.str().c_str(), 4096); + format_() = oss.str(); // Sanity check: verify that NumPy properly parses our buffer format string auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); @@ -348,7 +348,7 @@ struct npy_format_descriptor::value> private: static inline PyObject*& dtype_() { static PyObject *ptr = nullptr; return ptr; } - static inline char* format_() { static char s[4096]; return s; } + static inline std::string& format_() { static std::string s; return s; } }; // Extract name, offset and format descriptor for a struct field From 103d5eadc36ccc361474c7793bf4b588b16fce23 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 23:36:18 +0100 Subject: [PATCH 55/87] Remove redundant definition --- include/pybind11/numpy.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 10eb895328..1dc3de2462 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -27,7 +27,6 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; -object fix_dtype(object); template struct is_pod_struct { From f9c0defed7a764b1b1cfc02bce6c742d56e2664f Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 20 Jul 2016 00:19:24 +0100 Subject: [PATCH 56/87] Add numpy wrappers for char[] and std::array --- example/example20.cpp | 37 +++++++++++++++++++++++++++++++++++++ example/example20.py | 11 ++++++++++- example/example20.ref | 9 ++++++++- include/pybind11/numpy.h | 32 +++++++++++++++++++++++++++++--- 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/example/example20.cpp b/example/example20.cpp index 8b18c05d47..07849a099d 100644 --- a/example/example20.cpp +++ b/example/example20.cpp @@ -65,6 +65,19 @@ struct PartialNestedStruct { struct UnboundStruct { }; +struct StringStruct { + char a[3]; + std::array b; +}; + +std::ostream& operator<<(std::ostream& os, const StringStruct& v) { + os << "a='"; + for (size_t i = 0; i < 3 && v.a[i]; i++) os << v.a[i]; + os << "',b='"; + for (size_t i = 0; i < 3 && v.b[i]; i++) os << v.b[i]; + return os << "'"; +} + template py::array mkarray_via_buffer(size_t n) { return py::array(py::buffer_info(nullptr, sizeof(T), @@ -108,6 +121,25 @@ py::array_t create_partial_nested(size_t n) { return arr; } +py::array_t create_string_array(bool non_empty) { + auto arr = mkarray_via_buffer(non_empty ? 4 : 0); + if (non_empty) { + auto req = arr.request(); + auto ptr = static_cast(req.ptr); + for (size_t i = 0; i < req.size * req.itemsize; i++) + static_cast(req.ptr)[i] = 0; + ptr[1].a[0] = 'a'; ptr[1].b[0] = 'a'; + ptr[2].a[0] = 'a'; ptr[2].b[0] = 'a'; + ptr[3].a[0] = 'a'; ptr[3].b[0] = 'a'; + + ptr[2].a[1] = 'b'; ptr[2].b[1] = 'b'; + ptr[3].a[1] = 'b'; ptr[3].b[1] = 'b'; + + ptr[3].a[2] = 'c'; ptr[3].b[2] = 'c'; + } + return arr; +} + template void print_recarray(py::array_t arr) { auto req = arr.request(); @@ -122,6 +154,7 @@ void print_format_descriptors() { std::cout << py::format_descriptor::format() << std::endl; std::cout << py::format_descriptor::format() << std::endl; std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; } void print_dtypes() { @@ -133,6 +166,7 @@ void print_dtypes() { std::cout << to_str(py::dtype_of()) << std::endl; std::cout << to_str(py::dtype_of()) << std::endl; std::cout << to_str(py::dtype_of()) << std::endl; + std::cout << to_str(py::dtype_of()) << std::endl; } void init_ex20(py::module &m) { @@ -141,6 +175,7 @@ void init_ex20(py::module &m) { PYBIND11_NUMPY_DTYPE(NestedStruct, a, b); PYBIND11_NUMPY_DTYPE(PartialStruct, x, y, z); PYBIND11_NUMPY_DTYPE(PartialNestedStruct, a); + PYBIND11_NUMPY_DTYPE(StringStruct, a, b); m.def("create_rec_simple", &create_recarray); m.def("create_rec_packed", &create_recarray); @@ -153,6 +188,8 @@ void init_ex20(py::module &m) { m.def("print_rec_nested", &print_recarray); m.def("print_dtypes", &print_dtypes); m.def("get_format_unbound", &get_format_unbound); + m.def("create_string_array", &create_string_array); + m.def("print_string_array", &print_recarray); } #undef PYBIND11_PACKED diff --git a/example/example20.py b/example/example20.py index bb57590c35..34dfd83396 100644 --- a/example/example20.py +++ b/example/example20.py @@ -6,7 +6,7 @@ from example import ( create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound, - create_rec_partial, create_rec_partial_nested + create_rec_partial, create_rec_partial_nested, create_string_array, print_string_array ) @@ -72,3 +72,12 @@ def check_eq(arr, data, dtype): print_rec_nested(arr) assert create_rec_nested.__doc__.strip().endswith('numpy.ndarray[dtype=NestedStruct]') + +arr = create_string_array(True) +print(arr.dtype) +print_string_array(arr) +dtype = arr.dtype +assert arr['a'].tolist() == [b'', b'a', b'ab', b'abc'] +assert arr['b'].tolist() == [b'', b'a', b'ab', b'abc'] +arr = create_string_array(False) +assert dtype == arr.dtype diff --git a/example/example20.ref b/example/example20.ref index 72a6c18586..4f07ce445a 100644 --- a/example/example20.ref +++ b/example/example20.ref @@ -3,11 +3,13 @@ T{=?:x:=I:y:=f:z:} T{=T{=?:x:3x=I:y:=f:z:}:a:=T{=?:x:=I:y:=f:z:}:b:} T{=?:x:3x=I:y:=f:z:12x} T{8x=T{=?:x:3x=I:y:=f:z:12x}:a:8x} +T{=3s:a:=3s:b:} {'names':['x','y','z'], 'formats':['?',' #include +#include #include #include #include @@ -27,10 +28,14 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; +template struct is_std_array : std::false_type { }; +template struct is_std_array> : std::true_type { }; template struct is_pod_struct { enum { value = std::is_pod::value && // offsetof only works correctly for POD types + !std::is_array::value && + !is_std_array::value && !std::is_integral::value && !std::is_same::value && !std::is_same::value && @@ -221,9 +226,14 @@ template class array_t : public template struct format_descriptor::value>::type> { - static const char *format() { - return detail::npy_format_descriptor::format(); - } + static const char *format() { return detail::npy_format_descriptor::format(); } +}; + +template struct format_descriptor { + static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } +}; +template struct format_descriptor> { + static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } }; template @@ -268,6 +278,22 @@ DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #undef DECL_FMT +#define DECL_CHAR_FMT \ + static PYBIND11_DESCR name() { return _("S") + _(); } \ + static object dtype() { \ + auto& api = array::lookup_api(); \ + PyObject *descr = nullptr; \ + PYBIND11_DESCR fmt = _("S") + _(); \ + pybind11::str py_fmt(fmt.text()); \ + if (!api.PyArray_DescrConverter_(py_fmt.release().ptr(), &descr) || !descr) \ + pybind11_fail("NumPy: failed to create string dtype"); \ + return object(descr, false); \ + } \ + static const char *format() { PYBIND11_DESCR s = _() + _("s"); return s.text(); } +template struct npy_format_descriptor { DECL_CHAR_FMT }; +template struct npy_format_descriptor> { DECL_CHAR_FMT }; +#undef DECL_CHAR_FMT + struct field_descriptor { const char *name; size_t offset; From 5db82353f732ad258996b255a87ea6ce1e140192 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 20 Jul 2016 00:24:00 +0100 Subject: [PATCH 57/87] Rename example20 -> example-numpy-dtypes --- .gitignore | 1 + example/CMakeLists.txt | 2 +- example/{example20.cpp => example-numpy-dtypes.cpp} | 4 ++-- example/{example20.py => example-numpy-dtypes.py} | 0 example/{example20.ref => example-numpy-dtypes.ref} | 0 example/example.cpp | 4 ++-- 6 files changed, 6 insertions(+), 5 deletions(-) rename example/{example20.cpp => example-numpy-dtypes.cpp} (98%) rename example/{example20.py => example-numpy-dtypes.py} (100%) rename example/{example20.ref => example-numpy-dtypes.ref} (100%) diff --git a/.gitignore b/.gitignore index 4b9df18aa1..d1676c9b3a 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ MANIFEST .DS_Store /dist /build +/cmake/ diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index b6d3804973..2cc8f83269 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -26,7 +26,7 @@ set(PYBIND11_EXAMPLES example-stl-binder-vector.cpp example-eval.cpp example-custom-exceptions.cpp - example20.cpp + example-numpy-dtypes.cpp issues.cpp ) diff --git a/example/example20.cpp b/example/example-numpy-dtypes.cpp similarity index 98% rename from example/example20.cpp rename to example/example-numpy-dtypes.cpp index 07849a099d..2e25670f39 100644 --- a/example/example20.cpp +++ b/example/example-numpy-dtypes.cpp @@ -1,5 +1,5 @@ /* - example/example20.cpp -- Usage of structured numpy dtypes + example/example-numpy-dtypes.cpp -- Structured and compound NumPy dtypes Copyright (c) 2016 Ivan Smirnov @@ -169,7 +169,7 @@ void print_dtypes() { std::cout << to_str(py::dtype_of()) << std::endl; } -void init_ex20(py::module &m) { +void init_ex_numpy_dtypes(py::module &m) { PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z); PYBIND11_NUMPY_DTYPE(NestedStruct, a, b); diff --git a/example/example20.py b/example/example-numpy-dtypes.py similarity index 100% rename from example/example20.py rename to example/example-numpy-dtypes.py diff --git a/example/example20.ref b/example/example-numpy-dtypes.ref similarity index 100% rename from example/example20.ref rename to example/example-numpy-dtypes.ref diff --git a/example/example.cpp b/example/example.cpp index 819f69f449..bd6ac9b351 100644 --- a/example/example.cpp +++ b/example/example.cpp @@ -29,7 +29,7 @@ void init_ex_inheritance(py::module &); void init_ex_stl_binder_vector(py::module &); void init_ex_eval(py::module &); void init_ex_custom_exceptions(py::module &); -void init_ex20(py::module &); +void init_ex_numpy_dtypes(py::module &); void init_issues(py::module &); #if defined(PYBIND11_TEST_EIGEN) @@ -73,7 +73,7 @@ PYBIND11_PLUGIN(example) { init_ex_stl_binder_vector(m); init_ex_eval(m); init_ex_custom_exceptions(m); - init_ex20(m); + init_ex_numpy_dtypes(m); init_issues(m); #if defined(PYBIND11_TEST_EIGEN) From afb07e7e9268e0ef316c6dc58dca30599e878b6f Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 20 Jul 2016 00:26:18 +0100 Subject: [PATCH 58/87] Code reordering / cleanup only --- example/example-numpy-dtypes.py | 1 - include/pybind11/numpy.h | 33 ++++++++++++++++----------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/example/example-numpy-dtypes.py b/example/example-numpy-dtypes.py index 34dfd83396..68ea5c2cb1 100644 --- a/example/example-numpy-dtypes.py +++ b/example/example-numpy-dtypes.py @@ -42,7 +42,6 @@ def check_eq(arr, data, dtype): else: print_rec_packed(arr) - arr = create_rec_partial(3) print(arr.dtype) partial_dtype = arr.dtype diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 20fbfd86fa..d0cf1f6b7f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -27,22 +27,7 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; - -template struct is_std_array : std::false_type { }; -template struct is_std_array> : std::true_type { }; - -template -struct is_pod_struct { - enum { value = std::is_pod::value && // offsetof only works correctly for POD types - !std::is_array::value && - !is_std_array::value && - !std::is_integral::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same>::value && - !std::is_same>::value }; -}; +template struct is_pod_struct; } class array : public buffer { @@ -242,6 +227,21 @@ object dtype_of() { } NAMESPACE_BEGIN(detail) +template struct is_std_array : std::false_type { }; +template struct is_std_array> : std::true_type { }; + +template +struct is_pod_struct { + enum { value = std::is_pod::value && // offsetof only works correctly for POD types + !std::is_array::value && + !is_std_array::value && + !std::is_integral::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same>::value && + !std::is_same>::value }; +}; template struct npy_format_descriptor::value>::type> { private: @@ -302,7 +302,6 @@ struct field_descriptor { object descr; }; - template struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("struct"); } From 05cb58ade27ac74979efd7a64cb8d120744521dd Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 20 Jul 2016 00:54:57 +0100 Subject: [PATCH 59/87] Cleanup: move numpy API bindings out of py::array --- include/pybind11/numpy.h | 172 ++++++++++++++++++++------------------- 1 file changed, 87 insertions(+), 85 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d0cf1f6b7f..77f7e6f72e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -28,87 +28,94 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; template struct is_pod_struct; -} -class array : public buffer { -public: - struct API { - enum Entries { - API_PyArray_Type = 2, - API_PyArray_DescrFromType = 45, - API_PyArray_FromAny = 69, - API_PyArray_NewCopy = 85, - API_PyArray_NewFromDescr = 94, - API_PyArray_DescrNewFromType = 9, - API_PyArray_DescrConverter = 174, - API_PyArray_EquivTypes = 182, - API_PyArray_GetArrayParamsFromObject = 278, - - NPY_C_CONTIGUOUS_ = 0x0001, - NPY_F_CONTIGUOUS_ = 0x0002, - NPY_ARRAY_FORCECAST_ = 0x0010, - NPY_ENSURE_ARRAY_ = 0x0040, - NPY_BOOL_ = 0, - NPY_BYTE_, NPY_UBYTE_, - NPY_SHORT_, NPY_USHORT_, - NPY_INT_, NPY_UINT_, - NPY_LONG_, NPY_ULONG_, - NPY_LONGLONG_, NPY_ULONGLONG_, - NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, - NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, - NPY_OBJECT_ = 17, - NPY_STRING_, NPY_UNICODE_, NPY_VOID_ - }; - - static API lookup() { - module m = module::import("numpy.core.multiarray"); - object c = (object) m.attr("_ARRAY_API"); +struct npy_api { + enum constants { + NPY_C_CONTIGUOUS_ = 0x0001, + NPY_F_CONTIGUOUS_ = 0x0002, + NPY_ARRAY_FORCECAST_ = 0x0010, + NPY_ENSURE_ARRAY_ = 0x0040, + NPY_BOOL_ = 0, + NPY_BYTE_, NPY_UBYTE_, + NPY_SHORT_, NPY_USHORT_, + NPY_INT_, NPY_UINT_, + NPY_LONG_, NPY_ULONG_, + NPY_LONGLONG_, NPY_ULONGLONG_, + NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, + NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, + NPY_OBJECT_ = 17, + NPY_STRING_, NPY_UNICODE_, NPY_VOID_ + }; + + static npy_api& get() { + static npy_api api = lookup(); + return api; + } + + bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); } + + PyObject *(*PyArray_DescrFromType_)(int); + PyObject *(*PyArray_NewFromDescr_) + (PyTypeObject *, PyObject *, int, Py_intptr_t *, + Py_intptr_t *, void *, int, PyObject *); + PyObject *(*PyArray_DescrNewFromType_)(int); + PyObject *(*PyArray_NewCopy_)(PyObject *, int); + PyTypeObject *PyArray_Type_; + PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); + int (*PyArray_DescrConverter_) (PyObject *, PyObject **); + bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); + int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, + Py_ssize_t *, PyObject **, PyObject *); +private: + enum functions { + API_PyArray_Type = 2, + API_PyArray_DescrFromType = 45, + API_PyArray_FromAny = 69, + API_PyArray_NewCopy = 85, + API_PyArray_NewFromDescr = 94, + API_PyArray_DescrNewFromType = 9, + API_PyArray_DescrConverter = 174, + API_PyArray_EquivTypes = 182, + API_PyArray_GetArrayParamsFromObject = 278, + }; + + static npy_api lookup() { + module m = module::import("numpy.core.multiarray"); + object c = (object) m.attr("_ARRAY_API"); #if PY_MAJOR_VERSION >= 3 - void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr); + void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr); #else - void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); + void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); #endif - API api; + npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; - DECL_NPY_API(PyArray_Type); - DECL_NPY_API(PyArray_DescrFromType); - DECL_NPY_API(PyArray_FromAny); - DECL_NPY_API(PyArray_NewCopy); - DECL_NPY_API(PyArray_NewFromDescr); - DECL_NPY_API(PyArray_DescrNewFromType); - DECL_NPY_API(PyArray_DescrConverter); - DECL_NPY_API(PyArray_EquivTypes); - DECL_NPY_API(PyArray_GetArrayParamsFromObject); + DECL_NPY_API(PyArray_Type); + DECL_NPY_API(PyArray_DescrFromType); + DECL_NPY_API(PyArray_FromAny); + DECL_NPY_API(PyArray_NewCopy); + DECL_NPY_API(PyArray_NewFromDescr); + DECL_NPY_API(PyArray_DescrNewFromType); + DECL_NPY_API(PyArray_DescrConverter); + DECL_NPY_API(PyArray_EquivTypes); + DECL_NPY_API(PyArray_GetArrayParamsFromObject); #undef DECL_NPY_API - return api; - } - - bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); } - - PyObject *(*PyArray_DescrFromType_)(int); - PyObject *(*PyArray_NewFromDescr_) - (PyTypeObject *, PyObject *, int, Py_intptr_t *, - Py_intptr_t *, void *, int, PyObject *); - PyObject *(*PyArray_DescrNewFromType_)(int); - PyObject *(*PyArray_NewCopy_)(PyObject *, int); - PyTypeObject *PyArray_Type_; - PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); - int (*PyArray_DescrConverter_) (PyObject *, PyObject **); - bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); - int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, - Py_ssize_t *, PyObject **, PyObject *); - }; + return api; + } +}; +} - PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check_) +class array : public buffer { +public: + PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) enum { - c_style = API::NPY_C_CONTIGUOUS_, - f_style = API::NPY_F_CONTIGUOUS_, - forcecast = API::NPY_ARRAY_FORCECAST_ + c_style = detail::npy_api::NPY_C_CONTIGUOUS_, + f_style = detail::npy_api::NPY_F_CONTIGUOUS_, + forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; template array(size_t size, const Type *ptr) { - API& api = lookup_api(); + auto& api = detail::npy_api::get(); PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( @@ -121,7 +128,7 @@ class array : public buffer { } array(const buffer_info &info) { - auto& api = lookup_api(); + auto& api = detail::npy_api::get(); // _dtype_from_pep3118 returns dtypes with padding fields in, so we need to strip them auto numpy_internal = module::import("numpy.core._internal"); @@ -139,11 +146,6 @@ class array : public buffer { } protected: - static API &lookup_api() { - static API api = API::lookup(); - return api; - } - template friend struct detail::npy_format_descriptor; static object strip_padding_fields(object dtype) { @@ -183,7 +185,7 @@ class array : public buffer { args["itemsize"] = dtype.attr("itemsize").cast(); PyObject *descr = nullptr; - if (!lookup_api().PyArray_DescrConverter_(args.release().ptr(), &descr) || !descr) + if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &descr) || !descr) pybind11_fail("NumPy: failed to create structured dtype"); return object(descr, false); } @@ -198,10 +200,10 @@ template class array_t : public static PyObject *ensure(PyObject *ptr) { if (ptr == nullptr) return nullptr; - API &api = lookup_api(); + auto& api = detail::npy_api::get(); PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, - API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); Py_DECREF(ptr); @@ -246,12 +248,12 @@ struct is_pod_struct { template struct npy_format_descriptor::value>::type> { private: constexpr static const int values[8] = { - array::API::NPY_BYTE_, array::API::NPY_UBYTE_, array::API::NPY_SHORT_, array::API::NPY_USHORT_, - array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; + npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, + npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_ }; public: enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; static object dtype() { - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(value)) + if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) return object(ptr, true); pybind11_fail("Unsupported buffer format!"); } @@ -264,9 +266,9 @@ template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - enum { value = array::API::NumPyName }; \ + enum { value = npy_api::NumPyName }; \ static object dtype() { \ - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(value)) \ + if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \ return object(ptr, true); \ pybind11_fail("Unsupported buffer format!"); \ } \ @@ -281,7 +283,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #define DECL_CHAR_FMT \ static PYBIND11_DESCR name() { return _("S") + _(); } \ static object dtype() { \ - auto& api = array::lookup_api(); \ + auto& api = npy_api::get(); \ PyObject *descr = nullptr; \ PYBIND11_DESCR fmt = _("S") + _(); \ pybind11::str py_fmt(fmt.text()); \ @@ -319,7 +321,7 @@ struct npy_format_descriptor::value> } static void register_dtype(std::initializer_list fields) { - auto& api = array::lookup_api(); + auto& api = npy_api::get(); auto args = dict(); list names { }, offsets { }, formats { }; for (auto field : fields) { From 01f74095504211d2587ea3f48b2ffbe51b5ad724 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 23 Jul 2016 21:55:37 +0100 Subject: [PATCH 60/87] Initial implementation of py::dtype --- example/example-numpy-dtypes.cpp | 15 ++- include/pybind11/numpy.h | 178 ++++++++++++++++++------------- 2 files changed, 110 insertions(+), 83 deletions(-) diff --git a/example/example-numpy-dtypes.cpp b/example/example-numpy-dtypes.cpp index 2e25670f39..2c7cdc07c0 100644 --- a/example/example-numpy-dtypes.cpp +++ b/example/example-numpy-dtypes.cpp @@ -158,15 +158,12 @@ void print_format_descriptors() { } void print_dtypes() { - auto to_str = [](py::object obj) { - return (std::string) (py::str) ((py::object) obj.attr("__str__"))(); - }; - std::cout << to_str(py::dtype_of()) << std::endl; - std::cout << to_str(py::dtype_of()) << std::endl; - std::cout << to_str(py::dtype_of()) << std::endl; - std::cout << to_str(py::dtype_of()) << std::endl; - std::cout << to_str(py::dtype_of()) << std::endl; - std::cout << to_str(py::dtype_of()) << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; } void init_ex_numpy_dtypes(py::module &m) { diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 77f7e6f72e..3b52fa39b7 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -52,7 +52,12 @@ struct npy_api { return api; } - bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); } + bool PyArray_Check_(PyObject *obj) const { + return (bool) PyObject_TypeCheck(obj, PyArray_Type_); + } + bool PyArrayDescr_Check_(PyObject *obj) const { + return (bool) PyObject_TypeCheck(obj, PyArrayDescr_Type_); + } PyObject *(*PyArray_DescrFromType_)(int); PyObject *(*PyArray_NewFromDescr_) @@ -61,6 +66,7 @@ struct npy_api { PyObject *(*PyArray_DescrNewFromType_)(int); PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; + PyTypeObject *PyArrayDescr_Type_; PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); int (*PyArray_DescrConverter_) (PyObject *, PyObject **); bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); @@ -69,6 +75,7 @@ struct npy_api { private: enum functions { API_PyArray_Type = 2, + API_PyArrayDescr_Type = 3, API_PyArray_DescrFromType = 45, API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, @@ -90,6 +97,7 @@ struct npy_api { npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_Type); + DECL_NPY_API(PyArrayDescr_Type); DECL_NPY_API(PyArray_DescrFromType); DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_NewCopy); @@ -104,56 +112,55 @@ struct npy_api { }; } -class array : public buffer { +class dtype : public object { public: - PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) + PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); - enum { - c_style = detail::npy_api::NPY_C_CONTIGUOUS_, - f_style = detail::npy_api::NPY_F_CONTIGUOUS_, - forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ - }; + dtype(const buffer_info &info) { + dtype descr(_dtype_from_pep3118()(pybind11::str(info.format))); + m_ptr = descr.strip_padding().release().ptr(); + } - template array(size_t size, const Type *ptr) { - auto& api = detail::npy_api::get(); - PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); - Py_intptr_t shape = (Py_intptr_t) size; - object tmp = object(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); - if (!tmp) - pybind11_fail("NumPy: unable to create array!"); - if (ptr) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); - m_ptr = tmp.release().ptr(); + dtype(std::string format) { + m_ptr = from_args(pybind11::str(format)).release().ptr(); } - array(const buffer_info &info) { - auto& api = detail::npy_api::get(); + static dtype from_args(object args) { + // This is essentially the same as calling np.dtype() constructor in Python + PyObject *ptr = nullptr; + if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr) + pybind11_fail("NumPy: failed to create structured dtype"); + return object(ptr, false); + } - // _dtype_from_pep3118 returns dtypes with padding fields in, so we need to strip them - auto numpy_internal = module::import("numpy.core._internal"); - auto dtype_from_fmt = (object) numpy_internal.attr("_dtype_from_pep3118"); - auto dtype = strip_padding_fields(dtype_from_fmt(pybind11::str(info.format))); + template static dtype of() { + return detail::npy_format_descriptor::dtype(); + } - object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, dtype.release().ptr(), (int) info.ndim, (Py_intptr_t *) &info.shape[0], - (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); - if (!tmp) - pybind11_fail("NumPy: unable to create array!"); - if (info.ptr) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); - m_ptr = tmp.release().ptr(); + size_t itemsize() const { + return (size_t) attr("itemsize").cast(); } -protected: - template friend struct detail::npy_format_descriptor; + bool has_fields() const { + return attr("fields").cast().ptr() != Py_None; + } + + std::string kind() const { + return (std::string) attr("kind").cast(); + } + +private: + static object& _dtype_from_pep3118() { + static object obj = module::import("numpy.core._internal").attr("_dtype_from_pep3118"); + return obj; + } - static object strip_padding_fields(object dtype) { + dtype strip_padding() { // Recursively strip all void fields with empty names that are generated for // padding fields (as of NumPy v1.11). - auto fields = dtype.attr("fields").cast(); + auto fields = attr("fields").cast(); if (fields.ptr() == Py_None) - return dtype; + return *this; struct field_descr { pybind11::str name; object format; int_ offset; }; std::vector field_descriptors; @@ -162,11 +169,11 @@ class array : public buffer { for (auto field : items()) { auto spec = object(field, true).cast(); auto name = spec[0].cast(); - auto format = spec[1].cast()[0].cast(); + auto format = spec[1].cast()[0].cast(); auto offset = spec[1].cast()[1].cast(); - if (!len(name) && (std::string) dtype.attr("kind").cast() == "V") + if (!len(name) && format.kind() == "V") continue; - field_descriptors.push_back({name, strip_padding_fields(format), offset}); + field_descriptors.push_back({name, format.strip_padding(), offset}); } std::sort(field_descriptors.begin(), field_descriptors.end(), @@ -176,19 +183,57 @@ class array : public buffer { list names, formats, offsets; for (auto& descr : field_descriptors) { - names.append(descr.name); - formats.append(descr.format); - offsets.append(descr.offset); + names.append(descr.name); formats.append(descr.format); offsets.append(descr.offset); } auto args = dict(); args["names"] = names; args["formats"] = formats; args["offsets"] = offsets; - args["itemsize"] = dtype.attr("itemsize").cast(); + args["itemsize"] = (int_) itemsize(); + return dtype::from_args(args); + } +}; - PyObject *descr = nullptr; - if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &descr) || !descr) - pybind11_fail("NumPy: failed to create structured dtype"); - return object(descr, false); +class array : public buffer { +public: + PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) + + enum { + c_style = detail::npy_api::NPY_C_CONTIGUOUS_, + f_style = detail::npy_api::NPY_F_CONTIGUOUS_, + forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ + }; + + template array(size_t size, const Type *ptr) { + auto& api = detail::npy_api::get(); + auto descr = pybind11::dtype::of().release().ptr(); + Py_intptr_t shape = (Py_intptr_t) size; + object tmp = object(api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); + if (!tmp) + pybind11_fail("NumPy: unable to create array!"); + if (ptr) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + m_ptr = tmp.release().ptr(); + } + + array(const buffer_info &info) { + auto& api = detail::npy_api::get(); + auto descr = pybind11::dtype(info).release().ptr(); + object tmp(api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0], + (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); + if (!tmp) + pybind11_fail("NumPy: unable to create array!"); + if (info.ptr) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + m_ptr = tmp.release().ptr(); } + + pybind11::dtype dtype() { + return attr("dtype").cast(); + } + +protected: + template friend struct detail::npy_format_descriptor; }; template class array_t : public array { @@ -201,8 +246,7 @@ template class array_t : public if (ptr == nullptr) return nullptr; auto& api = detail::npy_api::get(); - PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); - PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, + PyObject *result = api.PyArray_FromAny_(ptr, pybind11::dtype::of().release().ptr(), 0, 0, detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -223,11 +267,6 @@ template struct format_descriptor> { static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } }; -template -object dtype_of() { - return detail::npy_format_descriptor::dtype(); -} - NAMESPACE_BEGIN(detail) template struct is_std_array : std::false_type { }; template struct is_std_array> : std::true_type { }; @@ -252,7 +291,7 @@ template struct npy_format_descriptor::value ? 1 : 0)] }; - static object dtype() { + static pybind11::dtype dtype() { if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) return object(ptr, true); pybind11_fail("Unsupported buffer format!"); @@ -267,7 +306,7 @@ template constexpr const int npy_format_descriptor< #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ enum { value = npy_api::NumPyName }; \ - static object dtype() { \ + static pybind11::dtype dtype() { \ if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \ return object(ptr, true); \ pybind11_fail("Unsupported buffer format!"); \ @@ -282,14 +321,9 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #define DECL_CHAR_FMT \ static PYBIND11_DESCR name() { return _("S") + _(); } \ - static object dtype() { \ - auto& api = npy_api::get(); \ - PyObject *descr = nullptr; \ + static pybind11::dtype dtype() { \ PYBIND11_DESCR fmt = _("S") + _(); \ - pybind11::str py_fmt(fmt.text()); \ - if (!api.PyArray_DescrConverter_(py_fmt.release().ptr(), &descr) || !descr) \ - pybind11_fail("NumPy: failed to create string dtype"); \ - return object(descr, false); \ + return pybind11::dtype::from_args(pybind11::str(fmt.text())); \ } \ static const char *format() { PYBIND11_DESCR s = _() + _("s"); return s.text(); } template struct npy_format_descriptor { DECL_CHAR_FMT }; @@ -301,14 +335,14 @@ struct field_descriptor { size_t offset; size_t size; const char *format; - object descr; + dtype descr; }; template struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("struct"); } - static object dtype() { + static pybind11::dtype dtype() { if (!dtype_()) pybind11_fail("NumPy: unsupported buffer format!"); return object(dtype_(), true); @@ -321,7 +355,6 @@ struct npy_format_descriptor::value> } static void register_dtype(std::initializer_list fields) { - auto& api = npy_api::get(); auto args = dict(); list names { }, offsets { }, formats { }; for (auto field : fields) { @@ -333,10 +366,7 @@ struct npy_format_descriptor::value> } args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; args["itemsize"] = int_(sizeof(T)); - // This is essentially the same as calling np.dtype() constructor in Python and passing - // it a dict of the form {'names': ..., 'formats': ..., 'offsets': ...}. - if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) - pybind11_fail("NumPy: failed to create structured dtype"); + dtype_() = pybind11::dtype::from_args(args).release().ptr(); // There is an existing bug in NumPy (as of v1.11): trailing bytes are // not encoded explicitly into the format string. This will supposedly @@ -366,9 +396,9 @@ struct npy_format_descriptor::value> format_() = oss.str(); // Sanity check: verify that NumPy properly parses our buffer format string + auto& api = npy_api::get(); auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); - auto fixed_dtype = array::strip_padding_fields(object(dtype_(), true)); - if (!api.PyArray_EquivTypes_(dtype_(), fixed_dtype.ptr())) + if (!api.PyArray_EquivTypes_(dtype_(), arr.dtype().ptr())) pybind11_fail("NumPy: invalid buffer descriptor!"); } From fc5620afa68a802b5953c2ef05b74611a908a054 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 17:34:53 +0100 Subject: [PATCH 61/87] Fix a segfault where func object wasn't released --- include/pybind11/numpy.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3b52fa39b7..3956c34c1d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -150,9 +150,10 @@ class dtype : public object { } private: - static object& _dtype_from_pep3118() { - static object obj = module::import("numpy.core._internal").attr("_dtype_from_pep3118"); - return obj; + static object _dtype_from_pep3118() { + static PyObject *obj = module::import("numpy.core._internal") + .attr("_dtype_from_pep3118").cast().release().ptr(); + return object(obj, true); } dtype strip_padding() { From d77bc8c343cb62c8a4b13fa50b523500099d59c6 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 17:51:35 +0100 Subject: [PATCH 62/87] Add dtype(names, offsets, formats, itemsize) ctor --- include/pybind11/numpy.h | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3956c34c1d..d048875910 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -125,6 +125,15 @@ class dtype : public object { m_ptr = from_args(pybind11::str(format)).release().ptr(); } + dtype(list names, list formats, list offsets, size_t itemsize) { + dict args; + args["names"] = names; + args["formats"] = formats; + args["offsets"] = offsets; + args["itemsize"] = int_(itemsize); + m_ptr = from_args(args).release().ptr(); + } + static dtype from_args(object args) { // This is essentially the same as calling np.dtype() constructor in Python PyObject *ptr = nullptr; @@ -184,12 +193,11 @@ class dtype : public object { list names, formats, offsets; for (auto& descr : field_descriptors) { - names.append(descr.name); formats.append(descr.format); offsets.append(descr.offset); + names.append(descr.name); + formats.append(descr.format); + offsets.append(descr.offset); } - auto args = dict(); - args["names"] = names; args["formats"] = formats; args["offsets"] = offsets; - args["itemsize"] = (int_) itemsize(); - return dtype::from_args(args); + return dtype(names, formats, offsets, itemsize()); } }; @@ -324,7 +332,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); static PYBIND11_DESCR name() { return _("S") + _(); } \ static pybind11::dtype dtype() { \ PYBIND11_DESCR fmt = _("S") + _(); \ - return pybind11::dtype::from_args(pybind11::str(fmt.text())); \ + return pybind11::dtype(fmt.text()); \ } \ static const char *format() { PYBIND11_DESCR s = _() + _("s"); return s.text(); } template struct npy_format_descriptor { DECL_CHAR_FMT }; @@ -356,18 +364,15 @@ struct npy_format_descriptor::value> } static void register_dtype(std::initializer_list fields) { - auto args = dict(); - list names { }, offsets { }, formats { }; + list names, formats, offsets; for (auto field : fields) { if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); names.append(str(field.name)); - offsets.append(int_(field.offset)); formats.append(field.descr); + offsets.append(int_(field.offset)); } - args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; - args["itemsize"] = int_(sizeof(T)); - dtype_() = pybind11::dtype::from_args(args).release().ptr(); + dtype_() = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr(); // There is an existing bug in NumPy (as of v1.11): trailing bytes are // not encoded explicitly into the format string. This will supposedly From 6bb0ee1186de21b59455b87e2e374e11e75cc48d Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 18:35:14 +0100 Subject: [PATCH 63/87] Add all possible ctors for py::array --- include/pybind11/numpy.h | 62 +++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d048875910..0d4aeaf63b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -211,12 +211,16 @@ class array : public buffer { forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; - template array(size_t size, const Type *ptr) { + array(const pybind11::dtype& dt, const std::vector& shape, + void *ptr, const std::vector& strides) { auto& api = detail::npy_api::get(); - auto descr = pybind11::dtype::of().release().ptr(); - Py_intptr_t shape = (Py_intptr_t) size; - object tmp = object(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); + auto ndim = shape.size(); + if (shape.size() != strides.size()) + pybind11_fail("NumPy: shape ndim doesn't match strides ndim"); + auto descr = dt; + object tmp(api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(), + (Py_intptr_t *) strides.data(), ptr, 0, nullptr), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); if (ptr) @@ -224,18 +228,30 @@ class array : public buffer { m_ptr = tmp.release().ptr(); } - array(const buffer_info &info) { - auto& api = detail::npy_api::get(); - auto descr = pybind11::dtype(info).release().ptr(); - object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0], - (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); - if (!tmp) - pybind11_fail("NumPy: unable to create array!"); - if (info.ptr) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); - m_ptr = tmp.release().ptr(); - } + array(const pybind11::dtype& dt, const std::vector& shape, void *ptr) + : array(dt, shape, ptr, default_strides(shape, dt.itemsize())) + { } + + array(const pybind11::dtype& dt, size_t size, void *ptr) + : array(dt, std::vector { size }, ptr) + { } + + template array(const std::vector& shape, + T* ptr, const std::vector& strides) + : array(pybind11::dtype::of(), shape, (void *) ptr, strides) + { } + + template array(const std::vector& shape, T* ptr) + : array(shape, ptr, default_strides(shape, sizeof(T))) + { } + + template array(size_t size, T* ptr) + : array(std::vector { size }, ptr) + { } + + array(const buffer_info &info) + : array(pybind11::dtype(info), info.shape, info.ptr, info.strides) + { } pybind11::dtype dtype() { return attr("dtype").cast(); @@ -243,6 +259,18 @@ class array : public buffer { protected: template friend struct detail::npy_format_descriptor; + + static std::vector default_strides(const std::vector& shape, size_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim); + if (ndim) { + std::fill(strides.begin(), strides.end(), itemsize); + for (size_t i = 0; i < ndim - 1; i++) + for (size_t j = 0; j < ndim - 1 - i; j++) + strides[j] *= shape[ndim - 1 - i]; + } + return strides; + } }; template class array_t : public array { From 6636ae9d4e6c9de5bc33a6f0fd40b72f90a5a888 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 18:54:53 +0100 Subject: [PATCH 64/87] Also add the new ctors to py::array_t --- include/pybind11/numpy.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0d4aeaf63b..d7b697f9a1 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -276,8 +276,21 @@ class array : public buffer { template class array_t : public array { public: PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); + array_t() : array() { } - array_t(const buffer_info& info) : array(info) {} + + array_t(const buffer_info& info) : array(info) { } + + array_t(const std::vector& shape, + T* ptr, const std::vector& strides) + : array(shape, ptr, strides) { } + + array_t(const std::vector& shape, T* ptr) + : array(shape, ptr) { } + + array_t(size_t size, T* ptr) + : array(size, ptr) { } + static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } static PyObject *ensure(PyObject *ptr) { if (ptr == nullptr) From 98ba98c06bbea58aa7e971b1cf87a118a0525433 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 20:29:44 +0100 Subject: [PATCH 65/87] Add a simplified buffer_info ctor for 1-D case --- include/pybind11/common.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/pybind11/common.h b/include/pybind11/common.h index 302d75a600..a915985053 100644 --- a/include/pybind11/common.h +++ b/include/pybind11/common.h @@ -210,6 +210,7 @@ struct buffer_info { std::vector strides; // Number of entries between adjacent entries (for each per dimension) buffer_info() : ptr(nullptr), view(nullptr) {} + buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t ndim, const std::vector &shape, const std::vector &strides) : ptr(ptr), itemsize(itemsize), size(1), format(format), @@ -218,6 +219,10 @@ struct buffer_info { size *= shape[i]; } + buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t size) + : buffer_info(ptr, itemsize, format, 1, std::vector { size }, + std::vector { itemsize }) { } + buffer_info(Py_buffer *view) : ptr(view->buf), itemsize((size_t) view->itemsize), size(1), format(view->format), ndim((size_t) view->ndim), shape((size_t) view->ndim), strides((size_t) view->ndim), view(view) { From e19980cc1082b26fc07a3fe0584015bb64ea688c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 20:30:17 +0100 Subject: [PATCH 66/87] Add tests for new array/array_t ctors --- example/example-numpy-dtypes.cpp | 35 ++++++++++++++++++++++++++++++++ example/example-numpy-dtypes.py | 8 +++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/example/example-numpy-dtypes.cpp b/example/example-numpy-dtypes.cpp index 2c7cdc07c0..7a4c1f6d84 100644 --- a/example/example-numpy-dtypes.cpp +++ b/example/example-numpy-dtypes.cpp @@ -166,6 +166,40 @@ void print_dtypes() { std::cout << (std::string) py::dtype::of().str() << std::endl; } +py::array_t test_array_ctors(int i) { + using arr_t = py::array_t; + + std::vector data { 1, 2, 3, 4, 5, 6 }; + std::vector shape { 3, 2 }; + std::vector strides { 8, 4 }; + + auto ptr = data.data(); + auto vptr = (void *) ptr; + auto dtype = py::dtype("int32"); + + py::buffer_info buf_ndim1(vptr, 4, "i", 6); + py::buffer_info buf_ndim2(vptr, 4, "i", 2, shape, strides); + + switch (i) { + // shape: (3, 2) + case 0: return arr_t(shape, ptr, strides); + case 1: return py::array(shape, ptr, strides); + case 2: return py::array(dtype, shape, vptr, strides); + case 3: return arr_t(shape, ptr); + case 4: return py::array(shape, ptr); + case 5: return py::array(dtype, shape, vptr); + case 6: return arr_t(buf_ndim2); + case 7: return py::array(buf_ndim2); + // shape: (6, ) + case 8: return arr_t(6, ptr); + case 9: return py::array(6, ptr); + case 10: return py::array(dtype, 6, vptr); + case 11: return arr_t(buf_ndim1); + case 12: return py::array(buf_ndim1); + } + return arr_t(); +} + void init_ex_numpy_dtypes(py::module &m) { PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z); @@ -187,6 +221,7 @@ void init_ex_numpy_dtypes(py::module &m) { m.def("get_format_unbound", &get_format_unbound); m.def("create_string_array", &create_string_array); m.def("print_string_array", &print_recarray); + m.def("test_array_ctors", &test_array_ctors); } #undef PYBIND11_PACKED diff --git a/example/example-numpy-dtypes.py b/example/example-numpy-dtypes.py index 68ea5c2cb1..930364afb8 100644 --- a/example/example-numpy-dtypes.py +++ b/example/example-numpy-dtypes.py @@ -6,7 +6,8 @@ from example import ( create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound, - create_rec_partial, create_rec_partial_nested, create_string_array, print_string_array + create_rec_partial, create_rec_partial_nested, create_string_array, print_string_array, + test_array_ctors ) @@ -80,3 +81,8 @@ def check_eq(arr, data, dtype): assert arr['b'].tolist() == [b'', b'a', b'ab', b'abc'] arr = create_string_array(False) assert dtype == arr.dtype + +data = np.arange(1, 7, dtype='int32') +for i in range(13): + expected = data if i >= 8 else data.reshape((3, 2)) + np.testing.assert_array_equal(test_array_ctors(i), expected) From 611e614619d398425133b70ad6286fcd08d497ba Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 23:52:42 +0100 Subject: [PATCH 67/87] Add tests for py::dtype ctors --- example/example-numpy-dtypes.cpp | 24 ++++++++++++++++++++++++ example/example-numpy-dtypes.py | 8 +++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/example/example-numpy-dtypes.cpp b/example/example-numpy-dtypes.cpp index 7a4c1f6d84..245de4fde6 100644 --- a/example/example-numpy-dtypes.cpp +++ b/example/example-numpy-dtypes.cpp @@ -200,6 +200,28 @@ py::array_t test_array_ctors(int i) { return arr_t(); } +py::list test_dtype_ctors() { + py::list list; + list.append(py::dtype("int32")); + list.append(py::dtype(std::string("float64"))); + list.append(py::dtype::from_args(py::str("bool"))); + py::list names, offsets, formats; + py::dict dict; + names.append(py::str("a")); names.append(py::str("b")); dict["names"] = names; + offsets.append(py::int_(1)); offsets.append(py::int_(10)); dict["offsets"] = offsets; + formats.append(py::dtype("int32")); formats.append(py::dtype("float64")); dict["formats"] = formats; + dict["itemsize"] = py::int_(20); + list.append(py::dtype::from_args(dict)); + list.append(py::dtype(names, formats, offsets, 20)); + list.append(py::dtype(py::buffer_info((void *) 0, 1, "I", 1))); + list.append(py::dtype(py::buffer_info((void *) 0, 1, "T{i:a:f:b:}", 1))); + return list; +} + +void test_dtype_methods() { + +} + void init_ex_numpy_dtypes(py::module &m) { PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z); @@ -222,6 +244,8 @@ void init_ex_numpy_dtypes(py::module &m) { m.def("create_string_array", &create_string_array); m.def("print_string_array", &print_recarray); m.def("test_array_ctors", &test_array_ctors); + m.def("test_dtype_ctors", &test_dtype_ctors); + m.def("test_dtype_methods", &test_dtype_methods); } #undef PYBIND11_PACKED diff --git a/example/example-numpy-dtypes.py b/example/example-numpy-dtypes.py index 930364afb8..f9c6653188 100644 --- a/example/example-numpy-dtypes.py +++ b/example/example-numpy-dtypes.py @@ -7,7 +7,7 @@ create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound, create_rec_partial, create_rec_partial_nested, create_string_array, print_string_array, - test_array_ctors + test_array_ctors, test_dtype_ctors ) @@ -86,3 +86,9 @@ def check_eq(arr, data, dtype): for i in range(13): expected = data if i >= 8 else data.reshape((3, 2)) np.testing.assert_array_equal(test_array_ctors(i), expected) + +d1 = np.dtype({'names': ['a', 'b'], 'formats': ['int32', 'float64'], + 'offsets': [1, 10], 'itemsize': 20}) +d2 = np.dtype([('a', 'i4'), ('b', 'f4')]) +assert test_dtype_ctors() == [np.dtype('int32'), np.dtype('float64'), + np.dtype('bool'), d1, d1, np.dtype('uint32'), d2] From 10af58fa77745478c78514eb58afc137e9d1d913 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 25 Jul 2016 00:15:07 +0100 Subject: [PATCH 68/87] Add a few more dtype tests --- example/example-numpy-dtypes.cpp | 10 ++++++++-- example/example-numpy-dtypes.py | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/example/example-numpy-dtypes.cpp b/example/example-numpy-dtypes.cpp index 245de4fde6..b5c0b06556 100644 --- a/example/example-numpy-dtypes.cpp +++ b/example/example-numpy-dtypes.cpp @@ -218,8 +218,14 @@ py::list test_dtype_ctors() { return list; } -void test_dtype_methods() { - +py::list test_dtype_methods() { + py::list list; + auto dt1 = py::dtype::of(); + auto dt2 = py::dtype::of(); + list.append(dt1); list.append(dt2); + list.append(py::bool_(dt1.has_fields())); list.append(py::bool_(dt2.has_fields())); + list.append(py::int_(dt1.itemsize())); list.append(py::int_(dt2.itemsize())); + return list; } void init_ex_numpy_dtypes(py::module &m) { diff --git a/example/example-numpy-dtypes.py b/example/example-numpy-dtypes.py index f9c6653188..2c90ed2863 100644 --- a/example/example-numpy-dtypes.py +++ b/example/example-numpy-dtypes.py @@ -7,7 +7,7 @@ create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound, create_rec_partial, create_rec_partial_nested, create_string_array, print_string_array, - test_array_ctors, test_dtype_ctors + test_array_ctors, test_dtype_ctors, test_dtype_methods ) @@ -92,3 +92,6 @@ def check_eq(arr, data, dtype): d2 = np.dtype([('a', 'i4'), ('b', 'f4')]) assert test_dtype_ctors() == [np.dtype('int32'), np.dtype('float64'), np.dtype('bool'), d1, d1, np.dtype('uint32'), d2] + +assert test_dtype_methods() == [np.dtype('int32'), simple_dtype, False, True, + np.dtype('int32').itemsize, simple_dtype.itemsize] From c6257f864112791e68241c6676b492a0fdd6e00b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 25 Jul 2016 00:46:39 +0100 Subject: [PATCH 69/87] Allow nullptr in array ctors wherever possible --- example/example-numpy-dtypes.cpp | 49 +++++++++++++++++++++++--------- example/example-numpy-dtypes.py | 9 ++++-- include/pybind11/numpy.h | 36 ++++++++++------------- 3 files changed, 57 insertions(+), 37 deletions(-) diff --git a/example/example-numpy-dtypes.cpp b/example/example-numpy-dtypes.cpp index b5c0b06556..d77945b1a7 100644 --- a/example/example-numpy-dtypes.cpp +++ b/example/example-numpy-dtypes.cpp @@ -178,24 +178,47 @@ py::array_t test_array_ctors(int i) { auto dtype = py::dtype("int32"); py::buffer_info buf_ndim1(vptr, 4, "i", 6); + py::buffer_info buf_ndim1_null(nullptr, 4, "i", 6); py::buffer_info buf_ndim2(vptr, 4, "i", 2, shape, strides); + py::buffer_info buf_ndim2_null(nullptr, 4, "i", 2, shape, strides); + + auto fill = [](py::array arr) { + auto req = arr.request(); + for (int i = 0; i < 6; i++) ((int32_t *) req.ptr)[i] = i + 1; + return arr; + }; switch (i) { // shape: (3, 2) - case 0: return arr_t(shape, ptr, strides); - case 1: return py::array(shape, ptr, strides); - case 2: return py::array(dtype, shape, vptr, strides); - case 3: return arr_t(shape, ptr); - case 4: return py::array(shape, ptr); - case 5: return py::array(dtype, shape, vptr); - case 6: return arr_t(buf_ndim2); - case 7: return py::array(buf_ndim2); + case 10: return arr_t(shape, strides, ptr); + case 11: return py::array(shape, strides, ptr); + case 12: return py::array(dtype, shape, strides, vptr); + case 13: return arr_t(shape, ptr); + case 14: return py::array(shape, ptr); + case 15: return py::array(dtype, shape, vptr); + case 16: return arr_t(buf_ndim2); + case 17: return py::array(buf_ndim2); + // shape: (3, 2) - post-fill + case 20: return fill(arr_t(shape, strides)); + case 21: return py::array(shape, strides, ptr); // can't have nullptr due to templated ctor + case 22: return fill(py::array(dtype, shape, strides)); + case 23: return fill(arr_t(shape)); + case 24: return py::array(shape, ptr); // can't have nullptr due to templated ctor + case 25: return fill(py::array(dtype, shape)); + case 26: return fill(arr_t(buf_ndim2_null)); + case 27: return fill(py::array(buf_ndim2_null)); + // shape: (6, ) + case 30: return arr_t(6, ptr); + case 31: return py::array(6, ptr); + case 32: return py::array(dtype, 6, vptr); + case 33: return arr_t(buf_ndim1); + case 34: return py::array(buf_ndim1); // shape: (6, ) - case 8: return arr_t(6, ptr); - case 9: return py::array(6, ptr); - case 10: return py::array(dtype, 6, vptr); - case 11: return arr_t(buf_ndim1); - case 12: return py::array(buf_ndim1); + case 40: return fill(arr_t(6)); + case 41: return py::array(6, ptr); // can't have nullptr due to templated ctor + case 42: return fill(py::array(dtype, 6)); + case 43: return fill(arr_t(buf_ndim1_null)); + case 44: return fill(py::array(buf_ndim1_null)); } return arr_t(); } diff --git a/example/example-numpy-dtypes.py b/example/example-numpy-dtypes.py index 2c90ed2863..7858acd461 100644 --- a/example/example-numpy-dtypes.py +++ b/example/example-numpy-dtypes.py @@ -83,9 +83,12 @@ def check_eq(arr, data, dtype): assert dtype == arr.dtype data = np.arange(1, 7, dtype='int32') -for i in range(13): - expected = data if i >= 8 else data.reshape((3, 2)) - np.testing.assert_array_equal(test_array_ctors(i), expected) +for i in range(8): + np.testing.assert_array_equal(test_array_ctors(10 + i), data.reshape((3, 2))) + np.testing.assert_array_equal(test_array_ctors(20 + i), data.reshape((3, 2))) +for i in range(5): + np.testing.assert_array_equal(test_array_ctors(30 + i), data) + np.testing.assert_array_equal(test_array_ctors(40 + i), data) d1 = np.dtype({'names': ['a', 'b'], 'formats': ['int32', 'float64'], 'offsets': [1, 10], 'itemsize': 20}) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d7b697f9a1..b9ee69e6c0 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -212,7 +212,7 @@ class array : public buffer { }; array(const pybind11::dtype& dt, const std::vector& shape, - void *ptr, const std::vector& strides) { + const std::vector& strides, void *ptr = nullptr) { auto& api = detail::npy_api::get(); auto ndim = shape.size(); if (shape.size() != strides.size()) @@ -228,30 +228,24 @@ class array : public buffer { m_ptr = tmp.release().ptr(); } - array(const pybind11::dtype& dt, const std::vector& shape, void *ptr) - : array(dt, shape, ptr, default_strides(shape, dt.itemsize())) - { } + array(const pybind11::dtype& dt, const std::vector& shape, void *ptr = nullptr) + : array(dt, shape, default_strides(shape, dt.itemsize()), ptr) { } - array(const pybind11::dtype& dt, size_t size, void *ptr) - : array(dt, std::vector { size }, ptr) - { } + array(const pybind11::dtype& dt, size_t size, void *ptr = nullptr) + : array(dt, std::vector { size }, ptr) { } template array(const std::vector& shape, - T* ptr, const std::vector& strides) - : array(pybind11::dtype::of(), shape, (void *) ptr, strides) - { } + const std::vector& strides, T* ptr) + : array(pybind11::dtype::of(), shape, strides, (void *) ptr) { } template array(const std::vector& shape, T* ptr) - : array(shape, ptr, default_strides(shape, sizeof(T))) - { } + : array(shape, default_strides(shape, sizeof(T)), ptr) { } template array(size_t size, T* ptr) - : array(std::vector { size }, ptr) - { } + : array(std::vector { size }, ptr) { } array(const buffer_info &info) - : array(pybind11::dtype(info), info.shape, info.ptr, info.strides) - { } + : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } pybind11::dtype dtype() { return attr("dtype").cast(); @@ -281,17 +275,17 @@ template class array_t : public array_t(const buffer_info& info) : array(info) { } - array_t(const std::vector& shape, - T* ptr, const std::vector& strides) - : array(shape, ptr, strides) { } + array_t(const std::vector& shape, const std::vector& strides, T* ptr = nullptr) + : array(shape, strides, ptr) { } - array_t(const std::vector& shape, T* ptr) + array_t(const std::vector& shape, T* ptr = nullptr) : array(shape, ptr) { } - array_t(size_t size, T* ptr) + array_t(size_t size, T* ptr = nullptr) : array(size, ptr) { } static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } + static PyObject *ensure(PyObject *ptr) { if (ptr == nullptr) return nullptr; From ad5ca6d4e64c8d8521567fe9ac982fafc793d021 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 25 Jul 2016 00:58:17 +0100 Subject: [PATCH 70/87] Added dtype from const char pointer ctor --- include/pybind11/numpy.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b9ee69e6c0..59530d4c2b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -125,6 +125,8 @@ class dtype : public object { m_ptr = from_args(pybind11::str(format)).release().ptr(); } + dtype(const char *format) : dtype(std::string(format)) { } + dtype(list names, list formats, list offsets, size_t itemsize) { dict args; args["names"] = names; From 3768b6abf9e2f8c745985f44050330bd864c0c4c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Tue, 2 Aug 2016 20:00:55 +0100 Subject: [PATCH 71/87] Use fully qualified name in PYBIND11_DESCR macro --- include/pybind11/descr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/descr.h b/include/pybind11/descr.h index 4123cc1b3e..e4a504e02d 100644 --- a/include/pybind11/descr.h +++ b/include/pybind11/descr.h @@ -181,7 +181,7 @@ PYBIND11_NOINLINE inline descr concat(descr &&d) { return d; } template PYBIND11_NOINLINE descr concat(descr &&d, Args&&... args) { return std::move(d) + _(", ") + concat(std::forward(args)...); } PYBIND11_NOINLINE inline descr type_descr(descr&& d) { return _("{") + std::move(d) + _("}"); } -#define PYBIND11_DESCR descr +#define PYBIND11_DESCR ::pybind11::detail::descr #endif NAMESPACE_END(detail) From 006d8b6621547c69abfe9b2d17d9296715b6c32d Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 12:38:11 +0100 Subject: [PATCH 72/87] Add casting operators between py::str / py::bytes --- example/example-python-types.cpp | 20 ++++++++ example/example-python-types.py | 5 ++ example/example-python-types.ref | 85 +++++++++++++++++--------------- include/pybind11/pytypes.h | 14 ++++++ 4 files changed, 84 insertions(+), 40 deletions(-) diff --git a/example/example-python-types.cpp b/example/example-python-types.cpp index 5decede9ae..429cf5eca8 100644 --- a/example/example-python-types.cpp +++ b/example/example-python-types.cpp @@ -139,6 +139,22 @@ class ExamplePythonTypes { throw std::runtime_error("This exception was intentionally thrown."); } + py::bytes get_bytes_from_string() { + return std::string("foo"); + } + + py::bytes get_bytes_from_str() { + return py::str(std::string("bar")); + } + + py::str get_str_from_string() { + return std::string("baz"); + } + + py::str get_str_from_bytes() { + return py::bytes(std::string("boo")); + } + static int value; static const int value2; }; @@ -167,6 +183,10 @@ void init_ex_python_types(py::module &m) { .def("pair_passthrough", &ExamplePythonTypes::pair_passthrough, "Return a pair in reversed order") .def("tuple_passthrough", &ExamplePythonTypes::tuple_passthrough, "Return a triple in reversed order") .def("throw_exception", &ExamplePythonTypes::throw_exception, "Throw an exception") + .def("get_bytes_from_string", &ExamplePythonTypes::get_bytes_from_string, "py::bytes from std::string") + .def("get_bytes_from_str", &ExamplePythonTypes::get_bytes_from_str, "py::bytes from py::str") + .def("get_str_from_string", &ExamplePythonTypes::get_str_from_string, "py::str from std::string") + .def("get_str_from_bytes", &ExamplePythonTypes::get_str_from_bytes, "py::str from py::bytes") .def_static("new_instance", &ExamplePythonTypes::new_instance, "Return an instance") .def_readwrite_static("value", &ExamplePythonTypes::value, "Static value member") .def_readonly_static("value2", &ExamplePythonTypes::value2, "Static value member (readonly)") diff --git a/example/example-python-types.py b/example/example-python-types.py index b77a2c531b..f743d88307 100755 --- a/example/example-python-types.py +++ b/example/example-python-types.py @@ -72,3 +72,8 @@ print("Instances not destroyed:", cstats.alive()) instance = None print("Instances not destroyed:", cstats.alive()) + +print(instance.get_bytes_from_string().decode()) +print(instance.get_bytes_from_str().decode()) +print(instance.get_str_from_string()) +print(instance.get_str_from_bytes()) diff --git a/example/example-python-types.ref b/example/example-python-types.ref index 052cce0ea4..6768d714ed 100644 --- a/example/example-python-types.ref +++ b/example/example-python-types.ref @@ -28,106 +28,106 @@ Help on class ExamplePythonTypes in module example class EExxaammpplleePPyytthhoonnTTyyppeess(__builtin__.object) | Example 2 documentation - | + | | Methods defined here: - | + | | ____iinniitt____(...) | x.__init__(...) initializes x; see help(type(x)) for signature - | + | | ggeett__aarrrraayy(...) - | + | | Signature : (example.ExamplePythonTypes) -> List[unicode[2]] | Return a C++ array - | + | | ggeett__ddiicctt(...) | Signature : (example.ExamplePythonTypes) -> dict - | + | | Return a Python dictionary - | + | | ggeett__ddiicctt__22(...) - | + | | Signature : (example.ExamplePythonTypes) -> Dict[unicode, unicode] | Return a C++ dictionary - | + | | ggeett__lliisstt(...) | Signature : (example.ExamplePythonTypes) -> list - | + | | Return a Python list - | + | | ggeett__lliisstt__22(...) - | + | | Signature : (example.ExamplePythonTypes) -> List[unicode] | Return a C++ list - | + | | ggeett__sseett(...) | Signature : (example.ExamplePythonTypes) -> set - | + | | Return a Python set - | + | | ggeett__sseett22(...) | Signature : (example.ExamplePythonTypes) -> set - | + | | Return a C++ set - | + | | ppaaiirr__ppaasssstthhrroouugghh(...) - | + | | Signature : (example.ExamplePythonTypes, Tuple[bool, unicode]) -> Tuple[unicode, bool] | Return a pair in reversed order - | + | | pprriinntt__aarrrraayy(...) - | + | | Signature : (example.ExamplePythonTypes, List[unicode[2]]) -> None | Print entries of a C++ array - | + | | pprriinntt__ddiicctt(...) - | + | | Signature : (example.ExamplePythonTypes, dict) -> None | Print entries of a Python dictionary - | + | | pprriinntt__ddiicctt__22(...) - | + | | Signature : (example.ExamplePythonTypes, Dict[unicode, unicode]) -> None | Print entries of a C++ dictionary - | + | | pprriinntt__lliisstt(...) - | + | | Signature : (example.ExamplePythonTypes, list) -> None | Print entries of a Python list - | + | | pprriinntt__lliisstt__22(...) - | + | | Signature : (example.ExamplePythonTypes, List[unicode]) -> None | Print entries of a C++ list - | + | | pprriinntt__sseett(...) - | + | | Signature : (example.ExamplePythonTypes, set) -> None | Print entries of a Python set - | + | | pprriinntt__sseett__22(...) - | + | | Signature : (example.ExamplePythonTypes, Set[unicode]) -> None | Print entries of a C++ set - | + | | tthhrrooww__eexxcceeppttiioonn(...) - | + | | Signature : (example.ExamplePythonTypes) -> None | Throw an exception - | + | | ttuuppllee__ppaasssstthhrroouugghh(...) - | + | | Signature : (example.ExamplePythonTypes, Tuple[bool, unicode, int]) -> Tuple[int, unicode, bool] | Return a triple in reversed order - | + | | ---------------------------------------------------------------------- | Data and other attributes defined here: - | + | | ____nneeww____ = | T.__new__(S, ...) -> a new object with type S, a subtype of T - | + | | nneeww__iinnssttaannccee = | Signature : () -> example.ExamplePythonTypes - | + | | Return an instance __name__(example) = example @@ -138,3 +138,8 @@ __module__(example.ExamplePythonTypes.get_set) = example Instances not destroyed: 1 ### ExamplePythonTypes @ 0x1045b80 destroyed Instances not destroyed: 0 +Destructing ExamplePythonTypes +foo +bar +baz +boo diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index db87b088a3..cc0e3a22a5 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -339,6 +339,8 @@ inline iterator handle::begin() const { return iterator(PyObject_GetIter(ptr()), inline iterator handle::end() const { return iterator(nullptr, false); } inline detail::args_proxy handle::operator*() const { return detail::args_proxy(*this); } +class bytes; + class str : public object { public: PYBIND11_OBJECT_DEFAULT(str, object, detail::PyUnicode_Check_Permissive) @@ -367,6 +369,8 @@ class str : public object { pybind11_fail("Unable to extract string contents! (invalid type)"); return std::string(buffer, (size_t) length); } + + operator bytes() const; }; inline pybind11::str handle::str() const { @@ -395,8 +399,18 @@ class bytes : public object { pybind11_fail("Unable to extract bytes contents!"); return std::string(buffer, (size_t) length); } + + operator pybind11::str() const; }; +inline str::operator bytes() const { + return bytes((std::string) *this); +} + +inline bytes::operator pybind11::str() const { + return pybind11::str((std::string) *this); +} + class none : public object { public: PYBIND11_OBJECT(none, object, detail::PyNone_Check) From 88239ef83d971587aa71ab670cb8138965455ce3 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 12:38:57 +0100 Subject: [PATCH 73/87] Don't use unittest in tests (Python 2 compat) --- example/example-numpy-dtypes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/example/example-numpy-dtypes.py b/example/example-numpy-dtypes.py index 7858acd461..4b83328315 100644 --- a/example/example-numpy-dtypes.py +++ b/example/example-numpy-dtypes.py @@ -1,7 +1,6 @@ #!/usr/bin/env python from __future__ import print_function -import unittest import numpy as np from example import ( create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, @@ -14,8 +13,11 @@ def check_eq(arr, data, dtype): np.testing.assert_equal(arr, np.array(data, dtype=dtype)) -unittest.TestCase().assertRaisesRegex( - RuntimeError, 'unsupported buffer format', get_format_unbound) +try: + get_format_unbound() + raise Exception +except RuntimeError as e: + assert 'unsupported buffer format' in str(e) print_format_descriptors() print_dtypes() From 1cdd171fbc5bad2e82075bc24e2f25375c7a1a47 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 12:39:16 +0100 Subject: [PATCH 74/87] Add PYBIND11_STR_TYPE to represent builtin `str` --- include/pybind11/common.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/pybind11/common.h b/include/pybind11/common.h index a915985053..9db37adc87 100644 --- a/include/pybind11/common.h +++ b/include/pybind11/common.h @@ -95,6 +95,7 @@ #define PYBIND11_STRING_NAME "str" #define PYBIND11_SLICE_OBJECT PyObject #define PYBIND11_FROM_STRING PyUnicode_FromString +#define PYBIND11_STR_TYPE ::pybind11::str #define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_base.ob_base.ob_type #define PYBIND11_PLUGIN_IMPL(name) \ extern "C" PYBIND11_EXPORT PyObject *PyInit_##name() @@ -113,6 +114,7 @@ #define PYBIND11_STRING_NAME "unicode" #define PYBIND11_SLICE_OBJECT PySliceObject #define PYBIND11_FROM_STRING PyString_FromString +#define PYBIND11_STR_TYPE ::pybind11::bytes #define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_type #define PYBIND11_PLUGIN_IMPL(name) \ extern "C" PYBIND11_EXPORT PyObject *init##name() From 61e3b0bd1580bde20e64514f534eb662a2ae1957 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 12:42:02 +0100 Subject: [PATCH 75/87] Use builtin str type for recarray field names --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 59530d4c2b..c71d9bbcae 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -117,7 +117,7 @@ class dtype : public object { PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); dtype(const buffer_info &info) { - dtype descr(_dtype_from_pep3118()(pybind11::str(info.format))); + dtype descr(_dtype_from_pep3118()(PYBIND11_STR_TYPE(info.format))); m_ptr = descr.strip_padding().release().ptr(); } @@ -174,7 +174,7 @@ class dtype : public object { if (fields.ptr() == Py_None) return *this; - struct field_descr { pybind11::str name; object format; int_ offset; }; + struct field_descr { PYBIND11_STR_TYPE name; object format; int_ offset; }; std::vector field_descriptors; auto items = fields.attr("items").cast(); @@ -185,7 +185,7 @@ class dtype : public object { auto offset = spec[1].cast()[1].cast(); if (!len(name) && format.kind() == "V") continue; - field_descriptors.push_back({name, format.strip_padding(), offset}); + field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(), offset}); } std::sort(field_descriptors.begin(), field_descriptors.end(), @@ -405,7 +405,7 @@ struct npy_format_descriptor::value> for (auto field : fields) { if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); - names.append(str(field.name)); + names.append(PYBIND11_STR_TYPE(field.name)); formats.append(field.descr); offsets.append(int_(field.offset)); } From 4611bcdd3665cdcd2a11bd5e95d2db1f5dcac25d Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 12:55:35 +0100 Subject: [PATCH 76/87] Fix rebasing problems in example-python-types --- example/example-python-types.py | 10 +++++----- example/example-python-types.ref | 7 +++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/example/example-python-types.py b/example/example-python-types.py index f743d88307..19df02ebde 100755 --- a/example/example-python-types.py +++ b/example/example-python-types.py @@ -66,14 +66,14 @@ print("__name__(example.ExamplePythonTypes.get_set) = %s" % ExamplePythonTypes.get_set.__name__) print("__module__(example.ExamplePythonTypes.get_set) = %s" % ExamplePythonTypes.get_set.__module__) +print(instance.get_bytes_from_string().decode()) +print(instance.get_bytes_from_str().decode()) +print(instance.get_str_from_string()) +print(instance.get_str_from_bytes()) + from example import ConstructorStats cstats = ConstructorStats.get(ExamplePythonTypes) print("Instances not destroyed:", cstats.alive()) instance = None print("Instances not destroyed:", cstats.alive()) - -print(instance.get_bytes_from_string().decode()) -print(instance.get_bytes_from_str().decode()) -print(instance.get_str_from_string()) -print(instance.get_str_from_bytes()) diff --git a/example/example-python-types.ref b/example/example-python-types.ref index 6768d714ed..ab89d7eb5c 100644 --- a/example/example-python-types.ref +++ b/example/example-python-types.ref @@ -135,11 +135,10 @@ __name__(example.ExamplePythonTypes) = ExamplePythonTypes __module__(example.ExamplePythonTypes) = example __name__(example.ExamplePythonTypes.get_set) = get_set __module__(example.ExamplePythonTypes.get_set) = example -Instances not destroyed: 1 -### ExamplePythonTypes @ 0x1045b80 destroyed -Instances not destroyed: 0 -Destructing ExamplePythonTypes foo bar baz boo +Instances not destroyed: 1 +### ExamplePythonTypes @ 0x1045b80 destroyed +Instances not destroyed: 0 From 0d7a015fb4addedfdf6e3e6402c0384e3c15f517 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 12:56:51 +0100 Subject: [PATCH 77/87] Update numpy docstring test to the new format --- example/example-numpy-dtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/example-numpy-dtypes.py b/example/example-numpy-dtypes.py index 4b83328315..06185914b2 100644 --- a/example/example-numpy-dtypes.py +++ b/example/example-numpy-dtypes.py @@ -73,7 +73,7 @@ def check_eq(arr, data, dtype): ((False, 2, 3.0), (True, 3, 4.5))], nested_dtype) print_rec_nested(arr) -assert create_rec_nested.__doc__.strip().endswith('numpy.ndarray[dtype=NestedStruct]') +assert create_rec_nested.__doc__.strip().endswith('numpy.ndarray[NestedStruct]') arr = create_string_array(True) print(arr.dtype) From f36ec97827d35ebf254e7b2598811917caed8913 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 13:14:51 +0100 Subject: [PATCH 78/87] Fix MSVC warnings in numpy example --- example/example-numpy-dtypes.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example/example-numpy-dtypes.cpp b/example/example-numpy-dtypes.cpp index d77945b1a7..dfde5d56b3 100644 --- a/example/example-numpy-dtypes.cpp +++ b/example/example-numpy-dtypes.cpp @@ -91,7 +91,7 @@ py::array_t create_recarray(size_t n) { auto req = arr.request(); auto ptr = static_cast(req.ptr); for (size_t i = 0; i < n; i++) { - ptr[i].x = i % 2; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f; + ptr[i].x = i % 2 != 0; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f; } return arr; } @@ -105,8 +105,8 @@ py::array_t create_nested(size_t n) { auto req = arr.request(); auto ptr = static_cast(req.ptr); for (size_t i = 0; i < n; i++) { - ptr[i].a.x = i % 2; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; - ptr[i].b.x = (i + 1) % 2; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f; + ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; + ptr[i].b.x = (i + 1) % 2 != 0; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f; } return arr; } @@ -116,7 +116,7 @@ py::array_t create_partial_nested(size_t n) { auto req = arr.request(); auto ptr = static_cast(req.ptr); for (size_t i = 0; i < n; i++) { - ptr[i].a.x = i % 2; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; + ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; } return arr; } From 245f77b4d504acac5336b3d4babb9a7169b8fbeb Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 13:20:36 +0100 Subject: [PATCH 79/87] Use uint64_t instead of long in numpy tests (MSVC) --- example/example-numpy-dtypes.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/example/example-numpy-dtypes.cpp b/example/example-numpy-dtypes.cpp index dfde5d56b3..e5292bba01 100644 --- a/example/example-numpy-dtypes.cpp +++ b/example/example-numpy-dtypes.cpp @@ -54,13 +54,13 @@ struct PartialStruct { bool x; uint32_t y; float z; - long dummy2; + uint64_t dummy2; }; struct PartialNestedStruct { - long dummy1; + uint64_t dummy1; PartialStruct a; - long dummy2; + uint64_t dummy2; }; struct UnboundStruct { }; From b65185906d1e1c1b9749b39ebe493b656297183d Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 13:28:56 +0100 Subject: [PATCH 80/87] Update the docs to use the new array ctor --- docs/advanced.rst | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index 5190e0b773..04dde6e068 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -1457,17 +1457,11 @@ simply using ``vectorize``). if (buf1.ndim != 1 || buf2.ndim != 1) throw std::runtime_error("Number of dimensions must be one"); - if (buf1.shape[0] != buf2.shape[0]) + if (buf1.size != buf2.size) throw std::runtime_error("Input shapes must match"); - auto result = py::array(py::buffer_info( - nullptr, /* Pointer to data (nullptr -> ask NumPy to allocate!) */ - sizeof(double), /* Size of one item */ - py::format_descriptor::format(), /* Buffer format */ - buf1.ndim, /* How many dimensions? */ - { buf1.shape[0] }, /* Number of elements for each dimension */ - { sizeof(double) } /* Strides for each dimension */ - )); + /* No pointer is passed, so NumPy will allocate the buffer */ + auto result = py::array_t(buf1.size); auto buf3 = result.request(); From 35c51c477bc3140e1b8c5cffd580f94434e50a90 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 13:52:27 +0100 Subject: [PATCH 81/87] A more strict bytes/str test --- example/example-python-types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/example-python-types.py b/example/example-python-types.py index 19df02ebde..1184f720a7 100755 --- a/example/example-python-types.py +++ b/example/example-python-types.py @@ -68,8 +68,8 @@ print(instance.get_bytes_from_string().decode()) print(instance.get_bytes_from_str().decode()) -print(instance.get_str_from_string()) -print(instance.get_str_from_bytes()) +print(instance.get_str_from_string().encode().decode()) +print(instance.get_str_from_bytes().encode().decode()) from example import ConstructorStats From fd6cede7e9affe25f967b4bb8f2ff02bb3885bee Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 15:46:46 +0100 Subject: [PATCH 82/87] Avoid extra allocations in operator str/bytes --- include/pybind11/pytypes.h | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index cc0e3a22a5..72c1573d78 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -364,8 +364,7 @@ class str : public object { } char *buffer; ssize_t length; - int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length); - if (err == -1) + if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length)) pybind11_fail("Unable to extract string contents! (invalid type)"); return std::string(buffer, (size_t) length); } @@ -394,8 +393,7 @@ class bytes : public object { operator std::string() const { char *buffer; ssize_t length; - int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length); - if (err == -1) + if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length)) pybind11_fail("Unable to extract bytes contents!"); return std::string(buffer, (size_t) length); } @@ -404,11 +402,31 @@ class bytes : public object { }; inline str::operator bytes() const { - return bytes((std::string) *this); + object temp = *this; + if (PyUnicode_Check(m_ptr)) { + temp = object(PyUnicode_AsUTF8String(m_ptr), false); + if (!temp) + pybind11_fail("Unable to extract string contents! (encoding issue)"); + } + char *buffer; + ssize_t length; + if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length)) + pybind11_fail("Unable to extract string contents! (invalid type)"); + auto obj = object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(buffer, length), false); + if (!obj) + pybind11_fail("Could not allocate bytes object!"); + return obj; } inline bytes::operator pybind11::str() const { - return pybind11::str((std::string) *this); + char *buffer; + ssize_t length; + if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length)) + pybind11_fail("Unable to extract bytes contents!"); + auto obj = object(PyUnicode_FromStringAndSize(buffer, (ssize_t) length), false); + if (!obj) + pybind11_fail("Could not allocate string object!"); + return obj; } class none : public object { From 1e1217817bda324181d49bc0acfab70cf5036677 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 18:52:39 +0100 Subject: [PATCH 83/87] Use explicit casts in str/bytes tests --- example/example-python-types.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example/example-python-types.cpp b/example/example-python-types.cpp index 429cf5eca8..52780cae44 100644 --- a/example/example-python-types.cpp +++ b/example/example-python-types.cpp @@ -140,19 +140,19 @@ class ExamplePythonTypes { } py::bytes get_bytes_from_string() { - return std::string("foo"); + return (py::bytes) std::string("foo"); } py::bytes get_bytes_from_str() { - return py::str(std::string("bar")); + return (py::bytes) py::str(std::string("bar")); } py::str get_str_from_string() { - return std::string("baz"); + return (py::str) std::string("baz"); } py::str get_str_from_bytes() { - return py::bytes(std::string("boo")); + return (py::str) py::bytes(std::string("boo")); } static int value; From 89ec7f3e794512edcbd1df9442c721168546d163 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 19:38:50 +0100 Subject: [PATCH 84/87] Add (const char *, size_t) ctors for str/bytes --- example/example-python-types.cpp | 4 ++-- include/pybind11/pytypes.h | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/example/example-python-types.cpp b/example/example-python-types.cpp index 52780cae44..00567931da 100644 --- a/example/example-python-types.cpp +++ b/example/example-python-types.cpp @@ -144,7 +144,7 @@ class ExamplePythonTypes { } py::bytes get_bytes_from_str() { - return (py::bytes) py::str(std::string("bar")); + return (py::bytes) py::str("bar", 3); } py::str get_str_from_string() { @@ -152,7 +152,7 @@ class ExamplePythonTypes { } py::str get_str_from_bytes() { - return (py::str) py::bytes(std::string("boo")); + return (py::str) py::bytes("boo", 3); } static int value; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 72c1573d78..5f9e39ef6c 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -345,8 +345,8 @@ class str : public object { public: PYBIND11_OBJECT_DEFAULT(str, object, detail::PyUnicode_Check_Permissive) - str(const std::string &s) - : object(PyUnicode_FromStringAndSize(s.c_str(), (ssize_t) s.length()), false) { + str(const char *c, size_t n) + : object(PyUnicode_FromStringAndSize(c, (ssize_t) n), false) { if (!m_ptr) pybind11_fail("Could not allocate string object!"); } @@ -355,6 +355,8 @@ class str : public object { if (!m_ptr) pybind11_fail("Could not allocate string object!"); } + str(const std::string &s) : str(s.data(), s.size()) { } + operator std::string() const { object temp = *this; if (PyUnicode_Check(m_ptr)) { @@ -385,11 +387,13 @@ class bytes : public object { public: PYBIND11_OBJECT_DEFAULT(bytes, object, PYBIND11_BYTES_CHECK) - bytes(const std::string &s) - : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(s.data(), (ssize_t) s.size()), false) { + bytes(const char *c, size_t n) + : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(c, (ssize_t) n), false) { if (!m_ptr) pybind11_fail("Could not allocate bytes object!"); } + bytes(const std::string &s) : bytes(s.data(), s.size()) { } + operator std::string() const { char *buffer; ssize_t length; From c22fe428eda0927be85bbf6cc1e3f9e62c5a526e Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 19:39:11 +0100 Subject: [PATCH 85/87] Change str/bytes cast operators to ctors --- include/pybind11/pytypes.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 5f9e39ef6c..b148c2a416 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -357,6 +357,8 @@ class str : public object { str(const std::string &s) : str(s.data(), s.size()) { } + str(const bytes &b); + operator std::string() const { object temp = *this; if (PyUnicode_Check(m_ptr)) { @@ -370,8 +372,6 @@ class str : public object { pybind11_fail("Unable to extract string contents! (invalid type)"); return std::string(buffer, (size_t) length); } - - operator bytes() const; }; inline pybind11::str handle::str() const { @@ -394,6 +394,8 @@ class bytes : public object { bytes(const std::string &s) : bytes(s.data(), s.size()) { } + bytes(const pybind11::str &s); + operator std::string() const { char *buffer; ssize_t length; @@ -401,14 +403,12 @@ class bytes : public object { pybind11_fail("Unable to extract bytes contents!"); return std::string(buffer, (size_t) length); } - - operator pybind11::str() const; }; -inline str::operator bytes() const { - object temp = *this; - if (PyUnicode_Check(m_ptr)) { - temp = object(PyUnicode_AsUTF8String(m_ptr), false); +inline bytes::bytes(const pybind11::str &s) { + object temp = s; + if (PyUnicode_Check(s.ptr())) { + temp = object(PyUnicode_AsUTF8String(s.ptr()), false); if (!temp) pybind11_fail("Unable to extract string contents! (encoding issue)"); } @@ -419,18 +419,18 @@ inline str::operator bytes() const { auto obj = object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(buffer, length), false); if (!obj) pybind11_fail("Could not allocate bytes object!"); - return obj; + m_ptr = obj.release().ptr(); } -inline bytes::operator pybind11::str() const { +inline str::str(const bytes& b) { char *buffer; ssize_t length; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length)) + if (PYBIND11_BYTES_AS_STRING_AND_SIZE(b.ptr(), &buffer, &length)) pybind11_fail("Unable to extract bytes contents!"); auto obj = object(PyUnicode_FromStringAndSize(buffer, (ssize_t) length), false); if (!obj) pybind11_fail("Could not allocate string object!"); - return obj; + m_ptr = obj.release().ptr(); } class none : public object { From 7dcbfe228e296890de7c67048fd4ce9389a950a9 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 20:00:15 +0100 Subject: [PATCH 86/87] Add a missing bytes ctor from const char * --- include/pybind11/pytypes.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index b148c2a416..d3895db57f 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -387,6 +387,11 @@ class bytes : public object { public: PYBIND11_OBJECT_DEFAULT(bytes, object, PYBIND11_BYTES_CHECK) + bytes(const char *c) + : object(PYBIND11_BYTES_FROM_STRING(c), false) { + if (!m_ptr) pybind11_fail("Could not allocate bytes object!"); + } + bytes(const char *c, size_t n) : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(c, (ssize_t) n), false) { if (!m_ptr) pybind11_fail("Could not allocate bytes object!"); From bccbc10a6504b1e11019656b678c25f12447d3dd Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 21:17:26 +0100 Subject: [PATCH 87/87] Update changelog and authors --- README.md | 5 +++-- docs/changelog.rst | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f9a8ed521a..b2129c1c9d 100644 --- a/README.md +++ b/README.md @@ -106,8 +106,9 @@ Tomasz Miąsko, Dean Moldovan, Ben Pritchard, Jason Rhinelander, -Boris Schäling, and -Pim Schellart. +Boris Schäling, +Pim Schellart, +Ivan Smirnov. ### License diff --git a/docs/changelog.rst b/docs/changelog.rst index 86ad18a8f5..def3ceb71e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,23 @@ Breaking changes queued for v2.0.0 (Not yet released) * ``make_iterator()`` improvements for better compatibility with various types (now uses prefix increment operator) * ``arg()`` now accepts a wider range of argument types for default values +* Added support for registering structured dtypes via ``PYBIND11_NUMPY_DTYPE()`` macro. +* Added ``PYBIND11_STR_TYPE`` macro which maps to the ``builtins.str`` type. +* Added a simplified ``buffer_info`` constructor for 1-dimensional buffers. +* Format descriptor strings should now be accessed via ``format_descriptor::format()`` + (for compatibility purposes, the old syntax ``format_descriptor::value`` will still + work for non-structured data types). +* Added a class wrapping NumPy array descriptors: ``dtype``. +* Added buffer/NumPy support for ``char[N]`` and ``std::array`` types. +* ``array`` gained new constructors accepting dtype objects. +* Added constructors for ``array`` and ``array_t`` explicitly accepting shape and + strides; if strides are not provided, they are deduced assuming C-contiguity. + Also added simplified constructors for 1-dimensional case. +* Added constructors for ``str`` from ``bytes`` and for ``bytes`` from ``str``. + This will do the UTF-8 decoding/encoding as required. +* Added constructors for ``str`` and ``bytes`` from zero-terminated char pointers, + and from char pointers and length. +* Added ``memoryview`` wrapper type which is constructible from ``buffer_info``. * Various minor improvements of library internals (no user-visible changes) 1.8.1 (July 12, 2016)