Merge pull request #308 from aldanor/recarray

py::dtype, buffer protocol improvements, structured types support

.gitignore

@@ -3,7 +3,7 @@ CMakeFiles
 Makefile
 cmake_install.cmake
 .DS_Store
-/example/example.so
+/example/example*.so
 /example/example.cpython*.so
 /example/example.pyd
 /example/example*.dll
@@ -31,3 +31,4 @@ MANIFEST
 .DS_Store
 /dist
 /build
+/cmake/

README.md

@@ -106,8 +106,9 @@ Tomasz Miąsko,
 Dean Moldovan,
 Ben Pritchard,
 Jason Rhinelander,
-Boris Schäling, and
-Pim Schellart.
+Boris Schäling,
+Pim Schellart,
+Ivan Smirnov.
 
 ### License
 

docs/advanced.rst

@@ -1224,12 +1224,12 @@ completely avoid copy operations with Python expressions like
     py::class_<Matrix>(m, "Matrix")
        .def_buffer([](Matrix &m) -> py::buffer_info {
             return py::buffer_info(
-                m.data(),                            /* Pointer to buffer */
-                sizeof(float),                       /* Size of one scalar */
-                py::format_descriptor<float>::value, /* Python struct-style format descriptor */
-                2,                                   /* Number of dimensions */
-                { m.rows(), m.cols() },              /* Buffer dimensions */
-                { sizeof(float) * m.rows(),          /* Strides (in bytes) for each index */
+                m.data(),                               /* Pointer to buffer */
+                sizeof(float),                          /* Size of one scalar */
+                py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
+                2,                                      /* Number of dimensions */
+                { m.rows(), m.cols() },                 /* Buffer dimensions */
+                { sizeof(float) * m.rows(),             /* Strides (in bytes) for each index */
                   sizeof(float) }
             );
         });
@@ -1273,7 +1273,7 @@ buffer objects (e.g. a NumPy matrix).
             py::buffer_info info = b.request();
 
             /* Some sanity checks ... */
-            if (info.format != py::format_descriptor<Scalar>::value)
+            if (info.format != py::format_descriptor<Scalar>::format())
                 throw std::runtime_error("Incompatible format: expected a double array!");
 
             if (info.ndim != 2)
@@ -1299,7 +1299,7 @@ as follows:
             m.data(),                /* Pointer to buffer */
             sizeof(Scalar),          /* Size of one scalar */
             /* Python struct-style format descriptor */
-            py::format_descriptor<Scalar>::value,
+            py::format_descriptor<Scalar>::format(),
             /* Number of dimensions */
             2,
             /* Buffer dimensions */
@@ -1358,6 +1358,30 @@ template paramenter, and it ensures that non-conforming arguments are converted
 into an array satisfying the specified requirements instead of trying the next
 function overload.
 
+NumPy structured types
+======================
+
+In order for ``py::array_t`` to work with structured (record) types, we first need
+to register the memory layout of the type. This can be done via ``PYBIND11_NUMPY_DTYPE``
+macro which expects the type followed by field names:
+
+.. code-block:: cpp
+
+    struct A {
+        int x;
+        double y;
+    };
+
+    struct B {
+        int z;
+        A a;
+    };
+
+    PYBIND11_NUMPY_DTYPE(A, x, y);
+    PYBIND11_NUMPY_DTYPE(B, z, a);
+
+    /* now both A and B can be used as template arguments to py::array_t */
+
 Vectorizing functions
 =====================
 
@@ -1433,17 +1457,11 @@ simply using ``vectorize``).
         if (buf1.ndim != 1 || buf2.ndim != 1)
             throw std::runtime_error("Number of dimensions must be one");
 
-        if (buf1.shape[0] != buf2.shape[0])
+        if (buf1.size != buf2.size)
             throw std::runtime_error("Input shapes must match");
 
-        auto result = py::array(py::buffer_info(
-            nullptr,            /* Pointer to data (nullptr -> ask NumPy to allocate!) */
-            sizeof(double),     /* Size of one item */
-            py::format_descriptor<double>::value, /* Buffer format */
-            buf1.ndim,          /* How many dimensions? */
-            { buf1.shape[0] },  /* Number of elements for each dimension */
-            { sizeof(double) }  /* Strides for each dimension */
-        ));
+        /* No pointer is passed, so NumPy will allocate the buffer */
+        auto result = py::array_t<double>(buf1.size);
 
         auto buf3 = result.request();
 
@@ -1830,4 +1848,3 @@ is always ``none``).
 
     // Evaluate the statements in an separate Python file on disk
     py::eval_file("script.py", scope);
-

docs/changelog.rst

@@ -27,6 +27,23 @@ Breaking changes queued for v2.0.0 (Not yet released)
 * ``make_iterator()`` improvements for better compatibility with various types
   (now uses prefix increment operator)
 * ``arg()`` now accepts a wider range of argument types for default values
+* Added support for registering structured dtypes via ``PYBIND11_NUMPY_DTYPE()`` macro.
+* Added ``PYBIND11_STR_TYPE`` macro which maps to the ``builtins.str`` type.
+* Added a simplified ``buffer_info`` constructor for 1-dimensional buffers.
+* Format descriptor strings should now be accessed via ``format_descriptor::format()``
+  (for compatibility purposes, the old syntax ``format_descriptor::value`` will still
+  work for non-structured data types).
+* Added a class wrapping NumPy array descriptors: ``dtype``.
+* Added buffer/NumPy support for ``char[N]`` and ``std::array<char, N>`` types.
+* ``array`` gained new constructors accepting dtype objects.
+* Added constructors for ``array`` and ``array_t`` explicitly accepting shape and
+  strides; if strides are not provided, they are deduced assuming C-contiguity.
+  Also added simplified constructors for 1-dimensional case.
+* Added constructors for ``str`` from ``bytes`` and for ``bytes`` from ``str``.
+  This will do the UTF-8 decoding/encoding as required.
+* Added constructors for ``str`` and ``bytes`` from zero-terminated char pointers,
+  and from char pointers and length.
+* Added ``memoryview`` wrapper type which is constructible from ``buffer_info``.
 * Various minor improvements of library internals (no user-visible changes)
 
 1.8.1 (July 12, 2016)

example/CMakeLists.txt

@@ -26,6 +26,7 @@ set(PYBIND11_EXAMPLES
   example-stl-binder-vector.cpp
   example-eval.cpp
   example-custom-exceptions.cpp
+  example-numpy-dtypes.cpp
   issues.cpp
 )
 
@@ -65,4 +66,3 @@ foreach(VALUE ${PYBIND11_EXAMPLES})
   string(REGEX REPLACE "^(.+).cpp$" "\\1" EXAMPLE_NAME "${VALUE}")
   add_test(NAME ${EXAMPLE_NAME} COMMAND ${RUN_TEST} ${EXAMPLE_NAME})
 endforeach()
-

example/example-buffers.cpp

@@ -81,7 +81,7 @@ void init_ex_buffers(py::module &m) {
         /// Construct from a buffer
         .def("__init__", [](Matrix &v, py::buffer b) {
             py::buffer_info info = b.request();
-            if (info.format != py::format_descriptor<float>::value || info.ndim != 2)
+            if (info.format != py::format_descriptor<float>::format() || info.ndim != 2)
                 throw std::runtime_error("Incompatible buffer format!");
             new (&v) Matrix(info.shape[0], info.shape[1]);
             memcpy(v.data(), info.ptr, sizeof(float) * v.rows() * v.cols());
@@ -104,12 +104,12 @@ void init_ex_buffers(py::module &m) {
        /// Provide buffer access
        .def_buffer([](Matrix &m) -> py::buffer_info {
             return py::buffer_info(
-                m.data(),                            /* Pointer to buffer */
-                sizeof(float),                       /* Size of one scalar */
-                py::format_descriptor<float>::value, /* Python struct-style format descriptor */
-                2,                                   /* Number of dimensions */
-                { m.rows(), m.cols() },              /* Buffer dimensions */
-                { sizeof(float) * m.rows(),          /* Strides (in bytes) for each index */
+                m.data(),                               /* Pointer to buffer */
+                sizeof(float),                          /* Size of one scalar */
+                py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
+                2,                                      /* Number of dimensions */
+                { m.rows(), m.cols() },                 /* Buffer dimensions */
+                { sizeof(float) * m.rows(),             /* Strides (in bytes) for each index */
                   sizeof(float) }
             );
         })