From 09ffb68b658225e53d099322427ba210bfa97700 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 19 Jun 2017 08:05:26 -0500 Subject: [PATCH] refactor pdal::plang::Array --- plugins/python/filters/CMakeLists.txt | 1 - plugins/python/plang/Array.cpp | 134 --------------------- plugins/python/plang/Array.hpp | 162 ++++++++++++++++++++++++-- python/pdal/PyPipeline.hpp | 1 - 4 files changed, 151 insertions(+), 147 deletions(-) diff --git a/plugins/python/filters/CMakeLists.txt b/plugins/python/filters/CMakeLists.txt index 556c7be1c1..d4c810aded 100644 --- a/plugins/python/filters/CMakeLists.txt +++ b/plugins/python/filters/CMakeLists.txt @@ -1,6 +1,5 @@ PDAL_ADD_PLUGIN(python_libname filter python FILES - ../plang/Array.cpp ../plang/Invocation.cpp ../plang/Environment.cpp ../plang/Redirector.cpp diff --git a/plugins/python/plang/Array.cpp b/plugins/python/plang/Array.cpp index 0b48f5f23b..3196c7d0ac 100644 --- a/plugins/python/plang/Array.cpp +++ b/plugins/python/plang/Array.cpp @@ -57,141 +57,7 @@ namespace pdal namespace plang { -Array::Array() - : m_py_array(0) -{ - auto initNumpy = []() - { -#undef NUMPY_IMPORT_ARRAY_RETVAL -#define NUMPY_IMPORT_ARRAY_RETVAL - import_array(); - }; - initNumpy(); -} - - -Array::~Array() -{ - cleanup(); -} - -void Array::cleanup() -{ - PyObject* p = (PyObject*)(m_py_array); - Py_XDECREF(p); - m_data_array.reset(); -} -PyObject* Array::buildNumpyDescription(PointViewPtr view) const -{ - - // Build up a numpy dtype dictionary - // - // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], - // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', 'NumberOfReturns', - // 'ScanDirectionFlag', 'EdgeOfFlightLine', 'Classification', - // 'ScanAngleRank', 'UserData', 'PointSourceId', 'GpsTime', 'Red', 'Green', - // 'Blue']} - // - std::stringstream oss; - Dimension::IdList dims = view->dims(); - - PyObject* dict = PyDict_New(); - PyObject* sizes = PyList_New(dims.size()); - PyObject* formats = PyList_New(dims.size()); - PyObject* titles = PyList_New(dims.size()); - - for (Dimension::IdList::size_type i=0; i < dims.size(); ++i) - { - Dimension::Id id = (dims[i]); - Dimension::Type t = view->dimType(id); - npy_intp stride = view->dimSize(id); - - std::string name = view->dimName(id); - - std::string kind("i"); - Dimension::BaseType b = Dimension::base(t); - if (b == Dimension::BaseType::Unsigned) - kind = "u"; - else if (b == Dimension::BaseType::Floating) - kind = "f"; - else - { - std::stringstream o; - oss << "unable to map kind '" << kind <<"' to PDAL dimension type"; - throw pdal::pdal_error(o.str()); - } - - oss << kind << stride; - PyObject* pySize = PyLong_FromLong(stride); - PyObject* pyTitle = PyUnicode_FromString(name.c_str()); - PyObject* pyFormat = PyUnicode_FromString(oss.str().c_str()); - - PyList_SetItem(sizes, i, pySize); - PyList_SetItem(titles, i, pyTitle); - PyList_SetItem(formats, i, pyFormat); - - oss.str(""); - } - - PyDict_SetItemString(dict, "names", titles); - PyDict_SetItemString(dict, "formats", formats); - -// PyObject* obj = PyUnicode_AsASCIIString(PyObject_Str(dict)); -// const char* s = PyBytes_AsString(obj); -// std::string output(s); -// std::cout << "array: " << output << std::endl; - return dict; -} -void Array::update(PointViewPtr view) -{ - typedef std::unique_ptr> DataPtr; - cleanup(); - int nd = 1; - Dimension::IdList dims = view->dims(); - npy_intp mydims = view->size(); - npy_intp* ndims = &mydims; - std::vector strides(dims.size()); - - - DataPtr pdata( new std::vector(view->pointSize()* view->size(), 0)); - - PyArray_Descr *dtype(0); - PyObject * dtype_dict = (PyObject*)buildNumpyDescription(view); - if (!dtype_dict) - throw pdal_error("Unable to build numpy dtype description dictionary"); - int did_convert = PyArray_DescrConverter(dtype_dict, &dtype); - if (did_convert == NPY_FAIL) - throw pdal_error("Unable to build numpy dtype"); - Py_XDECREF(dtype_dict); - -#ifdef NPY_ARRAY_CARRAY - int flags = NPY_ARRAY_CARRAY; -#else - int flags = NPY_CARRAY; -#endif - uint8_t* sp = pdata.get()->data(); - PyObject * pyArray = PyArray_NewFromDescr(&PyArray_Type, - dtype, - nd, - ndims, - 0, - sp, - flags, - NULL); - - // copy the data - uint8_t* p(sp); - DimTypeList types = view->dimTypes(); - for (PointId idx = 0; idx < view->size(); idx++) - { - p = sp + (view->pointSize() * idx); - view->getPackedPoint(types, idx, (char*)p); - } - - m_py_array = pyArray; - m_data_array = std::move(pdata); -} } // namespace plang diff --git a/plugins/python/plang/Array.hpp b/plugins/python/plang/Array.hpp index b1bcb423cc..520fb6f754 100644 --- a/plugins/python/plang/Array.hpp +++ b/plugins/python/plang/Array.hpp @@ -34,17 +34,24 @@ #pragma once -#include #include +#include -// forward declare PyObject so we don't need the python headers everywhere -// see: http://mail.python.org/pipermail/python-dev/2003-August/037601.html -#ifndef PyObject_HEAD -struct _object; -typedef _object PyObject; +#ifdef PDAL_COMPILER_MSVC +# pragma warning(disable: 4127) // conditional expression is constant #endif +#include +#undef toupper +#undef tolower +#undef isspace + + +#define PY_ARRAY_UNIQUE_SYMBOL PDALARRAY_ARRAY_API +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include + namespace pdal { namespace plang @@ -54,17 +61,150 @@ namespace plang class PDAL_DLL Array { public: - Array(); - ~Array(); - void update(PointViewPtr view); + Array() + : m_py_array(0) + { + auto initNumpy = []() + { +#undef NUMPY_IMPORT_ARRAY_RETVAL +#define NUMPY_IMPORT_ARRAY_RETVAL + import_array(); + }; + initNumpy(); + } + + ~Array() + { + cleanup(); + } + + + inline void update(PointViewPtr view) + { + typedef std::unique_ptr> DataPtr; + cleanup(); + int nd = 1; + Dimension::IdList dims = view->dims(); + npy_intp mydims = view->size(); + npy_intp* ndims = &mydims; + std::vector strides(dims.size()); + + + DataPtr pdata( new std::vector(view->pointSize()* view->size(), 0)); + + PyArray_Descr *dtype(0); + PyObject * dtype_dict = (PyObject*)buildNumpyDescription(view); + if (!dtype_dict) + throw pdal_error("Unable to build numpy dtype description dictionary"); + int did_convert = PyArray_DescrConverter(dtype_dict, &dtype); + if (did_convert == NPY_FAIL) + throw pdal_error("Unable to build numpy dtype"); + Py_XDECREF(dtype_dict); + +#ifdef NPY_ARRAY_CARRAY + int flags = NPY_ARRAY_CARRAY; +#else + int flags = NPY_CARRAY; +#endif + uint8_t* sp = pdata.get()->data(); + PyObject * pyArray = PyArray_NewFromDescr(&PyArray_Type, + dtype, + nd, + ndims, + 0, + sp, + flags, + NULL); + + // copy the data + uint8_t* p(sp); + DimTypeList types = view->dimTypes(); + for (PointId idx = 0; idx < view->size(); idx++) + { + p = sp + (view->pointSize() * idx); + view->getPackedPoint(types, idx, (char*)p); + } + + m_py_array = pyArray; + m_data_array = std::move(pdata); + + } + inline PyObject* getPythonArray() const { return m_py_array; } private: - void cleanup(); - PyObject* buildNumpyDescription(PointViewPtr view) const; + + inline void cleanup() + { + PyObject* p = (PyObject*)(m_py_array); + Py_XDECREF(p); + m_data_array.reset(); + } + + inline PyObject* buildNumpyDescription(PointViewPtr view) const + { + + // Build up a numpy dtype dictionary + // + // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], + // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', 'NumberOfReturns', + // 'ScanDirectionFlag', 'EdgeOfFlightLine', 'Classification', + // 'ScanAngleRank', 'UserData', 'PointSourceId', 'GpsTime', 'Red', 'Green', + // 'Blue']} + // + std::stringstream oss; + Dimension::IdList dims = view->dims(); + + PyObject* dict = PyDict_New(); + PyObject* sizes = PyList_New(dims.size()); + PyObject* formats = PyList_New(dims.size()); + PyObject* titles = PyList_New(dims.size()); + + for (Dimension::IdList::size_type i=0; i < dims.size(); ++i) + { + Dimension::Id id = (dims[i]); + Dimension::Type t = view->dimType(id); + npy_intp stride = view->dimSize(id); + + std::string name = view->dimName(id); + + std::string kind("i"); + Dimension::BaseType b = Dimension::base(t); + if (b == Dimension::BaseType::Unsigned) + kind = "u"; + else if (b == Dimension::BaseType::Floating) + kind = "f"; + else + { + std::stringstream o; + oss << "unable to map kind '" << kind <<"' to PDAL dimension type"; + throw pdal::pdal_error(o.str()); + } + + oss << kind << stride; + PyObject* pySize = PyLong_FromLong(stride); + PyObject* pyTitle = PyUnicode_FromString(name.c_str()); + PyObject* pyFormat = PyUnicode_FromString(oss.str().c_str()); + + PyList_SetItem(sizes, i, pySize); + PyList_SetItem(titles, i, pyTitle); + PyList_SetItem(formats, i, pyFormat); + + oss.str(""); + } + + PyDict_SetItemString(dict, "names", titles); + PyDict_SetItemString(dict, "formats", formats); + + // PyObject* obj = PyUnicode_AsASCIIString(PyObject_Str(dict)); + // const char* s = PyBytes_AsString(obj); + // std::string output(s); + // std::cout << "array: " << output << std::endl; + return dict; + } PyObject* m_py_array; std::unique_ptr > m_data_array; diff --git a/python/pdal/PyPipeline.hpp b/python/pdal/PyPipeline.hpp index e356f7ef3f..58f4e02a12 100644 --- a/python/pdal/PyPipeline.hpp +++ b/python/pdal/PyPipeline.hpp @@ -47,7 +47,6 @@ #undef isspace #define PY_ARRAY_UNIQUE_SYMBOL LIBPDALPYTHON_ARRAY_API -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include