Merge branch 'readers.numpy'

.gitignore

@@ -50,6 +50,13 @@ ipch/
 *.suo
 *.opensdf
 *.sln.docstates
+*.ilk
+*.exe
+*.lib
+*.pdb
+*.exp
+*.dll
+*.manifest
 
 #
 # intellij, clion

cmake/examples/hobu-config.sh

@@ -16,7 +16,7 @@ MATLAB_HOME=/Applications/MATLAB_R2017b.app
 export GDAL_HOME=/usr/local/opt/gdal2
 
 CONFIG="Unix Makefiles"
-#CONFIG="Ninja" /usr/local/Cellar/hdf5/1.8.16_1/
+CONFIG="Ninja"
 
 if ! [ -z "$1" ]; then
     CONFIG="$1"

doc/stages/readers.numpy.rst

@@ -0,0 +1,167 @@
+.. _readers.numpy:
+
+readers.numpy
+================================================================================
+
+PDAL has support for processing data using :ref:`filters.python`, but it is also
+convenient to read data from `Numpy`_ for processing in PDAL.
+
+`Numpy`_ supports saving files with the ``save`` method, usually with the
+extension ``.npy``. As of PDAL 1.7.0, ``.npz`` files were not yet supported.
+
+.. warning::
+
+    It is untested whether the version of Python PDAL was linked against and
+    the version that saved the ``.npy`` files can be mixed.
+
+Array Types
+--------------------------------------------------------------------------------
+
+:ref:`readers.numpy` supports reading data in two forms:
+
+* Arrays as named fields all of the same shape (from `laspy`_ for example)
+* 2-dimensional arrays
+
+
+Named Field Arrays
+................................................................................
+
+`laspy`_ provides its ``.points`` Numpy array as a bunch of named fields:
+
+::
+
+    import laspy
+    f = laspy.file.File('test/data/autzen/autzen.las')
+    print (f.points[0:1])
+
+::
+
+    array([ ((63608330, 84939865, 40735, 65, 73, 1, -11, 126, 7326,  245385.60820904),)],
+          dtype=[('point', [('X', '<i4'), ('Y', '<i4'), ('Z', '<i4'), ('intensity', '<u2'), ('flag_byte', 'u1'), ('raw_classification', 'u1'), ('scan_angle_rank', 'i1'), ('user_data', 'u1'), ('pt_src_id', '<u2'), ('gps_time', '<f8')])])
+
+:ref:`readers.numpy` supports reading these Numpy arrays and mapping applicable
+names to :ref:`dimensions` names. It will try to remove ``_``, ``-``, and ``space`` from
+the field name and use that as a dimension name if it can match. Types are also
+preserved when mapped to PDAL.
+
+
+Two-dimensional Arrays
+................................................................................
+
+Typical two-dimensional `Numpy`_ arrays are also supported, with options to allow
+you to map the values in the cells using the ``dimension`` option. Additionally,
+you can override the `Z` value for the entire array by using the ``assign_z``
+option to set a single `Z` value for the entire point cloud. Mapping the values to the
+``Z`` dimension using the ``dimension`` option is also allowed.
+
+
+::
+
+    f = open('./perlin.npy', 'rb')
+    data = np.load(f,)
+
+    data.shape
+    (100, 100)
+
+    data.dtype
+    dtype('float64')
+
+
+In this case, the cell locations are mapped to X and Y dimensions, the cell
+values are mapped to ``Intensity`` using the ``dimension`` option, and the Z
+values are assigned to 4 using the ``assign_z`` option.
+
+::
+
+    pdal info perlin.npy --readers.numpy.dimension=Intensity --readers.numpy.assign_z=4
+
+::
+
+    {
+      "filename": "perlin.npy",
+      "pdal_version": "1.6.0 (git-version: 897afd)",
+      "stats":
+      {
+            "statistic":
+            [
+              {
+                "average": 49.995,
+                "count": 10000,
+                "kurtosis": -1.201226882,
+                "maximum": 100,
+                "minimum": 0,
+                "name": "X",
+                "position": 0,
+                "skewness": -0.0001281084091,
+                "stddev": 29.16793715,
+                "variance": 850.7685575
+              },
+              {
+                "average": 50,
+                "count": 10000,
+                "kurtosis": -1.1996846,
+                "maximum": 100,
+                "minimum": 0,
+                "name": "Y",
+                "position": 1,
+                "skewness": -8.69273658e-05,
+                "stddev": 28.87401021,
+                "variance": 833.7084657
+              },
+              {
+                "average": 4,
+                "count": 10000,
+                "kurtosis": 9997,
+                "maximum": 4,
+                "minimum": 4,
+                "name": "Z",
+                "position": 2,
+                "skewness": 1.844674407e+21,
+                "stddev": 0.04000200015,
+                "variance": 0.001600160016
+              },
+              {
+                "average": 0.01112664759,
+                "count": 10000,
+                "kurtosis": -0.5634013693,
+                "maximum": 0.5189296418,
+                "minimum": -0.5189296418,
+                "name": "Intensity",
+                "position": 3,
+                "skewness": -0.1127124452,
+                "stddev": 0.2024120437,
+                "variance": 0.04097063545
+              }
+            ]
+          }
+        }
+
+.. _`Numpy`: http://www.numpy.org/
+.. _`laspy`: https://github.com/laspy/laspy
+
+.. plugin::
+
+.. streamable::
+
+Options
+-------
+
+filename
+  npy file to read [Required]
+
+dimension
+  Dimension name from :ref:`dimensions` to map raster values
+x
+  Dimension number (starting from 0) to map to the ``X`` PDAL :ref:`dimension <dimensions>`
+
+y
+  Dimension number (starting from 0) to map to the ``Y`` PDAL :ref:`dimension <dimensions>`
+
+z
+  Dimension number (starting from 0) to map to the ``Z`` PDAL :ref:`dimension <dimensions>`
+
+assign_z
+  A single value to override for ``Z`` values when ``dimension`` is used to assign the
+  Numpy values to another dimension
+
+.. _formatted: http://en.cppreference.com/w/cpp/string/basic_string/stof

doc/stages/readers.rst

@@ -28,6 +28,7 @@ like :ref:`readers.oci`, or a network service like :ref:`readers.greyhound`.
    readers.mbio
    readers.mrsid
    readers.nitf
+   readers.numpy
    readers.oci
    readers.optech
    readers.pcd
@@ -85,6 +86,9 @@ like :ref:`readers.oci`, or a network service like :ref:`readers.greyhound`.
 :ref:`readers.nitf`
     Read point cloud data (LAS or LAZ) wrapped in NITF 2.1 files.
 
+:ref:`readers.numpy`
+    Read point cloud data from Numpy ``.npy`` files.
+
 :ref:`readers.oci`
     Read data from Oracle point cloud databases.
 

pdal/PluginManager.cpp

@@ -271,7 +271,7 @@ bool PluginManager<T>::l_loadDynamic(const std::string& driverName)
     if (path.empty())
     {
         m_log->get(LogLevel::Debug) << "No plugin file found for driver '" <<
-            driverName << "." << std::endl;
+            driverName << "'." << std::endl;
         return false;
     }
     return loadByPath(path);

pdal/StageExtensions.cpp

@@ -70,6 +70,7 @@ R"PDALEXTENSIONS(
     "readers.icebridge" : "icebridge h5",
     "readers.matlab" : "mat",
     "writers.matlab" : "mat",
+    "readers.numpy" : "npy",
     "readers.nitf" : "nitf, nsf, ntf",
     "writers.nitf" : "nitf, nsf, ntf",
     "readers.pcd" : "pcd",

plugins/python/CMakeLists.txt

@@ -4,4 +4,32 @@
 
 include(${PDAL_CMAKE_DIR}/python.cmake)
 
+
+set(plang_srcs
+    ./plang/Invocation.cpp
+    ./plang/Environment.cpp
+    ./plang/Redirector.cpp
+    ./plang/Script.cpp
+)
+set(PDAL_PLANG_LIB_NAME pdal_plang)
+PDAL_ADD_LIBRARY(${PDAL_PLANG_LIB_NAME} ${plang_srcs} )
+set_target_properties(${PDAL_PLANG_LIB_NAME} PROPERTIES
+    VERSION "${PDAL_BUILD_VERSION}"
+    SOVERSION "${PDAL_API_VERSION}"
+    CLEAN_DIRECT_OUTPUT 1)
+target_link_libraries(${PDAL_PLANG_LIB_NAME} PUBLIC
+    ${PDAL_BASE_LIB_NAME}
+    ${PDAL_UTIL_LIB_NAME}
+    ${PYTHON_LIBRARY})
+target_include_directories(${PDAL_PLANG_LIB_NAME} PRIVATE
+    ${PYTHON_INCLUDE_DIR}
+    ${PROJECT_BINARY_DIR}/include)
+install(TARGETS ${PLANG_LIB_NAME}
+    RUNTIME DESTINATION ${PDAL_BIN_INSTALL_DIR}
+    LIBRARY DESTINATION ${PDAL_LIB_INSTALL_DIR}
+    ARCHIVE DESTINATION ${PDAL_LIB_INSTALL_DIR})
+
+
+
 add_subdirectory(filters)
+add_subdirectory(io)

plugins/python/filters/CMakeLists.txt

@@ -1,24 +1,19 @@
-set(PLANG ../plang/Invocation.cpp
-          ../plang/Environment.cpp
-          ../plang/Redirector.cpp
-          ../plang/Script.cpp)
 PDAL_ADD_PLUGIN(python_libname filter python
     FILES
-        ${PLANG}
         PythonFilter.cpp
     )
 target_include_directories(${python_libname} PRIVATE
     ${PYTHON_INCLUDE_DIR} ${PDAL_JSONCPP_INCLUDE_DIR})
 
 target_link_libraries(${python_libname} PUBLIC
-    ${PYTHON_LIBRARY} ${CMAKE_DL_LIBS})
+    ${PYTHON_LIBRARY} ${CMAKE_DL_LIBS} PRIVATE  ${PDAL_PLANG_LIB_NAME})
 
 if (WITH_TESTS)
 
     PDAL_ADD_TEST(pdal_filters_python_test
-        FILES ../test/PythonFilterTest.cpp ${PLANG}
+        FILES ../test/PythonFilterTest.cpp
         LINK_WITH ${programmable_libname} )
     target_link_libraries(pdal_filters_python_test PUBLIC
-        ${PYTHON_LIBRARY} ${CMAKE_DL_LIBS})
+        ${PYTHON_LIBRARY} ${CMAKE_DL_LIBS}  PRIVATE  ${PDAL_PLANG_LIB_NAME})
 
 endif()

plugins/python/io/CMakeLists.txt

@@ -0,0 +1,33 @@
+
+PDAL_ADD_PLUGIN(numpy_reader reader numpy
+    FILES
+        NumpyReader.cpp
+        ../plang/Environment.cpp
+        ../plang/Redirector.cpp
+    )
+
+target_include_directories(${numpy_reader} PRIVATE
+    ${PYTHON_INCLUDE_DIR})
+
+target_link_libraries(${numpy_reader} PUBLIC
+    ${PYTHON_LIBRARY})
+
+# Install headers so Python extension
+# can use them later
+
+install(DIRECTORY .
+    DESTINATION  include/pdal/io
+    FILES_MATCHING PATTERN "*.hpp"
+    PATTERN "CMakeFiles" EXCLUDE
+)
+
+if (WITH_TESTS)
+    PDAL_ADD_TEST(pdal_io_numpy_test
+        FILES
+            ../test/NumpyReaderTest.cpp
+            ../plang/Environment.cpp
+            ../plang/Redirector.cpp
+        LINK_WITH ${numpy_reader} )
+    target_link_libraries(pdal_io_numpy_test PUBLIC
+        ${PYTHON_LIBRARY})
+endif()