Merge remote-tracking branch 'origin/master'

.gitignore

@@ -50,6 +50,13 @@ ipch/
 *.suo
 *.opensdf
 *.sln.docstates
+*.ilk
+*.exe
+*.lib
+*.pdb
+*.exp
+*.dll
+*.manifest
 
 #
 # intellij, clion

apps/pdal.cpp

@@ -87,6 +87,7 @@ class App
     std::string m_showOptions;
     bool m_showJSON;
     std::string m_log;
+    bool m_logtiming;
 };
 
 
@@ -280,6 +281,7 @@ void App::addArgs(ProgramArgs& args)
         m_showOptions);
     args.add("log", "Log filename (accepts stderr, stdout, stdlog, devnull"
         " as special cases)", m_log, "stderr");
+    args.add("logtiming", "Turn on timing for log messages", m_logtiming);
     Arg& json = args.add("showjson", "List options or drivers as JSON output",
         m_showJSON);
     json.setHidden();
@@ -345,7 +347,7 @@ int App::execute(StringList& cmdArgs, LogPtr& log)
         return -1;
     }
 
-    log.reset(new Log("PDAL", m_log));
+    log.reset(new Log("PDAL", m_log, m_logtiming));
     if (m_logLevel != LogLevel::None)
         log->setLevel(m_logLevel);
     else if (m_debug)

cmake/examples/hobu-config.sh

@@ -16,7 +16,7 @@ MATLAB_HOME=/Applications/MATLAB_R2017b.app
 export GDAL_HOME=/usr/local/opt/gdal2
 
 CONFIG="Unix Makefiles"
-#CONFIG="Ninja" /usr/local/Cellar/hdf5/1.8.16_1/
+CONFIG="Ninja"
 
 if ! [ -z "$1" ]; then
     CONFIG="$1"

doc/stages/readers.numpy.rst

@@ -0,0 +1,167 @@
+.. _readers.numpy:
+
+readers.numpy
+================================================================================
+
+PDAL has support for processing data using :ref:`filters.python`, but it is also
+convenient to read data from `Numpy`_ for processing in PDAL.
+
+`Numpy`_ supports saving files with the ``save`` method, usually with the
+extension ``.npy``. As of PDAL 1.7.0, ``.npz`` files were not yet supported.
+
+.. warning::
+
+    It is untested whether the version of Python PDAL was linked against and
+    the version that saved the ``.npy`` files can be mixed.
+
+Array Types
+--------------------------------------------------------------------------------
+
+:ref:`readers.numpy` supports reading data in two forms:
+
+* Arrays as named fields all of the same shape (from `laspy`_ for example)
+* 2-dimensional arrays
+
+
+Named Field Arrays
+................................................................................
+
+`laspy`_ provides its ``.points`` Numpy array as a bunch of named fields:
+
+::
+
+    import laspy
+    f = laspy.file.File('test/data/autzen/autzen.las')
+    print (f.points[0:1])
+
+::
+
+    array([ ((63608330, 84939865, 40735, 65, 73, 1, -11, 126, 7326,  245385.60820904),)],
+          dtype=[('point', [('X', '<i4'), ('Y', '<i4'), ('Z', '<i4'), ('intensity', '<u2'), ('flag_byte', 'u1'), ('raw_classification', 'u1'), ('scan_angle_rank', 'i1'), ('user_data', 'u1'), ('pt_src_id', '<u2'), ('gps_time', '<f8')])])
+
+:ref:`readers.numpy` supports reading these Numpy arrays and mapping applicable
+names to :ref:`dimensions` names. It will try to remove ``_``, ``-``, and ``space`` from
+the field name and use that as a dimension name if it can match. Types are also
+preserved when mapped to PDAL.
+
+
+Two-dimensional Arrays
+................................................................................
+
+Typical two-dimensional `Numpy`_ arrays are also supported, with options to allow
+you to map the values in the cells using the ``dimension`` option. Additionally,
+you can override the `Z` value for the entire array by using the ``assign_z``
+option to set a single `Z` value for the entire point cloud. Mapping the values to the
+``Z`` dimension using the ``dimension`` option is also allowed.
+
+
+::
+
+    f = open('./perlin.npy', 'rb')
+    data = np.load(f,)
+
+    data.shape
+    (100, 100)
+
+    data.dtype
+    dtype('float64')
+
+
+In this case, the cell locations are mapped to X and Y dimensions, the cell
+values are mapped to ``Intensity`` using the ``dimension`` option, and the Z
+values are assigned to 4 using the ``assign_z`` option.
+
+::
+
+    pdal info perlin.npy --readers.numpy.dimension=Intensity --readers.numpy.assign_z=4
+
+::
+
+    {
+      "filename": "perlin.npy",
+      "pdal_version": "1.6.0 (git-version: 897afd)",
+      "stats":
+      {
+            "statistic":
+            [
+              {
+                "average": 49.995,
+                "count": 10000,
+                "kurtosis": -1.201226882,
+                "maximum": 100,
+                "minimum": 0,
+                "name": "X",
+                "position": 0,
+                "skewness": -0.0001281084091,
+                "stddev": 29.16793715,
+                "variance": 850.7685575
+              },
+              {
+                "average": 50,
+                "count": 10000,
+                "kurtosis": -1.1996846,
+                "maximum": 100,
+                "minimum": 0,
+                "name": "Y",
+                "position": 1,
+                "skewness": -8.69273658e-05,
+                "stddev": 28.87401021,
+                "variance": 833.7084657
+              },
+              {
+                "average": 4,
+                "count": 10000,
+                "kurtosis": 9997,
+                "maximum": 4,
+                "minimum": 4,
+                "name": "Z",
+                "position": 2,
+                "skewness": 1.844674407e+21,
+                "stddev": 0.04000200015,
+                "variance": 0.001600160016
+              },
+              {
+                "average": 0.01112664759,
+                "count": 10000,
+                "kurtosis": -0.5634013693,
+                "maximum": 0.5189296418,
+                "minimum": -0.5189296418,
+                "name": "Intensity",
+                "position": 3,
+                "skewness": -0.1127124452,
+                "stddev": 0.2024120437,
+                "variance": 0.04097063545
+              }
+            ]
+          }
+        }
+
+.. _`Numpy`: http://www.numpy.org/
+.. _`laspy`: https://github.com/laspy/laspy
+
+.. plugin::
+
+.. streamable::
+
+Options
+-------
+
+filename
+  npy file to read [Required]
+
+dimension
+  Dimension name from :ref:`dimensions` to map raster values
+x
+  Dimension number (starting from 0) to map to the ``X`` PDAL :ref:`dimension <dimensions>`
+
+y
+  Dimension number (starting from 0) to map to the ``Y`` PDAL :ref:`dimension <dimensions>`
+
+z
+  Dimension number (starting from 0) to map to the ``Z`` PDAL :ref:`dimension <dimensions>`
+
+assign_z
+  A single value to override for ``Z`` values when ``dimension`` is used to assign the
+  Numpy values to another dimension
+
+.. _formatted: http://en.cppreference.com/w/cpp/string/basic_string/stof

doc/stages/readers.rst

@@ -28,6 +28,7 @@ like :ref:`readers.oci`, or a network service like :ref:`readers.greyhound`.
    readers.mbio
    readers.mrsid
    readers.nitf
+   readers.numpy
    readers.oci
    readers.optech
    readers.pcd
@@ -85,6 +86,9 @@ like :ref:`readers.oci`, or a network service like :ref:`readers.greyhound`.
 :ref:`readers.nitf`
     Read point cloud data (LAS or LAZ) wrapped in NITF 2.1 files.
 
+:ref:`readers.numpy`
+    Read point cloud data from Numpy ``.npy`` files.
+
 :ref:`readers.oci`
     Read data from Oracle point cloud databases.
 

pdal/Log.cpp

@@ -41,10 +41,11 @@
 namespace pdal
 {
 
-Log::Log(std::string const& leaderString,
-         std::string const& outputName)
+Log::Log(std::string const& leaderString, std::string const& outputName,
+        bool timing)
     : m_level(LogLevel::Warning)
     , m_deleteStreamOnCleanup(false)
+    , m_timing(timing)
 {
 
     if (Utils::iequals(outputName, "stdlog"))
@@ -61,16 +62,20 @@ Log::Log(std::string const& leaderString,
         m_deleteStreamOnCleanup = true;
     }
     m_leaders.push(leaderString);
+    if (m_timing)
+        m_start = m_clock.now();
 }
 
 
-Log::Log(std::string const& leaderString,
-         std::ostream* v)
+Log::Log(std::string const& leaderString, std::ostream* v, bool timing)
     : m_level(LogLevel::Error)
     , m_deleteStreamOnCleanup(false)
+    , m_timing(timing)
 {
     m_log = v;
     m_leaders.push(leaderString);
+    if (m_timing)
+        m_start = m_clock.now();
 }
 
 
@@ -111,7 +116,10 @@ std::ostream& Log::get(LogLevel level)
         *m_log << "(" << l;
          if (l.size())
              *m_log << " ";
-         *m_log << getLevelString(level) <<") " <<
+         *m_log << getLevelString(level);
+         if (m_timing)
+             *m_log << " " << now();
+         *m_log <<") " <<
          std::string(incoming < nativeDebug ? 0 : incoming - nativeDebug,
              '\t');
         return *m_log;
@@ -138,4 +146,15 @@ std::string Log::getLevelString(LogLevel level) const
     }
 }
 
+std::string Log::now() const
+{
+    std::chrono::steady_clock::time_point end = m_clock.now();
+
+    std::chrono::duration<double> diff = end - m_start;
+    std::stringstream ss;
+
+    ss << std::fixed << std::setprecision(3) << diff.count();
+    return ss.str();
+}
+
 } // namespace

pdal/Log.hpp

@@ -37,6 +37,7 @@
 #include <cassert>
 #include <memory> // shared_ptr
 #include <stack>
+#include <chrono>
 
 #include <pdal/pdal_internal.hpp>
 #include <pdal/util/NullOStream.hpp>
@@ -56,13 +57,16 @@ class PDAL_DLL Log
     /// @param leaderString A string to presage all log entries with
     /// @param outputName A filename or one of 'stdout', 'stdlog', or 'stderr'
     ///                   to use for outputting log information.
-    Log(std::string const& leaderString, std::string const& outputName);
+    /// @param timing Set to true to get timing output with log messages.
+    Log(std::string const& leaderString, std::string const& outputName,
+        bool timing = false);
 
     /// Constructs a pdal::Log instance.
     /// @param leaderString A string to presage all log entries with
     /// @param v An existing std::ostream to use for logging (instead of the
     ///          the instance creating its own)
-    Log(std::string const& leaderString, std::ostream* v);
+    /// @param timing Set to true to get timing output with log messages.
+    Log(std::string const& leaderString, std::ostream* v, bool timing = false);
 
     /** @name Destructor
     */
@@ -136,13 +140,17 @@ class PDAL_DLL Log
     std::ostream *m_log;
 
 private:
-    Log(const Log&);
-    Log& operator =(const Log&);
+    Log(const Log&) = delete;
+    Log& operator =(const Log&) = delete;
+    std::string now() const;
 
     LogLevel m_level;
     bool m_deleteStreamOnCleanup;
     std::stack<std::string> m_leaders;
     NullOStream m_nullStream;
+    bool m_timing;
+    std::chrono::steady_clock m_clock;
+    std::chrono::steady_clock::time_point m_start;
 };
 
 typedef std::shared_ptr<Log> LogPtr;

pdal/StageExtensions.cpp

@@ -70,6 +70,7 @@ R"PDALEXTENSIONS(
     "readers.icebridge" : "icebridge h5",
     "readers.matlab" : "mat",
     "writers.matlab" : "mat",
+    "readers.numpy" : "npy",
     "readers.nitf" : "nitf, nsf, ntf",
     "writers.nitf" : "nitf, nsf, ntf",
     "readers.pcd" : "pcd",

plugins/python/CMakeLists.txt

@@ -5,3 +5,4 @@
 include(${PDAL_CMAKE_DIR}/python.cmake)
 
 add_subdirectory(filters)
+add_subdirectory(io)

plugins/python/filters/CMakeLists.txt

@@ -2,6 +2,7 @@ set(PLANG ../plang/Invocation.cpp
           ../plang/Environment.cpp
           ../plang/Redirector.cpp
           ../plang/Script.cpp)
+
 PDAL_ADD_PLUGIN(python_libname filter python
     FILES
         ${PLANG}
@@ -14,11 +15,11 @@ target_link_libraries(${python_libname} PUBLIC
     ${PYTHON_LIBRARY} ${CMAKE_DL_LIBS})
 
 if (WITH_TESTS)
-
     PDAL_ADD_TEST(pdal_filters_python_test
-        FILES ../test/PythonFilterTest.cpp ${PLANG}
+        FILES
+            ../test/PythonFilterTest.cpp
+            ${PLANG}
         LINK_WITH ${programmable_libname} )
     target_link_libraries(pdal_filters_python_test PUBLIC
         ${PYTHON_LIBRARY} ${CMAKE_DL_LIBS})
-
 endif()