Merge branch 'winfix' into numpy

CMakeLists.txt

@@ -81,20 +81,21 @@ set_directory_properties(PROPERTY ADDITIONAL_MAKE_CLEAN_FILES
 
 if(WIN32)
   add_definitions("-DPDAL_DLL_EXPORT=1")
-  foreach(config ${CMAKE_CONFIGURATION_TYPES})
-    string(TOUPPER ${config} CONFIG)
-    set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CONFIG} "${PDAL_OUTPUT_LIB_DIR}")
-    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG} "${PDAL_OUTPUT_BIN_DIR}")
-    # ---[ Windows requires DLLs (shared libraries) to be installed in
-    # ---[ the same directory as executables
-    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CONFIG} "${PDAL_OUTPUT_BIN_DIR}")
-  endforeach(config)
-else(WIN32)
-  set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${PDAL_OUTPUT_LIB_DIR}")
-  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${PDAL_OUTPUT_BIN_DIR}")
-  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${PDAL_OUTPUT_LIB_DIR}")
+  set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE} "${PDAL_OUTPUT_LIB_DIR}")
+  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE} "${PDAL_OUTPUT_BIN_DIR}")
+  # ---[ Windows requires DLLs (shared libraries) to be installed in
+  # ---[ the same directory as executables
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE} "${PDAL_OUTPUT_BIN_DIR}")
 endif(WIN32)
 
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${PDAL_OUTPUT_LIB_DIR}")
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${PDAL_OUTPUT_BIN_DIR}")
+if(WIN32)
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${PDAL_OUTPUT_BIN_DIR}")
+else()
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${PDAL_OUTPUT_LIB_DIR}")
+endif()
+
 # Choose package components
 
 include(${PDAL_CMAKE_DIR}/options.cmake)

apps/pdal.cpp

@@ -350,6 +350,7 @@ int App::execute(StringList& cmdArgs, LogPtr& log)
         log->setLevel(m_logLevel);
     else if (m_debug)
         log->setLevel(LogLevel::Debug);
+    log->get(LogLevel::Debug) << "Debugging..." << std::endl;
     PluginManager<Stage>::setLog(log);
     PluginManager<Kernel>::setLog(log);
 #ifndef _WIN32

appveyor.yml

@@ -35,38 +35,24 @@ install:
   - ps: mkdir C:\temp | out-null
   - ps: mkdir $env:OSGEO4W_ROOT | out-null
   # make an install directory for packacing
-  - ps: mkdir C:\pdalbin | out-null
   # get the OSGeo installer
   - ps: (new-object net.webclient).DownloadFile("http://download.osgeo.org/osgeo4w/osgeo4w-setup-x86_64.exe", "C:\temp\osgeo4w-setup.exe")
   # and install our dependencies
   - C:\temp\osgeo4w-setup.exe -q -k -r -A -s http://download.osgeo.org/osgeo4w/ -a x86_64 -P eigen,gdal,geos,hexer,iconv,laszip,libgeotiff,libpq,libtiff,libxml2,msys,nitro,laz-perf,proj,zlib,python3-core,python3-devel,python3-numpy,oci,oci-devel,laz-perf,jsoncpp,xz-devel,zstd -R %OSGEO4W_ROOT% > NUL
-  # call our PDAL install script
   - call %OSGEO4W_ROOT%\bin\o4w_env.bat
   - call %OSGEO4W_ROOT%\bin\py3_env.bat
   - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
-  - set PATH=%PATH%;C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files\7-Zip;C:\Program Files\Microsoft Windows Performance Toolkit\;C:\Program Files (x86)\Windows Kits\8.1\Windows Performance Toolkit\;C:\Tools\GitVersion;C:\Program Files (x86)\CMake\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;C:\Program Files\AppVeyor\BuildAgent\
-  - call .\\scripts\\appveyor\\config.cmd
+  - set PATH=%PATH%;C:\Program Files (x86)\CMake\bin
+  - call scripts\\appveyor\\config.cmd
 
 build_script:
-  - call .\\scripts\\appveyor\\build.cmd
-
-after_build:
-  - call %OSGEO4W_ROOT%\bin\o4w_env.bat
-  - call %OSGEO4W_ROOT%\bin\py3_env.bat
-  - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
-  - set PATH=%PATH%;C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files\7-Zip;C:\Program Files\Microsoft Windows Performance Toolkit\;C:\Program Files (x86)\Windows Kits\8.1\Windows Performance Toolkit\;C:\Tools\GitVersion;C:\Program Files (x86)\CMake\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;C:\Program Files\AppVeyor\BuildAgent\
-  - call .\\scripts\\appveyor\\install.cmd
-
+  - call scripts\\appveyor\\build.cmd
 
 test_script:
-  # FIXME: Clear PATH to avoid OSGeo4W loading incompatible DLLs
-  - set PATH=
-  - set PATH=%OSGEO4W_ROOT%\bin;C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files\7-Zip;C:\Program Files\Microsoft Windows Performance Toolkit\;C:\Program Files (x86)\Windows Kits\8.1\Windows Performance Toolkit\;C:\Tools\GitVersion;C:\Program Files (x86)\CMake\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;C:\Program Files\AppVeyor\BuildAgent\
-  - call %OSGEO4W_ROOT%\bin\o4w_env.bat
-  - call %OSGEO4W_ROOT%\bin\py3_env.bat
-  - set PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;%PATH%
-  - set PATH=%PATH%;C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files\7-Zip;C:\Program Files\Microsoft Windows Performance Toolkit\;C:\Program Files (x86)\Windows Kits\8.1\Windows Performance Toolkit\;C:\Tools\GitVersion;C:\Program Files (x86)\CMake\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\usr\bin;C:\Program Files\AppVeyor\BuildAgent\
-  - ctest -V --output-on-failure -C Release
+  - call scripts\\appveyor\\test.cmd
+
+deploy_script:
+  - call scripts\\appveyor\\install.cmd
 
 artifacts:
   - path: pdal-$(APPVEYOR_REPO_COMMIT).tar.bz2

doc/apps/info.rst

@@ -13,6 +13,8 @@ Displays information about a point cloud file, such as:
 * the plain text format should be reStructured text if possible to allow a user
   to retransform the output into whatever they want with ease
 
+Processing is performed with stream mode if possible.
+
 ::
 
     $ pdal info <input>
@@ -27,6 +29,7 @@ Displays information about a point cloud file, such as:
   --stats                   Dump stats on all points (reads entire dataset)
   --boundary                Compute a hexagonal hull/boundary of dataset
   --dimensions              Dimensions on which to compute statistics
+  --enumerate               Dimensions whose values should be enumerated
   --schema                  Dump the schema
   --pipeline-serialization  Output filename for pipeline serialization
   --summary                 Dump summary of the info

doc/apps/pipeline.rst

@@ -4,8 +4,9 @@
 pipeline
 ********************************************************************************
 
-The ``pipeline`` command is used to execute :ref:`pipeline` JSON. See
-:ref:`reading` or :ref:`pipeline` for more information.
+The ``pipeline`` command is used to execute :ref:`pipeline` JSON. The pipeline
+is run in stream mode if possible.  See :ref:`reading` or :ref:`pipeline` for
+more information.
 
 ::
 
@@ -21,7 +22,6 @@ The ``pipeline`` command is used to execute :ref:`pipeline` JSON. See
       progress information. The file/FIFO must exist. PDAL will not create the
       progress file.
   --stdin, -s               Read pipeline from standard input
-  --stream                  Attempt to run pipeline in streaming mode.
   --metadata                Metadata filename
 
 

doc/apps/translate.rst

@@ -6,7 +6,7 @@ translate
 
 The ``translate`` command can be used for simple conversion of files based on
 their file extensions. It can also be used for constructing pipelines directly
-from the command-line.
+from the command-line.  Processing is done with stream mode if possible.
 
 ::
 

doc/faq.rst

@@ -12,7 +12,7 @@ FAQ
   pronounced to rhyme with "GDAL".
 
   .. it is properly pronounced like the dog though :)
-
+|
 * Why do I get the error "Couldn't create ... stage of type ..."?
 
   In almost all cases this error occurs because you're trying to run a stage
@@ -36,6 +36,15 @@ FAQ
 
 .. index:: PCL
 
+* Why am I using 100GB of memory when trying to process a 10GB LAZ file?
+
+  If you're performing an operation that is using
+  :ref:`standard mode <processing_modes>`, PDAL will read all points into
+  memory at once.  Compressed files, like LAZ, can decompress to much larger
+  sizes before PDAL can process the data. Furthermore, some operations
+  (notably :ref:`DEM creation<writers.gdal>`) can use large amounts of
+  additional memory during processing before the output can be written.
+|
 * What is PDAL's relationship to PCL?
 
   PDAL is PCL's data translation cousin. PDAL is focused on providing a
@@ -55,7 +64,7 @@ FAQ
   with LASlib and LAStools. PDAL, on the other hand, aims to be
   a ultimate library and a set of tools for manipulating and processing
   point clouds and is easily extensible by its users.
-
+|
 * Are there any command line tools in PDAL similar to LAStools?
 
   Yes. The ``pdal`` command provides a wide range of features which go

doc/pipeline.rst

@@ -111,6 +111,28 @@ with the :ref:`writers.gdal` writer:
 .. _`UTM`: http://spatialreference.org/ref/epsg/nad83-utm-zone-16n/
 .. _`Geographic`: http://spatialreference.org/ref/epsg/4326/
 
+.. _processing_modes:
+
+Processing Modes
+--------------------------------------------------------------------------------
+
+PDAL process data in one of two ways: standard mode or stream mode.  With
+standard mode, all input is read into memory before it is processed.  Many
+algorithms require standard mode processing because they need access to
+all points. Operations that do sorting or require neighbors of points, for
+example, require access to all points.
+
+For operations that don't require access to all points, PDAL provides stream
+mode.  Stream mode processes points through a pipeline in chunks, which
+reduces memory requirements.
+
+When using :ref:`pdal translate<translate_command>` or
+:ref:`pdal pipeline<pipeline_command>`
+PDAL uses stream mode if possible.  If stream mode can't use used
+the applications fall back to standard mode processing.
+
+Users of the PDAL API can explicitly control the selection of the PDAL
+processing mode.
 
 Pipeline Objects
 --------------------------------------------------------------------------------

io/BpfReader.cpp

@@ -308,6 +308,9 @@ void BpfReader::done(PointTableRef)
 
 bool BpfReader::processOne(PointRef& point)
 {
+    if (eof() || m_index >= m_count)
+        return false;
+
     switch (m_header.m_pointFormat)
     {
     case BpfFormat::PointMajor:
@@ -320,7 +323,7 @@ bool BpfReader::processOne(PointRef& point)
         readByteMajor(point);
         break;
     }
-    return !eof() && (m_index < m_count);
+    return true;
 }
 
 

io/TextReader.cpp

@@ -90,7 +90,6 @@ QuickInfo TextReader::inspect()
 
 void TextReader::checkHeader(const std::string& header)
 {
-    std::cerr << "Header = " << header << "!\n";
     auto it = std::find_if(header.begin(), header.end(),
         [](char c){ return std::isalpha(c); });
     if (it == header.end())

kernels/InfoKernel.cpp

@@ -140,6 +140,8 @@ void InfoKernel::addSwitches(ProgramArgs& args)
         m_boundary);
     args.add("dimensions", "Dimensions on which to compute statistics",
         m_dimensions);
+    args.add("enumerate", "Dimensions whose values should be enumerated",
+        m_enumerate);
     args.add("schema", "Dump the schema", m_showSchema);
     args.add("pipeline-serialization", "Output filename for pipeline "
         "serialization", m_pipelineFile);
@@ -310,6 +312,8 @@ void InfoKernel::setup(const std::string& filename)
         Options filterOptions;
         if (m_dimensions.size())
             filterOptions.add({"dimensions", m_dimensions});
+        if (m_enumerate.size())
+            filterOptions.add({"enumerate", m_enumerate});
         m_statsStage = &m_manager.makeFilter("filters.stats", *stage,
             filterOptions);
         stage = m_statsStage;
@@ -345,15 +349,7 @@ MetadataNode InfoKernel::run(const std::string& filename)
     else
     {
         if (m_needPoints || m_showMetadata)
-        {
-            if (m_manager.pipelineStreamable())
-            {
-                FixedPointTable table(10000);
-                m_manager.executeStream(table);
-            }
-            else
-                m_manager.execute();
-        }
+            m_manager.execute();
         else
             m_manager.prepare();
         dump(root);

kernels/InfoKernel.hpp

@@ -82,6 +82,7 @@ class PDAL_DLL InfoKernel : public Kernel
     bool m_boundary;
     std::string m_pointIndexes;
     std::string m_dimensions;
+    std::string m_enumerate;
     std::string m_queryPoint;
     std::string m_pipelineFile;
     bool m_showSummary;

kernels/PipelineKernel.cpp

@@ -91,7 +91,7 @@ void PipelineKernel::addSwitches(ProgramArgs& args)
     args.add("pointcloudschema", "dump PointCloudSchema XML output",
         m_PointCloudSchemaOutput).setHidden();
     args.add("stdin,s", "Read pipeline from standard input", m_usestdin);
-    args.add("stream", "Attempt to run pipeline in streaming mode.", m_stream);
+    args.add("stream", "This option is obsolete.", m_stream);
     args.add("metadata", "Metadata filename", m_metadataFile);
 }
 
@@ -132,7 +132,7 @@ int PipelineKernel::execute()
     }
 
     m_manager.readPipeline(m_inputFile);
-    if (m_stream)
+    if (m_manager.pipelineStreamable())
     {
         FixedPointTable table(10000);
         m_manager.executeStream(table);

pdal/Stage.hpp

@@ -123,6 +123,12 @@ class PDAL_DLL Stage
     */
     PointViewSet execute(PointTableRef table);
 
+    virtual void execute(StreamPointTable& table)
+    {
+        throw pdal_error("Attempting to use stream mode with a non-streamable "
+            "stage.");
+    }
+
     /**
       Determine if a pipeline with this stage as a sink is streamable.
 

pdal/Streamable.cpp

@@ -98,7 +98,8 @@ void Streamable::execute(StreamPointTable& table)
     };
 
     if (!pipelineStreamable())
-        return;
+        throwError("Attempting to use stream mode with a stage that doesn't "
+            "support streaming.");
 
     SpatialReference srs;
     std::list<StreamableList> lists;

pdal/Streamable.hpp

@@ -61,7 +61,7 @@ class PDAL_DLL Streamable : public virtual Stage
         the same \ref table used in the \ref prepare function.
 
     */
-    void execute(StreamPointTable& table);
+    virtual void execute(StreamPointTable& table);
     using Stage::execute;
 
     /**

plugins/nitf/io/NitfWriter.cpp

@@ -102,9 +102,6 @@ void NitfWriter::addArgs(ProgramArgs& args)
 
 void NitfWriter::writeView(const PointViewPtr view)
 {
-    //ABELL - Think we can just get this from the LAS file header
-    //  when we're done.
-    view->calculateBounds(m_bounds);
     LasWriter::writeView(view);
 }
 
@@ -131,7 +128,7 @@ void NitfWriter::doneFile()
     buf->sgetn(bytes.data(), size);
     m_oss.clear();
     m_nitf.wrapData(bytes.data(), size);
-    m_nitf.setBounds(reprojectBoxToDD(m_srs, m_bounds));
+    m_nitf.setBounds(reprojectBoxToDD(m_srs, m_lasHeader.getBounds()));
 
     try
     {

plugins/nitf/io/tre_plugins.hpp

@@ -42,9 +42,10 @@
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wpragmas"
-#pragma GCC diagnostic ignored "-Wunused-private-field"
 #pragma GCC diagnostic ignored "-Wredundant-decls"
+#pragma GCC diagnostic ignored "-Wextra"
 #pragma GCC diagnostic ignored "-Wcast-qual"
+#pragma GCC diagnostic ignored "-Wunused-private-field"
 
 #include <nitro/c++/import/nitf.hpp>
 

scripts/appveyor/build.cmd

@@ -1 +1,3 @@
+pushd build
 nmake /f Makefile
+popd

scripts/appveyor/config.cmd

@@ -14,6 +14,7 @@ set PDAL_BUILD_TESTS=OFF
 
 ) ELSE (
 
+mkdir C:\pdalbin
 SET PDAL_PLUGIN_INSTALL_PATH="C:\pdalbin"
 set PDAL_BUILD_TESTS=ON
 
@@ -23,6 +24,9 @@ REM needed or else CMake won't find the Oracle library that OSGeo4W installs
 
 SET ORACLE_HOME="C:/OSGEO4W64/"
 
+mkdir build
+pushd build
+
 cmake -G "NMake Makefiles" ^
     -DBUILD_PLUGIN_CPD=OFF ^
     -DBUILD_PLUGIN_GREYHOUND=%PDAL_OPTIONAL_COMPONENTS% ^
@@ -34,12 +38,13 @@ cmake -G "NMake Makefiles" ^
     -DBUILD_PLUGIN_PCL=OFF ^
     -DBUILD_PLUGIN_PGPOINTCLOUD=%PDAL_OPTIONAL_COMPONENTS% ^
     -DBUILD_PLUGIN_SQLITE=%PDAL_OPTIONAL_COMPONENTS% ^
-	-DLIBLZMA_LIBRARY=%OSGEO4W_ROOT%\lib\liblzma.lib ^
+    -DLIBLZMA_LIBRARY=%OSGEO4W_ROOT%\lib\liblzma.lib ^
     -DBUILD_PLUGIN_RIVLIB=OFF ^
     -DBUILD_PLUGIN_PYTHON=%PDAL_OPTIONAL_COMPONENTS% ^
     -DENABLE_CTEST=OFF ^
     -DWITH_LAZPERF=ON ^
-	-DLazperf_DIR=%OSGEO4W_ROOT% ^
+    -DLazperf_DIR=%OSGEO4W_ROOT% ^
+    -DWITH_LZMA=ON ^
     -DWITH_LASZIP=ON ^
     -DWITH_TESTS=%PDAL_BUILD_TESTS% ^
 	-DPDAL_PLUGIN_INSTALL_PATH=%PDAL_PLUGIN_INSTALL_PATH% ^
@@ -60,6 +65,7 @@ cmake -G "NMake Makefiles" ^
     -DBUILD_SQLITE_TESTS=OFF ^
     -DCMAKE_BUILD_TYPE=RelWithDebInfo ^
     -DBUILD_OCI_TESTS=OFF ^
-    .
+    ..
 
+popd