Merge branch 'howard-windows-updates' into wktv1

PDAL · Mar 1, 2020 · b946255 · b946255
2 parents c4a673b + 558b537
commit b946255
Show file tree

Hide file tree

Showing 31 changed files with 538 additions and 338 deletions.
diff --git a/doc/stages/filters.python.rst b/doc/stages/filters.python.rst
@@ -59,6 +59,12 @@ filter and the ``outs`` array represents the points after filtering.
        "add_dimension": [ "NewDimensionOne", "NewDimensionTwo", "NewDimensionThree" ]
 
 
+   You can also specify the :ref:`type <types>` of the dimension using an ``=``.
+   ::
+
+       "add_dimension": "NewDimensionOne=uint8"
+
+
 Modification Example
 --------------------------------------------------------------------------------
 

diff --git a/doc/stages/readers.hdf.rst b/doc/stages/readers.hdf.rst
@@ -72,12 +72,13 @@ properly mapped and then outputs a LAS file.
 Common Use Cases
 ----------------
 
-A possible use case for this driver is reading NASA's ICESAT2 data.
-This example reads the X, Y, and Z coordinates from the ICESAT2
-ATL03 format and converts them into a LAS file.
+A possible use case for this driver is reading NASA's `ICESat-2 <https://icesat-2.gsfc.nasa.gov/>`__ data.
+This example reads the X, Y, and Z coordinates from the ICESat-2
+`ATL03 <https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL03_ATBD_r002.pdf>`__ format and converts them into a LAS file.
 
 .. note::
-    ICESAT2 data use EPSG:7912.
+    ICESat-2 data use `EPSG:7912 <https://epsg.io/7912>`__. ICESat-2 Data products documentation can be found `here <https://icesat-2.gsfc.nasa.gov/science/data-products>`_
+
 
 .. code-block:: json
 
@@ -99,9 +100,8 @@ ATL03 format and converts them into a LAS file.
     ] 
 
 
-`ICESAT2 Data products Documentation <https://icesat-2.gsfc.nasa.gov/science/data-products>`_
-~                                                                                              
-~             
+
+
 Options
 -------
 

diff --git a/doc/stages/writers.tiledb.rst b/doc/stages/writers.tiledb.rst
@@ -66,4 +66,31 @@ filters
   JSON array or object of compression filters for either `coords` or `attributes` of the form {coords/attributename : {"compression": name, compression_options: value, ...}} [Optional]
 
 
+By default TileDB will use the following set of compression filters for coordinates and attributes;
+
+.. code-block:: json
+
+  {
+      "coords":[
+          {"compression": "bit-shuffle"},
+          {"compression": "gzip", "compression_level": 9}
+      ],
+      "Intensity":{"compression": "bzip2", "compression_level": 5},
+      "ReturnNumber": {"compression": "zstd", "compression_level": 75},
+      "NumberOfReturns": {"compression": "zstd", "compression_level": 75},
+      "ScanDirectionFlag": {"compression": "bzip2", "compression_level": 5},
+      "EdgeOfFlightLine": {"compression": "bzip2", "compression_level": 5},
+      "Classification": {"compression": "gzip", "compression_level": 9},
+      "ScanAngleRank": {"compression": "bzip2", "compression_level": 5},
+      "UserData": {"compression": "gzip", "compression_level": 9},
+      "PointSourceId": {"compression": "bzip2"},
+      "Red": {"compression": "rle"},
+      "Green": {"compression": "rle"},
+      "Blue": {"compression": "rle"},
+      "GpsTime": [
+          {"compression": "bit-shuffle"},
+          {"compression": "zstd", "compression_level": 75}
+      ]
+  }
+
 .. _TileDB: https://tiledb.io
diff --git a/filters/HeadFilter.hpp b/filters/HeadFilter.hpp
@@ -35,7 +35,6 @@
 #pragma once
 
 #include <pdal/Filter.hpp>
-#include <pdal/PointViewIter.hpp>
 
 namespace pdal
 {

diff --git a/filters/RandomizeFilter.hpp b/filters/RandomizeFilter.hpp
@@ -36,7 +36,6 @@
 #pragma once
 
 #include <pdal/Filter.hpp>
-#include <pdal/PointViewIter.hpp>
 
 #include <random>
 

diff --git a/filters/SkewnessBalancingFilter.cpp b/filters/SkewnessBalancingFilter.cpp
@@ -34,7 +34,6 @@
 
 #include "SkewnessBalancingFilter.hpp"
 
-#include <pdal/PointViewIter.hpp>
 #include <pdal/util/ProgramArgs.hpp>
 
 namespace pdal
@@ -59,7 +58,7 @@ void SkewnessBalancingFilter::addDimensions(PointLayoutPtr layout)
 
 void SkewnessBalancingFilter::processGround(PointViewPtr view)
 {
-    auto cmp = [](const PointIdxRef& p1, const PointIdxRef& p2) {
+    auto cmp = [](const PointRef& p1, const PointRef& p2) {
         return p1.compare(Dimension::Id::Z, p2);
     };
     std::sort(view->begin(), view->end(), cmp);

diff --git a/filters/SortFilter.cpp b/filters/SortFilter.cpp
@@ -52,7 +52,8 @@ void SortFilter::addArgs(ProgramArgs& args)
 {
     args.add("dimension", "Dimension on which to sort", m_dimName).
         setPositional();
-    args.add("order", "Sort order ASC(ending) or DESC(ending)", m_order, SortOrder::ASC);
+    args.add("order", "Sort order ASC(ending) or DESC(ending)", m_order,
+        SortOrder::ASC);
 }
 
 void SortFilter::prepared(PointTableRef table)
@@ -64,7 +65,7 @@ void SortFilter::prepared(PointTableRef table)
 
 void SortFilter::filter(PointView& view)
 {
-    auto cmp = [this](const PointIdxRef& p1, const PointIdxRef& p2)
+    auto cmp = [this](const PointRef& p1, const PointRef& p2)
     {
         bool result = p1.compare(m_dim, p2);
         return (m_order == SortOrder::ASC) ? result : !result;

diff --git a/filters/SortFilter.hpp b/filters/SortFilter.hpp
@@ -35,7 +35,6 @@
 #pragma once
 
 #include <pdal/Filter.hpp>
-#include <pdal/PointViewIter.hpp>
 #include <pdal/util/ProgramArgs.hpp>
 
 namespace pdal

diff --git a/filters/StatsFilter.hpp b/filters/StatsFilter.hpp
@@ -89,7 +89,7 @@ typedef std::vector<double> DataVector;
     {
         if (M2 == 0 || m_cnt <= 2 || !m_advanced)
             return 0.0;
-        double c(m_cnt);
+        double c((double)m_cnt);
         return populationSkewness() * std::sqrt(c) * std::sqrt(c - 1) / (c - 2);
     }
     double skewness() const
@@ -112,14 +112,14 @@ typedef std::vector<double> DataVector;
     {
         if (M2 == 0 || m_cnt <= 3 || !m_advanced)
             return 0;
-        double c(m_cnt);
+        double c((double)m_cnt);
         return populationKurtosis() * (c + 1) * (c - 1) / ((c - 2) * (c - 3));
     }
     double sampleExcessKurtosis() const
     {
         if (M2 == 0 || m_cnt <= 3 || !m_advanced)
             return 0;
-        double c(m_cnt);
+        double c((double)m_cnt);
         return sampleKurtosis() - 3 * (c - 1) * (c - 1) / ((c - 2) * (c - 3));
     }
     double kurtosis() const

diff --git a/filters/TailFilter.hpp b/filters/TailFilter.hpp
@@ -35,7 +35,6 @@
 #pragma once
 
 #include <pdal/Filter.hpp>
-#include <pdal/PointViewIter.hpp>
 
 namespace pdal
 {

diff --git a/io/TextReader.cpp b/io/TextReader.cpp
@@ -197,7 +197,15 @@ void TextReader::initialize(PointTableRef table)
         checkHeader(header);
     }
 
-    parseHeader(header);
+    try
+    {
+        parseHeader(header);
+    }
+    catch( const pdal_error& )
+    {
+        Utils::closeFile(m_istream);
+        throw;
+    }
     Utils::closeFile(m_istream);
 }
 

diff --git a/kernels/InfoKernel.cpp b/kernels/InfoKernel.cpp
@@ -252,7 +252,11 @@ MetadataNode InfoKernel::run(const std::string& filename)
     t << std::put_time( std::localtime( &now ), "%FT%T%z" );
     root.add("reader", m_reader->getName());
     root.add("now", t.str());
-    root.add("file_size", pdal::FileUtils::fileSize(filename));
+
+    if (pdal::FileUtils::fileExists(filename) &&
+            (!pdal::FileUtils::isDirectory(filename))) // allow for s3 uris and directories
+        root.add("file_size", pdal::FileUtils::fileSize(filename));
+
     return root;
 }
 

diff --git a/pdal/PointContainer.hpp b/pdal/PointContainer.hpp
@@ -51,6 +51,16 @@ class PDAL_DLL PointContainer
         const void *val) = 0;
     virtual void getFieldInternal(Dimension::Id dim, PointId idx,
         void *val) const = 0;
+    virtual void swapItems(PointId id1, PointId id2)
+        { throw pdal_error("Can't swap items in this container."); }
+    virtual void setItem(PointId dst, PointId src)
+        { throw pdal_error("Can't set item in this container."); }
+    virtual bool compare(Dimension::Id dim, PointId id1, PointId id2) const
+        { throw pdal_error("Can't compare items in this container."); }
+    virtual PointId getTemp(PointId id)
+        { return id; }
+    virtual void freeTemp(PointId id)
+        {}
 public:
     virtual PointLayoutPtr layout() const = 0;
 };

diff --git a/pdal/PointRef.hpp b/pdal/PointRef.hpp
@@ -45,10 +45,40 @@ namespace pdal
 class PDAL_DLL PointRef
 {
 public:
+    PointRef() : m_container(nullptr), m_layout(nullptr), m_idx(0), m_tmp(false)
+    {}
+
     PointRef(PointContainer& container, PointId idx = 0) :
-        m_container(container), m_layout(*container.layout()), m_idx(idx)
+        m_container(&container), m_layout(container.layout()), m_idx(idx),
+        m_tmp(false)
     {}
 
+    PointRef(const PointRef& r) :
+        m_container(r.m_container), m_layout(r.m_layout), m_tmp(true)
+    {
+        m_idx = m_container->getTemp(r.m_idx);
+    }
+
+    ~PointRef()
+    {
+        if (m_tmp)
+            m_container->freeTemp(m_idx);
+    }
+
+    PointRef& operator=(const PointRef& r)
+    {
+        if (!m_container)
+        {
+            m_container = r.m_container;
+            m_layout = r.m_layout;
+            m_idx = m_container->getTemp(r.m_idx);
+            m_tmp = true;
+        }
+        else
+            m_container->setItem(m_idx, r.m_idx);
+        return *this;
+    }
+
     /**
       Determine if the point contains the specified dimension (defers to
       the layout).
@@ -57,7 +87,7 @@ class PDAL_DLL PointRef
       \return  Whether the point has a dimension.
     */
     bool hasDim(Dimension::Id dim) const
-    { return m_layout.hasDim(dim); }
+    { return m_layout->hasDim(dim); }
 
     /**
       Get the value of a field/dimension, converting it to the type as
@@ -73,9 +103,9 @@ class PDAL_DLL PointRef
         T val(0);
         bool success = true;
         Everything e;
-        Dimension::Type type = m_layout.dimDetail(dim)->type();
+        Dimension::Type type = m_layout->dimDetail(dim)->type();
 
-        m_container.getFieldInternal(dim, m_idx, &e);
+        m_container->getFieldInternal(dim, m_idx, &e);
         switch (type)
         {
         case Dimension::Type::Unsigned8:
@@ -134,7 +164,7 @@ class PDAL_DLL PointRef
     template<typename T>
     void setField(Dimension::Id dim, T val)
     {
-        Dimension::Type type = m_layout.dimDetail(dim)->type();
+        Dimension::Type type = m_layout->dimDetail(dim)->type();
         Everything e;
         bool success = false;
 
@@ -173,8 +203,9 @@ class PDAL_DLL PointRef
         case Dimension::Type::None:
             break;
         }
+
         if (success)
-            m_container.setFieldInternal(dim, m_idx, &e);
+            m_container->setFieldInternal(dim, m_idx, &e);
     }
 
     /**
@@ -227,13 +258,30 @@ class PDAL_DLL PointRef
         }
     }
 
+    bool compare(Dimension::Id dim, const PointRef& r) const
+    {
+        assert(m_container == r.m_container);
+        return m_container->compare(dim, m_idx, r.m_idx);
+    }
+
+    static void swap(const PointRef& r1, const PointRef& r2)
+    {
+        assert(r1.m_container == r2.m_container);
+        r1.m_container->swapItems(r1.m_idx, r2.m_idx);
+    }
 
 private:
-    PointContainer& m_container;
-    PointLayout& m_layout;
+    PointContainer *m_container;
+    PointLayout *m_layout;
     PointId m_idx;
+    bool m_tmp;
 };
 
+inline void swap(const PointRef& r1, const PointRef& r2)
+{
+    PointRef::swap(r1, r2);
+}
+
 
 /**
   Get the value of a dimension of a point.
@@ -357,10 +405,10 @@ inline MetadataNode PointRef::toMetadata() const
 */
 inline void PointRef::toMetadata(MetadataNode root) const
 {
-    for (Dimension::Id id : m_layout.dims())
+    for (Dimension::Id id : m_layout->dims())
     {
         double v = getFieldAs<double>(id);
-        root.add(m_layout.dimName(id), v);
+        root.add(m_layout->dimName(id), v);
     }
 }
 

diff --git a/pdal/PointView.cpp b/pdal/PointView.cpp
@@ -37,7 +37,6 @@
 #include <pdal/EigenUtils.hpp>
 #include <pdal/KDIndex.hpp>
 #include <pdal/PointView.hpp>
-#include <pdal/PointViewIter.hpp>
 #include <pdal/util/Algorithm.hpp>
 
 namespace pdal