Merge remote-tracking branch 'origin/master' into issue-2170

doc/stages/readers.ept.rst

@@ -0,0 +1,54 @@
+.. _readers.ept:
+
+readers.ept
+===========
+
+`Entwine Point Tile`_ (EPT) is a hierarchical octree-based point cloud format suitable for real-time rendering and lossless archival.  `Entwine`_ is a producer of this format.  The **EPT Reader** supports reading data from the EPT format, including spatially accelerated queries and file reconstruction queries.
+
+Sample EPT datasets of hundreds of billions of points in size may be viewed at http://potree.entwine.io and http://speck.ly.
+
+.. embed::
+
+Example
+--------------------------------------------------------------------------------
+
+This example downloads a small area around the the Statue of Liberty from the New York City data set (4.7 billion points) which can be viewed in its entirety in `Potree`_ or `Plasio`_.
+
+.. code-block:: json
+
+    {
+      "pipeline": [
+        {
+          "type": "readers.ept",
+          "filename": "http://na.entwine.io/nyc",
+          "bounds": "([-8242669, -8242529], [4966549, 4966674])"
+        },
+        "statue-of-liberty.las"
+      ]
+    }
+
+
+Options
+--------------------------------------------------------------------------------
+
+filename
+    EPT resource from which to read.  Because EPT resources do not have a file extension, to specify an EPT resource as a string, it must be prefixed with ``ept://``.  For example, ``pdal translate ept://http://na.entwine.io/autzen autzen.laz``. [Required]
+
+bounds
+    The extents of the resource to select in 2 or 3 dimensions, expressed as a string, e.g.: ``([xmin, xmax], [ymin, ymax], [zmin, zmax])``.  If omitted, the entire dataset will be selected.
+
+origin
+    EPT datasets are lossless aggregations of potentially multiple source files.  The *origin* options can be used to select all points from a single source file.  This option may be specified as a string or an integral ID.
+
+    The string form of this option selects a source file by its original file path.  This may be a substring instead of the entire path, but the string must uniquely select only one source file (via substring search).  For example, for an EPT dataset created from source files *one.las*, *two.las*, and *two.bpf*, `"one"` is a sufficient selector, but `"two"` is not.
+
+    The integral form of this option selects a source file by its ``OriginId`` dimension, which can be found via the files position in EPT metadata file ``entwine-files.json``.
+
+threads
+    Number of worker threads used to download and process EPT data.  A minimum of 4 will be used no matter what value is specified.
+
+.. _Entwine Point Tile: https://github.com/connormanning/entwine/blob/master/doc/entwine-point-tile.md
+.. _Entwine: https://entwine.io/
+.. _Potree: http://potree.entwine.io/data/nyc.html
+.. _Plasio: http://speck.ly/?s=http%3A%2F%2Fc%5B0-7%5D.greyhound.io&r=ept%3A%2F%2Fna.entwine.io%2Fnyc&ca=-0&ce=49.06&ct=-8239196%2C4958509.308%2C337&cd=42640.943&cmd=125978.13&ps=2&pa=0.1&ze=1&c0s=remote%3A%2F%2Fimagery%3Furl%3Dhttp%3A%2F%2Fserver.arcgisonline.com%2FArcGIS%2Frest%2Fservices%2FWorld_Imagery%2FMapServer%2Ftile%2F%7B%7Bz%7D%7D%2F%7B%7By%7D%7D%2F%7B%7Bx%7D%7D.jpg
+

doc/stages/readers.gdal.rst

@@ -85,3 +85,8 @@ filename
 
 count
     Maximum number of points to read [Optional]
+
+header
+    A comma-separated list of :ref:`dimensions` IDs to map
+    bands to. The length of the list must match the number
+    of bands in the raster.

filters/PMFFilter.cpp

@@ -60,7 +60,7 @@ struct PMFArgs
 {
     double m_cellSize;
     bool m_exponential;
-    DimRange m_ignored;
+    std::vector<DimRange> m_ignored;
     double m_initialDistance;
     StringList m_returns;
     double m_maxDistance;
@@ -106,7 +106,13 @@ void PMFFilter::prepared(PointTableRef table)
 {
     const PointLayoutPtr layout(table.layout());
 
-    m_args->m_ignored.m_id = layout->findDim(m_args->m_ignored.m_name);
+    for (auto& r : m_args->m_ignored)
+    {
+        r.m_id = layout->findDim(r.m_name);
+        if (r.m_id == Dimension::Id::Unknown)
+            throwError("Invalid dimension name in 'ignored' option: '" +
+                r.m_name + "'.");
+    }
 
     if (m_args->m_returns.size())
     {
@@ -141,11 +147,12 @@ PointViewSet PMFFilter::run(PointViewPtr input)
     // Segment input view into ignored/kept views.
     PointViewPtr ignoredView = input->makeNew();
     PointViewPtr keptView = input->makeNew();
-    if (m_args->m_ignored.m_id == Dimension::Id::Unknown)
+
+    if (m_args->m_ignored.empty())
         keptView->append(*input);
     else
-        Segmentation::ignoreDimRange(m_args->m_ignored, input, keptView,
-                                     ignoredView);
+        Segmentation::ignoreDimRanges(m_args->m_ignored,
+            input, keptView, ignoredView);
 
     // Classify remaining points with value of 1. processGround will mark ground
     // returns as 2.

filters/RangeFilter.cpp

@@ -73,41 +73,22 @@ RangeFilter::~RangeFilter()
 
 void RangeFilter::addArgs(ProgramArgs& args)
 {
-    args.add("limits", "Range limits", m_rangeSpec).setPositional();
-}
-
-
-void RangeFilter::initialize()
-{
-    // Would be better to have the range know how to read from an input stream.
-    for (auto const& r : m_rangeSpec)
-    {
-        try
-        {
-            DimRange range;
-            range.parse(r);
-            m_range_list.push_back(range);
-        }
-        catch (const DimRange::error& err)
-        {
-            throwError("Invalid 'limits' option: '" + r + "': " + err.what());
-        }
-    }
+    args.add("limits", "Range limits", m_ranges).setPositional();
 }
 
 
 void RangeFilter::prepared(PointTableRef table)
 {
     const PointLayoutPtr layout(table.layout());
 
-    for (auto& r : m_range_list)
+    for (auto& r : m_ranges)
     {
         r.m_id = layout->findDim(r.m_name);
         if (r.m_id == Dimension::Id::Unknown)
             throwError("Invalid dimension name in 'limits' option: '" +
                 r.m_name + "'.");
     }
-    std::sort(m_range_list.begin(), m_range_list.end());
+    std::sort(m_ranges.begin(), m_ranges.end());
 }
 
 
@@ -117,27 +98,7 @@ void RangeFilter::prepared(PointTableRef table)
 // common case.
 bool RangeFilter::processOne(PointRef& point)
 {
-    Dimension::Id lastId = m_range_list.front().m_id;
-    bool passes = false;
-    for (auto const& r : m_range_list)
-    {
-        // If we're at a new dimension, return false if we haven't passed
-        // the dimension, otherwise reset passes to false for the next
-        // dimension and keep checking.
-        if (r.m_id != lastId)
-        {
-            if (!passes)
-                return false;
-            lastId = r.m_id;
-            passes = false;
-        }
-        // If we've already passed this dimension, continue until we find
-        // a new dimension.
-        else if (passes)
-            continue;
-        passes = r.valuePasses(point.getFieldAs<double>(r.m_id));
-    }
-    return passes;
+    return DimRange::pointPasses(m_ranges, point);
 }
 
 

filters/RangeFilter.hpp

@@ -55,11 +55,9 @@ class PDAL_DLL RangeFilter : public Filter,  public Streamable
     std::string getName() const;
 
 private:
-    StringList m_rangeSpec;
-    std::vector<DimRange> m_range_list;
+    std::vector<DimRange> m_ranges;
 
     virtual void addArgs(ProgramArgs& args);
-    virtual void initialize();
     virtual void prepared(PointTableRef table);
     virtual bool processOne(PointRef& point);
     virtual PointViewSet run(PointViewPtr view);

filters/SMRFilter.cpp

@@ -82,7 +82,7 @@ struct SMRArgs
     double m_threshold;
     double m_cut;
     std::string m_dir;
-    DimRange m_ignored;
+    std::vector<DimRange> m_ignored;
     StringList m_returns;
 };
 
@@ -120,8 +120,13 @@ void SMRFilter::prepared(PointTableRef table)
 {
     const PointLayoutPtr layout(table.layout());
 
-    m_args->m_ignored.m_id = layout->findDim(m_args->m_ignored.m_name);
-
+    for (auto & r : m_args->m_ignored)
+    {
+        r.m_id = layout->findDim(r.m_name);
+        if (r.m_id == Dimension::Id::Unknown)
+            throwError("Invalid dimension name in 'ignored' option: '" +
+                r.m_name + "'.");
+    }
     if (m_args->m_returns.size())
     {
         for (auto& r : m_args->m_returns)
@@ -164,11 +169,11 @@ PointViewSet SMRFilter::run(PointViewPtr view)
     // Segment input view into ignored/kept views.
     PointViewPtr ignoredView = view->makeNew();
     PointViewPtr keptView = view->makeNew();
-    if (m_args->m_ignored.m_id == Dimension::Id::Unknown)
+    if (m_args->m_ignored.empty())
         keptView->append(*view);
     else
-        Segmentation::ignoreDimRange(m_args->m_ignored, view, keptView,
-                                     ignoredView);
+        Segmentation::ignoreDimRanges(m_args->m_ignored, view, keptView,
+            ignoredView);
 
     // Segment kept view into two views
     PointViewPtr firstView = keptView->makeNew();

filters/private/DimRange.cpp

@@ -126,6 +126,33 @@ bool DimRange::valuePasses(double v) const
     return !fail;
 }
 
+// Important - range list must be sorted.
+// This applies OR logic when there are multiple ranges for the same
+// dimension and AND logic for different dimensions.  It depends on
+// the range list being sorted such that ranges for the same dimension
+// are contiguous.
+bool DimRange::pointPasses(const std::vector<DimRange>& ranges, PointRef& point)
+{
+    Dimension::Id lastId = ranges.front().m_id;
+    bool passes = false;
+    for (auto const& r : ranges)
+    {
+        // If we're at a new dimension, return false if we haven't passed
+        // the dimension, otherwise reset lastId and keep checking.
+        if (r.m_id != lastId)
+        {
+            if (!passes)
+                return false;
+            lastId = r.m_id;
+        }
+        // If we've already passed this dimension, continue until we find
+        // a new dimension.
+        else if (passes)
+            continue;
+        passes = r.valuePasses(point.getFieldAs<double>(r.m_id));
+    }
+    return passes;
+}
 
 void DimRange::parse(const std::string& r)
 {

filters/private/DimRange.hpp

@@ -37,6 +37,8 @@
 #include <string>
 
 #include <pdal/Dimension.hpp>
+#include <pdal/PointRef.hpp>
+#include <pdal/util/ProgramArgs.hpp>
 
 namespace pdal
 {
@@ -69,6 +71,8 @@ struct DimRange
 
     void parse(const std::string& s);
     bool valuePasses(double d) const;
+    static bool pointPasses(const std::vector<DimRange>& ranges,
+        PointRef& point);
 
     std::string m_name;
     Dimension::Id m_id;
@@ -86,5 +90,72 @@ bool operator < (const DimRange& r1, const DimRange& r2);
 std::istream& operator>>(std::istream& in, DimRange& r);
 std::ostream& operator<<(std::ostream& out, const DimRange& r);
 
+template <>
+class VArg<DimRange> : public BaseVArg
+{
+public:
+    VArg(const std::string& longname, const std::string& shortname,
+        const std::string& description, std::vector<DimRange>& variable,
+        std::vector<DimRange> def) :
+        BaseVArg(longname, shortname, description), m_var(variable),
+        m_defaultVal(def)
+    {
+        m_var = def;
+        m_defaultProvided = true;
+    }
+
+    VArg(const std::string& longname, const std::string& shortname,
+        const std::string& description, std::vector<DimRange>& variable) :
+        BaseVArg(longname, shortname, description), m_var(variable)
+    {}
+
+    virtual void setValue(const std::string& s)
+    {
+        std::vector<std::string> slist = Utils::split2(s, ',');
+        for (auto& ts : slist)
+            Utils::trim(ts);
+
+        if (slist.empty())
+            throw arg_val_error("Missing value for argument '" + m_longname +
+                "'.");
+        m_rawVal = s;
+        if (!m_set)
+            m_var.clear();
+        for (auto& ts : slist)
+        {
+            DimRange dim;
+            dim.parse(ts);
+            m_var.push_back(dim);
+        }
+        m_set = true;
+    }
+
+    virtual void reset()
+    {
+        m_var = m_defaultVal;
+        m_set = false;
+        m_hidden = false;
+    }
+
+    virtual std::string defaultVal() const
+    {
+        std::string s;
+
+        for (size_t i = 0; i < m_defaultVal.size(); ++i)
+        {
+            std::ostringstream oss;
+
+            if (i > 0)
+                s += ", ";
+            oss << m_defaultVal[i];
+            s += oss.str();
+        }
+        return s;
+    }
+
+private:
+    std::vector<DimRange>& m_var;
+    std::vector<DimRange> m_defaultVal;
+};
 
 } // namespace pdal

filters/private/Segmentation.cpp

@@ -127,6 +127,21 @@ void ignoreDimRange(DimRange dr, PointViewPtr input, PointViewPtr keep,
     }
 }
 
+void ignoreDimRanges(std::vector<DimRange>& ranges, PointViewPtr input,
+    PointViewPtr keep, PointViewPtr ignore)
+{
+    std::sort(ranges.begin(), ranges.end());
+    PointRef point(*input, 0);
+    for (PointId i = 0; i < input->size(); ++i)
+    {
+        point.setPointId(i);
+        if (DimRange::pointPasses(ranges, point))
+            ignore->appendPoint(*input, i);
+        else
+            keep->appendPoint(*input, i);
+    }
+}
+
 void segmentLastReturns(PointViewPtr input, PointViewPtr last,
                         PointViewPtr other)
 {

filters/private/Segmentation.hpp

@@ -70,6 +70,8 @@ PDAL_DLL std::vector<std::vector<PointId>> extractClusters(PointView& view,
 
 PDAL_DLL void ignoreDimRange(DimRange dr, PointViewPtr input, PointViewPtr keep,
                              PointViewPtr ignore);
+PDAL_DLL void ignoreDimRanges(std::vector<DimRange>& ranges,
+    PointViewPtr input, PointViewPtr keep, PointViewPtr ignore);
 
 PDAL_DLL void segmentLastReturns(PointViewPtr input, PointViewPtr last,
                                  PointViewPtr other);