Merge branch 'ept-binary' into ept-zstandard

apps/pdal.cpp

@@ -161,18 +161,20 @@ void App::outputDrivers()
     else
     {
         NL::json j;
+        StageExtensions& extensions = PluginManager<Stage>::extensions();
         for (auto name : stages)
         {
             std::string description = PluginManager<Stage>::description(name);
             std::string link = PluginManager<Stage>::link(name);
             j.push_back(
-                { {"name", name},
-                  {"description", description},
-                  {"link", link}
+                { { "name", name },
+                  { "description", description },
+                  { "link", link },
+                  { "extensions", extensions.extensions(name) }
                 }
             );
         }
-        m_out << j;
+        m_out << std::setw(4) << j;
     }
 }
 

doc/references.rst

@@ -34,6 +34,8 @@ Reference
 
 .. [Alexa2003] Alexa, Marc, et al. "Computing and rendering point set surfaces." Visualization and Computer Graphics, IEEE Transactions on 9.1 (2003): 3-15.
 
+.. [Breunig2000] Breunig, M.M., Kriegel, H.-P., Ng, R.T., Sander, J., 2000. LOF: Identifying Density-Based Local Outliers. Proc. 2000 Acm Sigmod Int. Conf. Manag. Data 1–12.
+
 .. [Chen2012] Chen, Ziyue et al. “Upward-Fusion Urban DTM Generating Method Using Airborne Lidar Data.” ISPRS Journal of Photogrammetry and Remote Sensing 72 (2012): 121–130.
 
 .. [Cook1986] Cook, Robert L. "Stochastic sampling in computer graphics." *ACM Transactions on Graphics (TOG)* 5.1 (1986): 51-72.
@@ -42,6 +44,8 @@ Reference
 
 .. [Dippe1985] Dippé, Mark AZ, and Erling Henry Wold. "Antialiasing through stochastic sampling." *ACM Siggraph Computer Graphics* 19.3 (1985): 69-78.
 
+.. [Fischer2010] Fischer, Kaspar, Bernd Gärtner, and Martin Kutz. “Fast Smallest-Enclosing-Ball Computation in High Dimensions.” 26473 (2010): 630–641. Web.
+
 .. [Guinard2017] Guinard S., Landrieu L. "Weakly Supervised Segmented-Aided Classification of Urban Scenes From 3D LIDAR Point Clouds." Int. Arch. Photogramm. Remote Sens. Spatial Inf. Sci., XLII-1/W1, 151-157, 2017
 
 .. [Kazhdan2006] Kazhdan, Michael, Matthew Bolitho, and Hugues Hoppe. "Poisson surface reconstruction." Proceedings of the fourth Eurographics symposium on Geometry processing. Vol. 7. 2006.
@@ -56,4 +60,6 @@ Reference
 
 .. [Rusu2008] Rusu, Radu Bogdan, et al. "Towards 3D point cloud based object maps for household environments." Robotics and Autonomous Systems 56.11 (2008): 927-941.
 
+.. [Weyrich2004] Weyrich, T et al. “Post-Processing of Scanned 3D Surface Data.” Proceedings of Eurographics Symposium on Point-Based Graphics 2004 (2004): 85–94. Print.
+
 .. [Zhang2003] Zhang, Keqi, et al. "A progressive morphological filter for removing nonground measurements from airborne LIDAR data." Geoscience and Remote Sensing, IEEE Transactions on 41.4 (2003): 872-882.

doc/stages/filters.lof.rst

@@ -37,9 +37,9 @@ users of this filter should find instructive.
 Example
 -------
 
-The sample pipeline below uses computes the LOF with a neighborhood of 20
-neighbors, followed by a range filter to crop out points whose
-``LocalOutlierFactor`` exceeds 1.2 before writing the output.
+The sample pipeline below computes the LOF with a neighborhood of 20 neighbors,
+followed by a range filter to crop out points whose ``LocalOutlierFactor``
+exceeds 1.2 before writing the output.
 
 .. code-block:: json
 
@@ -62,5 +62,3 @@ Options
 _`minpts`
   The number of k nearest neighbors. [Default: 10]
 
-.. [Breunig2000] Breunig, M.M., Kriegel, H.-P., Ng, R.T., Sander, J., 2000. LOF: Identifying Density-Based Local Outliers. Proc. 2000 Acm Sigmod Int. Conf. Manag. Data 1–12.
-

doc/stages/filters.miniball.rst

@@ -0,0 +1,53 @@
+.. _filters.miniball:
+
+filters.miniball
+===============================================================================
+
+The **Miniball Criterion** was introduced in [Weyrich2004]_ and is based on the
+assumption that points that are distant to the cluster built by their
+k-neighborhood are likely to be outliers. First, the smallest enclosing ball is
+computed for the k-neighborhood, giving a center point and radius
+[Fischer2010]_. The miniball criterion is then computed by comparing the
+distance (from the current point to the miniball center) to the radius of the
+miniball.
+
+The author suggests that the Miniball Criterion is more robust than the
+:ref:`Plane Fit Criterion <filters.planefit>` around high-frequency details,
+but demonstrates poor outlier detection for points close to a smooth surface.
+
+The filter creates a single new dimension, ``Miniball``, that records the
+Miniball criterion for the current point.
+
+.. note::
+
+  To inspect the newly created, non-standard dimensions, be sure to write to an
+  output format that can support arbitrary dimensions, such as BPF.
+
+.. embed::
+
+Example
+-------
+
+The sample pipeline below computes the Miniball criterion with a neighborhood
+of 8 neighbors. We do not apply a fixed threshold to single out outliers based
+on the Miniball criterion as the range of values can vary from one dataset to
+another. In general, higher values indicate the likelihood of a point being an
+outlier.
+
+.. code-block:: json
+
+  [
+      "input.las",
+      {
+          "type":"filters.miniball",
+          "knn":8
+      }
+      "output.laz"
+  ]
+
+Options
+-------------------------------------------------------------------------------
+
+knn
+  The number of k nearest neighbors. [Default: 8]
+

doc/stages/filters.planefit.rst

@@ -0,0 +1,58 @@
+.. _filters.planefit:
+
+filters.planefit
+===============================================================================
+
+The **Plane Fit Criterion** was introduced in [Weyrich2004]_ and computes the
+deviation of a point from a manifold approximating its neighbors.  First, a
+plane is fit to each point's k-neighborhood by performing an eigenvalue
+decomposition. Next, the mean point to plane distance is computed by
+considering all points within the neighborhood. This is compared to the point
+to plane distance of the current point giving rise to the k-neighborhood. As
+the mean distance of the k-neighborhood approaches 0, the Plane Fit criterion
+will tend toward 1. As point to plane distance of the current point approaches
+0, the Plane Fit criterion will tend toward 0.
+
+The author suggests that the Plane Fit Criterion is well suited to outlier
+detection when considering noisy reconstructions of smooth surfaces, but
+produces poor results around small features and creases.
+
+The filter creates a single new dimension, ``PlaneFit``, that records the
+Plane Fit criterion for the current point.
+
+.. note::
+
+  To inspect the newly created, non-standard dimensions, be sure to write to an
+  output format that can support arbitrary dimensions, such as BPF.
+
+.. embed::
+
+Example
+-------
+
+The sample pipeline below computes the Plane Fit criterion with a neighborhood
+of 8 neighbors. We do not apply a fixed threshold to single out outliers based
+on the Plane Fit criterion as the range of values can vary from one dataset to
+another. In general, higher values indicate the likelihood of a point being an
+outlier.
+
+.. code-block:: json
+
+  [
+      "input.las",
+      {
+          "type":"filters.planefit",
+          "knn":8
+      }
+      "output.laz"
+  ]
+
+Options
+-------------------------------------------------------------------------------
+
+knn
+  The number of k nearest neighbors. [Default: 8]
+
+threads
+  The number of threads used for computing the plane fit criterion. [Default: 1]
+

doc/stages/filters.reciprocity.rst

@@ -0,0 +1,55 @@
+.. _filters.reciprocity:
+
+filters.reciprocity
+===============================================================================
+
+The **Nearest-Neighbor Reciprocity Criterion** was introduced in [Weyrich2004]_
+and is based on a simple assumption, that valid points may be in the
+k-neighborhood of an outlier, but the outlier will most likely not be part of
+the valid point's k-neighborhood.
+
+The author suggests that the Nearest-Neighbor Reciprocity Criterion is more
+robust than both the :ref:`Plane Fit <filters.planefit>` and :ref:`Miniball
+<filters.miniball>` Criterion, being equally sensitive around smooth and
+detailed regions. The criterion does however produce invalid reslts near
+manifold borders.
+
+The filter creates a single new dimension, ``Reciprocity``, that records the
+percentage of points(in the range 0 to 100) that are considered uni-directional
+neighbors of the current point. 
+
+.. note::
+
+  To inspect the newly created, non-standard dimensions, be sure to write to an
+  output format that can support arbitrary dimensions, such as BPF.
+
+.. embed::
+
+Example
+-------
+
+The sample pipeline below computes reciprocity with a neighborhood of 8
+neighbors, followed by a range filter to crop out points whose ``Reciprocity``
+percentage is less than 98% before writing the output.
+
+.. code-block:: json
+
+  [
+      "input.las",
+      {
+          "type":"filters.reciprocity",
+          "knn":8
+      },
+      {
+          "type":"filters.range",
+          "limits":"Reciprocity[:98.0]"
+      },
+      "output.laz"
+  ]
+
+Options
+-------------------------------------------------------------------------------
+
+knn
+  The number of k nearest neighbors. [Default: 8]
+

doc/stages/filters.shell.rst

@@ -0,0 +1,73 @@
+.. _filters.shell:
+
+filters.shell
+===================
+
+The shell filter allows you to run shell operations in-line 
+with PDAL pipeline tasks. This can be especially useful for 
+follow-on items or orchestration of complex workflows.
+
+.. embed::
+
+.. warning::
+
+	To use :ref:`filters.shell`, you must set ``PDAL_ALLOW_SHELL=1`` 
+	PDAL's execution environment. Without the environment variable 
+	set, every attempt at execution will result in the following 
+	error:
+
+		PDAL_ALLOW_SHELL environment variable not set, shell access is not allowed
+
+Example 
+---------
+
+GDAL processing operations applied to raster output from :ref:`writers.gdal` 
+are a common task. Applying these within the PDAL execution environment 
+can provide some convenience and allow downstream consumers to have deterministic 
+completion status of the task. The following task writes multiple elevation 
+models to disk and then uses the `gdaladdo <https://gdal.org/gdaladdo.html>`__ 
+command to construct overview bands for the data using average interpolation.
+
+.. code-block:: json
+
+    {
+      "pipeline":[
+        "autzen.las",
+        {
+          "type":"writers.gdal",
+          "filename" : "output-1m.tif",
+          "resolution" : "1.0"
+        },
+        {
+          "type":"writers.gdal",
+          "filename" : "output-2m.tif",
+          "resolution" : "2.0"
+        },
+        {
+          "type":"writers.gdal",
+          "filename" : "output-5m.tif",
+          "resolution" : "5.0"
+        },
+        {
+          "type":"filters.shell",
+          "command" : "gdaladdo -r average output-1m.tif 2 4 8 16"
+        },
+        {
+          "type":"filters.shell",
+          "command" : "gdaladdo -r average output-2m.tif 2 4 8 16"
+        }
+        {
+          "type":"filters.shell",
+          "command" : "gdaladdo -r average output-5m.tif 2 4 8 16"
+        }
+    	]
+    }
+
+
+Options
+-------
+
+command
+  The shell command to run. It is run in relation to the current 
+  working directory of the pipeline executing it.
+

doc/stages/writers.tiledb.rst

@@ -56,6 +56,9 @@ compression
 compression_level
   TileDB compression level for chosen compression [Optional]
 
+append
+  Append to an existing TileDB array with the same schema [Optional]
+
 stats
   Dump query stats to stdout [Optional]
 

filters/CovarianceFeaturesFilter.cpp

@@ -87,18 +87,18 @@ void CovarianceFeaturesFilter::filter(PointView& view)
     KD3Index& kdi = view.build3dIndex();
 
     point_count_t nloops = view.size();
-    std::vector<std::thread> threadPool(m_threads);
+    std::vector<std::thread> threadList(m_threads);
     for(int t = 0;t<m_threads;t++)
     {
-        threadPool[t] = std::thread(std::bind(
-                [&](const PointId start, const PointId end, const PointId t)
+        threadList[t] = std::thread(std::bind(
+                [&](const PointId start, const PointId end)
                 {
                     for(PointId i = start;i<end;i++)
                         setDimensionality(view, i, kdi);
                 },
-                t*nloops/m_threads,(t+1)==m_threads?nloops:(t+1)*nloops/m_threads,t));
+                t*nloops/m_threads,(t+1)==m_threads?nloops:(t+1)*nloops/m_threads));
     }
-    for (auto &t: threadPool)
+    for (auto &t: threadList)
         t.join();
 }