Merge remote-tracking branch 'origin/master' into issue-3313

.github/workflows/conda.yml

@@ -49,3 +49,12 @@ jobs:
       with:
         name: ${{ matrix.platform }}-conda-package
         path: ./pdal-feedstock/packages/
+
+    - name: Deploy to pdal-master Conda channel
+      if: github.ref == 'refs/heads/master'
+      shell: bash -l {0}
+      env:
+        ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
+      run: |
+          source ../scripts/ci/conda/upload.sh || true
+      working-directory: ./pdal-feedstock

.github/workflows/docker.yml

@@ -7,20 +7,30 @@ on: [push, pull_request, workflow_dispatch]
 jobs:
   containers:
     runs-on: ubuntu-latest
+    env:
+      PUSH_PACKAGES: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
     steps:
       - uses: actions/checkout@v2
+      - name: Lint
+        id: lint
+        run: |
+          echo "are we pushing packages" ${{ env.PUSH_PACKAGES }}
+          echo "event_name" ${{ github.event_name }}
+          echo "ref" ${{ github.ref }}
       - name: Setup Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
         with:
           version: latest
-      - name: Login to GitHub Container Registry
+      - if: ${{ env.PUSH_PACKAGES == 'true' }}
+        name: Login to GitHub Container Registry
         uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GHCR_TOKEN }}
-      - name: Login to Docker Hub
+      - if: ${{ env.PUSH_PACKAGES == 'true' }}
+        name: Login to Docker Hub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
@@ -37,14 +47,16 @@ jobs:
       - name: Build image
         uses: docker/build-push-action@v2
         with:
-          push: true
+          push: ${{ env.PUSH_PACKAGES == 'true' }}
           builder: ${{ steps.buildx.outputs.name }}
           context: .
           file: ./scripts/docker/ubuntu/Dockerfile
           platforms: linux/amd64
           tags: |
             docker.io/pdal/pdal:${{ steps.prep.outputs.VERSION }}
+            docker.io/pdal/pdal:latest
             ghcr.io/pdal/pdal:${{ steps.prep.outputs.VERSION }}
+            ghcr.io/pdal/pdal:latest
           labels: |
             org.opencontainers.image.title=${{ github.event.repository.name }}
             org.opencontainers.image.description=${{ github.event.repository.description }}

doc/index.rst

@@ -28,10 +28,10 @@ The entire website is available as a single PDF at http://pdal.io/PDAL.pdf
 News
 --------------------------------------------------------------------------------
 
-**03-21-2020**
+**09-09-2020**
 ................................................................................
 
-PDAL 2.1.0 has been released. You can :ref:`download <download>` the source
+PDAL 2.2.0 has been released. You can :ref:`download <download>` the source
 code or follow the :ref:`quickstart <quickstart>` to get going in a
 hurry with Conda.
 

doc/stages/readers.i3s.rst

@@ -3,7 +3,9 @@
 readers.i3s
 ===========
 
-`Indexed 3d Scene Layer (I3S)`_ is a specification created by Esri as a format for their 3D Scene Layer and scene services. The I3S reader handles RESTful webservices in an I3S file structure/format.
+`Indexed 3d Scene Layer (I3S)`_ is a specification created by Esri as a format for their
+3D Scene Layer and scene services. The I3S reader handles RESTful webservices in an I3S
+file structure/format.
 
 Example
 --------------------------------------------------------------------------------
@@ -15,16 +17,33 @@ This example will download the Autzen dataset from the ArcGIS scene server and o
       {
           "type": "readers.i3s",
           "filename": "https://tiles.arcgis.com/tiles/8cv2FuXuWSfF0nbL/arcgis/rest/services/AUTZEN_LiDAR/SceneServer",
-          "bounds": "([-123.075542,-123.06196],[44.049719,44.06278])"
+          "obb": {
+              "center": [
+                  636590,
+                  849216,
+                  460
+              ],
+              "halfSize": [
+                  590,
+                  281,
+                  60
+              ],
+              "quaternion":
+              [
+                  0,
+                  0,
+                  0,
+                  1
+              ]
+          }
       }
   ]
 
 .. code::
 
     pdal translate i3s://https://tiles.arcgis.com/tiles/8cv2FuXuWSfF0nbL/arcgis/rest/services/AUTZEN_LiDAR/SceneServer \
         autzen.las \
-        --readers.i3s.threads=64 \
-        --readers.i3s.bounds="([-123.075542,-123.06196],[44.049719,44.06278])"
+        --readers.i3s.threads=64
 
 Options
 --------------------------------------------------------------------------------
@@ -43,10 +62,9 @@ threads
 
     Example: ``--readers.i3s.threads=64``
 
-bounds
-    The bounds refers to the extents of the resource in X, Y, Z coordinates with the Z dimension being optional. This must be input as a string.
-
-    Example:``readers.i3s.bounds="([xmin,xmax],[ymin,ymax],[zmin,zmax])"``
+obb
+    An oriented bounding box used to filter the data being retrieved.  The obb
+    is specified as JSON exactly as described by the `I3S specification`_.
 
 dimensions
     Comma-separated list of dimensions that should be read.  Specify the
@@ -74,3 +92,4 @@ min_density and max_density
     Example: ``--readers.i3s.min_density=2 --readers.i3s.max_density=2.5``
 
 .. _Indexed 3d Scene Layer (I3S): https://github.com/Esri/i3s-spec/blob/master/format/Indexed%203d%20Scene%20Layer%20Format%20Specification.md
+.. _I3S specification: https://github.com/Esri/i3s-spec/blob/master/docs/2.0/obb.cmn.md

doc/stages/readers.las.rst

@@ -103,3 +103,10 @@ compression
   support for the decompressor being requested.  The LazPerf decompressor
   doesn't support version 1 LAZ files or version 1.4 of LAS. [Default: 'none']
 
+ignore_vlr
+  A comma-separated list of "userid/record_id" pairs specifying VLR records that should
+  not be loaded.
+
+fix_dims
+  Make invalid dimension names valid by converting disallowed characters to '_'. Only
+  applies to names specified in an extra-bytes VLR. [Default: true]

doc/stages/readers.slpk.rst

@@ -20,15 +20,31 @@ through PDAL's command line interface or through the pipeline.
       {
           "type": "readers.slpk",
           "filename": "PDAL/test/data/i3s/SMALL_AUTZEN_LAS_All.slpk",
-          "bounds": "([-123.075542,-123.06196],[44.049719,44.06278])"
+          "obb": {
+              "center": [
+                  636590,
+                  849216,
+                  460
+              ],
+              "halfSize": [
+                  590,
+                  281,
+                  60
+              ],
+              "quaternion":
+              [
+                  0,
+                  0,
+                  0,
+                  1
+              ]
+          }
       }
   ]
 
 ::
 
-    pdal traslate  PDAL/test/data/i3s/SMALL_AUTZEN_LAS_All.slpk \
-        autzen.las \
-        --readers.slpk.bounds="([-123.075542,-123.06196],[44.049719,44.06278])"``
+    pdal traslate  PDAL/test/data/i3s/SMALL_AUTZEN_LAS_All.slpk autzen.las
 
 Options
 --------------------------------------------------------------------------------
@@ -38,12 +54,9 @@ filename
 
 .. include:: reader_opts.rst
 
-bounds
-    The bounds refers to the extents of the resource in X, Y, Z
-    coordinates with the Z dimension being optional. This must be input
-    as a string.
-
-    Example:``readers.slpk.bounds="([xmin,xmax],[ymin,ymax],[zmin,zmax])"``
+obb
+    An oriented bounding box used to filter the data being retrieved.  The obb
+    is specified as JSON exactly as described by the `I3S specification`_.
 
 dimensions
     Comma-separated list of dimensions that should be read.  Specify the
@@ -76,3 +89,4 @@ min_density and max_density
     Example: ``--readers.slpk.min_density=2 --readers.slpk.max_density=2.5``
 
 .. _Scene Layer Packages (SLPK): https://github.com/Esri/i3s-spec/blob/master/format/Indexed%203d%20Scene%20Layer%20Format%20Specification.md#_8_1
+.. _I3S specification: https://github.com/Esri/i3s-spec/blob/master/docs/2.0/obb.cmn.md

doc/tutorial/iowa-entwine.rst

@@ -110,6 +110,12 @@ it a URL to the root of the resource in the ``filename`` option, and we also
 give it a ``bounds`` object to define the window in which we should select data
 from.
 
+.. note::
+
+    The full URL to the EPT root file (``ept.json``)) must be given
+    to the filename parameter for PDAL 2.2+. This was a change in
+    behavior of the :ref:`readers.ept` driver.
+
 The ``bounds`` object is in the form ``([minx, maxx], [miny, maxy])``.
 
 .. warning::

filters/StatsFilter.cpp

@@ -114,21 +114,60 @@ void Summary::computeGlobalStats()
 {
     auto compute_median = [](std::vector<double> vals)
     {
-        std::nth_element(vals.begin(), vals.begin()+vals.size()/2, vals.end());
-
-        return *(vals.begin()+vals.size()/2);
+        std::nth_element(vals.begin(), vals.begin() + vals.size() / 2, vals.end());
+        return *(vals.begin() + vals.size() / 2);
     };
 
     // TODO add quantiles
     m_median = compute_median(m_data);
     std::transform(m_data.begin(), m_data.end(), m_data.begin(),
        [this](double v) { return std::fabs(v - this->m_median); });
     m_mad = compute_median(m_data);
+}
 
+// Math comes from https://prod.sandia.gov/techlib-noauth/access-control.cgi/2008/086212.pdf
+// (Pebay paper from Sandia labs, 2008)
+bool Summary::merge(const Summary& s)
+{
+    if ((m_name != s.m_name) || (m_enumerate != s.m_enumerate) || (m_advanced != s.m_advanced))
+        return false;
+
+    double n1 = m_cnt;
+    double n2 = s.m_cnt;
+    double n = n1 + n2;
+    double nsq = n * n;
+    double n1n2 = m_cnt * s.m_cnt;
+    double n1sq = n1 * n1;
+    double n2sq = n2 * n2;
+    double ncube = n * n * n;
+    double deltaMean = s.M1 - M1;
+
+    if (n == 0)
+        return true;
+
+    double m1 = M1 + s.m_cnt * deltaMean / n;
+    double m2 = M2 + s.M2 + n1n2 * std::pow(deltaMean, 2) / n;
+    double m3 = M3 + s.M3 + n1n2 * (n1 - n2) * std::pow(deltaMean, 3) / nsq +
+        3 * (n1 * s.M2 - n2 * M2) * deltaMean / n;
+    double m4 = M4 + s.M4 +
+        n1n2 * (n1sq - n1n2 + n2sq) * std::pow(deltaMean, 4) / ncube +
+        6 * (n1sq * s.M2 + n2sq * M2) * std::pow(deltaMean, 2) / nsq +
+        4 * (n1 * s.M3 - n2 * M3) * deltaMean / n;
+
+    M1 = m1;
+    M2 = m2;
+    M3 = m3;
+    M4 = m4;
+    m_min = (std::min)(m_min, s.m_min);
+    m_max = (std::max)(m_max, s.m_max);
+    m_cnt = s.m_cnt + m_cnt;
+    m_data.insert(m_data.begin(), s.m_data.begin(), s.m_data.end());
+    for (auto p : s.m_values)
+        m_values[p.first] += p.second;
 
+    return true;
 }
 
-
 } // namespace stats
 
 using namespace stats;

filters/StatsFilter.hpp

@@ -61,6 +61,9 @@ typedef std::vector<double> DataVector;
         m_name(name), m_enumerate(enumerate), m_advanced(advanced)
     { reset(); }
 
+    // Merge another summary with this one. 'name', 'enumerate' and 'advanced' must match
+    // or false is returned and no merge occurs.
+    bool merge(const Summary& s);
     double minimum() const
         { return m_min; }
     double maximum() const

io/GeotiffSupport.cpp

@@ -39,6 +39,33 @@
 #include <geo_normalize.h>
 #include <geo_simpletags.h>
 
+namespace pdal
+{
+
+
+// Utility functor with accompanying to print GeoTIFF directory.
+struct geotiff_dir_printer
+{
+    geotiff_dir_printer() {}
+
+    std::string output() const { return m_oss.str(); }
+    std::string::size_type size() const { return m_oss.str().size(); }
+
+    void operator()(char* data, void* /*aux*/)
+    {
+
+        if (0 != data)
+        {
+            m_oss << data;
+        }
+    }
+
+private:
+    std::stringstream m_oss;
+};
+
+}
+
 PDAL_C_START
 
 // These functions are available from GDAL, but they
@@ -47,13 +74,19 @@ char PDAL_DLL * GTIFGetOGISDefn(GTIF*, GTIFDefn*);
 int PDAL_DLL GTIFSetFromOGISDefn(GTIF*, const char*);
 void VSIFree(void *data);
 
+int PDALGeoTIFFPrint(char* data, void* aux)
+{
+    pdal::geotiff_dir_printer* printer = reinterpret_cast<pdal::geotiff_dir_printer*>(aux);
+    (*printer)(data, 0);
+    return static_cast<int>(printer->size());
+}
+
 PDAL_C_END
 
 #include <io/LasVLR.hpp>
 
 namespace pdal
 {
-
 namespace
 {
 
@@ -152,6 +185,12 @@ GeotiffSrs::GeotiffSrs(const std::vector<uint8_t>& directoryRec,
             VSIFree(wkt);
         }
     }
+
+    geotiff_dir_printer geotiff_printer;
+    GTIFPrint(ctx.gtiff, PDALGeoTIFFPrint, &geotiff_printer);
+
+    m_gtiff_print_string = geotiff_printer.output();
+
 }
 
 

io/GeotiffSupport.hpp

@@ -59,9 +59,14 @@ class GeotiffSrs
         const std::vector<uint8_t>& asciiRec, LogPtr log);
     SpatialReference srs() const
         { return m_srs; }
+
+    std::string const& gtiffPrintString()
+        { return m_gtiff_print_string; }
+
 private:
     SpatialReference m_srs;
     LogPtr m_log;
+    std::string m_gtiff_print_string;
 
     void validateDirectory(const Entry *ent, size_t numEntries,
         size_t numDoubles, size_t asciiSize);