Merge remote-tracking branch 'origin/master' into issue-2513

apps/pdal.cpp

@@ -161,18 +161,20 @@ void App::outputDrivers()
     else
     {
         NL::json j;
+        StageExtensions& extensions = PluginManager<Stage>::extensions();
         for (auto name : stages)
         {
             std::string description = PluginManager<Stage>::description(name);
             std::string link = PluginManager<Stage>::link(name);
             j.push_back(
-                { {"name", name},
-                  {"description", description},
-                  {"link", link}
+                { { "name", name },
+                  { "description", description },
+                  { "link", link },
+                  { "extensions", extensions.extensions(name) }
                 }
             );
         }
-        m_out << j;
+        m_out << std::setw(4) << j;
     }
 }
 

cmake/policies.cmake

@@ -4,5 +4,8 @@
 if (CMAKE_MAJOR_VERSION GREATER 2)
     cmake_policy(SET CMP0022 NEW) # interface link libraries
     cmake_policy(SET CMP0042 NEW) # osx rpath
-    cmake_policy(SET CMP0075 NEW) # Mystical setting to eliminate warning
+    if (CMAKE_MAJOR_VERSION GREATER 3 AND
+        CMAKE_MINOR_VERSION GREATER 12)
+        cmake_policy(SET CMP0075 NEW) # Mystical setting to eliminate warning
+    endif()
 endif()

doc/stages/filters.shell.rst

@@ -0,0 +1,73 @@
+.. _filters.shell:
+
+filters.shell
+===================
+
+The shell filter allows you to run shell operations in-line 
+with PDAL pipeline tasks. This can be especially useful for 
+follow-on items or orchestration of complex workflows.
+
+.. embed::
+
+.. warning::
+
+	To use :ref:`filters.shell`, you must set ``PDAL_ALLOW_SHELL=1`` 
+	PDAL's execution environment. Without the environment variable 
+	set, every attempt at execution will result in the following 
+	error:
+
+		PDAL_ALLOW_SHELL environment variable not set, shell access is not allowed
+
+Example 
+---------
+
+GDAL processing operations applied to raster output from :ref:`writers.gdal` 
+are a common task. Applying these within the PDAL execution environment 
+can provide some convenience and allow downstream consumers to have deterministic 
+completion status of the task. The following task writes multiple elevation 
+models to disk and then uses the `gdaladdo <https://gdal.org/gdaladdo.html>`__ 
+command to construct overview bands for the data using average interpolation.
+
+.. code-block:: json
+
+    {
+      "pipeline":[
+        "autzen.las",
+        {
+          "type":"writers.gdal",
+          "filename" : "output-1m.tif",
+          "resolution" : "1.0"
+        },
+        {
+          "type":"writers.gdal",
+          "filename" : "output-2m.tif",
+          "resolution" : "2.0"
+        },
+        {
+          "type":"writers.gdal",
+          "filename" : "output-5m.tif",
+          "resolution" : "5.0"
+        },
+        {
+          "type":"filters.shell",
+          "command" : "gdaladdo -r average output-1m.tif 2 4 8 16"
+        },
+        {
+          "type":"filters.shell",
+          "command" : "gdaladdo -r average output-2m.tif 2 4 8 16"
+        }
+        {
+          "type":"filters.shell",
+          "command" : "gdaladdo -r average output-5m.tif 2 4 8 16"
+        }
+    	]
+    }
+
+
+Options
+-------
+
+command
+  The shell command to run. It is run in relation to the current 
+  working directory of the pipeline executing it.
+

doc/stages/writers.tiledb.rst

@@ -56,6 +56,9 @@ compression
 compression_level
   TileDB compression level for chosen compression [Optional]
 
+append
+  Append to an existing TileDB array with the same schema [Optional]
+
 stats
   Dump query stats to stdout [Optional]
 

filters/ShellFilter.cpp

@@ -0,0 +1,124 @@
+/******************************************************************************
+* Copyright (c) 2019, Howard Butler, howard@hobu.co
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#include "ShellFilter.hpp"
+
+#include <pdal/util/Algorithm.hpp>
+#include <pdal/util/ProgramArgs.hpp>
+
+#include <cstdio>
+#include <iostream>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <array>
+
+
+
+namespace pdal
+{
+
+static StaticPluginInfo const s_info
+{
+    "filters.shell",
+    "Execute a shell operation inline with PDAL pipeline steps",
+    "http://pdal.io/stages/filters.shell.html"
+};
+
+CREATE_STATIC_STAGE(ShellFilter, s_info)
+
+std::string ShellFilter::getName() const { return s_info.name; }
+
+void ShellFilter::addArgs(ProgramArgs& args)
+{
+    args.add("command", "Command to run", m_command).setPositional();
+}
+
+void ShellFilter::initialize()
+{
+	std::string allowed;
+    int set = Utils::getenv("PDAL_ALLOW_SHELL", allowed);
+	if (set == -1)
+		throw pdal::pdal_error("PDAL_ALLOW_SHELL environment variable not set, shell access is not allowed");
+}
+
+
+
+
+
+PointViewSet ShellFilter::run(PointViewPtr view)
+{
+    log()->get(LogLevel::Debug) << "running command : '" << m_command << "'"<< std::endl;
+
+    int status = Utils::run_shell_command(m_command.c_str(), m_command_output);
+    if (status)
+    {
+        std::stringstream msg;
+        msg << "Command '" << m_command << "' failed to execute";
+        msg << " with output '" << m_command_output <<"'";
+        throw pdal::pdal_error(msg.str());
+
+    }
+
+    log()->get(LogLevel::Debug) << "command output: '" << m_command_output << "'"<< std::endl;
+    log()->get(LogLevel::Debug) << "status: '" << status<< "'"<< std::endl;
+
+    PointViewSet views;
+    views.insert(view);
+    return views;
+}
+
+
+
+void ShellFilter::done(PointTableRef table)
+{
+    bool isJson = (m_command_output.find("{") != m_command_output.npos) ||
+                  (m_command_output.find("}") != m_command_output.npos);
+
+    if (isJson)
+        m_metadata.addWithType("output",
+                                m_command_output,
+                                "json",
+                                "Command output");
+    else
+        m_metadata.add("output", m_command_output, "Command output");
+
+}
+
+
+
+
+
+} // namespace pdal
+

filters/ShellFilter.hpp

@@ -0,0 +1,59 @@
+/******************************************************************************
+* Copyright (c) 2019, Howard Butler <hobu.inc@gmail.com>
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#pragma once
+
+#include <pdal/Filter.hpp>
+
+#include <vector>
+#include <string>
+
+namespace pdal
+{
+
+class PDAL_DLL ShellFilter : public Filter
+{
+public:
+    std::string getName() const override;
+    virtual void addArgs(ProgramArgs& args) override;
+    virtual void initialize() override;
+    virtual PointViewSet run(PointViewPtr view) override;
+
+private:
+    std::string m_command;
+    std::string m_command_output;
+    virtual void done(PointTableRef table) override;
+};
+
+} // namespace pdal