Add filters.shell that allows you to inline shell commands as part of…

… pipelines (#2430) * fix up my conda build script * add osx conda setup script * Add filters.shell * configuredpath instead of datapath * fix up my windows conda build setup * only allow filters.shell if PDAL_ALLOW_SHELL environment variable is set * add filters.shell doc
PDAL · Jun 13, 2019 · 9cb932b · 9cb932b
1 parent ac997bf
commit 9cb932b
Show file tree

Hide file tree

Showing 6 changed files with 327 additions and 0 deletions.
diff --git a/doc/stages/filters.shell.rst b/doc/stages/filters.shell.rst
@@ -0,0 +1,73 @@
+.. _filters.shell:
+
+filters.shell
+===================
+
+The shell filter allows you to run shell operations in-line 
+with PDAL pipeline tasks. This can be especially useful for 
+follow-on items or orchestration of complex workflows.
+
+.. embed::
+
+.. warning::
+
+	To use :ref:`filters.shell`, you must set ``PDAL_ALLOW_SHELL=1`` 
+	PDAL's execution environment. Without the environment variable 
+	set, every attempt at execution will result in the following 
+	error:
+
+		PDAL_ALLOW_SHELL environment variable not set, shell access is not allowed
+
+Example 
+---------
+
+GDAL processing operations applied to raster output from :ref:`writers.gdal` 
+are a common task. Applying these within the PDAL execution environment 
+can provide some convenience and allow downstream consumers to have deterministic 
+completion status of the task. The following task writes multiple elevation 
+models to disk and then uses the `gdaladdo <https://gdal.org/gdaladdo.html>`__ 
+command to construct overview bands for the data using average interpolation.
+
+.. code-block:: json
+
+    {
+      "pipeline":[
+        "autzen.las",
+        {
+          "type":"writers.gdal",
+          "filename" : "output-1m.tif",
+          "resolution" : "1.0"
+        },
+        {
+          "type":"writers.gdal",
+          "filename" : "output-2m.tif",
+          "resolution" : "2.0"
+        },
+        {
+          "type":"writers.gdal",
+          "filename" : "output-5m.tif",
+          "resolution" : "5.0"
+        },
+        {
+          "type":"filters.shell",
+          "command" : "gdaladdo -r average output-1m.tif 2 4 8 16"
+        },
+        {
+          "type":"filters.shell",
+          "command" : "gdaladdo -r average output-2m.tif 2 4 8 16"
+        }
+        {
+          "type":"filters.shell",
+          "command" : "gdaladdo -r average output-5m.tif 2 4 8 16"
+        }
+    	]
+    }
+
+
+Options
+-------
+
+command
+  The shell command to run. It is run in relation to the current 
+  working directory of the pipeline executing it.
+
diff --git a/filters/ShellFilter.cpp b/filters/ShellFilter.cpp
@@ -0,0 +1,124 @@
+/******************************************************************************
+* Copyright (c) 2019, Howard Butler, howard@hobu.co
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#include "ShellFilter.hpp"
+
+#include <pdal/util/Algorithm.hpp>
+#include <pdal/util/ProgramArgs.hpp>
+
+#include <cstdio>
+#include <iostream>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <array>
+
+
+
+namespace pdal
+{
+
+static StaticPluginInfo const s_info
+{
+    "filters.shell",
+    "Execute a shell operation inline with PDAL pipeline steps",
+    "http://pdal.io/stages/filters.shell.html"
+};
+
+CREATE_STATIC_STAGE(ShellFilter, s_info)
+
+std::string ShellFilter::getName() const { return s_info.name; }
+
+void ShellFilter::addArgs(ProgramArgs& args)
+{
+    args.add("command", "Command to run", m_command).setPositional();
+}
+
+void ShellFilter::initialize()
+{
+	std::string allowed;
+    int set = Utils::getenv("PDAL_ALLOW_SHELL", allowed);
+	if (set == -1)
+		throw pdal::pdal_error("PDAL_ALLOW_SHELL environment variable not set, shell access is not allowed");
+}
+
+
+
+
+
+PointViewSet ShellFilter::run(PointViewPtr view)
+{
+    log()->get(LogLevel::Debug) << "running command : '" << m_command << "'"<< std::endl;
+
+    int status = Utils::run_shell_command(m_command.c_str(), m_command_output);
+    if (status)
+    {
+        std::stringstream msg;
+        msg << "Command '" << m_command << "' failed to execute";
+        msg << " with output '" << m_command_output <<"'";
+        throw pdal::pdal_error(msg.str());
+
+    }
+
+    log()->get(LogLevel::Debug) << "command output: '" << m_command_output << "'"<< std::endl;
+    log()->get(LogLevel::Debug) << "status: '" << status<< "'"<< std::endl;
+
+    PointViewSet views;
+    views.insert(view);
+    return views;
+}
+
+
+
+void ShellFilter::done(PointTableRef table)
+{
+    bool isJson = (m_command_output.find("{") != m_command_output.npos) ||
+                  (m_command_output.find("}") != m_command_output.npos);
+
+    if (isJson)
+        m_metadata.addWithType("output",
+                                m_command_output,
+                                "json",
+                                "Command output");
+    else
+        m_metadata.add("output", m_command_output, "Command output");
+
+}
+
+
+
+
+
+} // namespace pdal
+
diff --git a/filters/ShellFilter.hpp b/filters/ShellFilter.hpp
@@ -0,0 +1,59 @@
+/******************************************************************************
+* Copyright (c) 2019, Howard Butler <hobu.inc@gmail.com>
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#pragma once
+
+#include <pdal/Filter.hpp>
+
+#include <vector>
+#include <string>
+
+namespace pdal
+{
+
+class PDAL_DLL ShellFilter : public Filter
+{
+public:
+    std::string getName() const override;
+    virtual void addArgs(ProgramArgs& args) override;
+    virtual void initialize() override;
+    virtual PointViewSet run(PointViewPtr view) override;
+
+private:
+    std::string m_command;
+    std::string m_command_output;
+    virtual void done(PointTableRef table) override;
+};
+
+} // namespace pdal
diff --git a/test/data/pipeline/shell.json.in b/test/data/pipeline/shell.json.in
@@ -0,0 +1,10 @@
+{
+  "pipeline":[
+    "@CMAKE_SOURCE_DIR@/test/data/autzen/autzen.las",
+    {
+      "type":"filters.shell",
+      "command":"gdalinfo -json @CMAKE_SOURCE_DIR@/test/data/gdal/int16.tif"
+    },
+    "@CMAKE_SOURCE_DIR@/test/temp/junk.las"
+  ]
+}
diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt
@@ -247,6 +247,8 @@ PDAL_ADD_TEST(pdal_filters_reprojection_test FILES
 PDAL_ADD_TEST(pdal_filters_range_test FILES filters/RangeFilterTest.cpp)
 PDAL_ADD_TEST(pdal_filters_randomize_test FILES filters/RandomizeFilterTest.cpp)
 PDAL_ADD_TEST(pdal_filters_returns_test FILES filters/ReturnsFilterTest.cpp)
+PDAL_ADD_TEST(pdal_filters_shell_test FILES filters/ShellFilterTest.cpp)
+
 PDAL_ADD_TEST(pdal_filters_smrf_test FILES filters/SMRFilterTest.cpp)
 PDAL_ADD_TEST(pdal_filters_sort_test
     FILES

diff --git a/test/unit/filters/ShellFilterTest.cpp b/test/unit/filters/ShellFilterTest.cpp
@@ -0,0 +1,59 @@
+/******************************************************************************
+* Copyright (c) 2018, Hobu Inc. (hobu@hobu.co)
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#include <pdal/pdal_test_main.hpp>
+
+#include <filters/ShellFilter.hpp>
+#include <pdal/StageFactory.hpp>
+
+#include "Support.hpp"
+
+using namespace pdal;
+
+TEST(ShellFilterTest, test_shell_filter)
+{
+    PipelineManager mgr;
+	Utils::setenv("PDAL_ALLOW_SHELL", "1");
+    mgr.readPipeline(Support::configuredpath("pipeline/shell.json"));
+
+    mgr.execute();
+    ConstPointTableRef table(mgr.pointTable());
+
+    PointViewSet viewSet = mgr.views();
+
+    EXPECT_EQ(viewSet.size(), 1u);
+    PointViewPtr view = *viewSet.begin();
+    EXPECT_EQ(view->size(), 106u);
+
+}