From 389ae91de0fac67064eccaf83417263edc157c7e Mon Sep 17 00:00:00 2001 From: Bradley J Chambers Date: Thu, 1 Sep 2016 13:36:04 -0400 Subject: [PATCH 1/3] Add MAD and IQR filters with user-selectable dimension and multiplier --- doc/stages/filters.iqr.rst | 51 +++++++++++++++ doc/stages/filters.mad.rst | 47 ++++++++++++++ filters/CMakeLists.txt | 2 + filters/iqr/CMakeLists.txt | 2 + filters/iqr/IQRFilter.cpp | 124 +++++++++++++++++++++++++++++++++++++ filters/iqr/IQRFilter.hpp | 75 ++++++++++++++++++++++ filters/mad/CMakeLists.txt | 2 + filters/mad/MADFilter.cpp | 124 +++++++++++++++++++++++++++++++++++++ filters/mad/MADFilter.hpp | 75 ++++++++++++++++++++++ src/StageFactory.cpp | 4 ++ 10 files changed, 506 insertions(+) create mode 100644 doc/stages/filters.iqr.rst create mode 100644 doc/stages/filters.mad.rst create mode 100644 filters/iqr/CMakeLists.txt create mode 100644 filters/iqr/IQRFilter.cpp create mode 100644 filters/iqr/IQRFilter.hpp create mode 100644 filters/mad/CMakeLists.txt create mode 100644 filters/mad/MADFilter.cpp create mode 100644 filters/mad/MADFilter.hpp diff --git a/doc/stages/filters.iqr.rst b/doc/stages/filters.iqr.rst new file mode 100644 index 0000000000..ba34fe7c78 --- /dev/null +++ b/doc/stages/filters.iqr.rst @@ -0,0 +1,51 @@ +.. _filters.iqr: + +filters.iqr +=============================================================================== + +The ``filters.iqr`` filter automatically crops the input point cloud based on +the distribution of points in the specified dimension. Specifically, we choose +the method of Interquartile Range (IQR). The IQR is defined as the range between +the first and third quartile (25th and 75th percentile). Upper and lower bounds +are determined by adding 1.5 times the IQR to the third quartile or subtracting +1.5 times the IQR from the first quartile. The multiplier, which defaults to +1.5, can be adjusted by the user. + +.. note:: + + This method can remove real data, especially ridges and valleys in rugged + terrain, or tall features such as towers and rooftops in flat terrain. While + the number of deviations can be adjusted to account for such content-specific + considerations, it must be used with care. + +Example +------- + +The sample pipeline below uses ``filters.iqr`` to automatically crop the Z +dimension and remove possible outliers. The multiplier to determine high/low +thresholds has been adjusted to be less agressive and to only crop those +outliers that are greater than the third quartile plus 3 times the IQR or are +less than the first quartile minus 3 times the IQR. + +.. code-block:: json + + { + "pipeline":[ + "input.las", + { + "type":"filters.iqr", + "dimension":"Z", + "k":3.0 + }, + "output.laz" + ] + } + +Options +------------------------------------------------------------------------------- + +k + The IQR multiplier used to determine upper/lower bounds. [Default: **1.5**] + +dimension + The name of the dimension to filter. diff --git a/doc/stages/filters.mad.rst b/doc/stages/filters.mad.rst new file mode 100644 index 0000000000..e9bbcf7da6 --- /dev/null +++ b/doc/stages/filters.mad.rst @@ -0,0 +1,47 @@ +.. _filters.mad: + +filters.mad +=============================================================================== + +The ``filters.mad`` filter automatically crops the input point cloud based on +the distribution of points in the specified dimension. Specifically, we choose +the method of median absolute deviation from the median (commonly referred to as +MAD), which is robust to outliers (as opposed to mean and standard deviation). + +.. note:: + + This method can remove real data, especially ridges and valleys in rugged + terrain, or tall features such as towers and rooftops in flat terrain. While + the number of deviations can be adjusted to account for such content-specific + considerations, it must be used with care. + +Example +------- + +The sample pipeline below uses ``filters.mad`` to automatically crop the Z +dimension and remove possible outliers. The number of deviations from the median +has been adjusted to be less agressive and to only crop those outliers that are +greater than four deviations from the median. + +.. code-block:: json + + { + "pipeline":[ + "input.las", + { + "type":"filters.mad", + "dimension":"Z", + "k":4.0 + }, + "output.laz" + ] + } + +Options +------------------------------------------------------------------------------- + +k + The number of deviations from the median. [Default: **2.0**] + +dimension + The name of the dimension to filter. diff --git a/filters/CMakeLists.txt b/filters/CMakeLists.txt index 78d0b25f6f..67473ed9b5 100644 --- a/filters/CMakeLists.txt +++ b/filters/CMakeLists.txt @@ -9,6 +9,8 @@ add_subdirectory(eigenvalues) add_subdirectory(estimaterank) add_subdirectory(ferry) add_subdirectory(hag) +add_subdirectory(iqr) +add_subdirectory(mad) add_subdirectory(merge) add_subdirectory(mongus) add_subdirectory(mortonorder) diff --git a/filters/iqr/CMakeLists.txt b/filters/iqr/CMakeLists.txt new file mode 100644 index 0000000000..5a1c3ae01a --- /dev/null +++ b/filters/iqr/CMakeLists.txt @@ -0,0 +1,2 @@ +PDAL_ADD_DRIVER(filter iqr "IQRFilter.cpp" "IQRFilter.hpp" objects) +set(PDAL_TARGET_OBJECTS ${PDAL_TARGET_OBJECTS} ${objects} PARENT_SCOPE) diff --git a/filters/iqr/IQRFilter.cpp b/filters/iqr/IQRFilter.cpp new file mode 100644 index 0000000000..a2952abd8c --- /dev/null +++ b/filters/iqr/IQRFilter.cpp @@ -0,0 +1,124 @@ +/****************************************************************************** +* Copyright (c) 2016, Bradley J Chambers (brad.chambers@gmail.com) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include "IQRFilter.hpp" + +#include + +#include +#include + +namespace pdal +{ + +static PluginInfo const s_info = + PluginInfo("filters.iqr", "Interquartile Range Filter", + "http://pdal.io/stages/filters.iqr.html"); + +CREATE_STATIC_PLUGIN(1, 0, IQRFilter, Filter, s_info) + +std::string IQRFilter::getName() const +{ + return s_info.name; +} + +void IQRFilter::addArgs(ProgramArgs& args) +{ + args.add("k", "Number of deviations", m_multiplier, 1.5); + args.add("dimension", "Dimension on which to calculate statistics", + m_dimName); +} + +// ready or prepared to make sure dim exists? does addArgs come first? (i think so) +void IQRFilter::prepared(PointTableRef table) +{ + PointLayoutPtr layout(table.layout()); + m_dimId = layout->findDim(m_dimName); + if (m_dimId == Dimension::Id::Unknown) + { + std::ostringstream oss; + oss << "Invalid dimension name in filters.iqr 'dimension' " + "option: '" << m_dimName << "'."; + throw pdal_error(oss.str()); + } +} + +PointViewSet IQRFilter::run(PointViewPtr view) +{ + using namespace Dimension; + + PointViewSet viewSet; + PointViewPtr output = view->makeNew(); + + auto quartile = [](std::vector vals, double percent) + { + std::nth_element(vals.begin(), vals.begin()+int(vals.size()*percent), vals.end()); + + return *(vals.begin()+int(vals.size()*percent)); + }; + + std::vector z(view->size()); + for (PointId j = 0; j < view->size(); ++j) + z[j] = view->getFieldAs(m_dimId, j); + + + double pc25 = quartile(z, 0.25); + log()->get(LogLevel::Debug) << "25th percentile: " << pc25 << std::endl; + + double pc75 = quartile(z, 0.75); + log()->get(LogLevel::Debug) << "75th percentile: " << pc75 << std::endl; + + double iqr = pc75-pc25; + log()->get(LogLevel::Debug) << "IQR: " << iqr << std::endl; + + double low_fence = pc25 - m_multiplier * iqr; + double hi_fence = pc75 + m_multiplier * iqr; + + for (PointId j = 0; j < view->size(); ++j) + { + double val = view->getFieldAs(m_dimId, j); + if (val > low_fence && val < hi_fence) + output->appendPoint(*view, j); + } + log()->get(LogLevel::Debug) << "Cropping " << m_dimName + << " in the range (" << low_fence + << "," << hi_fence << ")" << std::endl; + + viewSet.erase(view); + viewSet.insert(output); + + return viewSet; +} + +} // namespace pdal diff --git a/filters/iqr/IQRFilter.hpp b/filters/iqr/IQRFilter.hpp new file mode 100644 index 0000000000..03e9c54d37 --- /dev/null +++ b/filters/iqr/IQRFilter.hpp @@ -0,0 +1,75 @@ +/****************************************************************************** +* Copyright (c) 2016, Bradley J Chambers (brad.chambers@gmail.com) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#pragma once + +#include +#include + +#include + +extern "C" int32_t IQRFilter_ExitFunc(); +extern "C" PF_ExitFunc IQRFilter_InitPlugin(); + +namespace pdal +{ + +class ProgramArgs; +class PointTable; +class PointView; + +class PDAL_DLL IQRFilter : public Filter +{ +public: + IQRFilter() : Filter() + {} + + static void * create(); + static int32_t destroy(void *); + std::string getName() const; + +private: + double m_multiplier; + std::string m_dimName; + Dimension::Id m_dimId; + + virtual void addArgs(ProgramArgs& args); + virtual void prepared(PointTableRef table); + virtual PointViewSet run(PointViewPtr view); + + IQRFilter& operator=(const IQRFilter&); // not implemented + IQRFilter(const IQRFilter&); // not implemented +}; + +} // namespace pdal diff --git a/filters/mad/CMakeLists.txt b/filters/mad/CMakeLists.txt new file mode 100644 index 0000000000..c695efb363 --- /dev/null +++ b/filters/mad/CMakeLists.txt @@ -0,0 +1,2 @@ +PDAL_ADD_DRIVER(filter mad "MADFilter.cpp" "MADFilter.hpp" objects) +set(PDAL_TARGET_OBJECTS ${PDAL_TARGET_OBJECTS} ${objects} PARENT_SCOPE) diff --git a/filters/mad/MADFilter.cpp b/filters/mad/MADFilter.cpp new file mode 100644 index 0000000000..2a95acacac --- /dev/null +++ b/filters/mad/MADFilter.cpp @@ -0,0 +1,124 @@ +/****************************************************************************** +* Copyright (c) 2016, Bradley J Chambers (brad.chambers@gmail.com) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include "MADFilter.hpp" + +#include + +#include +#include + +namespace pdal +{ + +static PluginInfo const s_info = + PluginInfo("filters.mad", "Median Absolute Deviation Filter", + "http://pdal.io/stages/filters.mad.html"); + +CREATE_STATIC_PLUGIN(1, 0, MADFilter, Filter, s_info) + +std::string MADFilter::getName() const +{ + return s_info.name; +} + +void MADFilter::addArgs(ProgramArgs& args) +{ + args.add("k", "Number of deviations", m_multiplier, 2.0); + args.add("dimension", "Dimension on which to calculate statistics", + m_dimName); +} + +// ready or prepared to make sure dim exists? does addArgs come first? (i think so) +void MADFilter::prepared(PointTableRef table) +{ + PointLayoutPtr layout(table.layout()); + m_dimId = layout->findDim(m_dimName); + if (m_dimId == Dimension::Id::Unknown) + { + std::ostringstream oss; + oss << "Invalid dimension name in filters.mad 'dimension' " + "option: '" << m_dimName << "'."; + throw pdal_error(oss.str()); + } +} + +PointViewSet MADFilter::run(PointViewPtr view) +{ + using namespace Dimension; + + PointViewSet viewSet; + PointViewPtr output = view->makeNew(); + + auto median = [](std::vector vals) + { + std::nth_element(vals.begin(), vals.begin()+vals.size()/2, vals.end()); + + return *(vals.begin()+vals.size()/2); + }; + + std::vector z(view->size()); + for (PointId j = 0; j < view->size(); ++j) + z[j] = view->getFieldAs(m_dimId, j); + + double m = median(z); + log()->get(LogLevel::Debug) << "Median value: " << m << std::endl; + + std::vector a(view->size()); + for (PointId j = 0; j < view->size(); ++j) + a[j] = std::fabs(view->getFieldAs(Id::Z, j)-m); + + double mad = median(a)*1.4862; + log()->get(LogLevel::Debug) << "MAD: " << mad << std::endl; + + for (PointId j = 0; j < view->size(); ++j) + { + if (a[j]/mad < m_multiplier) + output->appendPoint(*view, j); + } + + double low_fence = m - m_multiplier * mad; + double hi_fence = m + m_multiplier * mad; + + log()->get(LogLevel::Debug) << "Cropping " << m_dimName + << " in the range (" << low_fence + << "," << hi_fence << ")" << std::endl; + + viewSet.erase(view); + viewSet.insert(output); + + return viewSet; +} + +} // namespace pdal diff --git a/filters/mad/MADFilter.hpp b/filters/mad/MADFilter.hpp new file mode 100644 index 0000000000..eee699810a --- /dev/null +++ b/filters/mad/MADFilter.hpp @@ -0,0 +1,75 @@ +/****************************************************************************** +* Copyright (c) 2016, Bradley J Chambers (brad.chambers@gmail.com) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#pragma once + +#include +#include + +#include + +extern "C" int32_t MADFilter_ExitFunc(); +extern "C" PF_ExitFunc MADFilter_InitPlugin(); + +namespace pdal +{ + +class ProgramArgs; +class PointTable; +class PointView; + +class PDAL_DLL MADFilter : public Filter +{ +public: + MADFilter() : Filter() + {} + + static void * create(); + static int32_t destroy(void *); + std::string getName() const; + +private: + double m_multiplier; + std::string m_dimName; + Dimension::Id m_dimId; + + virtual void addArgs(ProgramArgs& args); + virtual void prepared(PointTableRef table); + virtual PointViewSet run(PointViewPtr view); + + MADFilter& operator=(const MADFilter&); // not implemented + MADFilter(const MADFilter&); // not implemented +}; + +} // namespace pdal diff --git a/src/StageFactory.cpp b/src/StageFactory.cpp index ad4d1f2740..289feeecc6 100644 --- a/src/StageFactory.cpp +++ b/src/StageFactory.cpp @@ -48,6 +48,8 @@ #include #include #include +#include +#include #include #include #include @@ -235,6 +237,8 @@ StageFactory::StageFactory(bool no_plugins) PluginManager::initializePlugin(EstimateRankFilter_InitPlugin); PluginManager::initializePlugin(FerryFilter_InitPlugin); PluginManager::initializePlugin(HAGFilter_InitPlugin); + PluginManager::initializePlugin(IQRFilter_InitPlugin); + PluginManager::initializePlugin(MADFilter_InitPlugin); PluginManager::initializePlugin(MergeFilter_InitPlugin); PluginManager::initializePlugin(MongusFilter_InitPlugin); PluginManager::initializePlugin(MortonOrderFilter_InitPlugin); From 38b45cd569ba576acb75e99218e876636a9d4359 Mon Sep 17 00:00:00 2001 From: Bradley J Chambers Date: Thu, 1 Sep 2016 15:19:25 -0400 Subject: [PATCH 2/3] Remove old comment to self --- filters/iqr/IQRFilter.cpp | 1 - filters/mad/MADFilter.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/filters/iqr/IQRFilter.cpp b/filters/iqr/IQRFilter.cpp index a2952abd8c..de47a24ce2 100644 --- a/filters/iqr/IQRFilter.cpp +++ b/filters/iqr/IQRFilter.cpp @@ -60,7 +60,6 @@ void IQRFilter::addArgs(ProgramArgs& args) m_dimName); } -// ready or prepared to make sure dim exists? does addArgs come first? (i think so) void IQRFilter::prepared(PointTableRef table) { PointLayoutPtr layout(table.layout()); diff --git a/filters/mad/MADFilter.cpp b/filters/mad/MADFilter.cpp index 2a95acacac..59d79dfb1a 100644 --- a/filters/mad/MADFilter.cpp +++ b/filters/mad/MADFilter.cpp @@ -60,7 +60,6 @@ void MADFilter::addArgs(ProgramArgs& args) m_dimName); } -// ready or prepared to make sure dim exists? does addArgs come first? (i think so) void MADFilter::prepared(PointTableRef table) { PointLayoutPtr layout(table.layout()); From 6ce56dd9c01e6dd4f609df41bc807d9bdb66ad0f Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 2 Sep 2016 12:25:04 -0500 Subject: [PATCH 3/3] Push log leader for stages. Remove unused headers from Log.hpp. --- examples/writing-writer/MyWriter.cpp | 1 + include/pdal/Log.hpp | 31 ++++++++++++---- include/pdal/Stage.hpp | 14 +++++++ io/bpf/BpfWriter.cpp | 1 + io/las/LasWriter.cpp | 1 + io/sbet/SbetReader.cpp | 1 + plugins/cpd/test/CpdKernelTest.cpp | 1 + plugins/hexbin/kernel/DensityKernel.cpp | 1 + plugins/hexbin/test/HexbinFilterTest.cpp | 1 + .../icebridge/test/IcebridgeReaderTest.cpp | 1 + plugins/nitf/io/NitfReader.hpp | 1 + plugins/nitf/test/NitfWriterTest.cpp | 1 + plugins/oci/io/OciReader.cpp | 1 + plugins/p2g/io/P2gWriter.cpp | 1 + plugins/python/filters/PredicateFilter.cpp | 1 + src/Log.cpp | 6 +-- src/PipelineManager.cpp | 1 + src/Stage.cpp | 37 ++++++++++++++++--- test/unit/io/sbet/SbetReaderTest.cpp | 1 + test/unit/io/sbet/SbetWriterTest.cpp | 1 + 20 files changed, 88 insertions(+), 16 deletions(-) diff --git a/examples/writing-writer/MyWriter.cpp b/examples/writing-writer/MyWriter.cpp index ca83b6aa52..dc97d0bab7 100644 --- a/examples/writing-writer/MyWriter.cpp +++ b/examples/writing-writer/MyWriter.cpp @@ -2,6 +2,7 @@ #include "MyWriter.hpp" #include +#include #include namespace pdal diff --git a/include/pdal/Log.hpp b/include/pdal/Log.hpp index 5b8edf6194..287bf51ca5 100644 --- a/include/pdal/Log.hpp +++ b/include/pdal/Log.hpp @@ -34,18 +34,16 @@ #pragma once -#include +#include // shared_ptr +#include -#include -#include -#include +#include // Adapted from http://drdobbs.com/cpp/201804215 namespace pdal { - /// pdal::Log is a logging object that is provided by pdal::Stage to /// facilitate logging operations. class PDAL_DLL Log @@ -85,10 +83,27 @@ class PDAL_DLL Log m_level = v; } - /// Set the leader string. + /// Set the leader string (deprecated). /// \param[in] leader Leader string. void setLeader(const std::string& leader) - { m_leader = leader; } + { pushLeader(leader); } + + /// Push the leader string onto the stack. + /// \param leader Leader string + void pushLeader(const std::string& leader) + { m_leaders.push(leader); } + + /// Get the leader string. + /// \return The current leader string. + std::string leader() const + { return m_leaders.empty() ? std::string() : m_leaders.top(); } + + /// Pop the current leader string. + void popLeader() + { + if (!m_leaders.empty()) + m_leaders.pop(); + } /// @return A string representing the LogLevel std::string getLevelString(LogLevel v) const; @@ -126,7 +141,7 @@ class PDAL_DLL Log LogLevel m_level; bool m_deleteStreamOnCleanup; - std::string m_leader; + std::stack m_leaders; }; typedef std::shared_ptr LogPtr; diff --git a/include/pdal/Stage.hpp b/include/pdal/Stage.hpp index 7bc45da09a..bb40116b56 100644 --- a/include/pdal/Stage.hpp +++ b/include/pdal/Stage.hpp @@ -225,6 +225,18 @@ class PDAL_DLL Stage virtual LogPtr log() const { return m_log; } + /** + Push the stage's leader into the log. + */ + void pushLogLeader() const + { m_log->pushLeader(m_logLeader); } + + /** + Pop the stage's leader from the log. + */ + void popLogLeader() const + { m_log->popLeader(); } + /** Determine whether the stage is in debug mode or not. @@ -293,6 +305,7 @@ class PDAL_DLL Stage std::string m_logname; std::vector m_inputs; LogPtr m_log; + std::string m_logLeader; SpatialReference m_spatialReference; std::unique_ptr m_args; @@ -305,6 +318,7 @@ class PDAL_DLL Stage virtual void readerAddArgs(ProgramArgs& /*args*/) {} void l_addArgs(ProgramArgs& args); + void l_done(PointTableRef table); virtual void writerInitialize(PointTableRef /*table*/) {} diff --git a/io/bpf/BpfWriter.cpp b/io/bpf/BpfWriter.cpp index f7e4506ef5..61c440fa92 100644 --- a/io/bpf/BpfWriter.cpp +++ b/io/bpf/BpfWriter.cpp @@ -38,6 +38,7 @@ #include #include +#include #include #include diff --git a/io/las/LasWriter.cpp b/io/las/LasWriter.cpp index 233235da33..4704053ed7 100644 --- a/io/las/LasWriter.cpp +++ b/io/las/LasWriter.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include diff --git a/io/sbet/SbetReader.cpp b/io/sbet/SbetReader.cpp index 21cd0fe642..371ccdd54d 100644 --- a/io/sbet/SbetReader.cpp +++ b/io/sbet/SbetReader.cpp @@ -37,6 +37,7 @@ #include "SbetReader.hpp" #include +#include namespace pdal { diff --git a/plugins/cpd/test/CpdKernelTest.cpp b/plugins/cpd/test/CpdKernelTest.cpp index 44296779b1..dedaa63dbb 100644 --- a/plugins/cpd/test/CpdKernelTest.cpp +++ b/plugins/cpd/test/CpdKernelTest.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include "LasReader.hpp" #include "Support.hpp" diff --git a/plugins/hexbin/kernel/DensityKernel.cpp b/plugins/hexbin/kernel/DensityKernel.cpp index 0b0dd74e61..2b7cca47ab 100644 --- a/plugins/hexbin/kernel/DensityKernel.cpp +++ b/plugins/hexbin/kernel/DensityKernel.cpp @@ -39,6 +39,7 @@ #include #include #include +#include namespace pdal { diff --git a/plugins/hexbin/test/HexbinFilterTest.cpp b/plugins/hexbin/test/HexbinFilterTest.cpp index c3473ea5aa..63c26e528e 100644 --- a/plugins/hexbin/test/HexbinFilterTest.cpp +++ b/plugins/hexbin/test/HexbinFilterTest.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include "Support.hpp" diff --git a/plugins/icebridge/test/IcebridgeReaderTest.cpp b/plugins/icebridge/test/IcebridgeReaderTest.cpp index 34ac7505ab..a8d4173bb3 100644 --- a/plugins/icebridge/test/IcebridgeReaderTest.cpp +++ b/plugins/icebridge/test/IcebridgeReaderTest.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include "Support.hpp" diff --git a/plugins/nitf/io/NitfReader.hpp b/plugins/nitf/io/NitfReader.hpp index b9a25d8071..17737c3924 100644 --- a/plugins/nitf/io/NitfReader.hpp +++ b/plugins/nitf/io/NitfReader.hpp @@ -39,6 +39,7 @@ #include #include +#include namespace pdal { diff --git a/plugins/nitf/test/NitfWriterTest.cpp b/plugins/nitf/test/NitfWriterTest.cpp index 8900ac5d38..872990fdd6 100644 --- a/plugins/nitf/test/NitfWriterTest.cpp +++ b/plugins/nitf/test/NitfWriterTest.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include "Support.hpp" using namespace pdal; diff --git a/plugins/oci/io/OciReader.cpp b/plugins/oci/io/OciReader.cpp index 733df32f85..54dc996bfb 100644 --- a/plugins/oci/io/OciReader.cpp +++ b/plugins/oci/io/OciReader.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include "OciReader.hpp" diff --git a/plugins/p2g/io/P2gWriter.cpp b/plugins/p2g/io/P2gWriter.cpp index 4a10c6050e..56b3ff2e4d 100644 --- a/plugins/p2g/io/P2gWriter.cpp +++ b/plugins/p2g/io/P2gWriter.cpp @@ -35,6 +35,7 @@ #include "P2gWriter.hpp" #include #include +#include #include #include diff --git a/plugins/python/filters/PredicateFilter.cpp b/plugins/python/filters/PredicateFilter.cpp index 53b9f32d3c..f7a133afdf 100644 --- a/plugins/python/filters/PredicateFilter.cpp +++ b/plugins/python/filters/PredicateFilter.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include namespace pdal diff --git a/src/Log.cpp b/src/Log.cpp index 7ef176c0d3..f02982d6a4 100644 --- a/src/Log.cpp +++ b/src/Log.cpp @@ -45,7 +45,6 @@ Log::Log(std::string const& leaderString, std::string const& outputName) : m_level(LogLevel::Error) , m_deleteStreamOnCleanup(false) - , m_leader(leaderString) { makeNullStream(); @@ -62,6 +61,7 @@ Log::Log(std::string const& leaderString, m_log = Utils::createFile(outputName); m_deleteStreamOnCleanup = true; } + m_leaders.push(leaderString); } @@ -69,10 +69,10 @@ Log::Log(std::string const& leaderString, std::ostream* v) : m_level(LogLevel::Error) , m_deleteStreamOnCleanup(false) - , m_leader(leaderString) { m_log = v; makeNullStream(); + m_leaders.push(leaderString); } @@ -121,7 +121,7 @@ std::ostream& Log::get(LogLevel level) const auto nativeDebug(Utils::toNative(LogLevel::Debug)); if (incoming <= stored) { - *m_log << "(" << m_leader << " "<< getLevelString(level) <<": " << + *m_log << "(" << leader() << " "<< getLevelString(level) <<": " << incoming << "): " << std::string(incoming < nativeDebug ? 0 : incoming - nativeDebug, '\t'); diff --git a/src/PipelineManager.cpp b/src/PipelineManager.cpp index 0da337def5..e9f7f8962c 100644 --- a/src/PipelineManager.cpp +++ b/src/PipelineManager.cpp @@ -34,6 +34,7 @@ #include #include +#include #include "PipelineReaderXML.hpp" #include "PipelineReaderJSON.hpp" diff --git a/src/Stage.cpp b/src/Stage.cpp index a11fd085d8..0978fa5cf9 100644 --- a/src/Stage.cpp +++ b/src/Stage.cpp @@ -113,7 +113,10 @@ QuickInfo Stage::preview() { m_args.reset(new ProgramArgs); handleOptions(); - return inspect(); + pushLogLeader(); + QuickInfo qi = inspect(); + popLogLeader(); + return qi; } @@ -126,15 +129,18 @@ void Stage::prepare(PointTableRef table) prev->prepare(table); } handleOptions(); + pushLogLeader(); l_initialize(table); initialize(table); addDimensions(table.layout()); prepared(table); + popLogLeader(); } PointViewSet Stage::execute(PointTableRef table) { + pushLogLeader(); table.finalize(); PointViewSet views; @@ -194,7 +200,8 @@ PointViewSet Stage::execute(PointTableRef table) v->setSpatialReference(srs); outViews.insert(temp.begin(), temp.end()); } - done(table); + l_done(table); + popLogLeader(); return outViews; } @@ -281,6 +288,7 @@ void Stage::execute(StreamPointTable& table, std::list& stages) PointRef point(table, idx); point_count_t pointLimit = table.capacity(); + reader->pushLogLeader(); // When we get false back from a reader, we're done, so set // the point limit to the number of points processed in this loop // of the table. @@ -291,6 +299,7 @@ void Stage::execute(StreamPointTable& table, std::list& stages) if (finished) pointLimit = idx; } + reader->popLogLeader(); srs = reader->getSpatialReference(); if (!srs.empty()) table.setSpatialReference(srs); @@ -300,6 +309,7 @@ void Stage::execute(StreamPointTable& table, std::list& stages) // processed by subsequent filters. for (Stage *s : filters) { + s->pushLogLeader(); for (PointId idx = 0; idx < pointLimit; idx++) { if (skips[idx]) @@ -311,6 +321,7 @@ void Stage::execute(StreamPointTable& table, std::list& stages) srs = s->getSpatialReference(); if (!srs.empty()) table.setSpatialReference(srs); + s->popLogLeader(); } // Yes, vector is terrible. Can do something better later. @@ -320,9 +331,17 @@ void Stage::execute(StreamPointTable& table, std::list& stages) } for (Stage *s : stages) - s->done(table); + { + s->pushLogLeader(); + s->l_done(table); + s->popLogLeader(); + } } +void Stage::l_done(PointTableRef table) +{ + done(table); +} void Stage::l_addArgs(ProgramArgs& args) { @@ -336,14 +355,22 @@ void Stage::setupLog() LogLevel l(LogLevel::Error); if (m_log) + { l = m_log->getLevel(); + m_logLeader = m_log->leader(); + } if (!m_logname.empty()) - m_log.reset(new Log(getName(), m_logname)); + m_log.reset(new Log("", m_logname)); else if (!m_log) - m_log.reset(new Log(getName(), "stdlog")); + m_log.reset(new Log("", "stdlog")); m_log->setLevel(l); + // Add the stage name to the existing leader. + if (m_logLeader.size()) + m_logLeader += " "; + m_logLeader += getName(); + bool debug(l > LogLevel::Debug); gdal::ErrorHandler::getGlobalErrorHandler().set(m_log, debug); } diff --git a/test/unit/io/sbet/SbetReaderTest.cpp b/test/unit/io/sbet/SbetReaderTest.cpp index 181abd9f37..106722e17a 100644 --- a/test/unit/io/sbet/SbetReaderTest.cpp +++ b/test/unit/io/sbet/SbetReaderTest.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include diff --git a/test/unit/io/sbet/SbetWriterTest.cpp b/test/unit/io/sbet/SbetWriterTest.cpp index 7f5276ce1f..188c2b9117 100644 --- a/test/unit/io/sbet/SbetWriterTest.cpp +++ b/test/unit/io/sbet/SbetWriterTest.cpp @@ -33,6 +33,7 @@ ****************************************************************************/ #include +#include #include #include