From 7e723fc8f43c5672def93116245105fc6cc29d40 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Thu, 4 Feb 2016 13:58:10 -0600 Subject: [PATCH 1/4] Initial TextReader. --- include/pdal/util/Algorithm.hpp | 16 +++ io/text/CMakeLists.txt | 5 +- io/text/TextReader.cpp | 169 ++++++++++++++++++++++++++++++++ io/text/TextReader.hpp | 70 +++++++++++++ 4 files changed, 257 insertions(+), 3 deletions(-) create mode 100644 io/text/TextReader.cpp create mode 100644 io/text/TextReader.hpp diff --git a/include/pdal/util/Algorithm.hpp b/include/pdal/util/Algorithm.hpp index 0e92ff05ce..a4eb8a00ff 100644 --- a/include/pdal/util/Algorithm.hpp +++ b/include/pdal/util/Algorithm.hpp @@ -35,6 +35,8 @@ #pragma once #include +#include +#include namespace pdal { @@ -56,6 +58,19 @@ bool contains(const std::map& c, const KEY& v) } +template +void remove(CONTAINER& v, const VALUE& val) +{ + v.erase(std::remove(v.begin(), v.end(), val), v.end()); +} + + +template +void remove_if(CONTAINER& v, PREDICATE p) +{ + v.erase(std::remove_if(v.begin(), v.end(), p), v.end()); +} +/** template void remove(std::vector& v, const VALUE& val) { @@ -68,6 +83,7 @@ void remove_if(std::vector& v, PREDICATE p) { v.erase(std::remove_if(v.begin(), v.end(), p), v.end()); } +**/ } // namespace Utils } // namespace pdal diff --git a/io/text/CMakeLists.txt b/io/text/CMakeLists.txt index 61f6b1b700..6f7e112a10 100644 --- a/io/text/CMakeLists.txt +++ b/io/text/CMakeLists.txt @@ -2,14 +2,13 @@ # Text driver CMake configuration # -# -# Text Writer -# set(srcs + TextReader.cpp TextWriter.cpp ) set(incs + TextReader.hpp TextWriter.hpp ) diff --git a/io/text/TextReader.cpp b/io/text/TextReader.cpp new file mode 100644 index 0000000000..422ae7c4b5 --- /dev/null +++ b/io/text/TextReader.cpp @@ -0,0 +1,169 @@ +/****************************************************************************** +* Copyright (c) 2016, Hobu Inc., info@hobu.co +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include +#include + +#include "TextReader.hpp" + +namespace pdal +{ + +static PluginInfo const s_info = PluginInfo( + "readers.text", + "Text Reader", + "http://pdal.io/stages/readers.text.html" ); + +CREATE_STATIC_PLUGIN(1, 0, TextReader, Reader, s_info) + +std::string TextReader::getName() const { return s_info.name; } + +void TextReader::initialize(PointTableRef table) +{ + m_istream = FileUtils::openFile(m_filename); + if (!m_istream) + { + std::ostringstream oss; + oss << getName() << ": Unable to open text file '" << + m_filename << "'."; + throw pdal_error(oss.str()); + } + + std::string buf; + std::getline(*m_istream, buf); + + auto isspecial = [](char c) + { return (!std::isalnum(c) && c != ' '); }; + + // Scan string for some character not a number, space or letter. + for (size_t i = 0; i < buf.size(); ++i) + if (isspecial(buf[i])) + { + m_separator = buf[i]; + break; + } + + if (m_separator != ' ') + { + Utils::remove(buf, ' '); + m_dimNames = Utils::split(buf, m_separator); + } + else + m_dimNames = Utils::split2(buf, m_separator); + FileUtils::closeFile(m_istream); +} + + +void TextReader::addDimensions(PointLayoutPtr layout) +{ + for (auto name : m_dimNames) + { + Dimension::Id::Enum id = layout->registerOrAssignDim(name, + Dimension::Type::Double); + m_dims.push_back(id); + } +} + + +void TextReader::ready(PointTableRef table) +{ + m_istream = FileUtils::openFile(m_filename); + if (!m_istream) + { + std::ostringstream oss; + oss << getName() << ": Unable to open text file '" << + m_filename << "'."; + throw pdal_error(oss.str()); + } + + // Skip header line. + std::string buf; + std::getline(*m_istream, buf); +} + + +point_count_t TextReader::read(PointViewPtr view, point_count_t numPts) +{ + PointId idx = view->size(); + + point_count_t cnt = 0; + size_t line = 1; + while (m_istream->good() && cnt < numPts) + { + std::string buf; + StringList fields; + + std::getline(*m_istream, buf); + line++; + if (buf.empty()) + continue; + if (m_separator != ' ') + { + Utils::remove(buf, ' '); + fields = Utils::split(buf, m_separator); + } + else + fields = Utils::split2(buf, m_separator); + if (fields.size() != m_dims.size()) + log()->get(LogLevel::Error) << getName() << ": Line " << line << + " in '" << m_filename << "' contains " << fields.size() << + " fields when " << m_dims.size() << " were expected. Ingoring."; + + double d; + for (size_t i = 0; i < fields.size(); ++i) + { + if (!Utils::fromString(fields[i], d)) + { + log()->get(LogLevel::Error) << getName() << ": Can't convert " + "field '" << fields[i] << "' to numeric value on line " << + line << " in '" << m_filename << "'. Setting to 0."; + d = 0; + } + view->setField(m_dims[i], idx, d); + } + cnt++; + idx++; + } + return cnt; +} + + +void TextReader::done(PointTableRef table) +{ + FileUtils::closeFile(m_istream); +} + + +} // namespace pdal + diff --git a/io/text/TextReader.hpp b/io/text/TextReader.hpp new file mode 100644 index 0000000000..80e9d8511f --- /dev/null +++ b/io/text/TextReader.hpp @@ -0,0 +1,70 @@ +/****************************************************************************** +* Copyright (c) 2016, Hobu Inc. (info@hobu.co) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#pragma once + +#include + +#include + +extern "C" int32_t TextReader_ExitFunc(); +extern "C" PF_ExitFunc TextReader_InitPlugin(); + +namespace pdal +{ + +class PDAL_DLL TextReader : public Reader +{ +public: + static void * create(); + static int32_t destroy(void *); + std::string getName() const; + + TextReader() : m_separator(' '), m_istream(NULL) + {} + +private: + virtual void initialize(PointTableRef table); + virtual void addDimensions(PointLayoutPtr layout); + virtual void ready(PointTableRef table); + virtual point_count_t read(const PointViewPtr view, point_count_t numPts); + virtual void done(PointTableRef table); + + char m_separator; + std::istream *m_istream; + StringList m_dimNames; + Dimension::IdList m_dims; +}; + +} // namespace pdal From cf9d949c3269c7122a2c7f632046be0987cc3f2e Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 5 Feb 2016 14:51:51 -0600 Subject: [PATCH 2/4] TextReader with doc and test. --- doc/stages/readers.text.rst | 62 +++++++++++++++++ io/text/TextReader.cpp | 13 ++-- src/StageFactory.cpp | 5 +- test/data/text/utm17_1.txt | 11 ++++ test/data/text/utm17_2.txt | 11 ++++ test/data/text/utm17_3.txt | 16 +++++ test/unit/CMakeLists.txt | 1 + test/unit/io/text/TextReaderTest.cpp | 99 ++++++++++++++++++++++++++++ 8 files changed, 213 insertions(+), 5 deletions(-) create mode 100644 doc/stages/readers.text.rst create mode 100644 test/data/text/utm17_1.txt create mode 100644 test/data/text/utm17_2.txt create mode 100644 test/data/text/utm17_3.txt create mode 100644 test/unit/io/text/TextReaderTest.cpp diff --git a/doc/stages/readers.text.rst b/doc/stages/readers.text.rst new file mode 100644 index 0000000000..f55aa8f5a2 --- /dev/null +++ b/doc/stages/readers.text.rst @@ -0,0 +1,62 @@ +.. _readers.text: + +readers.text +============ + +The **text reader** reads data from ASCII text files. Each point is +represented in the file as a single line. Each line is expected to be divided +into a number of fields by a separator. Each field represents a value for +a point's dimension. Each value needs to be `formatted`_ properly for +C++ language double-precision values. + +The text reader expects a header line to 1) indicate the separator character +for the fields and 2) name the dimension for each field in the points. Any +single non-alphanumeric character can be used as a separator. +Each line in the file must contain the same number of fields as indicated by +dimension names in the header. Spaces are generally ignored in the input +unless used as a separator. When a space character is used as a separator, +any number of consecutive spaces are treated as single space. + +Blank lines after the header line are ignored. + +Example Input File +------------------ + +This input file contains X, Y and Z value for 10 points. + +:: + + X,Y,Z + 289814.15,4320978.61,170.76 + 289814.64,4320978.84,170.76 + 289815.12,4320979.06,170.75 + 289815.60,4320979.28,170.74 + 289816.08,4320979.50,170.68 + 289816.56,4320979.71,170.66 + 289817.03,4320979.92,170.63 + 289817.53,4320980.16,170.62 + 289818.01,4320980.38,170.61 + 289818.50,4320980.59,170.58 + +Example Pipeline +---------------- + +.. code-block:: xml + + + + + + + + + + + +Options +------- + +filename + text file to read [Required] + +.. _formatted: http://en.cppreference.com/w/cpp/string/basic_string/stof diff --git a/io/text/TextReader.cpp b/io/text/TextReader.cpp index 422ae7c4b5..94a7b57c21 100644 --- a/io/text/TextReader.cpp +++ b/io/text/TextReader.cpp @@ -136,18 +136,23 @@ point_count_t TextReader::read(PointViewPtr view, point_count_t numPts) else fields = Utils::split2(buf, m_separator); if (fields.size() != m_dims.size()) - log()->get(LogLevel::Error) << getName() << ": Line " << line << + { + log()->get(LogLevel::Error) << "Line " << line << " in '" << m_filename << "' contains " << fields.size() << - " fields when " << m_dims.size() << " were expected. Ingoring."; + " fields when " << m_dims.size() << " were expected. " + "Ignoring." << std::endl; + continue; + } double d; for (size_t i = 0; i < fields.size(); ++i) { if (!Utils::fromString(fields[i], d)) { - log()->get(LogLevel::Error) << getName() << ": Can't convert " + log()->get(LogLevel::Error) << "Can't convert " "field '" << fields[i] << "' to numeric value on line " << - line << " in '" << m_filename << "'. Setting to 0."; + line << " in '" << m_filename << "'. Setting to 0." << + std::endl; d = 0; } view->setField(m_dims[i], idx, d); diff --git a/src/StageFactory.cpp b/src/StageFactory.cpp index de2d001d4c..f4520407dc 100644 --- a/src/StageFactory.cpp +++ b/src/StageFactory.cpp @@ -63,6 +63,7 @@ #include #include #include +#include #include // writers @@ -110,7 +111,8 @@ std::string StageFactory::inferReaderDriver(const std::string& filename) drivers["sqlite"] = "readers.sqlite"; drivers["sid"] = "readers.mrsid"; drivers["tindex"] = "readers.tindex"; - drivers["txt"] = "readers.ilvis2"; +//ABELL - seems wrong? +// drivers["txt"] = "readers.ilvis2"; if (ext == "") return ""; @@ -216,6 +218,7 @@ StageFactory::StageFactory(bool no_plugins) PluginManager::initializePlugin(QfitReader_InitPlugin); PluginManager::initializePlugin(SbetReader_InitPlugin); PluginManager::initializePlugin(TerrasolidReader_InitPlugin); + PluginManager::initializePlugin(TextReader_InitPlugin); PluginManager::initializePlugin(TIndexReader_InitPlugin); // writers diff --git a/test/data/text/utm17_1.txt b/test/data/text/utm17_1.txt new file mode 100644 index 0000000000..dd176b5fe3 --- /dev/null +++ b/test/data/text/utm17_1.txt @@ -0,0 +1,11 @@ +X,Y,Z +289814.15,4320978.61,170.76 +289814.64,4320978.84,170.76 +289815.12,4320979.06,170.75 +289815.60,4320979.28,170.74 +289816.08,4320979.50,170.68 +289816.56,4320979.71,170.66 +289817.03,4320979.92,170.63 +289817.53,4320980.16,170.62 +289818.01,4320980.38,170.61 +289818.50,4320980.59,170.58 diff --git a/test/data/text/utm17_2.txt b/test/data/text/utm17_2.txt new file mode 100644 index 0000000000..afd67e60ff --- /dev/null +++ b/test/data/text/utm17_2.txt @@ -0,0 +1,11 @@ +X Y Z +289814.15 4320978.61 170.76 +289814.64 4320978.84 170.76 +289815.12 4320979.06 170.75 +289815.60 4320979.28 170.74 +289816.08 4320979.50 170.68 +289816.56 4320979.71 170.66 +289817.03 4320979.92 170.63 +289817.53 4320980.16 170.62 +289818.01 4320980.38 170.61 +289818.50 4320980.59 170.58 diff --git a/test/data/text/utm17_3.txt b/test/data/text/utm17_3.txt new file mode 100644 index 0000000000..6ed56b7307 --- /dev/null +++ b/test/data/text/utm17_3.txt @@ -0,0 +1,16 @@ +X, Y, Z + + + +289814.15 , 4320978.61 , 170.76,27 +289814.15 , 4320978.61 , 170.76 +289814.64 , 4320978.84 , 170.76 +289815.12 , 4320979.06 , 170.75 +289815.60 , 4320979.28 , 170.74 +289816.08 , 4320979.50 , 170.68 +289816.56 , 4320979.71 , 170.66 +289817.03 , 4320979.92 , 170.63 + +289817.53 , 4320980.16 , 170.62 +289818.01 , 4320980.38 , 170.61 +289818.50 , 4320980.59 , 170.58 diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index adcca1dc7f..febb042a32 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -96,6 +96,7 @@ PDAL_ADD_TEST(pdal_io_qfit_test FILES io/qfit/QFITReaderTest.cpp) PDAL_ADD_TEST(pdal_io_sbet_reader_test FILES io/sbet/SbetReaderTest.cpp) PDAL_ADD_TEST(pdal_io_sbet_writer_test FILES io/sbet/SbetWriterTest.cpp) PDAL_ADD_TEST(pdal_io_terrasolid_test FILES io/terrasolid/TerrasolidReaderTest.cpp) +PDAL_ADD_TEST(pdal_io_text_test FILES io/text/TextReaderTest.cpp) # # sources for the native filters diff --git a/test/unit/io/text/TextReaderTest.cpp b/test/unit/io/text/TextReaderTest.cpp new file mode 100644 index 0000000000..72266af992 --- /dev/null +++ b/test/unit/io/text/TextReaderTest.cpp @@ -0,0 +1,99 @@ +/****************************************************************************** + * Copyright (c) 2016, Hobu Inc. (info@hobu.co) + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Hobu, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + ****************************************************************************/ + +#include + +#include "Support.hpp" + +#include +#include + +using namespace pdal; + +void compareTextLas(const std::string& textFilename, + const std::string& lasFilename) +{ + TextReader t; + Options to; + to.add("filename", textFilename); + t.setOptions(to); + + LasReader l; + Options lo; + lo.add("filename", lasFilename); + l.setOptions(lo); + + PointTable tt; + t.prepare(tt); + PointViewSet ts = t.execute(tt); + EXPECT_EQ(ts.size(), 1U); + PointViewPtr tv = *ts.begin(); + + PointTable lt; + l.prepare(lt); + PointViewSet ls = l.execute(lt); + EXPECT_EQ(ls.size(), 1U); + PointViewPtr lv = *ls.begin(); + + EXPECT_EQ(tv->size(), lv->size()); + + // Validate some point data. + for (PointId i = 0; i < lv->size(); ++i) + { + EXPECT_DOUBLE_EQ(tv->getFieldAs(Dimension::Id::X, i), + lv->getFieldAs(Dimension::Id::X, i)); + EXPECT_DOUBLE_EQ(tv->getFieldAs(Dimension::Id::Y, i), + lv->getFieldAs(Dimension::Id::Y, i)); + EXPECT_DOUBLE_EQ(tv->getFieldAs(Dimension::Id::Z, i), + lv->getFieldAs(Dimension::Id::Z, i)); + } +} + +TEST(TextReaderTest, t1) +{ + compareTextLas(Support::datapath("text/utm17_1.txt"), + Support::datapath("las/utm17.las")); +} + +TEST(TextReaderTest, t2) +{ + compareTextLas(Support::datapath("text/utm17_2.txt"), + Support::datapath("las/utm17.las")); +} + +TEST(TextReaderTest, t3) +{ + compareTextLas(Support::datapath("text/utm17_3.txt"), + Support::datapath("las/utm17.las")); +} From 62b1d0e154e14c7181ee2bcf4342800a9a3930db Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 5 Feb 2016 15:06:12 -0600 Subject: [PATCH 3/4] Add doxygen. --- io/text/TextReader.hpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/io/text/TextReader.hpp b/io/text/TextReader.hpp index 80e9d8511f..82691e9545 100644 --- a/io/text/TextReader.hpp +++ b/io/text/TextReader.hpp @@ -55,12 +55,45 @@ class PDAL_DLL TextReader : public Reader {} private: + /** + Initialize the reader by opening the file and reading the header line. + Closes the file on completion. + + \param table Point table being initialized. + */ virtual void initialize(PointTableRef table); + + /** + Add dimensions found in the header line to the layout. + + \param layout Layout to which the dimenions are added. + */ virtual void addDimensions(PointLayoutPtr layout); + + /** + Reopen the file in preparation for reading. + + \param table Point table to make ready. + */ virtual void ready(PointTableRef table); + + /** + Read up to numPts points into the \ref view. + + \param view PointView in which to insert point data. + \param numPts Maximum number of points to read. + \return Number of points read. + */ virtual point_count_t read(const PointViewPtr view, point_count_t numPts); + + /** + Close input file. + + \param table PointTable we're done with. + */ virtual void done(PointTableRef table); +private: char m_separator; std::istream *m_istream; StringList m_dimNames; From 2e4b44070cccfd9ff13170ee33ada488241065cf Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 9 Feb 2016 17:34:03 -0600 Subject: [PATCH 4/4] PipelineReader is gone --- plugins/mrsid/test/MrsidTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/mrsid/test/MrsidTest.cpp b/plugins/mrsid/test/MrsidTest.cpp index 6101ecee73..aeea246614 100644 --- a/plugins/mrsid/test/MrsidTest.cpp +++ b/plugins/mrsid/test/MrsidTest.cpp @@ -37,7 +37,6 @@ #include #include -#include #include #include