diff --git a/doc/stages/readers.text.rst b/doc/stages/readers.text.rst
new file mode 100644
index 0000000000..f55aa8f5a2
--- /dev/null
+++ b/doc/stages/readers.text.rst
@@ -0,0 +1,62 @@
+.. _readers.text:
+
+readers.text
+============
+
+The **text reader** reads data from ASCII text files. Each point is
+represented in the file as a single line. Each line is expected to be divided
+into a number of fields by a separator. Each field represents a value for
+a point's dimension. Each value needs to be `formatted`_ properly for
+C++ language double-precision values.
+
+The text reader expects a header line to 1) indicate the separator character
+for the fields and 2) name the dimension for each field in the points. Any
+single non-alphanumeric character can be used as a separator.
+Each line in the file must contain the same number of fields as indicated by
+dimension names in the header. Spaces are generally ignored in the input
+unless used as a separator. When a space character is used as a separator,
+any number of consecutive spaces are treated as single space.
+
+Blank lines after the header line are ignored.
+
+Example Input File
+------------------
+
+This input file contains X, Y and Z value for 10 points.
+
+::
+
+ X,Y,Z
+ 289814.15,4320978.61,170.76
+ 289814.64,4320978.84,170.76
+ 289815.12,4320979.06,170.75
+ 289815.60,4320979.28,170.74
+ 289816.08,4320979.50,170.68
+ 289816.56,4320979.71,170.66
+ 289817.03,4320979.92,170.63
+ 289817.53,4320980.16,170.62
+ 289818.01,4320980.38,170.61
+ 289818.50,4320980.59,170.58
+
+Example Pipeline
+----------------
+
+.. code-block:: xml
+
+
+
+
+ outputfile.txt
+
+ inputfile.txt
+
+
+
+
+Options
+-------
+
+filename
+ text file to read [Required]
+
+.. _formatted: http://en.cppreference.com/w/cpp/string/basic_string/stof
diff --git a/include/pdal/util/Algorithm.hpp b/include/pdal/util/Algorithm.hpp
index 0e92ff05ce..a4eb8a00ff 100644
--- a/include/pdal/util/Algorithm.hpp
+++ b/include/pdal/util/Algorithm.hpp
@@ -35,6 +35,8 @@
#pragma once
#include
+#include
+#include
namespace pdal
{
@@ -56,6 +58,19 @@ bool contains(const std::map& c, const KEY& v)
}
+template
+void remove(CONTAINER& v, const VALUE& val)
+{
+ v.erase(std::remove(v.begin(), v.end(), val), v.end());
+}
+
+
+template
+void remove_if(CONTAINER& v, PREDICATE p)
+{
+ v.erase(std::remove_if(v.begin(), v.end(), p), v.end());
+}
+/**
template
void remove(std::vector& v, const VALUE& val)
{
@@ -68,6 +83,7 @@ void remove_if(std::vector& v, PREDICATE p)
{
v.erase(std::remove_if(v.begin(), v.end(), p), v.end());
}
+**/
} // namespace Utils
} // namespace pdal
diff --git a/io/text/CMakeLists.txt b/io/text/CMakeLists.txt
index 61f6b1b700..6f7e112a10 100644
--- a/io/text/CMakeLists.txt
+++ b/io/text/CMakeLists.txt
@@ -2,14 +2,13 @@
# Text driver CMake configuration
#
-#
-# Text Writer
-#
set(srcs
+ TextReader.cpp
TextWriter.cpp
)
set(incs
+ TextReader.hpp
TextWriter.hpp
)
diff --git a/io/text/TextReader.cpp b/io/text/TextReader.cpp
new file mode 100644
index 0000000000..94a7b57c21
--- /dev/null
+++ b/io/text/TextReader.cpp
@@ -0,0 +1,174 @@
+/******************************************************************************
+* Copyright (c) 2016, Hobu Inc., info@hobu.co
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+* * Redistributions of source code must retain the above copyright
+* notice, this list of conditions and the following disclaimer.
+* * Redistributions in binary form must reproduce the above copyright
+* notice, this list of conditions and the following disclaimer in
+* the documentation and/or other materials provided
+* with the distribution.
+* * Neither the name of Hobu, Inc. nor the
+* names of its contributors may be used to endorse or promote
+* products derived from this software without specific prior
+* written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#include
+#include
+
+#include "TextReader.hpp"
+
+namespace pdal
+{
+
+static PluginInfo const s_info = PluginInfo(
+ "readers.text",
+ "Text Reader",
+ "http://pdal.io/stages/readers.text.html" );
+
+CREATE_STATIC_PLUGIN(1, 0, TextReader, Reader, s_info)
+
+std::string TextReader::getName() const { return s_info.name; }
+
+void TextReader::initialize(PointTableRef table)
+{
+ m_istream = FileUtils::openFile(m_filename);
+ if (!m_istream)
+ {
+ std::ostringstream oss;
+ oss << getName() << ": Unable to open text file '" <<
+ m_filename << "'.";
+ throw pdal_error(oss.str());
+ }
+
+ std::string buf;
+ std::getline(*m_istream, buf);
+
+ auto isspecial = [](char c)
+ { return (!std::isalnum(c) && c != ' '); };
+
+ // Scan string for some character not a number, space or letter.
+ for (size_t i = 0; i < buf.size(); ++i)
+ if (isspecial(buf[i]))
+ {
+ m_separator = buf[i];
+ break;
+ }
+
+ if (m_separator != ' ')
+ {
+ Utils::remove(buf, ' ');
+ m_dimNames = Utils::split(buf, m_separator);
+ }
+ else
+ m_dimNames = Utils::split2(buf, m_separator);
+ FileUtils::closeFile(m_istream);
+}
+
+
+void TextReader::addDimensions(PointLayoutPtr layout)
+{
+ for (auto name : m_dimNames)
+ {
+ Dimension::Id::Enum id = layout->registerOrAssignDim(name,
+ Dimension::Type::Double);
+ m_dims.push_back(id);
+ }
+}
+
+
+void TextReader::ready(PointTableRef table)
+{
+ m_istream = FileUtils::openFile(m_filename);
+ if (!m_istream)
+ {
+ std::ostringstream oss;
+ oss << getName() << ": Unable to open text file '" <<
+ m_filename << "'.";
+ throw pdal_error(oss.str());
+ }
+
+ // Skip header line.
+ std::string buf;
+ std::getline(*m_istream, buf);
+}
+
+
+point_count_t TextReader::read(PointViewPtr view, point_count_t numPts)
+{
+ PointId idx = view->size();
+
+ point_count_t cnt = 0;
+ size_t line = 1;
+ while (m_istream->good() && cnt < numPts)
+ {
+ std::string buf;
+ StringList fields;
+
+ std::getline(*m_istream, buf);
+ line++;
+ if (buf.empty())
+ continue;
+ if (m_separator != ' ')
+ {
+ Utils::remove(buf, ' ');
+ fields = Utils::split(buf, m_separator);
+ }
+ else
+ fields = Utils::split2(buf, m_separator);
+ if (fields.size() != m_dims.size())
+ {
+ log()->get(LogLevel::Error) << "Line " << line <<
+ " in '" << m_filename << "' contains " << fields.size() <<
+ " fields when " << m_dims.size() << " were expected. "
+ "Ignoring." << std::endl;
+ continue;
+ }
+
+ double d;
+ for (size_t i = 0; i < fields.size(); ++i)
+ {
+ if (!Utils::fromString(fields[i], d))
+ {
+ log()->get(LogLevel::Error) << "Can't convert "
+ "field '" << fields[i] << "' to numeric value on line " <<
+ line << " in '" << m_filename << "'. Setting to 0." <<
+ std::endl;
+ d = 0;
+ }
+ view->setField(m_dims[i], idx, d);
+ }
+ cnt++;
+ idx++;
+ }
+ return cnt;
+}
+
+
+void TextReader::done(PointTableRef table)
+{
+ FileUtils::closeFile(m_istream);
+}
+
+
+} // namespace pdal
+
diff --git a/io/text/TextReader.hpp b/io/text/TextReader.hpp
new file mode 100644
index 0000000000..82691e9545
--- /dev/null
+++ b/io/text/TextReader.hpp
@@ -0,0 +1,103 @@
+/******************************************************************************
+* Copyright (c) 2016, Hobu Inc. (info@hobu.co)
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+* * Redistributions of source code must retain the above copyright
+* notice, this list of conditions and the following disclaimer.
+* * Redistributions in binary form must reproduce the above copyright
+* notice, this list of conditions and the following disclaimer in
+* the documentation and/or other materials provided
+* with the distribution.
+* * Neither the name of Hobu, Inc. nor the
+* names of its contributors may be used to endorse or promote
+* products derived from this software without specific prior
+* written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#pragma once
+
+#include
+
+#include
+
+extern "C" int32_t TextReader_ExitFunc();
+extern "C" PF_ExitFunc TextReader_InitPlugin();
+
+namespace pdal
+{
+
+class PDAL_DLL TextReader : public Reader
+{
+public:
+ static void * create();
+ static int32_t destroy(void *);
+ std::string getName() const;
+
+ TextReader() : m_separator(' '), m_istream(NULL)
+ {}
+
+private:
+ /**
+ Initialize the reader by opening the file and reading the header line.
+ Closes the file on completion.
+
+ \param table Point table being initialized.
+ */
+ virtual void initialize(PointTableRef table);
+
+ /**
+ Add dimensions found in the header line to the layout.
+
+ \param layout Layout to which the dimenions are added.
+ */
+ virtual void addDimensions(PointLayoutPtr layout);
+
+ /**
+ Reopen the file in preparation for reading.
+
+ \param table Point table to make ready.
+ */
+ virtual void ready(PointTableRef table);
+
+ /**
+ Read up to numPts points into the \ref view.
+
+ \param view PointView in which to insert point data.
+ \param numPts Maximum number of points to read.
+ \return Number of points read.
+ */
+ virtual point_count_t read(const PointViewPtr view, point_count_t numPts);
+
+ /**
+ Close input file.
+
+ \param table PointTable we're done with.
+ */
+ virtual void done(PointTableRef table);
+
+private:
+ char m_separator;
+ std::istream *m_istream;
+ StringList m_dimNames;
+ Dimension::IdList m_dims;
+};
+
+} // namespace pdal
diff --git a/src/StageFactory.cpp b/src/StageFactory.cpp
index de2d001d4c..f4520407dc 100644
--- a/src/StageFactory.cpp
+++ b/src/StageFactory.cpp
@@ -63,6 +63,7 @@
#include
#include
#include
+#include
#include
// writers
@@ -110,7 +111,8 @@ std::string StageFactory::inferReaderDriver(const std::string& filename)
drivers["sqlite"] = "readers.sqlite";
drivers["sid"] = "readers.mrsid";
drivers["tindex"] = "readers.tindex";
- drivers["txt"] = "readers.ilvis2";
+//ABELL - seems wrong?
+// drivers["txt"] = "readers.ilvis2";
if (ext == "")
return "";
@@ -216,6 +218,7 @@ StageFactory::StageFactory(bool no_plugins)
PluginManager::initializePlugin(QfitReader_InitPlugin);
PluginManager::initializePlugin(SbetReader_InitPlugin);
PluginManager::initializePlugin(TerrasolidReader_InitPlugin);
+ PluginManager::initializePlugin(TextReader_InitPlugin);
PluginManager::initializePlugin(TIndexReader_InitPlugin);
// writers
diff --git a/test/data/text/utm17_1.txt b/test/data/text/utm17_1.txt
new file mode 100644
index 0000000000..dd176b5fe3
--- /dev/null
+++ b/test/data/text/utm17_1.txt
@@ -0,0 +1,11 @@
+X,Y,Z
+289814.15,4320978.61,170.76
+289814.64,4320978.84,170.76
+289815.12,4320979.06,170.75
+289815.60,4320979.28,170.74
+289816.08,4320979.50,170.68
+289816.56,4320979.71,170.66
+289817.03,4320979.92,170.63
+289817.53,4320980.16,170.62
+289818.01,4320980.38,170.61
+289818.50,4320980.59,170.58
diff --git a/test/data/text/utm17_2.txt b/test/data/text/utm17_2.txt
new file mode 100644
index 0000000000..afd67e60ff
--- /dev/null
+++ b/test/data/text/utm17_2.txt
@@ -0,0 +1,11 @@
+X Y Z
+289814.15 4320978.61 170.76
+289814.64 4320978.84 170.76
+289815.12 4320979.06 170.75
+289815.60 4320979.28 170.74
+289816.08 4320979.50 170.68
+289816.56 4320979.71 170.66
+289817.03 4320979.92 170.63
+289817.53 4320980.16 170.62
+289818.01 4320980.38 170.61
+289818.50 4320980.59 170.58
diff --git a/test/data/text/utm17_3.txt b/test/data/text/utm17_3.txt
new file mode 100644
index 0000000000..6ed56b7307
--- /dev/null
+++ b/test/data/text/utm17_3.txt
@@ -0,0 +1,16 @@
+X, Y, Z
+
+
+
+289814.15 , 4320978.61 , 170.76,27
+289814.15 , 4320978.61 , 170.76
+289814.64 , 4320978.84 , 170.76
+289815.12 , 4320979.06 , 170.75
+289815.60 , 4320979.28 , 170.74
+289816.08 , 4320979.50 , 170.68
+289816.56 , 4320979.71 , 170.66
+289817.03 , 4320979.92 , 170.63
+
+289817.53 , 4320980.16 , 170.62
+289818.01 , 4320980.38 , 170.61
+289818.50 , 4320980.59 , 170.58
diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt
index a714512dea..54cc90c5be 100644
--- a/test/unit/CMakeLists.txt
+++ b/test/unit/CMakeLists.txt
@@ -97,6 +97,7 @@ PDAL_ADD_TEST(pdal_io_qfit_test FILES io/qfit/QFITReaderTest.cpp)
PDAL_ADD_TEST(pdal_io_sbet_reader_test FILES io/sbet/SbetReaderTest.cpp)
PDAL_ADD_TEST(pdal_io_sbet_writer_test FILES io/sbet/SbetWriterTest.cpp)
PDAL_ADD_TEST(pdal_io_terrasolid_test FILES io/terrasolid/TerrasolidReaderTest.cpp)
+PDAL_ADD_TEST(pdal_io_text_test FILES io/text/TextReaderTest.cpp)
#
# sources for the native filters
diff --git a/test/unit/io/text/TextReaderTest.cpp b/test/unit/io/text/TextReaderTest.cpp
new file mode 100644
index 0000000000..72266af992
--- /dev/null
+++ b/test/unit/io/text/TextReaderTest.cpp
@@ -0,0 +1,99 @@
+/******************************************************************************
+ * Copyright (c) 2016, Hobu Inc. (info@hobu.co)
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided
+ * with the distribution.
+ * * Neither the name of Hobu, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+ * OF SUCH DAMAGE.
+ ****************************************************************************/
+
+#include
+
+#include "Support.hpp"
+
+#include
+#include
+
+using namespace pdal;
+
+void compareTextLas(const std::string& textFilename,
+ const std::string& lasFilename)
+{
+ TextReader t;
+ Options to;
+ to.add("filename", textFilename);
+ t.setOptions(to);
+
+ LasReader l;
+ Options lo;
+ lo.add("filename", lasFilename);
+ l.setOptions(lo);
+
+ PointTable tt;
+ t.prepare(tt);
+ PointViewSet ts = t.execute(tt);
+ EXPECT_EQ(ts.size(), 1U);
+ PointViewPtr tv = *ts.begin();
+
+ PointTable lt;
+ l.prepare(lt);
+ PointViewSet ls = l.execute(lt);
+ EXPECT_EQ(ls.size(), 1U);
+ PointViewPtr lv = *ls.begin();
+
+ EXPECT_EQ(tv->size(), lv->size());
+
+ // Validate some point data.
+ for (PointId i = 0; i < lv->size(); ++i)
+ {
+ EXPECT_DOUBLE_EQ(tv->getFieldAs(Dimension::Id::X, i),
+ lv->getFieldAs(Dimension::Id::X, i));
+ EXPECT_DOUBLE_EQ(tv->getFieldAs(Dimension::Id::Y, i),
+ lv->getFieldAs(Dimension::Id::Y, i));
+ EXPECT_DOUBLE_EQ(tv->getFieldAs(Dimension::Id::Z, i),
+ lv->getFieldAs(Dimension::Id::Z, i));
+ }
+}
+
+TEST(TextReaderTest, t1)
+{
+ compareTextLas(Support::datapath("text/utm17_1.txt"),
+ Support::datapath("las/utm17.las"));
+}
+
+TEST(TextReaderTest, t2)
+{
+ compareTextLas(Support::datapath("text/utm17_2.txt"),
+ Support::datapath("las/utm17.las"));
+}
+
+TEST(TextReaderTest, t3)
+{
+ compareTextLas(Support::datapath("text/utm17_3.txt"),
+ Support::datapath("las/utm17.las"));
+}