Merge pull request #1100 from PDAL/textreader

TextReader

doc/stages/readers.text.rst

@@ -0,0 +1,62 @@
+.. _readers.text:
+
+readers.text
+============
+
+The **text reader** reads data from ASCII text files.  Each point is
+represented in the file as a single line.  Each line is expected to be divided
+into a number of fields by a separator.  Each field represents a value for
+a point's dimension.  Each value needs to be `formatted`_ properly for
+C++ language double-precision values.
+
+The text reader expects a header line to 1) indicate the separator character
+for the fields and 2) name the dimension for each field in the points.  Any
+single non-alphanumeric character can be used as a separator.
+Each line in the file must contain the same number of fields as indicated by
+dimension names in the header.  Spaces are generally ignored in the input
+unless used as a separator.  When a space character is used as a separator,
+any number of consecutive spaces are treated as single space.
+
+Blank lines after the header line are ignored.
+
+Example Input File
+------------------
+
+This input file contains X, Y and Z value for 10 points.
+
+::
+
+    X,Y,Z
+    289814.15,4320978.61,170.76
+    289814.64,4320978.84,170.76
+    289815.12,4320979.06,170.75
+    289815.60,4320979.28,170.74
+    289816.08,4320979.50,170.68
+    289816.56,4320979.71,170.66
+    289817.03,4320979.92,170.63
+    289817.53,4320980.16,170.62
+    289818.01,4320980.38,170.61
+    289818.50,4320980.59,170.58
+
+Example Pipeline
+----------------
+
+.. code-block:: xml
+
+  <?xml version="1.0" encoding="utf-8"?>
+  <Pipeline version="1.0">
+    <Writer type="writers.text">
+      <Option name="filename">outputfile.txt</Option>
+      <Reader type="readers.text">
+        <Option name="filename">inputfile.txt</Option>
+      </Reader>
+    </Writer>
+  </Pipeline>
+
+Options
+-------
+
+filename
+  text file to read [Required]
+
+.. _formatted: http://en.cppreference.com/w/cpp/string/basic_string/stof

include/pdal/util/Algorithm.hpp

@@ -35,6 +35,8 @@
 #pragma once
 
 #include <algorithm>
+#include <map>
+#include <vector>
 
 namespace pdal
 {
@@ -56,6 +58,19 @@ bool contains(const std::map<KEY, VALUE>& c, const KEY& v)
 }
 
 
+template<typename CONTAINER, typename VALUE>
+void remove(CONTAINER& v, const VALUE& val)
+{
+    v.erase(std::remove(v.begin(), v.end(), val), v.end());
+}
+
+
+template<typename CONTAINER, typename PREDICATE>
+void remove_if(CONTAINER& v, PREDICATE p)
+{
+    v.erase(std::remove_if(v.begin(), v.end(), p), v.end());
+}
+/**
 template<typename TYPE, typename VALUE>
 void remove(std::vector<TYPE>& v, const VALUE& val)
 {
@@ -68,6 +83,7 @@ void remove_if(std::vector<TYPE>& v, PREDICATE p)
 {
     v.erase(std::remove_if(v.begin(), v.end(), p), v.end());
 }
+**/
 
 } // namespace Utils
 } // namespace pdal

io/text/CMakeLists.txt

@@ -2,14 +2,13 @@
 # Text driver CMake configuration
 #
 
-#
-# Text Writer
-#
 set(srcs
+    TextReader.cpp
     TextWriter.cpp
 )
 
 set(incs
+    TextReader.hpp
     TextWriter.hpp
 )
 

io/text/TextReader.cpp

@@ -0,0 +1,174 @@
+/******************************************************************************
+* Copyright (c) 2016, Hobu Inc., info@hobu.co
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#include <pdal/util/Algorithm.hpp>
+#include <pdal/util/FileUtils.hpp>
+
+#include "TextReader.hpp"
+
+namespace pdal
+{
+
+static PluginInfo const s_info = PluginInfo(
+    "readers.text",
+    "Text Reader",
+    "http://pdal.io/stages/readers.text.html" );
+
+CREATE_STATIC_PLUGIN(1, 0, TextReader, Reader, s_info)
+
+std::string TextReader::getName() const { return s_info.name; }
+
+void TextReader::initialize(PointTableRef table)
+{
+    m_istream = FileUtils::openFile(m_filename);
+    if (!m_istream)
+    {
+        std::ostringstream oss;
+        oss << getName() << ": Unable to open text file '" <<
+            m_filename << "'.";
+        throw pdal_error(oss.str());
+    }
+
+    std::string buf;
+    std::getline(*m_istream, buf);
+
+    auto isspecial = [](char c)
+        { return (!std::isalnum(c) && c != ' '); };
+
+    // Scan string for some character not a number, space or letter.
+    for (size_t i = 0; i < buf.size(); ++i)
+        if (isspecial(buf[i]))
+        {
+            m_separator = buf[i];
+            break;
+        }
+
+    if (m_separator != ' ')
+    {
+        Utils::remove(buf, ' ');
+        m_dimNames = Utils::split(buf, m_separator);
+    }
+    else
+        m_dimNames = Utils::split2(buf, m_separator);
+    FileUtils::closeFile(m_istream);
+}
+
+
+void TextReader::addDimensions(PointLayoutPtr layout)
+{
+    for (auto name : m_dimNames)
+    {
+        Dimension::Id::Enum id = layout->registerOrAssignDim(name,
+            Dimension::Type::Double);
+        m_dims.push_back(id);
+    }
+}
+
+
+void TextReader::ready(PointTableRef table)
+{
+    m_istream = FileUtils::openFile(m_filename);
+    if (!m_istream)
+    {
+        std::ostringstream oss;
+        oss << getName() << ": Unable to open text file '" <<
+            m_filename << "'.";
+        throw pdal_error(oss.str());
+    }
+
+    // Skip header line.
+    std::string buf;
+    std::getline(*m_istream, buf);
+}
+
+
+point_count_t TextReader::read(PointViewPtr view, point_count_t numPts)
+{
+    PointId idx = view->size();
+
+    point_count_t cnt = 0;
+    size_t line = 1;
+    while (m_istream->good() && cnt < numPts)
+    {
+        std::string buf;
+        StringList fields;
+
+        std::getline(*m_istream, buf);
+        line++;
+        if (buf.empty())
+            continue;
+        if (m_separator != ' ')
+        {
+            Utils::remove(buf, ' ');
+            fields = Utils::split(buf, m_separator);
+        }
+        else
+            fields = Utils::split2(buf, m_separator);
+        if (fields.size() != m_dims.size())
+        {
+            log()->get(LogLevel::Error) << "Line " << line <<
+               " in '" << m_filename << "' contains " << fields.size() <<
+               " fields when " << m_dims.size() << " were expected.  "
+               "Ignoring." << std::endl;
+            continue;
+        }
+
+        double d;
+        for (size_t i = 0; i < fields.size(); ++i)
+        {
+            if (!Utils::fromString(fields[i], d))
+            {
+                log()->get(LogLevel::Error) << "Can't convert "
+                    "field '" << fields[i] << "' to numeric value on line " <<
+                    line << " in '" << m_filename << "'.  Setting to 0." <<
+                    std::endl;
+                d = 0;
+            }
+            view->setField(m_dims[i], idx, d);
+        }
+        cnt++;
+        idx++;
+    }
+    return cnt;
+}
+
+
+void TextReader::done(PointTableRef table)
+{
+    FileUtils::closeFile(m_istream);
+}
+
+
+} // namespace pdal
+

io/text/TextReader.hpp

@@ -0,0 +1,103 @@
+/******************************************************************************
+* Copyright (c) 2016, Hobu Inc. (info@hobu.co)
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#pragma once
+
+#include <istream>
+
+#include <pdal/Reader.hpp>
+
+extern "C" int32_t TextReader_ExitFunc();
+extern "C" PF_ExitFunc TextReader_InitPlugin();
+
+namespace pdal
+{
+
+class PDAL_DLL TextReader : public Reader
+{
+public:
+    static void * create();
+    static int32_t destroy(void *);
+    std::string getName() const;
+
+    TextReader() : m_separator(' '), m_istream(NULL)
+    {}
+
+private:
+    /**
+      Initialize the reader by opening the file and reading the header line.
+      Closes the file on completion.
+
+      \param table  Point table being initialized.
+    */
+    virtual void initialize(PointTableRef table);
+
+    /**
+      Add dimensions found in the header line to the layout.
+
+      \param layout  Layout to which the dimenions are added.
+    */
+    virtual void addDimensions(PointLayoutPtr layout);
+
+    /**
+      Reopen the file in preparation for reading.
+
+      \param table  Point table to make ready.
+    */
+    virtual void ready(PointTableRef table);
+
+    /**
+      Read up to numPts points into the \ref view.
+
+      \param view  PointView in which to insert point data.
+      \param numPts  Maximum number of points to read.
+      \return  Number of points read.
+    */
+    virtual point_count_t read(const PointViewPtr view, point_count_t numPts);
+
+    /**
+      Close input file.
+
+      \param table  PointTable we're done with.
+    */
+    virtual void done(PointTableRef table);
+
+private:
+    char m_separator;
+    std::istream *m_istream;
+    StringList m_dimNames;
+    Dimension::IdList m_dims;
+};
+
+} // namespace pdal