working readers.matlab

doc/stages/readers.matlab.rst

@@ -0,0 +1,51 @@
+.. _readers.matlab:
+
+readers.matlab
+==============
+
+The **Matlab Reader** supports readers Matlab ``.mat`` files. Data
+must be in a `Matlab struct`_, with field names that correspond to
+:ref:`dimensions` names. No ability to provide a name map is yet
+provided.
+
+Additionally, each array in the struct should ideally have the
+same number of points. The reader takes its number of points
+from the first array in the struct. If the array has fewer
+elements than the first array in the struct, the point's field
+beyond that number is set to zero.
+
+.. _`Matlab struct`: https://www.mathworks.com/help/matlab/ref/struct.html
+
+.. note::
+
+    The Matlab reader requires the Mat-File API from MathWorks, and it must be
+    explicitly enabled at compile time with the ``BUILD_PLUGIN_MATLAB=ON``
+    variable
+
+Example
+-------
+
+.. code-block:: json
+
+    {
+      "pipeline":[
+        {
+          "type":"readers.matlab",
+          "struct":"PDAL",
+          "filename":"autzen.mat"
+        },
+        {
+          "type":"writers.las",
+          "filename":"output.las"
+        }
+      ]
+    }
+
+Options
+-------
+
+filename
+  Output file name [REQUIRED]
+
+struct
+  Array structure name to read [OPTIONAL, defaults ``PDAL``]

pdal/StageFactory.cpp

@@ -130,6 +130,7 @@ StringList StageFactory::extensions(const std::string& driver)
         { "readers.rxp", { "rxp" } },
         { "readers.sbet", { "sbet" } },
         { "readers.sqlite", { "sqlite" } },
+        { "readers.matlab", { "mat" } },
         { "readers.mrsid", { "sid" } },
         { "readers.tindex", { "tindex" } },
         { "readers.text", { "csv", "txt" } },
@@ -163,6 +164,7 @@ std::string StageFactory::inferReaderDriver(const std::string& filename)
         { "icebridge", "readers.icebridge" },
         { "las", "readers.las" },
         { "laz", "readers.las" },
+        { "mat", "readers.matlab" },
         { "nitf", "readers.nitf" },
         { "nsf", "readers.nitf" },
         { "ntf", "readers.nitf" },

plugins/matlab/CMakeLists.txt

@@ -2,22 +2,16 @@
 
 include(${PDAL_CMAKE_DIR}/matlab.cmake)
 
-set(MLANG ./filters/Environment.cpp
-          ./filters/Script.cpp)
-
-PDAL_ADD_PLUGIN(libname writer matlab
+PDAL_ADD_PLUGIN(libname_writer writer matlab
     FILES
         ./filters/Script.cpp
         io/MatlabWriter.cpp
     LINK_WITH
         ${MATLAB_MX_LIBRARY} ${MATLAB_MAT_LIBRARY}
 )
-target_include_directories(${libname} PRIVATE
+target_include_directories(${libname_writer} PRIVATE
     ${MATLAB_INCLUDE_DIR})
 
-target_compile_definitions(${libname} PRIVATE -DHAVE_MATLAB=1)
-target_include_directories(${libname} PRIVATE ${MATLAB_INCLUDE_DIR})
-
 PDAL_ADD_PLUGIN(libname_reader reader matlab
     FILES
         ./filters/Script.cpp
@@ -28,11 +22,6 @@ PDAL_ADD_PLUGIN(libname_reader reader matlab
 target_include_directories(${libname_reader} PRIVATE
     ${MATLAB_INCLUDE_DIR})
 
-target_compile_definitions(${libname_reader} PRIVATE -DHAVE_MATLAB=1)
-target_include_directories(${libname_reader} PRIVATE ${MATLAB_INCLUDE_DIR})
-
-
-
 
 PDAL_ADD_PLUGIN(matlab_libname filter matlab
     FILES
@@ -46,17 +35,27 @@ target_include_directories(${matlab_libname} PRIVATE
     ${MATLAB_INCLUDE_DIR} )
 
 if (WITH_TESTS)
-    PDAL_ADD_TEST(matlabtest
+    PDAL_ADD_TEST(pdal_io_matlab_writer_test
         FILES
             test/MatlabWriterTest.cpp
         LINK_WITH
-           ${MATLAB_MX_LIBRARY} ${MATLAB_MAT_LIBRARY} ${libname}
+           ${MATLAB_MX_LIBRARY} ${MATLAB_MAT_LIBRARY} ${libname_writer}
     )
-    target_include_directories(matlabtest PRIVATE
+    target_include_directories(pdal_io_matlab_writer_test PRIVATE
         ${PDAL_IO_DIR}
         ${MATLAB_INCLUDE_DIR})
 
 
+    PDAL_ADD_TEST(pdal_io_matlab_reader_test
+        FILES
+            test/MatlabReaderTest.cpp
+        LINK_WITH
+           ${MATLAB_MX_LIBRARY} ${MATLAB_MAT_LIBRARY} ${libname_reader}
+    )
+    target_include_directories(pdal_io_matlab_reader_test PRIVATE
+        ${PDAL_IO_DIR}
+        ${MATLAB_INCLUDE_DIR})
+
     PDAL_ADD_TEST(pdal_filters_matlab_test
         FILES
             ./filters/Script.cpp

plugins/matlab/filters/Script.cpp

@@ -71,6 +71,36 @@ std::string Script::getLogicalMask(mxArray* array, LogPtr log)
     return output;
 }
 
+PointLayoutPtr Script::getStructLayout(mxArray* array, LogPtr log)
+{
+
+    PointLayoutPtr layout = PointLayoutPtr(new PointLayout());
+    std::vector<mxArray*> arrays;
+
+    mxClassID ml_id = mxGetClassID(array);
+    if (ml_id != mxSTRUCT_CLASS)
+        throw pdal::pdal_error("Selected array must be a Matlab struct array!");
+
+
+    int numFields = mxGetNumberOfFields(array);
+
+    if (!numFields)
+        throw pdal::pdal_error("Selected struct array must have fields!");
+
+    for (int i=0; i < numFields; ++i)
+    {
+        const char* fieldName = mxGetFieldNameByNumber(array, i);
+        mxArray* f = mxGetFieldByNumber(array, 0, i);
+        mxClassID mt = mxGetClassID(f);
+        Dimension::Type pt = Script::getPDALDataType(mt);
+        layout->registerOrAssignDim(fieldName, pt);
+    }
+
+    layout->finalize();
+    return layout;
+}
+
+
 void Script::getMatlabStruct(mxArray* array, PointViewPtr view, const Dimension::IdList& indims, LogPtr log)
 {
     std::vector<mxArray*> arrays;

plugins/matlab/filters/Script.hpp

@@ -58,6 +58,7 @@ class PDAL_DLL Script
     static mxArray* setMatlabStruct(PointViewPtr view, const Dimension::IdList& dims, LogPtr log);
 
     static void getMatlabStruct(mxArray* array, PointViewPtr view, const Dimension::IdList& dims, LogPtr log);
+    static PointLayoutPtr getStructLayout(mxArray* array, LogPtr log);
     static std::string getLogicalMask(mxArray* array, LogPtr log);
 
     std::string m_source;

plugins/matlab/io/MatlabReader.cpp

@@ -48,76 +48,79 @@ static PluginInfo const s_info = PluginInfo(
     "Matlab Reader",
     "http://pdal.io/stages/readers.matlab.html" );
 
-CREATE_STATIC_PLUGIN(1, 0, MatlabReader, Reader, s_info)
-
+CREATE_SHARED_PLUGIN(1, 0, MatlabReader, Reader, s_info)
 std::string MatlabReader::getName() const { return s_info.name; }
 
 void MatlabReader::initialize(PointTableRef table)
 {
-//     m_istream = Utils::openFile(m_filename);
-//     if (!m_istream)
-//         throwError("Unable to open text file '" + m_filename + "'.");
-//
-//     std::string buf;
-//     std::getline(*m_istream, buf);
-//
-//     auto isspecial = [](char c)
-//         { return (!std::isalnum(c) && c != ' '); };
-//
-//     // If the separator wasn't provided on the command line extract it
-//     // from the header line.
-//     if (m_separator == ' ')
-//     {
-//         // Scan string for some character not a number, space or letter.
-//         for (size_t i = 0; i < buf.size(); ++i)
-//             if (isspecial(buf[i]))
-//             {
-//                 m_separator = buf[i];
-//                 break;
-//             }
-//     }
-//
-//     if (m_separator != ' ')
-//         m_dimNames = Utils::split(buf, m_separator);
-//     else
-//         m_dimNames = Utils::split2(buf, m_separator);
-//     Utils::closeFile(m_istream);
+    m_matfile = matOpen(m_filename.c_str(), "r");
+    if (!m_matfile)
+        throwError("Could not open file '" + m_filename + "' for reading.");
+
+    m_pointIndex = 0;
+    m_numElements = 0;
+    m_numFields = 0;
 }
 
 
 void MatlabReader::addArgs(ProgramArgs& args)
 {
-//     args.add("separator", "Separator character that overrides special "
-//         "character in header line", m_separator, ' ');
+    args.add("struct", "Name of struct to read from file", m_structName, "PDAL");
 }
 
 
 void MatlabReader::addDimensions(PointLayoutPtr layout)
 {
-    m_dims.clear();
-//     for (auto name : m_dimNames)
-//     {
-//         Utils::trim(name);
-//         Dimension::Id id = layout->registerOrAssignDim(name,
-//             Dimension::Type::Double);
-//         if (Utils::contains(m_dims, id) && id != pdal::Dimension::Id::Unknown)
-//             throwError("Duplicate dimension '" + name +
-//                 "' detected in input file '" + m_filename + "'.");
-//         m_dims.push_back(id);
-//     }
-}
+    log()->get(LogLevel::Debug) << "Opening file" <<
+        " '" << m_filename << "'." << std::endl;
 
+    m_structArray = matGetVariable(m_matfile, m_structName.c_str());
+    if (!m_structArray)
+    {
+        std::ostringstream oss;
+        oss << "Array struct with name '" << m_structName << "' not found ";
+        oss << "in file '" << m_filename << "'";
+        throwError(oss.str());
+    }
+
+    // For now we read all of the fields in the given
+    // array structure
+    m_numFields = mxGetNumberOfFields(m_structArray);
+    if (!m_numFields)
+        throw pdal::pdal_error("Selected struct array must have fields!");
+
+    // Fetch the first array and determine number of elements
+    // it has. We're only going to read that many elements no
+    // matter what
+    mxArray* f = mxGetFieldByNumber(m_structArray, 0, 0);
+    if (!f)
+    {
+        throwError("Unable to fetch first array in array struct to determine number of elements!");
+    }
+    m_numElements = mxGetNumberOfElements(f);
+
+    // Get the dimensions and their types from the array struct
+    PointLayoutPtr matlabLayout = mlang::Script::getStructLayout(m_structArray, log());
+    const Dimension::IdList& dims = matlabLayout->dims();
+
+    int nDimensionNumber(0);
+    for(auto d: dims)
+    {
+        std::string dimName = matlabLayout->dimName(d);
+        const Dimension::Detail* detail = matlabLayout->dimDetail(d);
+        Dimension::Id id = detail->id();
+        Dimension::Type t = detail->type();
+        layout->registerDim(id, t);
+
+        // Keep a map of the Matlab dimension number to
+        // both the PDAL type and PDAL id
+        std::pair<int, int> pd = std::make_pair(nDimensionNumber, (int)id);
+        m_dimensionIdMap.insert(pd);
+        std::pair<int, int> pt = std::make_pair(nDimensionNumber, (int)t);
+        m_dimensionTypeMap.insert(pt);
+        nDimensionNumber++;
+    }
 
-void MatlabReader::ready(PointTableRef table)
-{
-//     m_istream = Utils::openFile(m_filename);
-//     if (!m_istream)
-//         throwError("Unable to open text file '" + m_filename + "'.");
-//
-//     // Skip header line.
-//     std::string buf;
-//     std::getline(*m_istream, buf);
-//     m_line = 1;
 }
 
 
@@ -140,26 +143,44 @@ point_count_t MatlabReader::read(PointViewPtr view, point_count_t numPts)
 
 bool MatlabReader::processOne(PointRef& point)
 {
-    double d;
-//     for (size_t i = 0; i < m_fields.size(); ++i)
-//     {
-//         if (!Utils::fromString(m_fields[i], d))
-//         {
-//             log()->get(LogLevel::Error) << "Can't convert "
-//                 "field '" << m_fields[i] << "' to numeric value on line " <<
-//                 m_line << " in '" << m_filename << "'.  Setting to 0." <<
-//                 std::endl;
-//             d = 0;
-//         }
-//         point.setField(m_dims[i], d);
-//     }
+    // We read them all
+    if (m_pointIndex == m_numElements)
+        return false;
+
+    for (int i=0; i < m_numFields; ++i)
+    {
+        Dimension::Id d = (Dimension::Id) m_dimensionIdMap[i];
+        Dimension::Type t = (Dimension::Type) m_dimensionTypeMap[i];
+
+        mxArray* f = mxGetFieldByNumber(m_structArray, 0, i);
+        if (!f)
+        {
+            std::ostringstream oss;
+            oss << "Unable to fetch array for point " << m_pointIndex;
+            throwError(oss.str());
+        }
+        PointId numElements = (PointId) mxGetNumberOfElements(f);
+        if (numElements >= m_pointIndex )
+        {
+            size_t size = mxGetElementSize(f);
+            char* p = (char*)mxGetData(f) + size;
+            point.setField(d, t, (void*)p);
+        }
+
+
+    }
+
+    m_pointIndex++;
+
     return true;
 }
 
 
-
 void MatlabReader::done(PointTableRef table)
 {
+    matClose(m_matfile);
+    getMetadata().addList("filename", m_filename);
+    getMetadata().add("struct", m_structName);
 }