readers.numpy test scaffolding

plugins/python/io/CMakeLists.txt

@@ -9,3 +9,13 @@ target_include_directories(${python_libname} PRIVATE
 target_link_libraries(${python_libname} PUBLIC
     ${PYTHON_LIBRARY} ${CMAKE_DL_LIBS} PRIVATE ${PDAL_PLANG_LIB_NAME})
 
+
+if (WITH_TESTS)
+
+    PDAL_ADD_TEST(pdal_io_numpy_test
+        FILES ../test/NumpyReaderTest.cpp
+        LINK_WITH ${python_libname} )
+    target_link_libraries(pdal_io_numpy_test PUBLIC
+        ${PYTHON_LIBRARY} ${CMAKE_DL_LIBS}  PRIVATE  ${PDAL_PLANG_LIB_NAME})
+
+endif()

plugins/python/io/NumpyReader.cpp

@@ -39,8 +39,17 @@
 #include <pdal/PDALUtils.hpp>
 #include <pdal/pdal_types.hpp>
 #include <pdal/util/ProgramArgs.hpp>
-#include "../plang/Invocation.hpp"
 
+#include <Python.h>
+#undef toupper
+#undef tolower
+#undef isspace
+
+#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+
+#define NO_IMPORT_ARRAY
+#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API
+#include <numpy/arrayobject.h>
 
 namespace pdal
 {
@@ -54,29 +63,11 @@ CREATE_SHARED_PLUGIN(1, 0, NumpyReader, Reader, s_info)
 
 std::string NumpyReader::getName() const { return s_info.name; }
 
-void NumpyReader::initialize()
-{
-
-}
-
-
-
-void NumpyReader::addArgs(ProgramArgs& args)
-{
-}
-
-
-
-void NumpyReader::addDimensions(PointLayoutPtr layout)
-{
-    log()->get(LogLevel::Debug) << "Adding dimensions in readers.numpy" << std::endl;
-}
-
 
 PyObject* load_npy(std::string const& filename)
 {
 
-    PyObject* py_filename =  PyUnicode_FromString(filename.c_str());
+    PyObject* py_filename =  PyString_FromString(filename.c_str());
     PyObject* numpy_module = PyImport_ImportModule("numpy");
     if (!numpy_module)
         throw pdal::pdal_error(plang::getTraceback());
@@ -85,7 +76,7 @@ PyObject* load_npy(std::string const& filename)
     if (!numpy_mod_dict)
         throw pdal::pdal_error(plang::getTraceback());
 
-    PyObject* loads_func = PyDict_GetItemString(numpy_mod_dict, "loads");
+    PyObject* loads_func = PyDict_GetItemString(numpy_mod_dict, "load");
     if (!loads_func)
         throw pdal::pdal_error(plang::getTraceback());
 
@@ -104,13 +95,77 @@ PyObject* load_npy(std::string const& filename)
     return array;
 }
 
+
+void NumpyReader::initialize()
+{
+    plang::Environment::get();
+    m_array = load_npy(m_filename);
+    if (!PyArray_Check(m_array))
+        throw pdal::pdal_error("Object in file  '" + m_filename +
+            "' is not a numpy array");
+}
+
+
+
+void NumpyReader::addArgs(ProgramArgs& args)
+{
+}
+
+
+
+void NumpyReader::addDimensions(PointLayoutPtr layout)
+{
+    using namespace Dimension;
+
+    // TODO pivot whether we are a 1d, 2d, or named arrays
+
+    PyArrayObject* arr = (PyArrayObject*)m_array;
+    PyArray_Descr *dtype = PyArray_DTYPE(arr);
+    if (!dtype)
+        throw pdal::pdal_error(plang::getTraceback());
+
+    // Get length of fields
+    Py_ssize_t count = PyDict_Size(dtype->fields);
+    log()->get(LogLevel::Debug) << "Adding " << count <<" dimensions" << std::endl;
+    std::cerr<< "Adding " << count <<" dimensions" << std::endl;
+
+
+//     PyObject* dtype = PyObject_GetAttrString(m_array, "dtype");
+//     if (!dtype)
+//         throw pdal::pdal_error(plang::getTraceback());
+//     PyObject* names = PyObject_GetAttrString(dtype, "names");
+//     if (!names)
+//         throw pdal::pdal_error(plang::getTraceback());
+//
+//     // Get length of names list
+//     Py_ssize_t count = PySequence_Size(names);
+//     log()->get(LogLevel::Debug) << "Adding " << count <<" dimensions" << std::endl;
+//     std::cerr<< "Adding " << count <<" dimensions" << std::endl;
+//
+//     for (int i=0; i < (int) count; i++)
+//     {
+//         char* name = PyString_AsString(PySequence_GetItem(names, i));
+//         Id id = Dimension::id(name);
+//         if (id != Id::Unknown)
+//         {
+//
+//         }
+//         log()->get(LogLevel::Debug) << "Adding dimension '" << name <<"'" << std::endl;
+//
+//     }
+//     layout->registerDim(Id::X, Type::Double);
+
+
+}
+
+
 void NumpyReader::ready(PointTableRef table)
 {
     // wake up python
-    static_cast<plang::Environment*>(plang::Environment::get())->set_stdout(log()->getLogStream());
 
-    // open .npy file
 
+    static_cast<plang::Environment*>(plang::Environment::get())->set_stdout(log()->getLogStream());
+    log()->get(LogLevel::Debug) << "Initializing Numpy array '" << m_filename <<"'" << std::endl;
 }
 
 

plugins/python/io/NumpyReader.hpp

@@ -38,6 +38,8 @@
 #include <pdal/pdal_features.hpp>
 #include <pdal/Reader.hpp>
 #include <pdal/Streamable.hpp>
+#include "../plang/Invocation.hpp"
+
 
 namespace pdal
 {
@@ -62,9 +64,8 @@ class PDAL_DLL NumpyReader : public Reader, public Streamable
     virtual point_count_t read(PointViewPtr view, point_count_t count);
     virtual bool processOne(PointRef& point);
     virtual void done(PointTableRef table);
-    virtual bool eof()
-        { return true; } //return m_index >= getNumPoints(); }
 
+    PyObject* m_array;
 
     NumpyReader& operator=(const NumpyReader&); // not implemented
     NumpyReader(const NumpyReader&); // not implemented

plugins/python/plang/Environment.cpp

@@ -345,6 +345,42 @@ int Environment::getPythonDataType(Dimension::Type t)
 }
 
 
+Dimension::Type Environment::getPDALDataType(int t)
+{
+    using namespace Dimension;
+
+    switch (t)
+    {
+    case NPY_FLOAT:
+        return Type::Float;
+    case NPY_DOUBLE:
+        return Type::Double;
+    case NPY_BYTE:
+        return Type::Signed8;
+    case NPY_SHORT:
+        return Type::Signed16;
+    case NPY_INT:
+        return Type::Signed32;
+    case NPY_LONGLONG:
+        return Type::Signed64;
+    case NPY_UBYTE:
+        return Type::Unsigned8;
+    case NPY_USHORT:
+        return Type::Unsigned16;
+    case NPY_UINT:
+        return Type::Unsigned32;
+    case NPY_ULONGLONG:
+        return Type::Unsigned64;
+    default:
+        return Type::None;
+    }
+    assert(0);
+
+    return Type::None;
+}
+
+
+
 
 } // namespace plang
 } // namespace pdal

plugins/python/plang/Environment.hpp

@@ -79,6 +79,7 @@ class PDAL_DLL Environment
     static EnvironmentPtr get();
 
     static int getPythonDataType(Dimension::Type t);
+    static pdal::Dimension::Type getPDALDataType(int t);
 
 private:
     Redirector m_redirector;

plugins/python/test/NumpyReaderTest.cpp

@@ -0,0 +1,80 @@
+/******************************************************************************
+* Copyright (c) 2018, Howard Butler (howard@hobu.co)
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#include <pdal/pdal_test_main.hpp>
+
+#include <pdal/PipelineManager.hpp>
+#include <pdal/StageFactory.hpp>
+#include <filters/StatsFilter.hpp>
+
+#include "../io/NumpyReader.hpp"
+
+#include <pdal/StageWrapper.hpp>
+
+#include "Support.hpp"
+
+using namespace pdal;
+
+
+class NumpyReaderTest : public ::testing::Test
+{
+public:
+    virtual void SetUp()
+    {
+        pdal::plang::Environment::get();
+    }
+
+};
+
+TEST_F(NumpyReaderTest, NumpyReaderTest_read)
+{
+    StageFactory f;
+
+    Options ops;
+//         Support::temppath("mylog_three.txt"),
+    ops.add("filename", Support::datapath("plang/1.2-with-color.npy"));
+
+    NumpyReader reader;
+    reader.setOptions(ops);
+
+    PointTable table;
+
+    reader.prepare(table);
+    reader.log()->setLevel(LogLevel::Debug1);
+    PointViewSet viewSet = reader.execute(table);
+    EXPECT_EQ(viewSet.size(), 1065u);
+    PointViewPtr view = *viewSet.begin();
+
+}
+

plugins/python/test/make-test-data.py

@@ -0,0 +1,10 @@
+
+import laspy
+
+f = laspy.file.File('../../../test/data/las/1.2-with-color.las')
+
+import numpy
+import pdb;pdb.set_trace()
+out = open('../../../test/data/plang/1.2-with-color.npy','wb')
+numpy.save(out, f.points['point'], allow_pickle=False)
+out.close()