Reads one chunk - wip

PDAL · Feb 7, 2020 · c865e88 · c865e88
1 parent 408fed5
commit c865e88
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 4 deletions.
diff --git a/plugins/hdf/io/Hdf5Handler.cpp b/plugins/hdf/io/Hdf5Handler.cpp
@@ -65,6 +65,7 @@ using namespace hdf5;
 Hdf5Handler::Hdf5Handler()
     : m_numPoints(0)
     , m_columnDataMap()
+    , m_chunkOffset(0)
 { }
 
 DimInfo::DimInfo(
@@ -134,7 +135,18 @@ void Hdf5Handler::initialize(
         H5::DataSet dset = m_h5File.get()->openDataSet(datasetName);
         H5::DataSpace dspace = dset.getSpace();
         m_numPoints = dspace.getSelectNpoints();
-        std::cout << "--" << m_numPoints << "--" << std::endl;
+        if(dset.getCreatePlist().getLayout() == H5D_CHUNKED) {
+            int dimensionality = dset.getCreatePlist().getChunk(1, &m_chunkSize);
+            if(dimensionality != 1)
+                std::cout << "Something is wrong" << std::endl <<
+                    "expected dimensionality 1, got " << dimensionality
+                    << std::endl;
+        } else {
+            std::cout << "Dataset not chunked; proceeding to read one element at a time" << std::endl;
+            m_chunkSize = 1;
+        }
+        std::cout << "Chunk size: " << m_chunkSize << std::endl;
+        std::cout << "Num points: " << m_numPoints << std::endl;
         std::cout << "Number of dataspace dimensions: " << dspace.getSimpleExtentNdims() << std::endl;
         H5::DataType dtype = dset.getDataType();
         H5T_class_t vauge_type = dtype.getClass();
@@ -160,8 +172,13 @@ void Hdf5Handler::initialize(
         } else {
             throw error("Unkown type: " + vauge_type);
         }
+        // m_buf = malloc(dtype.getSize() * m_chunkSize); //TODO free
         m_buf = malloc(dtype.getSize() * m_numPoints); //TODO free
-        dset.read(m_buf, dtype);
+        std::cout << "Chunk offset: " << m_chunkOffset << std::endl;
+        // dspace.selectElements(H5S_SELECT_SET, m_chunkSize, &m_chunkOffset);
+        dspace.selectHyperslab(H5S_SELECT_SET, &m_chunkSize, &m_chunkOffset);
+        // H5::DataSpace mspace( 1, &m_chunkOffset);
+        dset.read(m_buf, dtype, H5::DataSpace::ALL, dspace);
     }
     catch (const H5::Exception&)
     {
@@ -174,6 +191,13 @@ void Hdf5Handler::close()
     m_h5File->close();
 }
 
+void *Hdf5Handler::getNextChunk() {
+    // m_dset.read(m_buf, m_dset.getDataType(), memspace, thing);
+    // m_dset.read(m_buf, m_dset.getDataType());
+    m_chunkOffset += m_chunkSize;
+    return m_buf;
+}
+
 uint64_t Hdf5Handler::getNumPoints() const
 {
     return m_numPoints;

diff --git a/plugins/hdf/io/Hdf5Handler.hpp b/plugins/hdf/io/Hdf5Handler.hpp
@@ -114,6 +114,7 @@ class Hdf5Handler
     void close();
 
     void *getBuffer();
+    void *getNextChunk();
 
     uint64_t getNumPoints() const;
     std::vector<pdal::hdf5::DimInfo> getDimensionInfos();
@@ -123,6 +124,7 @@ class Hdf5Handler
             const std::string& dataSetName,
             const hsize_t numEntries,
             const hsize_t offset) const;
+    hsize_t m_chunkSize;
 
 private:
     struct ColumnData
@@ -152,6 +154,11 @@ class Hdf5Handler
     hsize_t getColumnNumEntries(const std::string& dataSetName) const;
     const ColumnData& getColumnData(const std::string& dataSetName) const;
     void *m_buf;
+    void *m_nextVal;
+    // hsize_t m_chunkSize;
+    H5::DataSet m_dset;
+    H5::DataSpace m_dspace;
+    hsize_t m_chunkOffset;
 
     std::unique_ptr<H5::H5File> m_h5File;
     uint64_t m_numPoints;

diff --git a/plugins/hdf/io/HdfReader.cpp b/plugins/hdf/io/HdfReader.cpp
@@ -99,6 +99,9 @@ void HdfReader::addDimensions(PointLayoutPtr layout)
         std::cout << unsigned(m_infos[i].id) << " : ";
         m_infos[i].id = layout->registerOrAssignDim(m_infos[i].name, m_infos[i].pdal_type);
         std::cout << unsigned(m_infos[i].id) << std::endl;
+        std::cout << unsigned(m_infos[i].pdal_type) << std::endl;
+        std::cout << unsigned(pdal::Dimension::Type::Double) << std::endl;
+        std::cout << unsigned(pdal::Dimension::Type::Signed64) << std::endl;
         // ); // TODO: add correct type
     };
     for(auto info: m_infos) {
@@ -146,8 +149,10 @@ point_count_t HdfReader::read(PointViewPtr view, point_count_t count)
     for(auto info : m_infos) {
         PointId nextId = startId;
         std::cout << (unsigned)info.id << ": ";
-        for(uint64_t pi = 0; pi < m_hdf5Handler.getNumPoints(); pi++) {
-            void *p = m_hdf5Handler.getBuffer() + pi*point_size + info.offset;
+        void *buf = m_hdf5Handler.getNextChunk();
+        // for(uint64_t pi = 0; pi < m_hdf5Handler.getNumPoints(); pi++) {
+        for(uint64_t pi = 0; pi < m_hdf5Handler.m_chunkSize; pi++) {
+            void *p = buf + pi*point_size + info.offset;
             if(pi == 0) std::cout<< Dimension::interpretationName(info.pdal_type) <<std::endl;
             switch(info.pdal_type) {
                 case Dimension::Type::Double: