Skip to content

Commit

Permalink
Make chunkSize a property of dimension, rather than file
Browse files Browse the repository at this point in the history
  • Loading branch information
Ryan Pals committed Feb 12, 2020
1 parent 9132331 commit dfdf56c
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 33 deletions.
30 changes: 16 additions & 14 deletions plugins/hdf/io/Hdf5Handler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,25 +48,29 @@ void Hdf5Handler::setLog(pdal::LogPtr log) {

DimInfo::DimInfo(
const std::string& dimName,
H5::IntType int_type)
H5::IntType int_type,
hsize_t chunkSize)
: name(dimName)
, hdf_type(H5T_INTEGER)
, endianness(int_type.getOrder())
, sign(int_type.getSign())
, size(int_type.getSize())
, chunkSize(chunkSize)
, pdal_type(sign == H5T_SGN_2 ?
Dimension::Type(unsigned(Dimension::BaseType::Signed) | int_type.getSize()) :
Dimension::Type(unsigned(Dimension::BaseType::Unsigned) | int_type.getSize()))
{ }

DimInfo::DimInfo(
const std::string& dimName,
H5::FloatType float_type)
H5::FloatType float_type,
hsize_t chunkSize)
: name(dimName)
, hdf_type(H5T_FLOAT)
, endianness(float_type.getOrder())
, sign(H5T_SGN_ERROR)
, size(float_type.getSize())
, chunkSize(chunkSize)
, pdal_type(Dimension::Type(unsigned(Dimension::BaseType::Floating) | float_type.getSize()))
{ }

Expand All @@ -86,6 +90,7 @@ void Hdf5Handler::initialize(
std::vector<hsize_t> m_chunkOffset();

for(const auto& [dimName, datasetName] : map.items()) {
hsize_t chunkSize;
m_logger->get(LogLevel::Info) << "Opening dataset '"
<< datasetName << "' with dimension name '" << dimName
<< "'" << std::endl;
Expand All @@ -96,14 +101,14 @@ void Hdf5Handler::initialize(
m_numPoints = dspace.getSelectNpoints();
H5::DSetCreatPropList plist = dset.getCreatePlist();
if(plist.getLayout() == H5D_CHUNKED) {
int dimensionality = plist.getChunk(1, &m_chunkSize);
int dimensionality = plist.getChunk(1, &chunkSize);
if(dimensionality != 1)
throw pdal_error("Only 1-dimensional arrays are supported.");
} else {
m_logger->get(LogLevel::Warning) << "Dataset not chunked; proceeding to read 1024 elements at a time" << std::endl;
m_chunkSize = 1024;
chunkSize = 1024;
}
m_logger->get(LogLevel::Info) << "Chunk size: " << m_chunkSize << std::endl;
m_logger->get(LogLevel::Info) << "Chunk size: " << chunkSize << std::endl;
m_logger->get(LogLevel::Info) << "Num points: " << m_numPoints << std::endl;
H5::DataType dtype = dset.getDataType();
H5T_class_t vague_type = dtype.getClass();
Expand All @@ -113,19 +118,19 @@ void Hdf5Handler::initialize(
}
else if(vague_type == H5T_INTEGER) {
m_dimInfos.push_back(
DimInfo(dimName, dset.getIntType())
DimInfo(dimName, dset.getIntType(), chunkSize)
);
}
else if(vague_type == H5T_FLOAT) {
m_dimInfos.push_back(
DimInfo(dimName, dset.getFloatType())
DimInfo(dimName, dset.getFloatType(), chunkSize)
);
} else {
throw pdal_error("Unkown type: " + vague_type);
}
m_chunkOffsets.push_back(0);
m_buffers.push_back(std::vector<uint8_t>());
m_buffers.at(index).resize(m_chunkSize*dtype.getSize());
m_buffers.at(index).resize(chunkSize*dtype.getSize());
index++;
}
}
Expand All @@ -138,7 +143,8 @@ void Hdf5Handler::close()

uint8_t *Hdf5Handler::getNextChunk(int index) {
hsize_t elementsRemaining = m_numPoints - m_chunkOffsets.at(index);
hsize_t selectionSize = std::min(elementsRemaining, m_chunkSize);
hsize_t chunkSize = m_dimInfos.at(0).chunkSize;
hsize_t selectionSize = std::min(elementsRemaining, chunkSize);

H5::DataSpace memspace(1, &selectionSize);
m_dspaces.at(index).selectHyperslab(H5S_SELECT_SET, &selectionSize, &m_chunkOffsets.at(index));
Expand All @@ -147,7 +153,7 @@ uint8_t *Hdf5Handler::getNextChunk(int index) {
m_dsets.at(index).getDataType(),
memspace,
m_dspaces.at(index) );
m_chunkOffsets.at(index) += m_chunkSize;
m_chunkOffsets.at(index) += chunkSize;
return data.data();
}

Expand All @@ -163,9 +169,5 @@ std::vector<pdal::hdf5::DimInfo> Hdf5Handler::getDimensionInfos() {
}


hsize_t Hdf5Handler::getChunkSize() {
return m_chunkSize;
}

} // namespace pdal

21 changes: 3 additions & 18 deletions plugins/hdf/io/Hdf5Handler.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,28 +50,18 @@ namespace pdal

namespace hdf5
{
struct Hdf5ColumnData
{
Hdf5ColumnData(const std::string& name, const H5::PredType predType)
: name(name)
, predType(predType)
{ }

const std::string name;
const H5::PredType predType;
};

struct DimInfo
{
DimInfo(const std::string& dimName, H5::IntType int_type);
DimInfo(const std::string& dimName, H5::IntType int_type, hsize_t chunkSize);

DimInfo(const std::string& dimName, H5::FloatType float_type);
DimInfo(const std::string& dimName, H5::FloatType float_type, hsize_t chunkSize);

std::string name;
H5T_class_t hdf_type;
H5T_order_t endianness;
H5T_sign_t sign;
size_t size;
hsize_t chunkSize;
Dimension::Type pdal_type;
Dimension::Id id = Dimension::Id::Unknown;
};
Expand All @@ -83,7 +73,6 @@ class Hdf5Handler
void initialize(
const std::string& filename,
const NL::json& map);
// const std::vector<hdf5::Hdf5ColumnData>& columns);
void close();

uint8_t *getNextChunk(int index);
Expand All @@ -92,14 +81,10 @@ class Hdf5Handler
std::vector<pdal::hdf5::DimInfo> getDimensionInfos();

void setLog(pdal::LogPtr log);
hsize_t getChunkSize();

private:
std::vector<pdal::hdf5::DimInfo> m_dimInfos;
std::vector<std::vector<uint8_t>> m_buffers;
hsize_t m_chunkSize;
// H5::DataSet m_dset;
// H5::DataSpace m_dspace;
std::vector<H5::DataSet> m_dsets;
std::vector<H5::DataSpace> m_dspaces;

Expand Down
2 changes: 1 addition & 1 deletion plugins/hdf/io/HdfReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ point_count_t HdfReader::read(PointViewPtr view, point_count_t count)
log()->get(LogLevel::Info) << "type info: " << pdal::Dimension::interpretationName(info.pdal_type) << std::endl;
nextId=0;
for(uint64_t pi = 0; pi < m_hdf5Handler.getNumPoints(); pi++) {
int bufIndex = pi % m_hdf5Handler.getChunkSize();
int bufIndex = pi % info.chunkSize;
if(bufIndex == 0) {
buf = m_hdf5Handler.getNextChunk(index);
}
Expand Down

0 comments on commit dfdf56c

Please sign in to comment.