Skip to content

Commit

Permalink
Address feedback
Browse files Browse the repository at this point in the history
  • Loading branch information
Ryan Pals committed Feb 14, 2020
1 parent 3ca185d commit 138ab4c
Show file tree
Hide file tree
Showing 5 changed files with 46 additions and 42 deletions.
22 changes: 15 additions & 7 deletions doc/stages/readers.hdf.rst
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,26 @@ properly mapped and then outputs a LAS file.
]
This example reads the X, Y, and Z coordinates from the ICESAT2
ATL03 format and converts them into a LAS file.

.. note::
All dimensions must be simple numeric HDF datasets with
equal lengths. Compound types, enum types, string types,
etc. are not supported.


.. warning::
The HDF reader does not set an SRS.


Common Use Cases
----------------

A common use case for this data is reading NASA's ICESAT2 data.
This example reads the X, Y, and Z coordinates from the ICESAT2
ATL03 format and converts them into a LAS file.

.. note::
ICESAT2 data use EPSG:7912.

.. code-block:: json
[
Expand Down Expand Up @@ -85,9 +96,6 @@ Options

.. include:: reader_opts.rst

map
dimensions
A JSON map with PDAL dimension names as the keys and HDF dataset paths as the values.

metadata
TBD

9 changes: 6 additions & 3 deletions plugins/hdf/io/Hdf5Handler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ DimInfo::DimInfo(

void Hdf5Handler::initialize(
const std::string& filename,
const NL::json& map)
const std::map<std::string,std::string>& map)
{
try
{
Expand All @@ -89,8 +89,11 @@ void Hdf5Handler::initialize(
int index = 0;
std::vector<hsize_t> m_chunkOffset;

for(const auto& [dimName, datasetName] : map.items()) {
for( auto const& entry : map) {
std::string const& dimName = entry.first;
std::string const& datasetName = entry.second;
hsize_t chunkSize;

m_logger->get(LogLevel::Info) << "Opening dataset '"
<< datasetName << "' with dimension name '" << dimName
<< "'" << std::endl;
Expand Down Expand Up @@ -150,7 +153,7 @@ void Hdf5Handler::close()

uint8_t *Hdf5Handler::getNextChunk(int index) {
hsize_t elementsRemaining = m_numPoints - m_chunkOffsets.at(index);
hsize_t chunkSize = m_dimInfos.at(0).chunkSize;
hsize_t chunkSize = m_dimInfos.at(index).chunkSize;
hsize_t selectionSize = std::min(elementsRemaining, chunkSize);

H5::DataSpace memspace(1, &selectionSize);
Expand Down
2 changes: 1 addition & 1 deletion plugins/hdf/io/Hdf5Handler.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ class Hdf5Handler
public:
void initialize(
const std::string& filename,
const NL::json& map);
const std::map<std::string,std::string>& map);
void close();

uint8_t *getNextChunk(int index);
Expand Down
35 changes: 14 additions & 21 deletions plugins/hdf/io/HdfReader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -130,40 +130,38 @@ bool HdfReader::processOne(PointRef& point)

void HdfReader::addArgs(ProgramArgs& args)
{
args.add("map", "Map of HDF path to PDAL dimension", m_pathDimMap);
args.add("dimensions", "Map of HDF path to PDAL dimension", m_pathDimJson);
}

void HdfReader::initialize()
{
for(const auto& [key, value] : m_pathDimMap.items()) {
log()->get(LogLevel::Info) << "Key: " << key << ", Val: " << value <<std::endl;
}
validateMap();

// ICESat-2 datasets are EPSG:7912
setSpatialReference(SpatialReference("EPSG:7912"));
parseDimensions();
}

void HdfReader::validateMap()
void HdfReader::parseDimensions()
{
log()->get(LogLevel::Info) << "**JSON map**" << std::endl;
log()->get(LogLevel::Info) << m_pathDimMap << std::endl;
log()->get(LogLevel::Info) << m_pathDimJson << std::endl;

if(m_pathDimMap.is_null()) {
if(m_pathDimJson.is_null()) {
throw pdal_error("Required option 'map' was not set");
} else if(!m_pathDimMap.is_object()) {
throw pdal_error("Option 'map' must be a JSON object, not a " +
std::string(m_pathDimMap.type_name()));
} else if(!m_pathDimJson.is_object()) {
throw pdal_error("Option 'dimensions' must be a JSON object, not a " +
std::string(m_pathDimJson.type_name()));
}

for(auto& [dimName, datasetName] : m_pathDimMap.items()) {
for(auto& entry : m_pathDimJson.items()) {
std::string dimName = entry.key();
auto datasetName = entry.value();
log()->get(LogLevel::Info) << "Key: " << dimName << ", Value: "
<< datasetName << ", Type: " << datasetName.type_name() << std::endl;

if(!datasetName.is_string()) {
throw pdal_error("Every value in 'map' must be a string. Key '"
throw pdal_error("Every value in 'dimensions' must be a string. Key '"
+ dimName + "' has value with type '" +
std::string(datasetName.type_name()) + "'");
} else {
m_pathDimMap[dimName] = datasetName;
}
}
}
Expand All @@ -174,9 +172,4 @@ void HdfReader::done(PointTableRef table)
}


bool HdfReader::eof()
{
return m_index >= m_hdf5Handler->getNumPoints();
}

} // namespace pdal
20 changes: 10 additions & 10 deletions plugins/hdf/io/HdfReader.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -63,16 +63,16 @@ class PDAL_DLL HdfReader : public pdal::Reader, public pdal::Streamable
std::unique_ptr<Hdf5Handler> m_hdf5Handler;
point_count_t m_index;

virtual void addDimensions(PointLayoutPtr layout);
virtual void addArgs(ProgramArgs& args);
virtual void initialize();
virtual void ready(PointTableRef table);
virtual point_count_t read(PointViewPtr view, point_count_t count);
virtual bool processOne(PointRef& point);
virtual void done(PointTableRef table);
virtual bool eof();
virtual void addDimensions(PointLayoutPtr layout) override;
virtual void addArgs(ProgramArgs& args) override;
virtual void initialize() override;
virtual void ready(PointTableRef table) override;
virtual point_count_t read(PointViewPtr view, point_count_t count) override;
virtual bool processOne(PointRef& point) override;
virtual void done(PointTableRef table) override;

NL::json m_pathDimMap;
NL::json m_pathDimJson;
std::map<std::string,std::string> m_pathDimMap;
Dimension::IdList m_idlist;
struct BufferInfo {
BufferInfo(const hdf5::DimInfo& info);
Expand All @@ -82,7 +82,7 @@ class PDAL_DLL HdfReader : public pdal::Reader, public pdal::Streamable
std::vector<BufferInfo> m_info;
// std::vector<std::unique_ptr<hdf5::DimInfo>> m_infos;
// std::vector<uint8_t *> m_bufs;
void validateMap();
void parseDimensions();

HdfReader& operator=(const HdfReader&); // Not implemented.
HdfReader(const HdfReader&); // Not implemented.
Expand Down

0 comments on commit 138ab4c

Please sign in to comment.