Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

match type of timestep for h5 engine to size_t (same as adios Variabl… #3776

Merged
merged 5 commits into from
Aug 23, 2023
Merged

match type of timestep for h5 engine to size_t (same as adios Variabl… #3776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
29bf70f
match type of timestep for h5 engine to size_t (same as adios Variabl…
guj Aug 22, 2023
702e298
clang-format
guj Aug 22, 2023
3907568
removing warning from auto
guj Aug 22, 2023
56f06b1
removed comments and cleaned up more code
guj Aug 23, 2023
3502c55
removed a comment
guj Aug 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
51 changes: 5 additions & 46 deletions source/adios2/engine/hdf5/HDF5ReaderP.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,14 +77,7 @@ void HDF5ReaderP::Init()
m_H5File.ParseParameters(m_IO);

/*
int ts = m_H5File.GetNumAdiosSteps();

if (ts == 0)
{
helper::Throw<std::runtime_error>( "Engine", "HDF5ReaderP", "Init",
"This h5 file is NOT written by ADIOS2");
}
*/
*/
m_H5File.ReadAttrToIO(m_IO);
if (!m_InStreamMode)
{
Expand All @@ -103,37 +96,6 @@ void HDF5ReaderP::Init()
template <class T>
size_t HDF5ReaderP::ReadDataset(hid_t dataSetId, hid_t h5Type, Variable<T> &variable, T *values)
{
/*
size_t variableStart = variable.m_StepsStart;

if ((m_H5File.m_IsGeneratedByAdios) && (ts >= 0))
{
try
{
m_H5File.SetAdiosStep(variableStart + ts);
}
catch (std::exception &e)
{
printf("[Not fatal] %s\n", e.what());
return 0;
//break;
}
}

std::vector<hid_t> chain;
if (!m_H5File.OpenDataset(variable.m_Name, chain)) {
return -1;
}

hid_t dataSetId = chain.back();
interop::HDF5DatasetGuard g(chain);
//hid_t dataSetId =
// H5Dopen(m_H5File.m_GroupId, variable.m_Name.c_str(), H5P_DEFAULT);
if (dataSetId < 0)
{
return 0;
}
*/
hid_t fileSpace = H5Dget_space(dataSetId);
interop::HDF5TypeGuard g_fs(fileSpace, interop::E_H5_SPACE);

Expand Down Expand Up @@ -248,8 +210,8 @@ void HDF5ReaderP::UseHDFRead(Variable<T> &variable, T *data, hid_t h5Type)
}

T *values = data;
unsigned int ts = 0;
// T *values = data;

size_t ts = 0;
size_t variableStart = variable.m_StepsStart;
/*
// looks like m_StepsStart is defaulted to be 0 now.
Expand Down Expand Up @@ -278,8 +240,7 @@ void HDF5ReaderP::UseHDFRead(Variable<T> &variable, T *data, hid_t h5Type)
}
hid_t dataSetId = chain.back();
interop::HDF5DatasetGuard g(chain);
// hid_t dataSetId =
// H5Dopen(m_H5File.m_GroupId, variable.m_Name.c_str(), H5P_DEFAULT);

if (dataSetId < 0)
{
return;
Expand All @@ -291,8 +252,6 @@ void HDF5ReaderP::UseHDFRead(Variable<T> &variable, T *data, hid_t h5Type)
{
break;
}
// H5Sclose(fileSpace);
// H5Dclose(dataSetId);

ts++;
values += slabsize;
Expand All @@ -304,7 +263,7 @@ void HDF5ReaderP::UseHDFRead(Variable<T> &variable, T *data, hid_t h5Type)

StepStatus HDF5ReaderP::BeginStep(StepMode mode, const float timeoutSeconds)
{
const unsigned int ts = m_H5File.GetNumAdiosSteps();
const size_t ts = m_H5File.GetNumAdiosSteps();

if (m_StreamAt >= ts)
{
Expand Down
3 changes: 2 additions & 1 deletion source/adios2/engine/hdf5/HDF5ReaderP.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ class HDF5ReaderP : public Engine
void Init() final;

bool m_InStreamMode = false; // default is not streaming, i.e. set var timestep range
unsigned int m_StreamAt = 0; // stream step counter
// unsigned int m_StreamAt = 0; // stream step counter
size_t m_StreamAt = 0;
#define declare_type(T) \
void DoGetSync(Variable<T> &, T *) final; \
void DoGetDeferred(Variable<T> &, T *) final; \
Expand Down
101 changes: 35 additions & 66 deletions source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include <complex>
#include <ios>
#include <iostream>
#include <limits>
#include <mutex>
#include <stdexcept>
#include <vector>
Expand Down Expand Up @@ -85,6 +86,14 @@ HDF5Common::HDF5Common()
H5T_NATIVE_LDOUBLE);

m_PropertyTxfID = H5Pcreate(H5P_DATASET_XFER);

size_t size_t_max = (size_t)-1;
if (size_t_max <= std::numeric_limits<unsigned int>::max())
m_TimeStepH5T = H5T_NATIVE_UINT;
else if (size_t_max <= std::numeric_limits<unsigned long>::max())
m_TimeStepH5T = H5T_NATIVE_ULONG;

// default is m_TimeStepH5T = H5T_NATIVE_ULLONG;
}

HDF5Common::~HDF5Common() { Close(); }
Expand Down Expand Up @@ -279,32 +288,35 @@ void HDF5Common::WriteAdiosSteps()

hid_t s = H5Screate(H5S_SCALAR);
hid_t attr = H5Aexists(m_FileId, ATTRNAME_NUM_STEPS.c_str());

if (0 == attr)
attr = H5Acreate(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5T_NATIVE_UINT, s, H5P_DEFAULT,
attr = H5Acreate(m_FileId, ATTRNAME_NUM_STEPS.c_str(), m_TimeStepH5T, s, H5P_DEFAULT,
H5P_DEFAULT);
else
attr = H5Aopen(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5P_DEFAULT);

unsigned int totalAdiosSteps = m_CurrentAdiosStep + 1;
size_t totalAdiosSteps = m_CurrentAdiosStep + 1;

if (m_GroupId < 0)
{
totalAdiosSteps = m_CurrentAdiosStep;
}

H5Awrite(attr, H5T_NATIVE_UINT, &totalAdiosSteps);
H5Awrite(attr, m_TimeStepH5T, &totalAdiosSteps);

H5Sclose(s);
H5Aclose(attr);
}

unsigned int HDF5Common::GetAdiosStep() const { return m_NumAdiosSteps; }
size_t HDF5Common::GetAdiosStep() const { return m_NumAdiosSteps; }

unsigned int HDF5Common::GetNumAdiosSteps()
// A valid file should have at least 1 step.
size_t HDF5Common::GetNumAdiosSteps()
{
if (m_WriteMode)
{
return static_cast<unsigned int>(-1);
// return static_cast<unsigned int>(-1);
return 0;
}

if (m_FileId < 0)
Expand All @@ -329,7 +341,7 @@ unsigned int HDF5Common::GetNumAdiosSteps()
{
hid_t attr = H5Aopen(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5P_DEFAULT);

H5Aread(attr, H5T_NATIVE_UINT, &m_NumAdiosSteps);
H5Aread(attr, m_TimeStepH5T, &m_NumAdiosSteps);
H5Aclose(attr);
}
}
Expand All @@ -347,22 +359,20 @@ void HDF5Common::ReadAllVariables(core::IO &io)
}

GetNumAdiosSteps();
unsigned int i = 0;

for (i = 0; i < m_NumAdiosSteps; i++)
for (size_t i = 0; i < m_NumAdiosSteps; i++)
{
ReadVariables(i, io);
}
}

void HDF5Common::FindVarsFromH5(core::IO &io, hid_t top_id, const char *gname, const char *heritage,
unsigned int ts)
size_t ts)
{
// int i = 0;
// std::string stepStr;
hsize_t numObj;

// StaticGetAdiosStepString(stepStr, ts);
hid_t gid = H5Gopen2(top_id, gname, H5P_DEFAULT);
HDF5TypeGuard g(gid, E_H5_GROUP);
/// if (gid > 0) {
Expand Down Expand Up @@ -419,52 +429,17 @@ void HDF5Common::FindVarsFromH5(core::IO &io, hid_t top_id, const char *gname, c
}
}
}
/*
void HDF5Common::ReadNativeGroup(hid_t hid, IO& io)
{
H5G_info_t group_info;
herr_t result = H5Gget_info(hid, &group_info);

if (result < 0) {
// error
helper::Throw<std::ios_base::failure>( "Toolkit", "interop::hdf5::HDF5Common",
"ReadNativeGroup", "Unable to get group info");
}

if (group_info.nlinks == 0) {
return;
}

char tmpstr[1024];

hsize_t idx;
for (idx=0; idx<group_info.nlinks; idx++) {
int currType = H5Gget_objtype_by_idx(hid, idx);
if (currType < 0) {
helper::Throw<std::ios_base::failure>( "Toolkit", "interop::hdf5::HDF5Common",
"ReadNativeGroup", "unable to get type info of idx"+idx);
}


ssize_t curr= H5Gget_objname_by_idx(hid, idx, tmpstr, sizeof(tmpstr));
if (curr > 0) { // got a name
std::cout<<" ... printing a name: "<<tmpstr<<",
type:[0=G/1=D/2=T/3=L/4=UDL]"<<currType<<std::endl;
}
}
}
*/

// read variables from the input timestep
void HDF5Common::ReadVariables(unsigned int ts, core::IO &io)
void HDF5Common::ReadVariables(size_t ts, core::IO &io)
{
std::string stepStr;
hsize_t numObj;

StaticGetAdiosStepString(stepStr, ts);
hid_t gid = H5Gopen2(m_FileId, stepStr.c_str(), H5P_DEFAULT);
HDF5TypeGuard g(gid, E_H5_GROUP);
/// if (gid > 0) {

herr_t ret = H5Gget_num_objs(gid, &numObj);
if (ret >= 0)
{
Expand Down Expand Up @@ -500,12 +475,9 @@ void HDF5Common::ReadVariables(unsigned int ts, core::IO &io)
}
}
}
/// H5Gclose(gid);
///}
}

void HDF5Common::AddSingleString(core::IO &io, std::string const &name, hid_t datasetId,
unsigned int ts)
void HDF5Common::AddSingleString(core::IO &io, std::string const &name, hid_t datasetId, size_t ts)
{
try
{
Expand All @@ -530,8 +502,7 @@ void HDF5Common::AddSingleString(core::IO &io, std::string const &name, hid_t da
}
}

void HDF5Common::AddVarString(core::IO &io, std::string const &name, hid_t datasetId,
unsigned int ts)
void HDF5Common::AddVarString(core::IO &io, std::string const &name, hid_t datasetId, size_t ts)
{
core::Variable<std::string> *v = io.InquireVariable<std::string>(name);
if (v != NULL)
Expand All @@ -548,7 +519,6 @@ void HDF5Common::AddVarString(core::IO &io, std::string const &name, hid_t datas
H5Sget_simple_extent_dims(dspace, dims.data(), NULL);
H5Sclose(dspace);

// if ( (ndims > 1) || ( (ndims == 1) && (dims[0] > 1) ) ) {
if ((ndims > 0))
{
bool isSingleElement = true;
Expand Down Expand Up @@ -613,7 +583,7 @@ void HDF5Common::AddVarString(core::IO &io, std::string const &name, hid_t datas
}

template <class T>
void HDF5Common::AddVar(core::IO &io, std::string const &name, hid_t datasetId, unsigned int ts)
void HDF5Common::AddVar(core::IO &io, std::string const &name, hid_t datasetId, size_t ts)
{
core::Variable<T> *v = io.InquireVariable<T>(name);
if (NULL == v)
Expand Down Expand Up @@ -678,7 +648,7 @@ void HDF5Common::AddVar(core::IO &io, std::string const &name, hid_t datasetId,
}

void HDF5Common::CreateVar(core::IO &io, hid_t datasetId, std::string const &nameSuggested,
unsigned int ts)
size_t ts)
{
std::string name;
ReadADIOSName(datasetId, name);
Expand Down Expand Up @@ -798,7 +768,7 @@ void HDF5Common::Close()
m_GroupId = -1;
}

void HDF5Common::SetAdiosStep(int step)
void HDF5Common::SetAdiosStep(size_t step)
{
if (m_WriteMode)
helper::Throw<std::ios_base::failure>("Toolkit", "interop::hdf5::HDF5Common",
Expand All @@ -811,13 +781,12 @@ void HDF5Common::SetAdiosStep(int step)

GetNumAdiosSteps();

unsigned int ustep = static_cast<unsigned int>(step);
if (ustep >= m_NumAdiosSteps)
if (step >= m_NumAdiosSteps)
helper::Throw<std::ios_base::failure>("Toolkit", "interop::hdf5::HDF5Common",
"SetAdiosStep",
"given time step is more than actual known steps");

if (m_CurrentAdiosStep == ustep)
if (m_CurrentAdiosStep == step)
{
return;
}
Expand All @@ -835,7 +804,7 @@ void HDF5Common::SetAdiosStep(int step)
"ERROR: unable to open HDF5 group " + stepName + ", in call to Open");
}

m_CurrentAdiosStep = ustep;
m_CurrentAdiosStep = step;
}

//
Expand Down Expand Up @@ -896,11 +865,11 @@ void HDF5Common::Advance()
}
else
{
if (m_NumAdiosSteps == 0)
if (0 == m_NumAdiosSteps)
{
GetNumAdiosSteps();
}
if (m_CurrentAdiosStep + 1 >= m_NumAdiosSteps)
if ((m_CurrentAdiosStep + 1) >= m_NumAdiosSteps)
{
return;
}
Expand Down Expand Up @@ -1467,7 +1436,7 @@ void HDF5Common::LocateAttrParent(const std::string &attrName, std::vector<std::
if (list.size() >= 1)
{
std::string ts;
for (unsigned int i = 0; i < m_CurrentAdiosStep; i++)
for (size_t i = 0; i < m_CurrentAdiosStep; i++)
{
StaticGetAdiosStepString(ts, i);
for (size_t j = 0; j < list.size() - 1; j++)
Expand Down Expand Up @@ -1725,7 +1694,7 @@ void HDF5Common::ReadNativeAttrToIO(core::IO &io, hid_t datasetId, std::string c
}
}

void HDF5Common::StaticGetAdiosStepString(std::string &stepName, int ts)
void HDF5Common::StaticGetAdiosStepString(std::string &stepName, size_t ts)
{
stepName = "/Step" + std::to_string(ts);
}
Expand Down