Merge pull request #976 from pnorbert/fix_hdf5

Fix hdf5
ornladios · Nov 1, 2018 · 939bdf7 · 939bdf7
2 parents 8333e4b + 1ebacbc
commit 939bdf7
Show file tree

Hide file tree

Showing 14 changed files with 1,811 additions and 55 deletions.
diff --git a/source/adios2/core/Stream.cpp b/source/adios2/core/Stream.cpp
@@ -21,7 +21,7 @@ namespace core
 Stream::Stream(const std::string &name, const Mode mode, MPI_Comm comm,
                const std::string engineType, const std::string hostLanguage)
 : m_Name(name), m_ADIOS(std::make_shared<ADIOS>(comm, DebugON, hostLanguage)),
-  m_IO(&m_ADIOS->DeclareIO(name)), m_Mode(mode)
+  m_IO(&m_ADIOS->DeclareIO(name)), m_Mode(mode), m_EngineType(engineType)
 {
     if (mode == adios2::Mode::Read)
     {
@@ -105,10 +105,12 @@ void Stream::CheckOpen()
 {
     if (m_Engine == nullptr)
     {
+        m_IO->SetEngine(m_EngineType);
         m_Engine = &m_IO->Open(m_Name, m_Mode);
         if (m_Mode == adios2::Mode::Write)
         {
             m_Engine->BeginStep();
+            m_StepStatus = true;
         }
     }
 }

diff --git a/source/adios2/core/Stream.h b/source/adios2/core/Stream.h
@@ -108,9 +108,14 @@ class Stream
     /** mode to open engine at first read/write */
     Mode m_Mode;
 
+    /** Sets engine type to be opened at first read/write */
+    std::string m_EngineType = "BPFile";
+
     /** internal flag to check if getstep was called */
     bool m_FirstStep = true;
 
+    bool m_StepStatus = false;
+
     template <class T>
     std::vector<T> GetCommon(Variable<T> &variable);
 

diff --git a/source/adios2/core/Stream.tcc b/source/adios2/core/Stream.tcc
@@ -44,13 +44,18 @@ void Stream::Write(const std::string &name, const T *data, const Dims &shape,
     }
 
     CheckOpen();
+    if (!m_StepStatus)
+    {
+        m_Engine->BeginStep();
+        m_StepStatus = true;
+    }
 
     m_Engine->Put(*variable, data, adios2::Mode::Sync);
 
     if (endStep)
     {
         m_Engine->EndStep();
-        m_Engine->BeginStep();
+        m_StepStatus = false;
     }
 }
 

diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
@@ -255,14 +255,18 @@ void HDF5ReaderP::UseHDFRead(Variable<T> &variable, T *data, hid_t h5Type)
 
 StepStatus HDF5ReaderP::BeginStep(StepMode mode, const float timeoutSeconds)
 {
-    // printf(".... in begin step: \n");
     m_InStreamMode = true;
     int ts = m_H5File.GetNumAdiosSteps();
+
     if (m_StreamAt >= ts)
     {
         return StepStatus::EndOfStream;
     }
 
+    m_IO.RemoveAllVariables();
+    m_IO.RemoveAllAttributes();
+    m_H5File.ReadVariables(m_StreamAt, m_IO);
+
     return StepStatus::OK;
 }
 

diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.tcc b/source/adios2/engine/hdf5/HDF5ReaderP.tcc
@@ -31,6 +31,11 @@ void HDF5ReaderP::GetSyncCommon(Variable<T> &variable, T *data)
     */
     hid_t h5Type = m_H5File.GetHDF5Type<T>();
     //    UseHDFRead(variable.m_Name, data, h5Type);
+    if (m_InStreamMode)
+    {
+        variable.m_StepsStart = m_StreamAt;
+        variable.m_StepsCount = 1;
+    }
     UseHDFRead(variable, data, h5Type);
 }
 

diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
@@ -1054,7 +1054,7 @@ void HDF5Common::LocateAttrParent(const std::string &attrName,
     std::string token;
     while ((pos = s.find(delimiter)) != std::string::npos)
     {
-        if (pos > 1)
+        if (pos > 0)
         { // "///a/b/c" == "a/b/c"
             token = s.substr(0, pos);
             list.push_back(token);

diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt
@@ -12,9 +12,14 @@ if(ADIOS2_HAVE_MPI)
       EXEC_WRAPPER
       ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS})
     python_add_test(NAME PythonBPWriteReadTypes ${test_parameters} 
-    				SCRIPT TestBPWriteReadTypes.py)
+                    SCRIPT TestBPWriteReadTypes.py)
     python_add_test(NAME PythonBPWriteReadHighLevelAPI ${test_parameters} 
-    				SCRIPT TestBPWriteTypesHighLevelAPI.py)
+                    SCRIPT TestBPWriteTypesHighLevelAPI.py)
     python_add_test(NAME PythonBPReadMultisteps ${test_parameters} 
-    				SCRIPT TestBPReadMultisteps.py)
+                    SCRIPT TestBPReadMultisteps.py)
+    if (ADIOS2_HAVE_HDF5)
+        python_add_test(NAME PythonBPWriteReadHighLevelAPI_HDF5 
+                        SCRIPT TestBPWriteTypesHighLevelAPI_HDF5.py)
+    endif(ADIOS2_HAVE_HDF5)
 endif()
+
diff --git a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py
@@ -10,6 +10,7 @@
 #      Author: William F Godoy godoywf@ornl.gov
 
 
+import sys
 from adios2NPTypes import SmallTestData
 from mpi4py import MPI
 import numpy as np
@@ -30,24 +31,25 @@
 # Writer
 with adios2.open("types_np.bp", "w", comm) as fw:
 
-    if(rank == 0):
-        fw.write("tag", "Testing ADIOS2 high-level API")
-        fw.write("gvarI8", np.array(data.I8[0]))
-        fw.write("gvarI16", np.array(data.I16[0]))
-        fw.write("gvarI32", np.array(data.I32[0]))
-        fw.write("gvarI64", np.array(data.I64[0]))
-        fw.write("gvarU8", np.array(data.U8[0]))
-        fw.write("gvarU16", np.array(data.U16[0]))
-        fw.write("gvarU32", np.array(data.U32[0]))
-        fw.write("gvarU64", np.array(data.U64[0]))
-        fw.write("gvarR32", np.array(data.R32[0]))
-        fw.write("gvarR64", np.array(data.R64[0]))
-
     for i in range(0, 5):
 
-        print("i: " + str(i))
+        data.update(rank, i, size)
+
+        if(rank == 0 and i == 0):
+            fw.write("tag", "Testing ADIOS2 high-level API")
+            fw.write("gvarI8", np.array(data.I8[0]))
+            fw.write("gvarI16", np.array(data.I16[0]))
+            fw.write("gvarI32", np.array(data.I32[0]))
+            fw.write("gvarI64", np.array(data.I64[0]))
+            fw.write("gvarU8", np.array(data.U8[0]))
+            fw.write("gvarU16", np.array(data.U16[0]))
+            fw.write("gvarU32", np.array(data.U32[0]))
+            fw.write("gvarU64", np.array(data.U64[0]))
+            fw.write("gvarR32", np.array(data.R32[0]))
+            fw.write("gvarR64", np.array(data.R64[0]))
+
+        print("Write Step: " + str(i) + " rank: " + str(rank))
         fw.write("steps", "Step:" + str(i))
-
         fw.write("varI8", data.I8, shape, start, count)
         fw.write("varI16", data.I16, shape, start, count)
         fw.write("varI32", data.I32, shape, start, count)
@@ -59,13 +61,20 @@
         fw.write("varR32", data.R32, shape, start, count)
         fw.write("varR64", data.R64, shape, start, count, endl=True)
 
+comm.Barrier()
+sys.stdout.flush()
+
 # Reader
+data = SmallTestData()
+
 with adios2.open("types_np.bp", "r", comm) as fr:
 
     for fr_step in fr:
 
         step = fr_step.currentstep()
-        print("Step: " + str(step))
+        data.update(rank, step, size)
+
+        print("Reader Step: " + str(step))
 
         step_vars = fr_step.availablevariables()
 
@@ -101,24 +110,21 @@
             if(inI32[0] != data.I32[0]):
                 raise ValueError('gvarI32 read failed')
 
-            if(inU16[0] != data.U16[0]):
-                raise ValueError('gvarU16 read failed')
-
             if(inI64[0] != data.I64[0]):
                 raise ValueError('gvarI64 read failed')
 
-            if(inU64[0] != data.U64[0]):
-                raise ValueError('gvarU64 read failed')
-
-            if(inI8[0] != data.I8[0]):
-                raise ValueError('gvarI8 read failed')
-
             if(inU8[0] != data.U8[0]):
                 raise ValueError('gvarU8 read failed')
 
+            if(inU16[0] != data.U16[0]):
+                raise ValueError('gvarU16 read failed')
+
             if(inU32[0] != data.U32[0]):
                 raise ValueError('gvarU32 read failed')
 
+            if(inU64[0] != data.U64[0]):
+                raise ValueError('gvarU64 read failed')
+
             if(inR32[0] != data.R32[0]):
                 raise ValueError('gvarR32 read failed')