Merge pull request #3732 from scottwittenburg/ci-add-hdf5-to-windows

ci: Add HDF5 to a windows build
ornladios · Sep 5, 2023 · fe61f77 · fe61f77
2 parents a021f45 + 2e37421
commit fe61f77
Show file tree

Hide file tree

Showing 13 changed files with 170 additions and 36 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -287,6 +287,11 @@ if(BUILD_SHARED_LIBS AND ADIOS2_RUN_INSTALL_TEST)
   endif()
 endif()
 
+if(MSVC AND BUILD_SHARED_LIBS AND ADIOS2_HAVE_HDF5)
+  # See note about building with visual studio and shared libs, here:
+  #     https://github.com/HDFGroup/hdf5/blob/develop/release_docs/USING_HDF5_VS.txt
+  add_definitions(-DH5_BUILT_AS_DYNAMIC_LIB=1)
+endif()
 
 #------------------------------------------------------------------------------#
 # Third party libraries

diff --git a/examples/heatTransfer/write/IO_hdf5_a.cpp b/examples/heatTransfer/write/IO_hdf5_a.cpp
@@ -49,7 +49,7 @@ void IO::write(int step, const HeatTransfer &ht, const Settings &s, MPI_Comm com
     double total_time = MPI_Wtime() - time_start;
     uint64_t adios_totalsize = 2 * sizeof(int) + 2 * s.ndx * s.ndy * sizeof(double);
     uint64_t sizeMB = adios_totalsize * s.nproc / 1024 / 1024 / 1024; // size in MB
-    uint64_t mbs = sizeMB / total_time;
+    double mbs = sizeMB / total_time;
     if (s.rank == 0)
         std::cout << "Step " << step << ": " << m_outputfilename << " " << sizeMB << " "
                   << total_time << "" << mbs << std::endl;

diff --git a/examples/hello/hdf5Reader/helloHDF5Reader.cpp b/examples/hello/hdf5Reader/helloHDF5Reader.cpp
@@ -23,9 +23,9 @@ void ReadData(adios2::IO h5IO, adios2::Engine &h5Reader, const std::string &name
 
     if (var)
     {
-        int nDims = var.Shape().size();
+        size_t nDims = var.Shape().size();
         size_t totalSize = 1;
-        for (int i = 0; i < nDims; i++)
+        for (size_t i = 0; i < nDims; i++)
         {
             totalSize *= var.Shape()[i];
         }

diff --git a/examples/hello/hdf5Reader/helloHDF5Reader_nompi.cpp b/examples/hello/hdf5Reader/helloHDF5Reader_nompi.cpp
@@ -22,9 +22,9 @@ void ReadData(adios2::IO h5IO, adios2::Engine h5Reader, const std::string &name)
 
     if (var)
     {
-        int nDims = var.Shape().size();
+        size_t nDims = var.Shape().size();
         size_t totalSize = 1;
-        for (int i = 0; i < nDims; i++)
+        for (size_t i = 0; i < nDims; i++)
         {
             totalSize *= var.Shape()[i];
         }

diff --git a/scripts/ci/cmake/ci-win2022-vs2022-ompi.cmake b/scripts/ci/cmake/ci-win2022-vs2022-ompi.cmake
@@ -3,9 +3,15 @@ set(ENV{CXX} cl)
 set(ENV{CFLAGS} /WX)
 set(ENV{CXXFLAGS} /WX)
 
+# Tests need to find hdf5.dll
+set(ENV{PATH} "$ENV{PATH};C:/hdf5/HDF5-1.14.2.1-win64/bin")
+
 set(dashboard_cache "
 ADIOS2_USE_Fortran:BOOL=OFF
 ADIOS2_USE_MPI:BOOL=ON
+ADIOS2_USE_HDF5:STRING=ON
+ADIOS2_USE_HDF5_VOL:STRING=OFF
+HDF5_ROOT:PATH=C:/hdf5/HDF5-1.14.2.1-win64
 ")
 
 set(CTEST_CMAKE_GENERATOR "Visual Studio 17 2022")

diff --git a/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake b/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake
@@ -3,9 +3,15 @@ set(ENV{CXX} cl)
 set(ENV{CFLAGS} /WX)
 set(ENV{CXXFLAGS} /WX)
 
+# Tests need to find hdf5.dll
+set(ENV{PATH} "$ENV{PATH};C:/hdf5/HDF5-1.14.2.1-win64/bin")
+
 set(dashboard_cache "
 ADIOS2_USE_Fortran:BOOL=OFF
 ADIOS2_USE_MPI:BOOL=OFF
+ADIOS2_USE_HDF5:STRING=ON
+ADIOS2_USE_HDF5_VOL:STRING=OFF
+HDF5_ROOT:PATH=C:/hdf5/HDF5-1.14.2.1-win64
 ")
 
 set(CTEST_CMAKE_GENERATOR "Visual Studio 17 2022")

diff --git a/scripts/ci/gh-actions/windows-setup.ps1 b/scripts/ci/gh-actions/windows-setup.ps1
@@ -56,3 +56,27 @@ if($Env:GH_YML_MATRIX_PARALLEL -eq "ompi")
   Write-Host "::endgroup::"
 }
 
+if ($Env:GH_YML_MATRIX_COMPILER -eq "vs2022")
+{
+  $url = "https://github.com/HDFGroup/hdf5/releases/download/hdf5-1_14_1-2/hdf5-1_14_1-2-win_vs2022.zip"
+
+  $tempdir    = $Env:RUNNER_TEMP
+  $hdf5Zip    = Join-Path $tempdir hdf5-1_14_1-2-win_vs2022.zip
+  $installDir  = "C:\hdf5"
+  $extractDir = Join-Path $tempdir hdf5
+  $innerZip   = Join-Path $extractDir HDF5-1.14.2.1-win64.zip
+
+  Write-Host "::group::Downloading windows HDF5 release $url"
+  Invoke-WebRequest $url -OutFile $hdf5Zip
+  Write-Host "::endgroup::"
+
+  Write-Host "::group::Extracting outer windows HDF5 release $hdf5Zip"
+  Expand-Archive $hdf5Zip -DestinationPath $tempDir -Force
+  Write-Host "::endgroup::"
+
+  Write-Host "::group::Extracting inner windows HDF5 release $innerZip"
+  Expand-Archive $innerZip -DestinationPath $installDir -Force
+  Write-Host "::endgroup::"
+
+  # Now the HDF5_DIR can be set to: C:\hdf5\HDF5-1.14.2.1-win64\cmake
+}
diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
@@ -13,6 +13,8 @@
 
 #include "adios2/helper/adiosFunctions.h" //CSVToVector
 #include "adios2/helper/adiosFunctions.h" //IsHDF5
+#include <limits>
+#include <stdexcept>
 #include <vector>
 
 namespace adios2
@@ -145,10 +147,18 @@ size_t HDF5ReaderP::ReadDataset(hid_t dataSetId, hid_t h5Type, Variable<T> &vari
         if (ret < 0)
             return 0;
 
-        hid_t memDataSpace = H5Screate_simple(ndims, count.data(), NULL);
+        size_t max_int = static_cast<size_t>(std::numeric_limits<int>::max());
+        if (ndims > max_int)
+        {
+            helper::Throw<std::overflow_error>("Engine", "HDF5ReaderP", "ReadDataset",
+                                               "Number of dimensions is too large to be "
+                                               "represented by an int");
+        }
+
+        hid_t memDataSpace = H5Screate_simple(static_cast<int>(ndims), count.data(), NULL);
         interop::HDF5TypeGuard g_mds(memDataSpace, interop::E_H5_SPACE);
 
-        int elementsRead = 1;
+        size_t elementsRead = 1;
         for (size_t i = 0u; i < ndims; i++)
         {
             elementsRead *= count[i];

diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
@@ -135,8 +135,18 @@ void HDF5Common::ParseParameters(core::IO &io)
 
             m_ChunkPID = H5Pcreate(H5P_DATASET_CREATE);
             m_ChunkDim = chunkDim.size();
+
+            size_t max_int = static_cast<size_t>(std::numeric_limits<int>::max());
+            if (m_ChunkDim > max_int)
+            {
+                helper::Throw<std::overflow_error>("Toolkit", "interop::hdf5::HDF5Common",
+                                                   "ParseParameters",
+                                                   "chunkDim.size() is "
+                                                   "too large to be represented by an int");
+            }
+
             if (m_ChunkDim > 0)
-                H5Pset_chunk(m_ChunkPID, chunkDim.size(), chunkDim.data());
+                H5Pset_chunk(m_ChunkPID, static_cast<int>(m_ChunkDim), chunkDim.data());
         }
     }
 
@@ -383,8 +393,8 @@ void HDF5Common::FindVarsFromH5(core::IO &io, hid_t top_id, const char *gname, c
         char name[100];
         for (k = 0; k < numObj; k++)
         {
-            ret = H5Gget_objname_by_idx(gid, (hsize_t)k, name, sizeof(name));
-            if (ret >= 0)
+            ssize_t result = H5Gget_objname_by_idx(gid, (hsize_t)k, name, sizeof(name));
+            if (result >= 0)
             {
                 int currType = H5Gget_objtype_by_idx(gid, k);
                 if ((currType == H5G_DATASET) || (currType == H5G_TYPE))
@@ -447,8 +457,8 @@ void HDF5Common::ReadVariables(size_t ts, core::IO &io)
         char name[50];
         for (k = 0; k < numObj; k++)
         {
-            ret = H5Gget_objname_by_idx(gid, (hsize_t)k, name, sizeof(name));
-            if (ret >= 0)
+            ssize_t result = H5Gget_objname_by_idx(gid, (hsize_t)k, name, sizeof(name));
+            if (result >= 0)
             {
                 int currType = H5Gget_objtype_by_idx(gid, k);
                 if (currType == H5G_GROUP)
@@ -497,8 +507,8 @@ void HDF5Common::AddSingleString(core::IO &io, std::string const &name, hid_t da
     {
         // invalid variable, do not define
         printf("WARNING: IO is not accepting definition of variable: %s. "
-               "Skipping. \n",
-               name.c_str());
+               "\nException: %s. \nSkipping. \n",
+               name.c_str(), e.what());
     }
 }
 
@@ -568,9 +578,9 @@ void HDF5Common::AddVarString(core::IO &io, std::string const &name, hid_t datas
                     {
                         // invalid variable, do not define
                         printf("WARNING: IO is not accepting definition of "
-                               "variable: %s. "
+                               "variable: %s. \nException: %s. \n"
                                "Skipping. \n",
-                               name.c_str());
+                               name.c_str(), e.what());
                     }
                 }
             return;
@@ -632,8 +642,8 @@ void HDF5Common::AddVar(core::IO &io, std::string const &name, hid_t datasetId,
         {
             // invalid variable, do not define
             printf("WARNING: IO is not accepting definition of variable: %s. "
-                   "Skipping. \n",
-                   name.c_str());
+                   "\nException: %s. \nSkipping. \n",
+                   name.c_str(), e.what());
         }
     }
     else

diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.h b/source/adios2/toolkit/interop/hdf5/HDF5Common.h
@@ -261,7 +261,7 @@ class HDF5Common
     hid_t m_TimeStepH5T = H5T_NATIVE_ULLONG;
     ;
     hid_t m_ChunkPID;
-    int m_ChunkDim;
+    size_t m_ChunkDim;
     std::set<std::string> m_ChunkVarNames;
     bool m_OrderByC = true; // C or fortran
 

diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc b/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc
@@ -25,7 +25,7 @@ namespace interop
 template <class T>
 void HDF5Common::DefineDataset(core::Variable<T> &variable)
 {
-    int dimSize = std::max(variable.m_Shape.size(), variable.m_Count.size());
+    size_t dimSize = std::max(variable.m_Shape.size(), variable.m_Count.size());
     hid_t h5Type = GetHDF5Type<T>();
 
     if (dimSize == 0)
@@ -43,7 +43,15 @@ void HDF5Common::DefineDataset(core::Variable<T> &variable)
     std::vector<hsize_t> dimsf, count, offset;
     GetHDF5SpaceSpec(variable, dimsf, count, offset);
 
-    hid_t fileSpace = H5Screate_simple(dimSize, dimsf.data(), NULL);
+    size_t max_int = static_cast<size_t>(std::numeric_limits<int>::max());
+    if (dimSize > max_int)
+    {
+        helper::Throw<std::overflow_error>("Toolkit", "interop::hdf5::HDF5Common", "DefineDataset",
+                                           "dimSize is too large "
+                                           "to be represented by an int");
+    }
+
+    hid_t fileSpace = H5Screate_simple(static_cast<int>(dimSize), dimsf.data(), NULL);
     HDF5TypeGuard fs(fileSpace, E_H5_SPACE);
 
     std::vector<hid_t> chain;
@@ -110,7 +118,7 @@ template <class T>
 void HDF5Common::Write(core::Variable<T> &variable, const T *values)
 {
     CheckWriteGroup();
-    int dimSize = std::max(variable.m_Shape.size(), variable.m_Count.size());
+    size_t dimSize = std::max(variable.m_Shape.size(), variable.m_Count.size());
     hid_t h5Type = GetHDF5Type<T>();
 
     if (std::is_same<T, std::string>::value)
@@ -173,7 +181,15 @@ void HDF5Common::Write(core::Variable<T> &variable, const T *values)
     std::vector<hsize_t> dimsf, count, offset;
     GetHDF5SpaceSpec(variable, dimsf, count, offset);
 
-    hid_t fileSpace = H5Screate_simple(dimSize, dimsf.data(), NULL);
+    size_t max_int = static_cast<size_t>(std::numeric_limits<int>::max());
+    if (dimSize > max_int)
+    {
+        helper::Throw<std::overflow_error>("Toolkit", "interop::hdf5::HDF5Common", "Write",
+                                           "dimSize is too large "
+                                           "to be represented by an int");
+    }
+
+    hid_t fileSpace = H5Screate_simple(static_cast<int>(dimSize), dimsf.data(), NULL);
 #ifndef RELAY_DEFINE_TO_HDF5 // RELAY_DEFINE_TO_HDF5 = variables in io are
                              // created at begin_step
     std::vector<hid_t> chain;
@@ -187,7 +203,7 @@ void HDF5Common::Write(core::Variable<T> &variable, const T *values)
     {
     }
 #endif
-    hid_t memSpace = H5Screate_simple(dimSize, count.data(), NULL);
+    hid_t memSpace = H5Screate_simple(static_cast<int>(dimSize), count.data(), NULL);
 
     // Select hyperslab
     fileSpace = H5Dget_space(dsetID);

diff --git a/testing/adios2/engine/SmallTestData.h b/testing/adios2/engine/SmallTestData.h
@@ -68,19 +68,22 @@ struct SmallTestData
     std::array<bool, 10> TF = {{true, false, true, true, false, false, true, false, false, true}};
 };
 
-SmallTestData generateNewSmallTestData(SmallTestData in, int step, int rank, int size)
+SmallTestData generateNewSmallTestData(SmallTestData in, size_t step, size_t rank, size_t size)
 {
-    int j = rank + 1 + step * size;
-    std::for_each(in.I8.begin(), in.I8.end(), [&](int8_t &v) { v += j; });
-    std::for_each(in.I16.begin(), in.I16.end(), [&](int16_t &v) { v += j; });
-    std::for_each(in.I32.begin(), in.I32.end(), [&](int32_t &v) { v += j; });
-    std::for_each(in.I64.begin(), in.I64.end(), [&](int64_t &v) { v += j; });
-    std::for_each(in.U8.begin(), in.U8.end(), [&](uint8_t &v) { v += j; });
-    std::for_each(in.U16.begin(), in.U16.end(), [&](uint16_t &v) { v += j; });
-    std::for_each(in.U32.begin(), in.U32.end(), [&](uint32_t &v) { v += j; });
-    std::for_each(in.U64.begin(), in.U64.end(), [&](uint64_t &v) { v += j; });
-    std::for_each(in.R32.begin(), in.R32.end(), [&](float &v) { v += j; });
-    std::for_each(in.R64.begin(), in.R64.end(), [&](double &v) { v += j; });
+    size_t j = rank + 1 + step * size;
+    std::for_each(in.I8.begin(), in.I8.end(), [&](int8_t &v) { v += static_cast<int8_t>(j); });
+    std::for_each(in.I16.begin(), in.I16.end(), [&](int16_t &v) { v += static_cast<int16_t>(j); });
+    std::for_each(in.I32.begin(), in.I32.end(), [&](int32_t &v) { v += static_cast<int32_t>(j); });
+    std::for_each(in.I64.begin(), in.I64.end(), [&](int64_t &v) { v += static_cast<int64_t>(j); });
+    std::for_each(in.U8.begin(), in.U8.end(), [&](uint8_t &v) { v += static_cast<uint8_t>(j); });
+    std::for_each(in.U16.begin(), in.U16.end(),
+                  [&](uint16_t &v) { v += static_cast<uint16_t>(j); });
+    std::for_each(in.U32.begin(), in.U32.end(),
+                  [&](uint32_t &v) { v += static_cast<uint32_t>(j); });
+    std::for_each(in.U64.begin(), in.U64.end(),
+                  [&](uint64_t &v) { v += static_cast<uint64_t>(j); });
+    std::for_each(in.R32.begin(), in.R32.end(), [&](float &v) { v += static_cast<float>(j); });
+    std::for_each(in.R64.begin(), in.R64.end(), [&](double &v) { v += static_cast<double>(j); });
     std::for_each(in.R128.begin(), in.R128.end(), [&](long double &v) { v += j; });
 
     std::for_each(in.CR32.begin(), in.CR32.end(), [&](std::complex<float> &v) {