Write to (compressed) HDF5 (#58)

* add hdf5 support

* switch to level 3 compression

* try adding hdf5 to CLI?

* fix mpi version error

* ignore unused parameters when necessary

* add more unused params and try to link hdf5

.github/workflows/clang.yml

@@ -13,7 +13,7 @@ jobs:
   build_clang_single:
     name: Clang@7.0 C++17 SP NOMPI Debug [lib]
     runs-on: ubuntu-20.04
-    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor"}
+    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wno-unused-parameter"}
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies
@@ -35,7 +35,7 @@ jobs:
   build_clang_mixed:
     name: Clang@7.0 C++17 SP Particles DP Mesh Debug
     runs-on: ubuntu-20.04
-    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -O1 -Wnon-virtual-dtor"}
+    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -O1 -Wnon-virtual-dtor -Wno-unused-parameter"}
       # It's too slow with -O0
     steps:
     - uses: actions/checkout@v3
@@ -58,7 +58,7 @@ jobs:
   build_clang_double:
     name: Clang@7.0 C++17 DP Particles DP Mesh Debug
     runs-on: ubuntu-20.04
-    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -O1 -Wnon-virtual-dtor"}
+    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -O1 -Wnon-virtual-dtor -Wno-unused-parameter"}
       # It's too slow with -O0
     steps:
     - uses: actions/checkout@v3
@@ -74,4 +74,4 @@ jobs:
             -DAMReX_MPI=OFF                           \
             -DCMAKE_C_COMPILER=$(which clang)         \
             -DCMAKE_CXX_COMPILER=$(which clang++)
-        make -j 2
+        make -j 2

.github/workflows/cuda.yml

@@ -50,6 +50,8 @@ jobs:
         source /etc/profile.d/modules.sh
         module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11
 
+        export HDF5_ROOT=/usr/lib/x86_64-linux-gnu/hdf5/openmpi
+
         which nvcc || echo "nvcc not in PATH!"
         which nvc++ || echo "nvc++ not in PATH!"
         which nvc || echo "nvc not in PATH!"
@@ -70,4 +72,4 @@ jobs:
             -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
             -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
 
-        cmake --build build -j 2
+        cmake --build build -j 2

.github/workflows/dependencies/dependencies.sh

@@ -13,4 +13,5 @@ sudo apt-get install -y --no-install-recommends\
     build-essential \
     g++ gfortran    \
     libopenmpi-dev  \
-    openmpi-bin
+    openmpi-bin     \
+    libhdf5-openmpi-dev

.github/workflows/dependencies/dependencies_clang7.sh

@@ -11,4 +11,5 @@ sudo apt-get update
 
 sudo apt-get install -y  \
     build-essential      \
-    clang-7 gfortran
+    clang-7 gfortran     \
+    libhdf5-dev

.github/workflows/dependencies/dependencies_gcc10.sh

@@ -14,4 +14,5 @@ sudo apt-get install -y --no-install-recommends \
     build-essential    \
     g++-10 gfortran-10 \
     libopenmpi-dev     \
-    openmpi-bin
+    openmpi-bin        \
+    libhdf5-openmpi-dev

.github/workflows/dependencies/dependencies_gcc8.sh

@@ -14,4 +14,5 @@ sudo apt-get install -y --no-install-recommends \
     build-essential    \
     g++-8 gfortran-8   \
     libopenmpi-dev     \
-    openmpi-bin
+    openmpi-bin        \
+    libhdf5-openmpi-dev

.github/workflows/dependencies/dependencies_gcc8_nompi.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright 2020-2022 The AMReX Community
+#
+# License: BSD-3-Clause-LBNL
+# Authors: Axel Huebl
+
+set -eu -o pipefail
+
+sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+sudo apt-get update
+
+sudo apt-get install -y --no-install-recommends \
+    build-essential    \
+    g++-8 gfortran-8   \
+    libopenmpi-dev     \
+    openmpi-bin        \
+    libhdf5-dev

.github/workflows/dependencies/dependencies_hip.sh

@@ -61,7 +61,8 @@ sudo apt-get install -y --no-install-recommends \
     roctracer-dev   \
     rocprofiler-dev \
     rocrand-dev     \
-    rocprim-dev
+    rocprim-dev     \
+    libhdf5-openmpi-dev
 
 # hiprand-dev is a new package that does not exist in old versions
 sudo apt-get install -y --no-install-recommends hiprand-dev || true

.github/workflows/dependencies/dependencies_nvcc11.sh

@@ -16,7 +16,8 @@ sudo apt-get install -y \
     libopenmpi-dev      \
     openmpi-bin         \
     pkg-config          \
-    wget
+    wget                \
+    libhdf5-openmpi-dev
 
 sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
 echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" \

.github/workflows/dependencies/dependencies_nvhpc21-11.sh

@@ -15,7 +15,8 @@ sudo apt-get install -y \
     environment-modules \
     gnupg               \
     pkg-config          \
-    wget
+    wget                \
+    libhdf5-openmpi-dev
 
 echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | \
   sudo tee /etc/apt/sources.list.d/nvhpc.list

.github/workflows/gcc.yml

@@ -15,7 +15,7 @@ jobs:
   library:
     name: GNU@8.4 C++17 Release
     runs-on: ubuntu-20.04
-    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs"}
+    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs -Wno-unused-parameter"}
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies
@@ -35,7 +35,7 @@ jobs:
   build_debug:
     name: GNU@9.3 C++17 Debug
     runs-on: ubuntu-20.04
-    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -O1 -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs"}
+    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -O1 -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs -Wno-unused-parameter"}
       # It's too slow with -O0
     steps:
     - uses: actions/checkout@v3
@@ -52,7 +52,7 @@ jobs:
   tests_cxx20:
     name: GNU@10.1 C++20 OMP
     runs-on: ubuntu-20.04
-    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs"}
+    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs -Wno-unused-parameter"}
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies
@@ -79,11 +79,11 @@ jobs:
   tests-nonmpi:
     name: GNU@8.4 C++17 NOMPI
     runs-on: ubuntu-20.04
-    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs"}
+    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs -Wno-unused-parameter"}
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies
-      run: .github/workflows/dependencies/dependencies_gcc8.sh
+      run: .github/workflows/dependencies/dependencies_gcc8_nompi.sh
     - name: Build & Install
       run: |
         mkdir build
@@ -99,4 +99,4 @@ jobs:
             -DCMAKE_C_COMPILER=$(which gcc-8)     \
             -DCMAKE_CXX_COMPILER=$(which g++-8)   \
             -DCMAKE_Fortran_COMPILER=$(which gfortran-8)
-        make -j 2
+        make -j 2

.github/workflows/hip.yml

@@ -28,7 +28,7 @@ jobs:
       #    /opt/rocm-6.1.0/include/rocprim/device/detail/device_adjacent_difference.hpp:198:26: error: token is not a valid binary operator in a preprocessor subexpression
       #      198 | #if defined(__gfx1102__) or defined(__gfx1030__)
       #          |     ~~~~~~~~~~~~~~~~~~~~~^~
-      env: {CXXFLAGS: "-Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor"}
+      env: {CXXFLAGS: "-Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wno-unused-parameter"}
       run: |
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
@@ -77,7 +77,7 @@ jobs:
       #    /opt/rocm-6.1.0/include/rocprim/device/detail/device_adjacent_difference.hpp:198:26: error: token is not a valid binary operator in a preprocessor subexpression
       #      198 | #if defined(__gfx1102__) or defined(__gfx1030__)
       #          |     ~~~~~~~~~~~~~~~~~~~~~^~
-      env: {CXXFLAGS: "-Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor"}
+      env: {CXXFLAGS: "-Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wno-unused-parameter"}
       run: |
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
@@ -101,4 +101,4 @@ jobs:
             -DCMAKE_C_COMPILER=$(which clang)             \
             -DCMAKE_CXX_COMPILER=$(which hipcc)           \
             -DCMAKE_CXX_STANDARD=17
-        cmake --build build_full_legacywrapper -j 2
+        cmake --build build_full_legacywrapper -j 2

CMakeLists.txt

@@ -15,6 +15,7 @@ set(AMReX_TINY_PROFILE    ON)
 set(AMReX_SPACEDIM 2 CACHE INTERNAL "")
 set(AMReX_PRECISION SINGLE CACHE INTERNAL "")
 set(AMReX_PARTICLES_PRECISION SINGLE CACHE INTERNAL "")
+set(AMReX_HDF5          TRUE)
 
 #
 # Fetch amrex repo

src/IO.cpp

@@ -88,12 +88,13 @@ void writePlotFile (const AgentContainer& pc, /*!< Agent (particle) container */
         plt_varnames.push_back("Tract");
         plt_varnames.push_back("comm");
 
-        WriteSingleLevelPlotfile(   amrex::Concatenate("plt", step, 5),
-                                    output_mf,
-                                    plt_varnames,
-                                    pc.ParticleGeom(0),
-                                    cur_time,
-                                    step );
+        WriteSingleLevelPlotfileHDF5MultiDset(  amrex::Concatenate("plt", step, 5),
+                                                output_mf,
+                                                plt_varnames,
+                                                pc.ParticleGeom(0),
+                                                cur_time,
+                                                step,
+                                                "ZLIB@3" );
     }
 
     {
@@ -135,12 +136,13 @@ void writePlotFile (const AgentContainer& pc, /*!< Agent (particle) container */
             }
         }
 
-        pc.WritePlotFile(   amrex::Concatenate("plt", step, 5),
-                            "agents",
-                            write_real_comp,
-                            write_int_comp,
-                            real_varnames,
-                            int_varnames );
+        pc.WritePlotFileHDF5(   amrex::Concatenate("plt", step, 5),
+                                "agents",
+                                write_real_comp,
+                                write_int_comp,
+                                real_varnames,
+                                int_varnames,
+                                "ZLIB@3" );
     }
 }
 

utilities/plotMovie/generate_frames.py

@@ -20,6 +20,8 @@
 
 import numpy as np
 
+import h5py
+
 import yt
 from yt.frontends import boxlib
 from yt.frontends.boxlib.data_structures import AMReXDataset
@@ -30,8 +32,6 @@
 import pandas as pd
 import geopandas as gpd
 from shapely.geometry import shape
-import shapefile
-# import fiona
 
 import os
 import sys
@@ -81,6 +81,33 @@ def get_raw(county):
     ds.close()
     return raw_df
 
+def get_raw_data_hdf5(name: str):
+    f = h5py.File(name, 'r')
+    found = 0
+    i = 0
+    while found < 2:
+        if f.attrs['component_' + str(i)] == b'FIPS':
+            fips_idx = i
+            found += 1
+        if f.attrs['component_' + str(i)] == b'infected':
+            inf_idx = i
+            found += 1
+        i += 1
+
+    fips = f['level_0']['data:datatype=' + str(fips_idx)][()]
+    infs = f['level_0']['data:datatype=' + str(inf_idx)][()]
+    unique_fips = np.unique(fips).astype(int)
+
+    def get_raw(county):
+        mask = fips == county
+        return np.log(1 + infs[mask].sum())
+
+    raw_df = pd.DataFrame()
+    raw_df["FIPS"] = unique_fips
+    raw_df["per"] = raw_df.apply(lambda row: get_raw(row["FIPS"]), axis=1)
+    f.close()
+    return raw_df
+
 # example: prefix = "../data/San_Francisco_Bay_Region_2020_Census_Tracts/region_2020_censustract"
 def get_gdf(prefix: str):
     """Generates a GeoDataFrame from .shp, .dbf, and .prj files.
@@ -145,7 +172,7 @@ def generate_plot(per_df, gdf, vmin = None, vmax = None, crop_usa = False):
 
     argc = len(sys.argv)
     data_dir = sys.argv[1] if argc > 1 else "/global/cfs/projectdirs/m3623/test/output_usa/"
-    data_names = sorted([os.path.join(data_dir, f) for f in os.listdir(data_dir) if f.startswith("plt")])
+    data_names = sorted([os.path.join(data_dir, f) for f in os.listdir(data_dir) if f.endswith(".h5")])
 
     # BA: data/San_Francisco_Bay_Region_2020_Census_Tracts/region_2020_censustract
     # CA: data/CA_2020_Census_Tracts/tl_2020_06_tract
@@ -158,6 +185,6 @@ def generate_plot(per_df, gdf, vmin = None, vmax = None, crop_usa = False):
     for i in range(len(data_names)):
         # vmin and vmax are endpoints for color range; 16 > log(population of LA) is a safe upper bound
         # for per-capita, endpoints should be set to much less
-        fig = generate_plot(get_raw_data(data_names[i]), gdf, vmin=0, vmax=16, crop_usa = crop_usa)
+        fig = generate_plot(get_raw_data_hdf5(data_names[i]), gdf, vmin=0, vmax=16, crop_usa = crop_usa)
         fig.savefig(output_dir + "frame{:05d}".format(i))
         plt.close(fig)