Add ability to read N-D arrays from HDF5 files (#1129)

* If we have unsigned long long, let us convert it to an array. * Support reading hyperslabs of N-D HDF5 arrays into linear memory arrays. * Documentation, including emphasis of HDF5 array ordering (fastest-varying rightmost, like C++ nested arrays)
LLNL · Jul 21, 2023 · c381cc0 · c381cc0
1 parent 97dd029
commit c381cc0
Show file tree

Hide file tree

Showing 10 changed files with 692 additions and 48 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,11 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s
 
 ## [Unreleased]
 
+### Added
+
+#### Relay
+- Added ability to read N-dimensional hyperslabs from HDF5 leaf arrays into linear memory arrays.
+
 ### Fixed
 
 #### Blueprint

diff --git a/src/docs/sphinx/relay_io.rst b/src/docs/sphinx/relay_io.rst
@@ -348,4 +348,31 @@ HDF5 I/O Options
 You can verify using ``h5stat`` that the data set was written to the hdf5 file using chunking and
 compression.
 
+HDF5 Hyperslabs
+++++++++++++++++
 
+HDF5 can store N-dimensional arrays.  Conduit can read subsets of these arrays, called "hyperslabs" in HDF5 terminology (see the HDF5 `tutorial`_).  The following example uses the HDF5 API to create a file containing a 2D array.  Then it uses Conduit to read a subset of the 2D array into memory.  Users can specify array **size** (element count), **offset**, and **stride**; Conduit does not support element blocks.  To match naming conventions elsewhere in Conduit, users may equivalently use the names **sizes**, **offsets**, and **strides**, as shown in the example.
+
+.. _tutorial: https://portal.hdfgroup.org/display/HDF5/Reading+From+or+Writing+To+a+Subset+of+a+Dataset
+
+Conduit reads an array from the HDF5 file into a linear buffer in memory.  Conduit does not support HDF5's idea of an in-memory destination hyperslab.
+
+* **C++ Example:**
+
+.. literalinclude:: ../../tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp
+   :start-after: BEGIN_EXAMPLE("relay_io_example_hdf5_interface_read_ndarray")
+   :end-before:  END_EXAMPLE("relay_io_example_hdf5_interface_read_ndarray")
+   :language: cpp
+   :dedent: 4
+
+* **Output:**
+
+.. literalinclude:: t_conduit_docs_relay_io_hdf5_examples_out.txt
+   :start-after: BEGIN_EXAMPLE("relay_io_example_hdf5_interface_read_ndarray")
+   :end-before:  END_EXAMPLE("relay_io_example_hdf5_interface_read_ndarray")
+
+.. warning::
+   HDF5 parameter arrays are "fastest varying rightmost", like indexes to a C
+   or C++ nested array.  If a user takes paper and pencil and writes down
+   p = (4, 5, 2), that commonly specifies a point p at x=4, y=5, z=2.  To
+   specify an offset at point p, the user has to pass the array [2, 5, 4].
diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt
diff --git a/src/libs/conduit/conduit_node.cpp b/src/libs/conduit/conduit_node.cpp
@@ -10924,6 +10924,84 @@ Node::to_long_array(Node &res) const
     }
 }
 
+//---------------------------------------------------------------------------//
+#ifdef CONDUIT_HAS_LONG_LONG
+//---------------------------------------------------------------------------//
+void
+Node::to_signed_long_long_array(Node &res) const
+{
+    res.set(DataType::c_signed_long_long(dtype().number_of_elements()));
+
+    signed_long_long_array res_array = res.as_signed_long_long_array();
+
+    switch(dtype().id())
+    {
+        /* ints */
+        case DataType::INT8_ID:
+        {
+            res_array.set(this->as_int8_array());
+            break;
+        }
+        case DataType::INT16_ID:
+        {
+            res_array.set(this->as_int16_array());
+            break;
+        }
+        case DataType::INT32_ID:
+        {
+            res_array.set(this->as_int32_array());
+            break;
+        }
+        case DataType::INT64_ID:
+        {
+            res_array.set(this->as_int64_array());
+            break;
+        }
+        /* uints */
+        case DataType::UINT8_ID:
+        {
+            res_array.set(this->as_uint8_array());
+            break;
+        }
+        case DataType::UINT16_ID:
+        {
+            res_array.set(this->as_uint16_array());
+            break;
+        }
+        case DataType::UINT32_ID:
+        {
+            res_array.set(this->as_uint32_array());
+            break;
+        }
+        case DataType::UINT64_ID:
+        {
+            res_array.set(this->as_uint64_array());
+            break;
+        }
+        /* floats */
+        case DataType::FLOAT32_ID:
+        {
+            res_array.set(this->as_float32_array());
+            break;
+        }
+        case DataType::FLOAT64_ID:
+        {
+            res_array.set(this->as_float64_array());
+            break;
+        }
+        default:
+        {
+            // error
+            CONDUIT_ERROR("Cannot convert non numeric "
+                        << dtype().name()
+                        << " type to unsigned_long_array.");
+        }
+    }
+}
+//---------------------------------------------------------------------------//
+#endif
+//---------------------------------------------------------------------------//
+
 
 //---------------------------------------------------------------------------//
 /// convert array to c unsigned integer arrays
@@ -11219,6 +11297,84 @@ Node::to_unsigned_long_array(Node &res) const
     }
 }
 
+//---------------------------------------------------------------------------//
+#ifdef CONDUIT_HAS_LONG_LONG
+//---------------------------------------------------------------------------//
+void
+Node::to_unsigned_long_long_array(Node &res) const
+{
+    res.set(DataType::c_unsigned_long_long(dtype().number_of_elements()));
+
+    unsigned_long_long_array res_array = res.as_unsigned_long_long_array();
+
+    switch(dtype().id())
+    {
+        /* ints */
+        case DataType::INT8_ID:
+        {
+            res_array.set(this->as_int8_array());
+            break;
+        }
+        case DataType::INT16_ID:
+        {
+            res_array.set(this->as_int16_array());
+            break;
+        }
+        case DataType::INT32_ID:
+        {
+            res_array.set(this->as_int32_array());
+            break;
+        }
+        case DataType::INT64_ID:
+        {
+            res_array.set(this->as_int64_array());
+            break;
+        }
+        /* uints */
+        case DataType::UINT8_ID:
+        {
+            res_array.set(this->as_uint8_array());
+            break;
+        }
+        case DataType::UINT16_ID:
+        {
+            res_array.set(this->as_uint16_array());
+            break;
+        }
+        case DataType::UINT32_ID:
+        {
+            res_array.set(this->as_uint32_array());
+            break;
+        }
+        case DataType::UINT64_ID:
+        {
+            res_array.set(this->as_uint64_array());
+            break;
+        }
+        /* floats */
+        case DataType::FLOAT32_ID:
+        {
+            res_array.set(this->as_float32_array());
+            break;
+        }
+        case DataType::FLOAT64_ID:
+        {
+            res_array.set(this->as_float64_array());
+            break;
+        }
+        default:
+        {
+            // error
+            CONDUIT_ERROR("Cannot convert non numeric "
+                        << dtype().name()
+                        << " type to unsigned_long_array.");
+        }
+    }
+}
+//---------------------------------------------------------------------------//
+#endif
+//---------------------------------------------------------------------------//
+
 /// convert array to c floating point arrays
 //---------------------------------------------------------------------------//
 void

diff --git a/src/libs/conduit/conduit_node.hpp b/src/libs/conduit/conduit_node.hpp
@@ -3243,12 +3243,20 @@ class CONDUIT_API Node
     void    to_signed_int_array(Node &res)   const;
     void    to_signed_long_array(Node &res)  const;
 
+#ifdef CONDUIT_HAS_LONG_LONG
+    void    to_signed_long_long_array(Node &res)  const;
+#endif
+
     /// convert to c unsigned integer types
     void    to_unsigned_char_array(Node &res)  const;
     void    to_unsigned_short_array(Node &res) const;
     void    to_unsigned_int_array(Node &res)   const;
     void    to_unsigned_long_array(Node &res)  const;
 
+#ifdef CONDUIT_HAS_LONG_LONG
+    void    to_unsigned_long_long_array(Node &res)  const;
+#endif
+
     /// convert to c floating point types
     void    to_float_array(Node &res) const;
     void    to_double_array(Node &res) const;