Subfiling

h5py/_hl/files.py

@@ -115,6 +115,7 @@ def registered_drivers():
 def make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0, locking,
               page_buf_size, min_meta_keep, min_raw_keep,
               alignment_threshold, alignment_interval, meta_block_size,
+              ioc_selection, stripe_size, stripe_count, ioc_thread_pool_size,
               **kwds):
     """ Set up a file access property list """
     plist = h5p.create(h5p.FILE_ACCESS)
@@ -176,9 +177,36 @@ def make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0, locking,
     else:
         set_fapl(plist, **kwds)
 
+    # Subfiling parameter setting must be done after calling H5Pset_mpi_params or H5P_set_fapl_mpio
+    if ioc_selection or stripe_size or stripe_count or ioc_thread_pool_size:
+        make_subfiling_fapl(plist, ioc_thread_pool_size, ioc_selection, stripe_count, stripe_size)
+
     return plist
 
 
+def make_subfiling_fapl(plist, ioc_thread_pool_size, ioc_selection, stripe_count, stripe_size):
+    ioc_config = plist.get_fapl_ioc()
+    subf_config = plist.get_fapl_subfiling()
+    if ioc_thread_pool_size:
+        ioc_config.thread_pool_size = ioc_thread_pool_size
+    if stripe_size:
+        subf_config.stripe_size = stripe_size * 1024 * 1024
+    if stripe_count:
+        subf_config.stripe_count = stripe_count
+    if ioc_selection:
+        if ioc_selection == "one_per_node":
+            ioc_selection_enum = h5fd.IOC_ONE_PER_NODE
+        elif ioc_selection == "every_nth_rank":
+            ioc_selection_enum = h5fd.IOC_EVERY_NTH_RANK
+        elif ioc_selection == "total":
+            ioc_selection_enum = h5fd.IOC_TOTAL
+        else:
+            raise NotImplementedError("Unsupported IO concentrator allocation mode.")
+        subf_config.ioc_selection = ioc_selection_enum
+    plist.set_fapl_ioc(subf_config.ioc_fapl_id, ioc_config)
+    plist.set_fapl_subfiling(subf_config)
+
+
 def make_fcpl(track_order=False, fs_strategy=None, fs_persist=False,
               fs_threshold=1, fs_page_size=None):
     """ Set up a file creation property list """
@@ -376,7 +404,8 @@ def __init__(self, name, mode='r', driver=None, libver=None, userblock_size=None
                  rdcc_nslots=None, rdcc_nbytes=None, rdcc_w0=None, track_order=None,
                  fs_strategy=None, fs_persist=False, fs_threshold=1, fs_page_size=None,
                  page_buf_size=None, min_meta_keep=0, min_raw_keep=0, locking=None,
-                 alignment_threshold=1, alignment_interval=1, meta_block_size=None, **kwds):
+                 alignment_threshold=1, alignment_interval=1, meta_block_size=None,
+                 ioc_selection=None, stripe_size=None, stripe_count=None, ioc_thread_pool_size=None, **kwds):
         """Create a new file object.
 
         See the h5py user guide for a detailed explanation of the options.
@@ -493,6 +522,22 @@ def __init__(self, name, mode='r', driver=None, libver=None, userblock_size=None
             Set the current minimum size, in bytes, of new metadata block allocations.
             See https://portal.hdfgroup.org/display/HDF5/H5P_SET_META_BLOCK_SIZE
 
+        ioc_selection
+            Set IO concentrator method for different allocations of MPI ranks as I/O concentrators. Can be set
+            to h5py.h5fd.IOC_<method> constants (e.g. IOC_ONE_PER_NODE)
+            See https://docs.hdfgroup.org/hdf5/develop/_h5_f_dsubfiling_8h.html#a2bcf2d531a0668895308692b0c1108d7
+
+        stripe_count
+            Set the parameter for Subfiling stripe count.
+            See https://docs.hdfgroup.org/hdf5/develop/struct_h5_f_d__subfiling__params__t.html
+
+        stripe_size
+            Set the parameter for Subfiling stripe size.
+            See https://docs.hdfgroup.org/hdf5/develop/struct_h5_f_d__subfiling__params__t.html
+
+        ioc_thread_pool_size
+            Set the parameter for the IOC.
+
         Additional keywords
             Passed on to the selected file driver.
         """
@@ -560,6 +605,8 @@ def __init__(self, name, mode='r', driver=None, libver=None, userblock_size=None
                                  alignment_threshold=alignment_threshold,
                                  alignment_interval=alignment_interval,
                                  meta_block_size=meta_block_size,
+                                 ioc_selection=ioc_selection, stripe_size=stripe_size, stripe_count=stripe_count,
+                                 ioc_thread_pool_size=ioc_thread_pool_size,
                                  **kwds)
                 fcpl = make_fcpl(track_order=track_order, fs_strategy=fs_strategy,
                                  fs_persist=fs_persist, fs_threshold=fs_threshold,

h5py/api_functions.txt

@@ -329,6 +329,15 @@ hdf5:
   1.10.7-1.10.99 herr_t H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled)
   1.12.1 herr_t H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled)
 
+  # Subfiling
+  MPI 1.14.0 herr_t H5Pset_fapl_subfiling(hid_t fapl_id, const H5FD_subfiling_config_t *vfd_config)
+  MPI 1.14.0 herr_t H5Pget_fapl_subfiling(hid_t fapl_id, H5FD_subfiling_config_t *config_out)
+  MPI 1.14.0 herr_t H5Pset_fapl_ioc(hid_t fapl_id, const H5FD_ioc_config_t *vfd_config)
+  MPI 1.14.0 herr_t H5Pget_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *config_out)
+
+
+
+
 
   # Dataset creation
   herr_t        H5Pset_layout(hid_t plist, int layout)

h5py/api_types_hdf5.pxd

@@ -359,6 +359,40 @@ cdef extern from "hdf5.h":
       char    secret_key[129]
 
     unsigned int H5FD_CURR_ROS3_FAPL_T_VERSION # version of struct
+
+  # 1.14 Subfiling support
+  IF MPI and HDF5_VERSION >= (1, 14, 0):
+    unsigned int H5FD_IOC_CURR_FAPL_VERSION
+    unsigned int H5FD_SUBFILING_CURR_FAPL_VERSION
+    unsigned int H5FD_IOC_FAPL_MAGIC
+    unsigned int H5FD_SUBFILING_FAPL_MAGIC
+
+    # IO Concentrator structs, the IO Concentrator is used by subfiling driver
+    ctypedef enum H5FD_subfiling_ioc_select_t:
+      SELECT_IOC_ONE_PER_NODE = 0,
+      SELECT_IOC_EVERY_NTH_RANK,
+      SELECT_IOC_WITH_CONFIG,
+      SELECT_IOC_TOTAL,
+      ioc_selection_options
+
+    ctypedef struct H5FD_ioc_config_t:
+      uint32_t magic
+      uint32_t version
+      int32_t thread_pool_size
+
+    # Structs for configuring subfiling
+    ctypedef struct H5FD_subfiling_params_t:
+      H5FD_subfiling_ioc_select_t ioc_selection
+      int64_t stripe_size
+      int32_t stripe_count
+
+    ctypedef struct H5FD_subfiling_config_t:
+      uint32_t magic
+      uint32_t version
+      hid_t ioc_fapl_id
+      hbool_t require_ioc
+      H5FD_subfiling_params_t shared_cfg
+
 # === H5G - Groups API ========================================================
 
   ctypedef enum H5G_link_t:
@@ -832,8 +866,6 @@ cdef extern from "hdf5.h":
   ctypedef herr_t (*H5A_operator2_t)(hid_t location_id, char *attr_name,
           H5A_info_t *ainfo, void *op_data) except 2
 
-
-
 #  === H5AC - Attribute Cache configuration API ================================
 
 

h5py/h5fd.pxd

@@ -12,3 +12,11 @@
 # licenses/hdf5.txt for the full HDF5 software license.
 
 from .defs cimport *
+include "config.pxi"
+
+IF MPI and HDF5_VERSION >= (1, 14, 0):
+    cdef class IOCConfig:
+        cdef H5FD_ioc_config_t ioc_config
+
+    cdef class SubfilingConfig:
+        cdef H5FD_subfiling_config_t subf_config

h5py/h5fd.pyx

@@ -220,3 +220,113 @@ IF HDF5_VERSION >= (1, 14, 0):
     info.version = H5FD_CLASS_VERSION
 
 fileobj_driver = H5FDregister(&info)
+
+
+
+
+# Subfiling configuration classes
+
+IF MPI and HDF5_VERSION >= (1, 14, 0):
+    IOC_ONE_PER_NODE = SELECT_IOC_ONE_PER_NODE
+    IOC_EVERY_NTH_RANK = SELECT_IOC_EVERY_NTH_RANK
+    IOC_WITH_CONFIG = SELECT_IOC_WITH_CONFIG
+    IOC_TOTAL = SELECT_IOC_TOTAL
+    """
+        Low-level HDF5 "H5FD" IO concentrator configuration interface.
+    """
+    cdef class IOCConfig:
+        """Represents H5FD_ioc_config_t objects
+
+        """
+
+        #cdef H5FD_ioc_config_t
+        #     /* general configuration fields: */
+        def __cinit__(self):
+            self.ioc_config.magic = H5FD_IOC_FAPL_MAGIC
+            self.ioc_config.version = H5FD_IOC_CURR_FAPL_VERSION
+
+        property magic:
+            def __get__(self):
+                return self.ioc_config.magic
+            def __set__(self, int val):
+                self.ioc_config.magic = val
+
+        property version:
+            def __get__(self):
+                return self.ioc_config.version
+            def __set__(self, int val):
+                self.ioc_config.version = val
+
+        property thread_pool_size:
+            def __get__(self):
+                return self.ioc_config.thread_pool_size
+            def __set__(self, int val):
+                self.ioc_config.thread_pool_size = val
+
+    """
+            Low-level HDF5 "H5FD" Subfiling configuration interface.
+        """
+    cdef class SubfilingConfig:
+        """Represents H5FD_subfiling_config_t  objects
+
+        """
+
+        #cdef H5FD_subfiling_config_t
+        #     /* general configuration fields: */
+        def __cinit__(self):
+            self.subf_config.magic = H5FD_SUBFILING_FAPL_MAGIC
+            self.subf_config.version = H5FD_SUBFILING_CURR_FAPL_VERSION
+
+        property magic:
+            def __get__(self):
+                return self.subf_config.magic
+            def __set__(self, int val):
+                self.subf_config.magic = val
+
+        property version:
+            def __get__(self):
+                return self.subf_config.version
+            def __set__(self, int val):
+                self.subf_config.version = val
+
+        property ioc_fapl_id:
+            def __get__(self):
+                return self.subf_config.ioc_fapl_id
+            def __set__(self, int val):
+                self.subf_config.ioc_fapl_id = val
+
+        property require_ioc:
+            def __get__(self):
+                return self.subf_config.require_ioc
+            def __set__(self, int val):
+                self.subf_config.require_ioc = val
+
+        property shared_cfg:
+            def __get__(self):
+                return self.subf_config.shared_cfg
+            def __set__(self, val):
+                self.subf_config.shared_cfg = val
+
+        property stripe_size:
+            def __get__(self):
+                cdef H5FD_subfiling_params_t * shared_cfg = &self.subf_config.shared_cfg
+                return shared_cfg.stripe_size
+            def __set__(self, long val):
+                cdef H5FD_subfiling_params_t * shared_cfg = &self.subf_config.shared_cfg
+                shared_cfg.stripe_size = val
+
+        property stripe_count:
+            def __get__(self):
+                cdef H5FD_subfiling_params_t * shared_cfg = &self.subf_config.shared_cfg
+                return shared_cfg.stripe_count
+            def __set__(self, int val):
+                cdef H5FD_subfiling_params_t * shared_cfg = &self.subf_config.shared_cfg
+                shared_cfg.stripe_count = val
+
+        property ioc_selection:
+            def __get__(self):
+                cdef H5FD_subfiling_params_t * shared_cfg = &self.subf_config.shared_cfg
+                return shared_cfg.ioc_selection
+            def __set__(self, val):
+                cdef H5FD_subfiling_params_t * shared_cfg = &self.subf_config.shared_cfg
+                shared_cfg.ioc_selection = val

h5py/h5p.pyx

@@ -27,6 +27,8 @@ from numpy cimport ndarray, import_array
 from .h5t cimport TypeID, py_create
 from .h5s cimport SpaceID
 from .h5ac cimport CacheConfig
+IF MPI and HDF5_VERSION >= (1, 14, 0):
+    from .h5fd cimport IOCConfig, SubfilingConfig
 
 # Python level imports
 from ._objects import phil, with_phil
@@ -1502,6 +1504,30 @@ cdef class PropFAID(PropInstanceID):
             H5Pset_file_locking(
                 self.id, <hbool_t>use_file_locking, <hbool_t>ignore_when_disabled)
 
+    IF MPI and HDF5_VERSION >= (1, 14, 0):
+        @with_phil
+        def set_fapl_ioc(self, ioc_fapl_id, IOCConfig config not None):
+            H5Pset_fapl_ioc(ioc_fapl_id, &config.ioc_config)
+
+        @with_phil
+        def get_fapl_ioc(self):
+            cdef IOCConfig config = IOCConfig()
+            H5Pget_fapl_ioc(self.id, &config.ioc_config)
+
+            return config
+
+        @with_phil
+        def set_fapl_subfiling(self, SubfilingConfig config not None):
+            H5Pset_fapl_subfiling(self.id, &config.subf_config)
+
+        @with_phil
+        def get_fapl_subfiling(self):
+            cdef SubfilingConfig config = SubfilingConfig()
+            H5Pget_fapl_subfiling(self.id, &config.subf_config)
+
+            return config
+
+
 
 # Link creation
 cdef class PropLCID(PropCreateID):