Merge 8467f2c into aca6586

hera_pspec/container.py

@@ -3,6 +3,7 @@
 from hera_pspec import uvpspec, version, utils
 import argparse
 import time
+from functools import wraps
 
 
 def transactional(fn):
@@ -11,6 +12,7 @@ def transactional(fn):
     opened and then closed again for every operation. This is done when 
     keep_open = False.
     """
+    @wraps(fn)
     def wrapper(*args, **kwargs):
         psc = args[0] # self object
 
@@ -42,11 +44,15 @@ class PSpecContainer(object):
     Container class for managing multiple UVPSpec objects.
     """
 
-    def __init__(self, filename, mode='r', keep_open=True, tsleep=0.5, maxiter=2):
+    def __init__(self, filename, mode='r', keep_open=True, swmr=False, tsleep=0.5, maxiter=2):
         """
         Manage a collection of UVPSpec objects that are stored in a structured
         HDF5 file.
 
+        Note: one should not create new groups or datasets with SWMR. See page 6 of
+        https://support.hdfgroup.org/HDF5/docNewFeatures/SWMR/HDF5_SWMR_Users_Guide.pdf
+        for SWMR limitations.
+
         Parameters
         ----------
         filename : str
@@ -61,13 +67,14 @@ def __init__(self, filename, mode='r', keep_open=True, tsleep=0.5, maxiter=2):
             Whether the HDF5 file should be kept open, or opened and then 
             closed again each time an operation is performed. Setting 
             `keep_open=False` is helpful for multi-process access patterns.
-            
-            This feature uses the Single-Writer Multiple-Reader (SWMR) feature 
-            of HDF5. Note that SWMR can only be used on POSIX-compliant 
-            filesystems, and so may not work on some network filesystems.
-            
             Default: True (keep file open).
 
+        swmr : bool, optional
+            Enable Single-Writer Multiple-Reader (SWMR) feature of HDF5.
+            Note that SWMR can only be used on POSIX-compliant 
+            filesystems, and so may not work on some network filesystems.
+            Default: False (do not use SWMR)
+
         tsleep : float, optional
             Time to wait in seconds after each attempt at opening the file.
 
@@ -80,6 +87,7 @@ def __init__(self, filename, mode='r', keep_open=True, tsleep=0.5, maxiter=2):
         self.mode = mode
         self.tsleep = tsleep
         self.maxiter = maxiter
+        self.swmr = swmr
         if mode not in ['r', 'rw']:
             raise ValueError("Must set mode to either 'r' or 'rw'.")
 
@@ -107,7 +115,7 @@ def _open(self):
             mode = 'a'
         else:
             mode = 'r'
-        if self.mode == 'r':
+        if self.swmr and self.mode == 'r':
             swmr = True
         else:
             swmr = False
@@ -130,7 +138,8 @@ def _open(self):
                         # Enable single writer, multiple reader mode on HDF5 file. 
                         # This allows multiple handles to exist for the same file 
                         # at the same time, as long as only one is in rw mode
-                        self.data.swmr_mode = True
+                        if self.swmr:
+                            self.data.swmr_mode = True
                     except ValueError:
                         pass
                 break
@@ -262,6 +271,9 @@ def set_pspec(self, group, psname, pspec, overwrite=False):
         if self.mode == 'r':
             raise IOError("HDF5 file was opened read-only; cannot write to file.")
 
+        if self.swmr:
+            print("Warning: HDF5 forbids creating new groups or datasets with SWMR")
+
         if isinstance(group, (tuple, list, dict)):
             raise ValueError("Only one group can be specified at a time.")
 

hera_pspec/grouping.py

@@ -819,7 +819,7 @@ def bootstrap_resampled_error(uvp, blpair_groups=None, time_avg=False, Nsamples=
 
 def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Nsamples=1000, seed=0,
                   normal_std=True, robust_std=True, cintervals=None, keep_samples=False,
-                  bl_error_tol=1.0, overwrite=False, add_to_history='', verbose=True):
+                  bl_error_tol=1.0, overwrite=False, add_to_history='', verbose=True, maxiter=40):
     """
     Run bootstrap resampling on a PSpecContainer object to estimate errorbars.
     For each group/spectrum specified in the PSpecContainer, this function produces
@@ -828,7 +828,9 @@ def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Ns
        (3.) series of bootstrap resamples of UVPSpec average (optional)
 
     The output of 1. and 2. are placed in a *_avg spectrum, while the output of 3.
-    is placed in *_bs0, *_bs1, *_bs2 etc. objects. 
+    is placed in *_bs0, *_bs1, *_bs2 etc. objects.
+
+    Note: PSpecContainers should nont be opened in SWMR mode for this function.
 
     Parameters:
     -----------
@@ -882,14 +884,19 @@ def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Ns
 
     verbose : bool
         If True, report feedback to stdout.
+
+    maxiter : int
+        Maximum number of attempts to open the PSpecContainer. 0.5 sec wait per attempt.
     """
     from hera_pspec import uvpspec
     from hera_pspec import PSpecContainer
     # type check
     if isinstance(filename, (str, np.str)):
-        psc = PSpecContainer(filename)
+        # open in transactional mode
+        psc = PSpecContainer(filename, mode='rw', keep_open=False, swmr=False, tsleep=0.5, maxiter=maxiter)
     elif isinstance(filename, PSpecContainer):
         psc = filename
+        assert not psc.swmr, "PSpecContainer should not be in SWMR mode"
     else:
         raise AssertionError("filename must be a PSpecContainer or filepath to one")
 
@@ -900,7 +907,7 @@ def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Ns
     # get spectra if not fed
     all_spectra = utils.flatten([ [os.path.join(grp, s) 
                                    for s in psc.spectra(grp)] 
-                                 for grp in groups ])
+                                   for grp in groups])
     if spectra is None:
         spectra = all_spectra
     else: