propagate pspecdata cal to uvp history

hera_pspec/grouping.py

@@ -819,7 +819,7 @@ def bootstrap_resampled_error(uvp, blpair_groups=None, time_avg=False, Nsamples=
 
 def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Nsamples=1000, seed=0,
                   normal_std=True, robust_std=True, cintervals=None, keep_samples=False,
-                  bl_error_tol=1.0, overwrite=False, add_to_history='', verbose=True, maxiter=40):
+                  bl_error_tol=1.0, overwrite=False, add_to_history='', verbose=True, maxiter=1):
     """
     Run bootstrap resampling on a PSpecContainer object to estimate errorbars.
     For each group/spectrum specified in the PSpecContainer, this function produces
@@ -830,7 +830,7 @@ def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Ns
     The output of 1. and 2. are placed in a *_avg spectrum, while the output of 3.
     is placed in *_bs0, *_bs1, *_bs2 etc. objects.
 
-    Note: PSpecContainers should nont be opened in SWMR mode for this function.
+    Note: PSpecContainers should not be opened in SWMR mode for this function.
 
     Parameters:
     -----------
@@ -885,8 +885,10 @@ def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Ns
     verbose : bool
         If True, report feedback to stdout.
 
-    maxiter : int
-        Maximum number of attempts to open the PSpecContainer. 0.5 sec wait per attempt.
+    maxiter : int, optional, default=1
+        Maximum number of attempts to open the PSpecContainer by a single process.
+        0.5 sec wait per attempt. Useful in the case of multiprocesses bootstrapping
+        different groups of the same container.
     """
     from hera_pspec import uvpspec
     from hera_pspec import PSpecContainer

hera_pspec/pspecdata.py

@@ -13,6 +13,7 @@
 import ast
 import glob
 import warnings
+import json
 
 
 class PSpecData(object):
@@ -50,7 +51,8 @@ def __init__(self, dsets=[], wgts=None, dsets_std=None, labels=None,
             Default: None.
 
         cals : list of UVCal objects, optional
-            Calibration objects to apply to data.
+            Calibration objects to apply to data. One per dset or
+            one for all dsets.
 
         cal_flag : bool, optional
             If True, propagate flags from calibration into data
@@ -205,8 +207,10 @@ def add(self, dsets, wgts, labels=None, dsets_std=None, cals=None, cal_flag=True
             if cal is not None:
                 if dset is not None:
                     uvutils.uvcalibrate(dset, cal, inplace=True, prop_flags=cal_flag, flag_missing=cal_flag)
+                    dset.extra_keywords['calibration'] = cal.extra_keywords.get('filename', '""')
                 if dset_std is not None:
                     uvutils.uvcalibrate(dset_std, cal, inplace=True, prop_flags=cal_flag, flag_missing=cal_flag)
+                    dset_std.extra_keywords['calibration'] = cal.extra_keywords.get('filename', '""')
 
         # Append to list
         self.dsets += dsets
@@ -2529,8 +2533,10 @@ def pspec(self, bls1, bls2, dsets, pols, n_dlys=None,
         uvp.weighting = input_data_weight
         uvp.vis_units, uvp.norm_units = self.units(little_h=little_h)
         uvp.telescope_location = dset1.telescope_location
-        filename1 = getattr(dset1.extra_keywords, 'filename', None)
-        filename2 = getattr(dset2.extra_keywords, 'filename', None)
+        filename1 = json.loads(dset1.extra_keywords.get('filename', '""'))
+        cal1 = json.loads(dset1.extra_keywords.get('calibration', '""'))
+        filename2 = json.loads(dset2.extra_keywords.get('filename', '""'))
+        cal2 = json.loads(dset2.extra_keywords.get('calibration', '""'))
         label1 = self.labels[self.dset_idx(dsets[0])]
         label2 = self.labels[self.dset_idx(dsets[1])]
         uvp.labels = sorted(set([label1, label2]))
@@ -2540,11 +2546,11 @@ def pspec(self, bls1, bls2, dsets, pols, n_dlys=None,
                             * uvp.labels.index(label2)
         uvp.labels = np.array(uvp.labels, np.str)
         uvp.history = "UVPSpec written on {} with hera_pspec git hash {}\n{}\n" \
-                      "dataset1: filename: {}, label: {}, history:\n{}\n{}\n" \
-                      "dataset2: filename: {}, label: {}, history:\n{}\n{}\n" \
+                      "dataset1: filename: {}, label: {}, cal: {}, history:\n{}\n{}\n" \
+                      "dataset2: filename: {}, label: {}, cal: {}, history:\n{}\n{}\n" \
                       "".format(datetime.datetime.utcnow(), version.git_hash, '-'*20,
-                                filename1, label1, dset1.history, '-'*20,
-                                filename2, label2, dset2.history, '-'*20)
+                                filename1, label1, cal1, dset1.history, '-'*20,
+                                filename2, label2, cal2, dset2.history, '-'*20)
         uvp.taper = taper
         uvp.norm = norm
 
@@ -3335,7 +3341,7 @@ def _load_dsets(fnames, bls=None, pols=None, logf=None, verbose=True,
     for i, dset in enumerate(fnames):
         utils.log("Reading {} / {} datasets...".format(i+1, Ndsets), 
                   f=logf, lvl=1, verbose=verbose)
-        
+
         # read data
         uvd = UVData()
         if isinstance(dset, (str, np.str)):
@@ -3344,6 +3350,7 @@ def _load_dsets(fnames, bls=None, pols=None, logf=None, verbose=True,
             dfiles = dset
         uvd.read(dfiles, bls=bls, polarizations=pols, 
                  file_type=file_type)
+        uvd.extra_keywords['filename'] = json.dumps(dfiles)
         dsets.append(uvd)
 
     return dsets
@@ -3379,6 +3386,7 @@ def _load_cals(cnames, logf=None, verbose=True):
             uvc.read_calfits(glob.glob(cfile))
         else:
             uvc.read_calfits(cfile)
+        uvc.extra_keywords['filename'] = json.dumps(cfile)
         cals.append(uvc)
 
     return cals

hera_pspec/tests/test_pspecdata.py

@@ -1657,11 +1657,15 @@ def test_pspec_run():
                              pol_pairs=[('xx', 'xx')], interleave_times=False,
                              file_type='uvh5', spw_ranges=[(100, 150)], cal_flag=True)
     psc = container.PSpecContainer('./out.h5', 'rw')
+    uvp = psc.get_pspec('dset0_dset1', 'dset0_x_dset1')
     # test calibration flags were propagated to test that cal was applied
     assert ds.dsets[0].flag_array.any()
     assert ds.dsets[1].flag_array.any()
     assert ds.dsets_std[0].flag_array.any()
     assert ds.dsets_std[1].flag_array.any()
+    assert ds.dsets[0].extra_keywords['filename'] is not '""'
+    assert ds.dsets[0].extra_keywords['calibration'] is not '""'
+    assert 'cal: /' in uvp.history
 
     # test exceptions
     nt.assert_raises(AssertionError, pspecdata.pspec_run, 'foo', "./out.h5")