diff --git a/hera_pspec/grouping.py b/hera_pspec/grouping.py index be750682..4cc77d98 100644 --- a/hera_pspec/grouping.py +++ b/hera_pspec/grouping.py @@ -819,7 +819,7 @@ def bootstrap_resampled_error(uvp, blpair_groups=None, time_avg=False, Nsamples= def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Nsamples=1000, seed=0, normal_std=True, robust_std=True, cintervals=None, keep_samples=False, - bl_error_tol=1.0, overwrite=False, add_to_history='', verbose=True, maxiter=40): + bl_error_tol=1.0, overwrite=False, add_to_history='', verbose=True, maxiter=1): """ Run bootstrap resampling on a PSpecContainer object to estimate errorbars. For each group/spectrum specified in the PSpecContainer, this function produces @@ -830,7 +830,7 @@ def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Ns The output of 1. and 2. are placed in a *_avg spectrum, while the output of 3. is placed in *_bs0, *_bs1, *_bs2 etc. objects. - Note: PSpecContainers should nont be opened in SWMR mode for this function. + Note: PSpecContainers should not be opened in SWMR mode for this function. Parameters: ----------- @@ -885,8 +885,10 @@ def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Ns verbose : bool If True, report feedback to stdout. - maxiter : int - Maximum number of attempts to open the PSpecContainer. 0.5 sec wait per attempt. + maxiter : int, optional, default=1 + Maximum number of attempts to open the PSpecContainer by a single process. + 0.5 sec wait per attempt. Useful in the case of multiprocesses bootstrapping + different groups of the same container. """ from hera_pspec import uvpspec from hera_pspec import PSpecContainer diff --git a/hera_pspec/pspecdata.py b/hera_pspec/pspecdata.py index 23061903..a06cbd82 100644 --- a/hera_pspec/pspecdata.py +++ b/hera_pspec/pspecdata.py @@ -13,6 +13,7 @@ import ast import glob import warnings +import json class PSpecData(object): @@ -50,7 +51,8 @@ def __init__(self, dsets=[], wgts=None, dsets_std=None, labels=None, Default: None. cals : list of UVCal objects, optional - Calibration objects to apply to data. + Calibration objects to apply to data. One per dset or + one for all dsets. cal_flag : bool, optional If True, propagate flags from calibration into data @@ -205,8 +207,10 @@ def add(self, dsets, wgts, labels=None, dsets_std=None, cals=None, cal_flag=True if cal is not None: if dset is not None: uvutils.uvcalibrate(dset, cal, inplace=True, prop_flags=cal_flag, flag_missing=cal_flag) + dset.extra_keywords['calibration'] = cal.extra_keywords.get('filename', '""') if dset_std is not None: uvutils.uvcalibrate(dset_std, cal, inplace=True, prop_flags=cal_flag, flag_missing=cal_flag) + dset_std.extra_keywords['calibration'] = cal.extra_keywords.get('filename', '""') # Append to list self.dsets += dsets @@ -2529,8 +2533,10 @@ def pspec(self, bls1, bls2, dsets, pols, n_dlys=None, uvp.weighting = input_data_weight uvp.vis_units, uvp.norm_units = self.units(little_h=little_h) uvp.telescope_location = dset1.telescope_location - filename1 = getattr(dset1.extra_keywords, 'filename', None) - filename2 = getattr(dset2.extra_keywords, 'filename', None) + filename1 = json.loads(dset1.extra_keywords.get('filename', '""')) + cal1 = json.loads(dset1.extra_keywords.get('calibration', '""')) + filename2 = json.loads(dset2.extra_keywords.get('filename', '""')) + cal2 = json.loads(dset2.extra_keywords.get('calibration', '""')) label1 = self.labels[self.dset_idx(dsets[0])] label2 = self.labels[self.dset_idx(dsets[1])] uvp.labels = sorted(set([label1, label2])) @@ -2540,11 +2546,11 @@ def pspec(self, bls1, bls2, dsets, pols, n_dlys=None, * uvp.labels.index(label2) uvp.labels = np.array(uvp.labels, np.str) uvp.history = "UVPSpec written on {} with hera_pspec git hash {}\n{}\n" \ - "dataset1: filename: {}, label: {}, history:\n{}\n{}\n" \ - "dataset2: filename: {}, label: {}, history:\n{}\n{}\n" \ + "dataset1: filename: {}, label: {}, cal: {}, history:\n{}\n{}\n" \ + "dataset2: filename: {}, label: {}, cal: {}, history:\n{}\n{}\n" \ "".format(datetime.datetime.utcnow(), version.git_hash, '-'*20, - filename1, label1, dset1.history, '-'*20, - filename2, label2, dset2.history, '-'*20) + filename1, label1, cal1, dset1.history, '-'*20, + filename2, label2, cal2, dset2.history, '-'*20) uvp.taper = taper uvp.norm = norm @@ -3335,7 +3341,7 @@ def _load_dsets(fnames, bls=None, pols=None, logf=None, verbose=True, for i, dset in enumerate(fnames): utils.log("Reading {} / {} datasets...".format(i+1, Ndsets), f=logf, lvl=1, verbose=verbose) - + # read data uvd = UVData() if isinstance(dset, (str, np.str)): @@ -3344,6 +3350,7 @@ def _load_dsets(fnames, bls=None, pols=None, logf=None, verbose=True, dfiles = dset uvd.read(dfiles, bls=bls, polarizations=pols, file_type=file_type) + uvd.extra_keywords['filename'] = json.dumps(dfiles) dsets.append(uvd) return dsets @@ -3379,6 +3386,7 @@ def _load_cals(cnames, logf=None, verbose=True): uvc.read_calfits(glob.glob(cfile)) else: uvc.read_calfits(cfile) + uvc.extra_keywords['filename'] = json.dumps(cfile) cals.append(uvc) return cals diff --git a/hera_pspec/tests/test_pspecdata.py b/hera_pspec/tests/test_pspecdata.py index 5b233fd6..36161c48 100644 --- a/hera_pspec/tests/test_pspecdata.py +++ b/hera_pspec/tests/test_pspecdata.py @@ -1657,11 +1657,15 @@ def test_pspec_run(): pol_pairs=[('xx', 'xx')], interleave_times=False, file_type='uvh5', spw_ranges=[(100, 150)], cal_flag=True) psc = container.PSpecContainer('./out.h5', 'rw') + uvp = psc.get_pspec('dset0_dset1', 'dset0_x_dset1') # test calibration flags were propagated to test that cal was applied assert ds.dsets[0].flag_array.any() assert ds.dsets[1].flag_array.any() assert ds.dsets_std[0].flag_array.any() assert ds.dsets_std[1].flag_array.any() + assert ds.dsets[0].extra_keywords['filename'] is not '""' + assert ds.dsets[0].extra_keywords['calibration'] is not '""' + assert 'cal: /' in uvp.history # test exceptions nt.assert_raises(AssertionError, pspecdata.pspec_run, 'foo', "./out.h5")