Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add an option to be able to ignore N-sample while LST-binning #932

Merged
merged 6 commits into from
May 27, 2024
Merged

Add an option to be able to ignore N-sample while LST-binning #932

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2d62541
Add a switch in lst_bin to allow for weight_by_nsamples or not
Kai-FengChen Jan 31, 2024
68f65da
Add tests related to weight_by_nsamples
Kai-FengChen Jan 31, 2024
2f59937
Add weight_by_nsamples in argparser
Kai-FengChen Jan 31, 2024
3f2f1f6
Change from weight_by_nsamples to weight_only_by_flags
Kai-FengChen Feb 2, 2024
592911b
Modify tests after changing weight_by_nsamples to weight_only_by_flags
Kai-FengChen Feb 2, 2024
b1288c5
Resolve merge conflict for the old lst binner.
Kai-FengChen May 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions hera_cal/lstbin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
DeprecationWarning,
)


def baselines_same_across_nights(data_list):
"""
Check whether the sets of baselines in the datacontainers are consistent.
Expand Down Expand Up @@ -61,7 +60,7 @@ def baselines_same_across_nights(data_list):


def lst_bin(data_list, lst_list, flags_list=None, nsamples_list=None, dlst=None, begin_lst=None, lst_low=None,
lst_hi=None, flag_thresh=0.7, atol=1e-10, median=False, truncate_empty=True,
lst_hi=None, flag_thresh=0.7, atol=1e-10, median=False, truncate_empty=True, weight_only_by_flags=False,
sig_clip=False, sigma=4.0, min_N=4, flag_below_min_N=False, return_no_avg=False, antpos=None,
rephase=False, freq_array=None, lat=-30.72152, verbose=True, bl_list=None):
"""
Expand Down Expand Up @@ -94,6 +93,7 @@ def lst_bin(data_list, lst_list, flags_list=None, nsamples_list=None, dlst=None,
median=True is not supported when nsamples_list is not None.
truncate_empty : type=boolean, if True, truncate output time bins that have
no averaged data in them.
weight_only_by_flags : type=boolean, if True, weight by flags during lst binning instead of nsamples;
sig_clip : type=boolean, if True, perform a sigma clipping algorithm of the LST bins on the
real and imag components separately. Resultant clip flags are OR'd between real and imag.
Warning: This is considerably slow.
Expand Down Expand Up @@ -399,13 +399,17 @@ def lst_bin(data_list, lst_list, flags_list=None, nsamples_list=None, dlst=None,
# (inverse variance weighted sum).
isfinite = np.isfinite(d)
n[~isfinite] = 0.0

norm = np.sum(n, axis=0).clip(1e-99, np.inf)
if weight_only_by_flags:
wgt = np.ones_like(n)
wgt[~isfinite] = 0.0
else:
wgt = np.copy(n)
norm = np.sum(wgt, axis=0).clip(1e-99, np.inf)
mask = norm > 0
real_avg_t = np.full(d.shape[1:], np.nan)
imag_avg_t = np.full(d.shape[1:], np.nan)

sm = np.nansum(d * n, axis=0)
sm = np.nansum(d * wgt, axis=0)
sm[mask] /= norm[mask]

real_avg_t[mask] = sm.real[mask]
Expand Down Expand Up @@ -541,6 +545,7 @@ def lst_bin_arg_parser():
a.add_argument("--outdir", default=None, type=str, help="directory for writing output")
a.add_argument("--overwrite", default=False, action='store_true', help="overwrite output files")
a.add_argument("--lst_start", type=float, default=None, help="starting LST for binner as it sweeps across 2pi LST. Default is first LST of first file.")
a.add_argument("--weight_only_by_flags", default=False, action='store_true', help="Weight by flags instead of nsamples during LST binning. Default is False.")
a.add_argument("--sig_clip", default=False, action='store_true', help="perform robust sigma clipping before binning")
a.add_argument("--sigma", type=float, default=4.0, help="sigma threshold for sigma clipping")
a.add_argument("--min_N", type=int, default=4, help="minimum number of points in bin needed to proceed with sigma clipping")
Expand Down Expand Up @@ -643,7 +648,7 @@ def config_lst_bin_files(data_files, dlst=None, atol=1e-10, lst_start=None, verb

def lst_bin_files(data_files, input_cals=None, dlst=None, verbose=True, ntimes_per_file=60,
file_ext="{type}.{time:7.5f}.uvh5", outdir=None, overwrite=False, history='', lst_start=None,
atol=1e-6, sig_clip=True, sigma=5.0, min_N=5, flag_below_min_N=False, rephase=False,
atol=1e-6, weight_only_by_flags=False, sig_clip=True, sigma=5.0, min_N=5, flag_below_min_N=False, rephase=False,
output_file_select=None, Nbls_to_load=None, ignore_flags=False, average_redundant_baselines=False,
bl_error_tol=1.0, include_autos=True, ex_ant_yaml_files=None, Nblgroups=None, **kwargs):
"""
Expand Down Expand Up @@ -673,6 +678,7 @@ def lst_bin_files(data_files, input_cals=None, dlst=None, verbose=True, ntimes_p
overwrite : type=bool, if True overwrite output files
history : history to insert into output files
rephase : type=bool, if True, rephase data points in LST bin to center of bin
weight_only_by_flags : type=boolean, if True, weight by flags during lst binning instead of nsamples;
bin_kwargs : type=dictionary, keyword arguments for lst_bin.
atol : type=float, absolute tolerance for LST bin float comparison
output_file_select : type=int or integer list, list of integer indices of the output files to run on.
Expand Down Expand Up @@ -921,7 +927,8 @@ def lst_bin_files(data_files, input_cals=None, dlst=None, verbose=True, ntimes_p
utils.echo("About to run LST binning...")
(bin_lst, bin_data, flag_data, std_data,
num_data) = lst_bin(data_list, lst_list, flags_list=flgs_list, dlst=dlst, begin_lst=begin_lst,
lst_low=fmin, lst_hi=fmax, truncate_empty=False, sig_clip=sig_clip, nsamples_list=nsamples_list,
lst_low=fmin, lst_hi=fmax, truncate_empty=False, sig_clip=sig_clip,
weight_only_by_flags=weight_only_by_flags, nsamples_list=nsamples_list,
sigma=sigma, min_N=min_N, flag_below_min_N=flag_below_min_N, rephase=rephase, freq_array=freq_array,
antpos=antpos, bl_list=all_blgroup_baselines)
# append to lists
Expand Down
Loading