This repository has been archived by the owner on Aug 28, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 13
/
LB-Split-Calibrators.parset
executable file
·220 lines (183 loc) · 10.2 KB
/
LB-Split-Calibrators.parset
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
##################################################################
## LOFAR long baseline calibration pipeline parset ##
## - This pipeline requires prefactor V3 ##
## to have already been run on both calibrator ##
## and target, and LB-Delay-Calibration on the target data ##
## - The working and runtime directory are expected ##
## to be the same. ##
## - Please report any issues at ##
## https://github.com/lmorabit/long_baseline_pipeline ##
##################################################################
##########################################################
## Please update these parameters. ##
##########################################################
# software paths
! long_baseline_pipeline_dir = /home/morabito/software/long_baseline_pipeline_new
! prefactor_dir = /home/morabito/software/prefactorV3
! losoto_directory = /home/morabito/software/losotoV2
! aoflagger = /soft/aoflagger-new/aoflagger/bin/aoflagger
! lofar_directory = $LOFARROOT
##########################################################
## Optional DDF parameters ##
##########################################################
! data_col = DATA ## set to CORRECTED_DATA to use DDF solutions
##########################################################
## These parameters may need to be updated. ##
##########################################################
## catalogue information
! image_cat = {{ job_directory }}/../LB-Delay-Calibration/lotss_catalogue.csv
! delay_cat = {{ job_directory }}/../LB-Delay-Calibration/delay_calibrators.csv
! subtract_cat = {{ job_directory }}/../LB-Delay-Calibration/subtract_sources.csv
## global information -- do not change
! base_directory = input.output.working_directory
! job_directory = input.output.job_directory
## Runtime setup -- will depend on your computing cluster
! num_proc_per_node = input.output.max_per_node ## number of processes to use per step per node
! num_proc_per_node_limit = 4 ## number of processes to use per step per node for tasks with high i/o (dppp or cp) or memory (eg calibration)
! max_dppp_threads = 2 ## number of threads per process for NDPPP
! error_tolerance = False ## False = stop if any subband fails, True = keep going
##########################################################
## Changing these will probably break something. ##
##########################################################
## global information
! lb_delaycal_directory = {{ job_directory }}/../LB-Delay-Calibration/
! scripts = {{ long_baseline_pipeline_dir }}/bin
! prefactor_scripts = {{ prefactor_dir }}/scripts
## Averaging parameters
! cal_shift_avg_timestep = 4 ## Calibrator time averaging: output should be 8 seconds
! cal_shift_avg_freqstep = 4 ## Calibrator freq averaging: output should be 2 channels per subband
! tgt_shift_avg_timestep = 16 ## Target time averaging: output should be 16 seconds
! tgt_shift_avg_freqstep = 16 ## Target freq averaging: output should be 1 channel per subband
## Station combination
! phaseup_command = {ST001:'CS*'} ## 'CS*' for all core stations (or select stations you want to use)
! filter_command = '!CS*&*' ## remove the core stations after combination
## Closure phase stations
! closure_phase_stations = 'DE601;DE605;ST001' ## list of three stations separated by ;
! closure_phase_file = {{ job_directory }}/closure_phases.txt ## pipeline expects this location - do not changes
## Configuration for long baseline pipeline
! lin2circ_executable = {{ scripts }}/lin2circ.py
! firstnpy_file = {{ long_baseline_pipeline_dir }}/data/first_2008.simple.npy
! rfistrategy = HBAdefault
! max_length = 60 ## amount of subbands to concatenate for full-bandwidth flagging (for calibrator you can take all SBs, for target stick to about 148)
##
## END CONFIGURATION
##########################################################
## Steps to run. ##
##########################################################
pipeline.steps = [ setup, selfcal ]
pipeline.steps.setup = [ targetdata_map, targetdata_map_list, parallel_split, caldata_map, cal_model, h5parm_map, sourcedb_map ]
pipeline.steps.selfcal = [ tecsolve, plot_tec, apply_tec, cal_loop3 ]
###########################################################################
##################################################################################################################
## ##
## BEGIN PIPELINE: DO NOT UPDATE BELOW THIS LINE! ##
## ##
##################################################################################################################
# generate a mapfile of the delay-calibrated subbands
targetdata_map.control.kind = plugin
targetdata_map.control.type = createMapfile
targetdata_map.control.method = mapfile_from_folder
targetdata_map.control.mapfile_dir = input.output.mapfile_dir
targetdata_map.control.filename = targetdata_map.mapfile
targetdata_map.control.folder = {{ lb_delaycal_directory }}
targetdata_map.control.pattern = L*msdpppconcat
# convert to a mapfile with all subbands in one list
targetdata_map_list.control.kind = plugin
targetdata_map_list.control.type = createMapfile
targetdata_map_list.control.method = mapfile_all_to_one
targetdata_map_list.control.mapfile_dir = input.output.mapfile_dir
targetdata_map_list.control.filename = combine_data_tar_map.mapfile
targetdata_map_list.control.mapfile_in = targetdata_map.output.mapfile
# parallel split the lbcs calibrators
parallel_split.control.type = pythonplugin
parallel_split.control.max_per_node = {{ num_proc_per_node }}
parallel_split.control.executable = {{ scripts }}/parallel_split.py
parallel_split.argument.flags = [targetdata_map_list.output.mapfile,{{ delay_cat }}]
parallel_split.argument.phaseup_cmd = {{ phaseup_command }}
parallel_split.argument.filter_cmd = {{ filter_command }}
parallel_split.argument.ncpu = {{ num_proc_per_node }}
parallel_split.argument.datacol = {{ data_col }}
parallel_split.argument.timestep = {{ cal_shift_avg_timestep }}
parallel_split.argument.freqstep = {{ cal_shift_avg_freqstep }}
parallel_split.argument.nsbs = 999
# calibrator mapfile
caldata_map.control.kind = plugin
caldata_map.control.type = createMapfile
caldata_map.control.method = mapfile_from_folder
caldata_map.control.mapfile_dir = input.output.mapfile_dir
caldata_map.control.filename = caldata_map.mapfile
caldata_map.control.folder = {{ job_directory }}
caldata_map.control.pattern = S*phasecal.MS
# generate point source models with skynet
cal_model.control.type = pythonplugin
cal_model.control.executable = {{ scripts }}/skynet.py
cal_model.control.mapfile_in = caldata_map.output.mapfile
cal_model.control.inputkey = msin
cal_model.argument.flags = [msin]
cal_model.argument.delayCalFile = {{ delay_cat }}
# parmdb mapfile
h5parm_map.control.kind = plugin
h5parm_map.control.type = createMapfile
h5parm_map.control.method = add_suffix_to_file
h5parm_map.control.mapfile_in = caldata_map.output.mapfile
h5parm_map.control.add_suffix_to_file = _tec.h5
h5parm_map.control.mapfile_dir = input.output.mapfile_dir
h5parm_map.control.filename = h5parm_map.mapfile
# sourcedb mapfile
sourcedb_map.control.kind = plugin
sourcedb_map.control.type = createMapfile
sourcedb_map.control.method = add_suffix_to_file
sourcedb_map.control.mapfile_in = caldata_map.output.mapfile
sourcedb_map.control.add_suffix_to_file = /sky
sourcedb_map.control.mapfile_dir = input.output.mapfile_dir
sourcedb_map.control.filename = sourcedb_map.mapfile
# run the tecsolve on them
tecsolve.control.type = dppp
tecsolve.control.inplace = True
tecsolve.control.max_per_node = {{ num_proc_per_node_limit }}
tecsolve.control.error_tolerance = {{ error_tolerance }}
tecsolve.control.mapfiles_in = [caldata_map.output.mapfile,h5parm_map.output.mapfile,sourcedb_map.output.mapfile]
tecsolve.control.inputkeys = [msfiles,h5parms,skymodels]
tecsolve.argument.numthreads = {{ max_dppp_threads }}
tecsolve.argument.msin = msfiles
tecsolve.argument.msin.datacolumn = DATA
tecsolve.argument.steps = [gaincal]
tecsolve.argument.gaincal.caltype = tec
tecsolve.argument.gaincal.solint = 5
tecsolve.argument.gaincal.nchan = 10
tecsolve.argument.gaincal.directions = [ME0]
tecsolve.argument.gaincal.parmdb = h5parms
tecsolve.argument.gaincal.sourcedb = skymodels
# plot the tec solutions
plot_tec.control.type = pythonplugin
plot_tec.control.max_per_node = 1
plot_tec.control.executable = {{ scripts }}/plot_tec.py
plot_tec.argument.flags = [targetdata_map_list.output.mapfile]
# apply the solutions
apply_tec.control.type = dppp
apply_tec.control.max_per_node = {{ num_proc_per_node_limit }}
apply_tec.control.error_tolerance = {{ error_tolerance }}
apply_tec.control.mapfiles_in = [caldata_map.output.mapfile,h5parm_map.output.mapfile]
apply_tec.control.inputkeys = [msfiles,h5parms]
apply_tec.argument.msin = msfiles
apply_tec.argument.numthreads = {{ max_dppp_threads }}
apply_tec.argument.msin.datacolumn = DATA
apply_tec.argument.msout.datacolumn = DATA
apply_tec.argument.steps = [applycal]
apply_tec.argument.applycal.type = applycal
apply_tec.argument.applycal.correction = tec000
apply_tec.argument.applycal.parmdb = h5parms
# run loop3
cal_loop3.control.type = pythonplugin
cal_loop3.control.max_per_node = {{ num_proc_per_node }}
cal_loop3.control.executable = {{ scripts }}/loop3B_v1.py
cal_loop3.argument.flags = [apply_tec.output.mapfile]
cal_loop3.argument.strategy = 'P30,P30,P30,A500,A450,A400'
cal_loop3.argument.startmod =
cal_loop3.argument.ith = 5.0
cal_loop3.argument.bandwidth = '8MHz'
##################################################################
# ##
# END OF PIPELINE ##
# ##
##################################################################