Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

desi_proc_tilenight processes skysub when --laststeps=skysub #2015

Merged
merged 4 commits into from Mar 16, 2023
Merged

desi_proc_tilenight processes skysub when --laststeps=skysub #2015

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7a9d122
desi_proc_tilenight processes skysub with --laststeps=skysub
marcelo-alvarez Mar 4, 2023
2b4dcbe
Merge branch 'main' into tilenight-laststep
marcelo-alvarez Mar 10, 2023
40f4d69
same submit_night good_exps when laststeps is None
marcelo-alvarez Mar 10, 2023
d7baf6a
laststep of skysub only to sframes
marcelo-alvarez Mar 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 9 additions & 2 deletions py/desispec/scripts/proc_tilenight.py
View file
Open in desktop
Expand Up @@ -70,6 +70,13 @@ def main(args=None, comm=None):
if comm is not None:
exptable = comm.bcast(exptable, root=0)

#- Set the eligible laststep values for different processing steps
if args.laststeps is None:
prestd_laststeps = ['all', 'fluxcalib', 'skysub']
jntpst_laststeps = ['all', 'fluxcalib']
else:
prestd_laststeps = jntpst_laststeps = args.laststeps

#- Determine expids and cameras for a tile night
keep = exptable['OBSTYPE'] == 'science'
keep &= exptable['TILEID'] == int(args.tileid)
Expand Down Expand Up @@ -100,9 +107,9 @@ def main(args=None, comm=None):
prestd_camwords[expids[i]] = camword

laststep = str(exptable['LASTSTEP'][i]).lower()
if laststep in ('all', 'fluxcalib', 'skysub'):
if laststep in prestd_laststeps:
prestdstar_expids.append(expid)
if laststep in ('all', 'fluxcalib'):
if laststep in jntpst_laststeps:
stdstar_expids.append(expid)
poststdstar_expids.append(expid)

Expand Down
6 changes: 4 additions & 2 deletions py/desispec/scripts/submit_night.py
View file
Open in desktop
Expand Up @@ -150,7 +150,8 @@ def submit_night(night, proc_obstypes=None, z_submit_types=None, queue='realtime

## If laststeps not defined, default is only LASTSTEP=='all' exposures for non-tilenight runs
if laststeps is None:
laststeps = ['all',]
if not use_tilenight:
laststeps = ['all',]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when laststeps=None and use_tilenight=True, this block leaves laststeps=None. This results in line 232 not selecting any exposures to process because nothing is in None:

    good_exps = np.isin(np.array(etable['LASTSTEP']).astype(str), laststeps)

For example:

$> desi_run_night --dry-run -n 20230210 --z-submit-types cumulative --all-tiles
Processing the following obstypes: ['bias', 'dark', 'arc', 'flat', 'science', 'twilight', 'sci', 'dither']
Redshift fitting with redshift group types: ['cumulative']
Processing exposures with the following LASTSTEP's: None
INFO:tableio.py:334:load_table: Found table: /global/cfs/cdirs/desi/users/sjbailey/spectro/redux/pr2015/exposure_tables/202302/exposure_table_20230210.csv
INFO:tableio.py:337:load_table: Table /global/cfs/cdirs/desi/users/sjbailey/spectro/redux/pr2015/processing_tables/processing_table_pr2015-20230210.csv not found, creating new table of type proctable
INFO:submit_night.py:247:submit_night: Submitting cumulative redshifts for 0/0 tiles for which 20230210 is the last night: []
Completed submission of exposures for night 20230210.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sbailey thanks for catching this. The most recent commit preserves the previous (correct) assignment to good_exps when laststeps is None, while also passing alaststeps value of None to submit_tilenight_and_redshifts.

else:
laststep_options = get_last_step_options()
for laststep in laststeps:
Expand Down Expand Up @@ -400,7 +401,8 @@ def submit_night(night, proc_obstypes=None, z_submit_types=None, queue='realtime
check_for_outputs=check_for_outputs,
resubmit_partial_complete=resubmit_partial_complete,
z_submit_types=cur_z_submit_types,
system_name=system_name,use_specter=use_specter)
system_name=system_name,use_specter=use_specter,
laststeps=laststeps)
else:
## If running redshifts and there is a future exposure of the same tile
## then only run per exposure redshifts until then
Expand Down
19 changes: 15 additions & 4 deletions py/desispec/workflow/desi_proc_funcs.py
View file
Open in desktop
Expand Up @@ -134,10 +134,17 @@ def add_desi_proc_tilenight_terms(parser):
"""
parser.add_argument("-t", "--tileid", type=str, help="Tile ID")
parser.add_argument("-d", "--dryrun", action="store_true", help="show commands only, do not run")
parser.add_argument("--laststeps", type=str, default='all',
help="Comma separated list of LASTSTEP's to process "
parser.add_argument("--laststeps", type=str, default=None,
help="Comma separated list of LASTSTEP values "
+ "(e.g. all, skysub, fluxcalib, ignore); "
+ "by default we only process 'all'.")
+ "by default, exposures with LASTSTEP "
+ "'all' and 'fluxcalib' will be processed "
+ "to the poststdstar step, and those with "
+ "LASTSTEP 'skysub' to the prestdstar step. "
+ "If specified, exposures with LASTSTEP "
+ "values in the list will be processed to the "
+ "poststdstar step, and all others will not "
+ "be processed at all.")

return parser

Expand Down Expand Up @@ -873,7 +880,7 @@ def create_desi_proc_batch_script(night, exp, cameras, jobdesc, queue, runtime=N

def create_desi_proc_tilenight_batch_script(night, exp, tileid, ncameras, queue, runtime=None, batch_opts=None,
system_name=None, mpistdstars=True, use_specter=False,
no_gpu=False,
no_gpu=False, laststeps=None
):
"""
Generate a SLURM batch script to be submitted to the slurm scheduler to run desi_proc.
Expand All @@ -884,6 +891,7 @@ def create_desi_proc_tilenight_batch_script(night, exp, tileid, ncameras, queue,
tileid: str or int. The tile id for the data.
ncameras: int. The number of cameras used for joint fitting.
queue: str. Queue to be used.
laststeps (list of str, optional): A list of laststeps to pass as the laststeps argument to tilenight.

Options:
runtime: str. Timeout wall clock time.
Expand Down Expand Up @@ -976,6 +984,9 @@ def create_desi_proc_tilenight_batch_script(night, exp, tileid, ncameras, queue,
elif use_specter:
cmd += f' --use-specter'

if laststeps is not None:
cmd += f' --laststeps="{",".join(laststeps)}"'

cmd += f' --timingfile {timingfile}'

fx.write(f'# running a tile-night\n')
Expand Down
24 changes: 15 additions & 9 deletions py/desispec/workflow/procfuncs.py
View file
Open in desktop
Expand Up @@ -198,7 +198,7 @@ def check_for_outputs_on_disk(prow, resubmit_partial_complete=True):

def create_and_submit(prow, queue='realtime', reservation=None, dry_run=0, joint=False,
strictly_successful=False, check_for_outputs=True, resubmit_partial_complete=True,
system_name=None,use_specter=False):
system_name=None,use_specter=False,laststeps=None):
"""
Wrapper script that takes a processing table row and three modifier keywords, creates a submission script for the
compute nodes, and then submits that script to the Slurm scheduler with appropriate dependencies.
Expand All @@ -225,6 +225,7 @@ def create_and_submit(prow, queue='realtime', reservation=None, dry_run=0, joint
remaining cameras not found to exist.
system_name (str): batch system name, e.g. cori-haswell or perlmutter-gpu
use_specter (bool, optional): Default is False. If True, use specter, otherwise use gpu_specter by default.
laststeps (list of str, optional): A list of laststeps to pass as the laststeps argument to tilenight.

Returns:
Table.Row or dict: The same prow type and keywords as input except with modified values updated to reflect
Expand All @@ -241,7 +242,7 @@ def create_and_submit(prow, queue='realtime', reservation=None, dry_run=0, joint
if prow['STATUS'].upper() == 'COMPLETED':
return prow

prow = create_batch_script(prow, queue=queue, dry_run=dry_run, joint=joint, system_name=system_name, use_specter=use_specter)
prow = create_batch_script(prow, queue=queue, dry_run=dry_run, joint=joint, system_name=system_name, use_specter=use_specter,laststeps=laststeps)
prow = submit_batch_script(prow, reservation=reservation, dry_run=dry_run, strictly_successful=strictly_successful)
## If resubmitted partial, the PROCCAMWORD and SCRIPTNAME will correspond to the pruned values. But we want to
## retain the full job's value, so get those from the old job.
Expand Down Expand Up @@ -320,7 +321,7 @@ def desi_proc_joint_fit_command(prow, queue=None):
cmd += f' -e {expid_str}'
return cmd

def create_batch_script(prow, queue='realtime', dry_run=0, joint=False, system_name=None, use_specter=False):
def create_batch_script(prow, queue='realtime', dry_run=0, joint=False, system_name=None, use_specter=False, laststeps=None):
"""
Wrapper script that takes a processing table row and three modifier keywords and creates a submission script for the
compute nodes.
Expand All @@ -336,6 +337,7 @@ def create_batch_script(prow, queue='realtime', dry_run=0, joint=False, system_n
run with desi_proc_joint_fit when not using tilenight. Default is False.
system_name (str): batch system name, e.g. cori-haswell or perlmutter-gpu
use_specter, bool, optional. Default is False. If True, use specter, otherwise use gpu_specter by default.
laststeps (list of str, optional): A list of laststeps to pass as the laststeps argument to tilenight.

Returns:
Table.Row or dict: The same prow type and keywords as input except with modified values updated values for
Expand Down Expand Up @@ -399,7 +401,8 @@ def create_batch_script(prow, queue='realtime', dry_run=0, joint=False, system_n
queue=queue,
mpistdstars=True,
use_specter=use_specter,
system_name=system_name)
system_name=system_name,
laststeps=laststeps)
else:
log.info("Running: {}".format(cmd.split()))
scriptpathname = create_desi_proc_batch_script(
Expand Down Expand Up @@ -1148,7 +1151,7 @@ def submit_redshifts(ptable, prows, tnight, internal_id, queue, reservation,
#########################################
def submit_tilenight(ptable, prows, calibjobs, internal_id, queue, reservation,
dry_run=0, strictly_successful=False, resubmit_partial_complete=True,
system_name=None,use_specter=False):
system_name=None,use_specter=False,laststeps=None):
"""
Given a set of prows, this generates a processing table row, creates a batch script, and submits the appropriate
tilenight job given by descriptor. The returned ptable has all of these rows added to the
Expand All @@ -1175,6 +1178,7 @@ def submit_tilenight(ptable, prows, calibjobs, internal_id, queue, reservation,
remaining cameras not found to exist.
system_name (str): batch system name, e.g. cori-haswell or perlmutter-gpu
use_specter (bool, optional): Default is False. If True, use specter, otherwise use gpu_specter by default.
laststeps (list of str, optional): A list of laststeps to pass as the laststeps argument to tilenight.

Returns:
tuple: A tuple containing:
Expand All @@ -1195,7 +1199,7 @@ def submit_tilenight(ptable, prows, calibjobs, internal_id, queue, reservation,
tnight_prow = create_and_submit(tnight_prow, queue=queue, reservation=reservation, dry_run=dry_run,
strictly_successful=strictly_successful, check_for_outputs=False,
resubmit_partial_complete=resubmit_partial_complete, system_name=system_name,
use_specter=use_specter)
use_specter=use_specter,laststeps=laststeps)
ptable.add_row(tnight_prow)

return ptable, tnight_prow, internal_id
Expand Down Expand Up @@ -1520,7 +1524,8 @@ def checkfor_and_submit_joint_job(ptable, arcs, flats, sciences, calibjobs,
def submit_tilenight_and_redshifts(ptable, sciences, calibjobs, lasttype, internal_id, dry_run=0,
queue='realtime', reservation=None, strictly_successful=False,
check_for_outputs=True, resubmit_partial_complete=True,
z_submit_types=None, system_name=None,use_specter=False):
z_submit_types=None, system_name=None,use_specter=False,
laststeps=None):
"""
Takes all the state-ful data from daily processing and determines whether a tilenight job needs to be submitted.

Expand Down Expand Up @@ -1550,7 +1555,7 @@ def submit_tilenight_and_redshifts(ptable, sciences, calibjobs, lasttype, intern
exposure. If not specified or None, then no redshifts are submitted.
system_name (str): batch system name, e.g. cori-haswell, cori-knl, permutter-gpu
use_specter (bool, optional): Default is False. If True, use specter, otherwise use gpu_specter by default.

laststeps (list of str, optional): A list of laststeps to pass as the laststeps argument to tilenight.
Returns:
tuple: A tuple containing:

Expand All @@ -1565,7 +1570,8 @@ def submit_tilenight_and_redshifts(ptable, sciences, calibjobs, lasttype, intern
queue=queue, reservation=reservation,
dry_run=dry_run, strictly_successful=strictly_successful,
resubmit_partial_complete=resubmit_partial_complete,
system_name=system_name,use_specter=use_specter
system_name=system_name,use_specter=use_specter,
laststeps=laststeps
)

ptable, internal_id = submit_redshifts(ptable, sciences, tnight, internal_id,
Expand Down