Merge branch 'main' into wrap_rrdesi

bin/desi_daily_proc_manager

@@ -168,7 +168,7 @@ if __name__ == '__main__':
                              z_submit_types=args.z_submit_types,
                              tab_filetype=args.table_file_type, camword=camword,
                              badcamword=badcamword, badamps=args.badamps, queue=args.queue,
-                            dry_run_level=args.dry_run_level, dry_run=args.dry_run, no_redshifts=args.no_redshifts,
+                             dry_run_level=args.dry_run_level, dry_run=args.dry_run, no_redshifts=args.no_redshifts,
                              override_night=args.override_night, exps_to_ignore=exps_to_ignore,
                              continue_looping_debug=args.continue_looping_debug,
                              data_cadence_time=args.data_cadence_time, queue_cadence_time=args.queue_cadence_time,

bin/desi_proc_night

@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import argparse
+
+import os, sys
+from desispec.scripts.proc_night import proc_night
+
+def parse_args():
+    """
+    Creates an arguments parser for desi_proc_night to be given to scripts.proc_night
+    """
+    parser = argparse.ArgumentParser(description="Submit data for processing "
+                                                 "with the DESI data pipeline.")
+
+    parser.add_argument("-n","--night", type=int, default=None, help="The night you want processed.")
+    parser.add_argument("--daily", action="store_true", help="If set the "
+                        + "the pipeline operates in daily mode, updating"
+                        + " the exposure_table, not submitting the last"
+                        + " tile, waiting for a complete set of cals, etc.")
+    parser.add_argument("--proc-obstypes", type=str, default=None, required=False,
+                        help="The basic data obstypes to submit for processing. " +
+                             "E.g. science, dark, twilight, flat, arc, zero.")
+    parser.add_argument("--z-submit-types", type=str, default='cumulative', required=False,
+                        help="The group types of redshifts that should be submitted with each exposure. If not "+
+                             "specified, default  is 'cumulative'. If "+
+                             "'false' or 'None' then no redshifts are submitted")
+    parser.add_argument("--tiles", type=str, required=False, default=None,
+                        help="Comma separated list of TILEIDs to include; use -99 to include arcs/flats")
+    parser.add_argument("--surveys", type=str, required=False, default=None,
+                        help="Comma separated list of surveys to include (e.g. sv1,sv3 or main); "+
+                             "use --proc-obstypes to filter out arcs/flats if desired")
+    parser.add_argument("--science-laststeps", type=str, required=False, default=None,
+                        help="Comma separated list of LASTSTEP's to process "
+                             + "(e.g. all, skysub, fluxcalib, ignore); "
+                             + "by default we only process 'all'.")
+    # File and dir defs
+    #parser.add_argument("-s", "--specprod", type=str, required=False, default=None,
+    #                    help="Subdirectory under DESI_SPECTRO_REDUX to write the output files. "+\
+    #                         "Overwrites the environment variable SPECPROD")
+    parser.add_argument("-q", "--queue", type=str, required=False, default=None,
+                        help="The queue to submit jobs to. Default is realtime.")
+    parser.add_argument("-r", "--reservation", type=str, required=False, default=None,
+                        help="The reservation to submit jobs to. If None, it is not submitted to a reservation.")
+    parser.add_argument("--system-name", type=str, required=False, default=None,
+                        help="Batch system name, e.g. cori-haswell, cori-knl, perlmutter-gpu, ...")
+    parser.add_argument("--exp-table-pathname", type=str, required=False, default=None,
+                        help="Directory and file name where the output exposure table should be saved.")
+    parser.add_argument("--proc-table-pathname", type=str, required=False, default=None,
+                        help="Directory and file name where the output processing table should be saved.")
+    parser.add_argument("--update-exptable", action="store_true", help="If true then the "
+                       + "exposure table is updated. The default is False.")
+    parser.add_argument("--complete-tiles-thrunight", type=int, required=False, default=None,
+                        help="Only tiles completed on or before the supplied YYYYMMDD are "
+                        +"considered complete and will be processed. None will process "
+                        +"all completed tiles.")
+    parser.add_argument("--sub-wait-time", type=float, required=False, default=0.5,
+                        help="Time to wait between submissions.")
+    # Code Flags
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Perform a dry run where no jobs are actually created or submitted. Overwritten if "+
+                        "dry-run-level is defined as nonzero.")
+    parser.add_argument("--dry-run-level", type=int, default=0, required=False,
+                        help="""If nonzero, this is a simulated run.
+                                If level=1 the scripts will be written but not submitted.
+                                If level=2, scripts will not be written or submitted but processing_table is created.
+                                if level=3, no output files are written at all.
+                                Logging will remain the same for testing as though scripts are being submitted.
+                                Default is 0 (false).""")
+    parser.add_argument("--no-redshifts", action="store_true",
+                        help="Whether to submit redshifts or not. If set, redshifts are not submitted.")
+    parser.add_argument("--ignore-proc-table-failures", action="store_true",
+                        help="Give this flag if you want to submit jobs even if "+
+                        " there are incomplete jobs in the existing proc table."+
+                        " Only to be used after vetting proc table failures.")
+    parser.add_argument("--dont-check-job-outputs", action="store_true",
+                        help="If all files for a pending job exist and this is False, then the script will not be "+
+                             "submitted. If some files exist and this is True, only the "+
+                             "subset of the cameras without the final data products will be generated and submitted.")
+    parser.add_argument("--dont-resubmit-partial-jobs", action="store_true",
+                        help="Must be False if --dont-check-job-outputs is False. If False, jobs with some prior data "+
+                             "are pruned using PROCCAMWORD to only process the remaining cameras not found to exist.")
+    parser.add_argument("--all-tiles", action="store_true",
+                        help="Set to NOT restrict to completed tiles as defined by the table pointed to by specstatus_path.")
+    parser.add_argument("--all-cumulatives", action="store_true",
+                        help="Set to run cumulative redshifts for all tiles"
+                             + "even if the tile has observations on a later night.")
+    parser.add_argument("--specstatus-path", type=str, required=False, default=None,
+                        help="Location of the surveyops specstatus table. Default is "+
+                             "$DESI_SURVEYOPS/ops/tiles-specstatus.ecsv")
+    parser.add_argument("--use-specter", action="store_true",
+                        help="Use specter. Default is to use gpu_specter")
+    parser.add_argument("--no-cte-flats", action="store_true",
+                        help="If flag set then cte flat exposures are "
+                             + "not processed for cte identification.")
+    parser.add_argument("--dont-require-cals", action="store_true",
+                        help="If set then the code requires either a valid set "
+                             + "of calibrations or a valid override file to link"
+                             + " to calibrations in order to proceed with "
+                             + "science processing.")
+    parser.add_argument("--psf-linking-without-fflat", action="store_true",
+                        help="If set then the code will NOT raise an error "
+                             + "if asked to link psfnight calibrations "
+                             + "without fiberflatnight calibrations.")
+    parser.add_argument("--still-acquiring", action='store_true',
+                        help="for testing --daily mode, assume more data is still coming even if "
+                             + "outside of normal observing hours.")
+
+    args = parser.parse_args()
+
+    # convert str lists to actual lists
+    if args.proc_obstypes is not None:
+        args.proc_obstypes = [pobstype.strip().lower() for pobstype in
+                              args.proc_obstypes.split(',')]
+
+    if args.tiles is not None:
+        args.tiles = [int(tileid) for tileid in args.tiles.split(',')]
+
+    if args.surveys is not None:
+        args.surveys = [survey.strip().lower() for survey in args.surveys.split(',')]
+
+    if args.science_laststeps is not None:
+        args.science_laststeps = [laststep.strip().lower() for laststep in
+                                  args.science_laststeps.split(',')]
+
+    if args.z_submit_types.lower() in ['false', 'none']:
+        args.z_submit_types = None
+        args.no_redshifts = True
+    else:
+        args.z_submit_types = [ztype.strip().lower() for ztype in
+                               args.z_submit_types.split(',')]
+
+    return args
+
+if __name__ == '__main__':
+    args = parse_args()
+    proc_night(**args.__dict__)

bin/desi_update_exposure_tables → bin/desi_reformat_exposure_tables

@@ -4,12 +4,12 @@
 import argparse
 
 ## Import some helper functions, you can see their definitions by uncomenting the bash shell command
-from desispec.scripts.updateexptables import update_exposure_tables
+from desispec.scripts.reformat_exptables import update_exposure_tables
 
 
 def get_parser():
     """
-    Creates an arguments parser for the desi_update_exposure_tables script
+    Creates an arguments parser for the desi_reformat_exposure_tables script
     """
     parser = argparse.ArgumentParser(usage = "{prog} [options]")
     parser.add_argument("-n", "--nights", type=str,  default=None, help="nights as comma separated string")

bin/desi_resubmit_queue_failures

@@ -13,7 +13,7 @@ import glob
 ## Import some helper functions, you can see their definitions by uncomenting the bash shell command
 from desispec.workflow.tableio import load_table, write_table
 from desispec.workflow.proctable import get_processing_table_pathname
-from desispec.workflow.procfuncs import update_and_recurvsively_submit
+from desispec.workflow.processing import update_and_recurvsively_submit
 from desispec.workflow.queue import get_resubmission_states
 
 def parse_args():  # options=None):

bin/desi_update_exposure_table

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import argparse
+
+## Import some helper functions, you can see their definitions by uncomenting the bash shell command
+from desispec.scripts.update_exptable import update_exposure_table
+
+
+def get_parser():
+    """
+    Creates an ArgumentParser object for the desi_update_exposure_table script
+    """
+    parser = argparse.ArgumentParser(usage = "{prog} [options]")
+
+    parser.add_argument('-n', '--night', type=int,
+                        help='8 digit night, e.g. 20200314, of data to run on. '
+                             'If None, it runs on the current night.')
+    parser.add_argument('-s', '--specprod', type=str,
+                        help='The name of the current production. If used, this '
+                             'will overwrite the SPECPROD environment variable.')
+    parser.add_argument('--exp-table-pathname', type=str,
+                        help='Full path and filename where the exposure tables are stored, '
+                             'WITHOUT the monthly directory included.')
+    parser.add_argument('--path_to_data', type=str,
+                        help='Path to the raw data.')
+    parser.add_argument('--exp_obstypes', type=str,
+                        help='Exposure OBSTYPE(s) separated by commas.')
+    parser.add_argument('-c', '--camword', type=str,
+                        help='Camword that alters the set of cameras for processing.')
+    parser.add_argument('--badcamword', type=str,
+                        help='Camword that will be removed from the camword if '
+                             'given, or the camword inferred from the data if camword is not given.')
+    parser.add_argument('--badamps', type=str,
+                        help='Comma separated list of bad amplifiers that should '
+                             'not be processed.')
+    parser.add_argument('--exps-to-ignore', nargs='+', type=int,
+                        help='List of exposure id\'s that should not be processed.')
+    parser.add_argument('--dry-run-level', type=int, default=0,
+                        help='Simulated run level. 1: scripts will be written '
+                             'or submitted. 2: scripts will not be written or'
+                             ' submitted. Default is 0 (false).')
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='True if you want more verbose output, '
+                             'false otherwise.')
+
+    return parser
+
+
+
+if __name__ == '__main__':
+    parser = get_parser()
+    args = parser.parse_args()
+
+    update_exposure_table(**args.__dict__)

doc/api.rst

@@ -536,6 +536,9 @@ desispec API
 .. automodule:: desispec.scripts.proc_joint_fit
     :members:
 
+.. automodule:: desispec.scripts.proc_night
+    :members:
+
 .. automodule:: desispec.scripts.proc_tilenight
     :members:
 
@@ -572,6 +575,9 @@ desispec API
 .. automodule:: desispec.scripts.quicklook
     :members:
 
+.. automodule:: desispec.scripts.reformat_exptables
+    :members:
+
 .. automodule:: desispec.scripts.rejectcosmics
     :members:
 
@@ -614,10 +620,10 @@ desispec API
 .. automodule:: desispec.scripts.trace_shifts
     :members:
 
-.. automodule:: desispec.scripts.update_spectra
+.. automodule:: desispec.scripts.update_exptable
     :members:
-
-.. automodule:: desispec.scripts.updateexptables
+       
+.. automodule:: desispec.scripts.update_spectra
     :members:
 
 .. automodule:: desispec.scripts.zcatalog
@@ -680,6 +686,9 @@ desispec API
 .. automodule:: desispec.workflow.batch
     :members:
 
+.. automodule:: desispec.workflow.calibration_selection
+    :members:
+
 .. automodule:: desispec.workflow.desi_proc_funcs
     :members:
 
@@ -689,7 +698,7 @@ desispec API
 .. automodule:: desispec.workflow.proc_dashboard_funcs
     :members:
 
-.. automodule:: desispec.workflow.procfuncs
+.. automodule:: desispec.workflow.processing
     :members:
 
 .. automodule:: desispec.workflow.proctable
@@ -704,6 +713,9 @@ desispec API
 .. automodule:: desispec.workflow.schedule
     :members:
 
+.. automodule:: desispec.workflow.science_selection
+    :members:
+
 .. automodule:: desispec.workflow.tableio
     :members:
 

py/desispec/calibfinder.py

@@ -214,6 +214,10 @@ def __init__(self,headers,yaml_file=None, fallback_on_dark_not_found=False) :
                         # it happens with the current version of fitsio
                         # if the value = 'None'.
                         pass
+
+        # save header in object for reference
+        self.header = header
+
         if "CAMERA" not in header :
             log.error("no 'CAMERA' keyword in header, cannot find calib")
             log.error("header is:")

py/desispec/ccdcalib.py

@@ -822,8 +822,8 @@ def compare_dark(preprocfile1, preprocfile2, ny=8, nx=40):
     """Compare preprocessed dark images based on different dark models
 
     Args:
-        preprocfile1: filepath to bias model made from OBSTYPE=ZERO exposures
-        preprocfile2: filepath to bias model made from OBSTYPE=ZERO exposures
+        preprocfile1: filepath to preprocessed (i.e. dark subtracted) DARK image
+        preprocfile2: filepath to preprocessed (i.e. dark subtracted) DARK image
 
     Options:
         ny (even int): number of patches in y (row) direction
@@ -850,10 +850,22 @@ def compare_dark(preprocfile1, preprocfile2, ny=8, nx=40):
         log.critical(msg)
         raise ValueError(msg)
 
-    #- calculate differences per-amp, thus //2
-    ny_groups = ny//2
-    nx_groups = nx//2
-
+    #- calculate differences per-amp
+    readout_mode = get_readout_mode(preprochdr1)
+    if readout_mode == '4Amp':
+        ny_groups = ny//2
+        nx_groups = nx//2
+    elif readout_mode == '2AmpLeftRight':
+        ny_groups = ny
+        nx_groups = nx//2
+    elif readout_mode == '2AmpUpDown':
+        ny_groups = ny//2
+        nx_groups = nx
+    else:
+        msg = f'Unknown {readout_mode=}'
+        log.critical(msg)
+        raise ValueError(msg)
+
     median_diff1 = list()
     median_diff2 = list()
 
@@ -884,8 +896,16 @@ def compare_dark(preprocfile1, preprocfile2, ny=8, nx=40):
     #- put back into 2D array by amp
     d1 = median_diff1
     d2 = median_diff2
-    mdiff1 = np.vstack([np.hstack([d1[2],d1[3]]), np.hstack([d1[0],d1[2]])])
-    mdiff2 = np.vstack([np.hstack([d2[2],d2[3]]), np.hstack([d2[0],d2[2]])])
+
+    if readout_mode == '4Amp':
+        mdiff1 = np.vstack([np.hstack([d1[2],d1[3]]), np.hstack([d1[0],d1[2]])])
+        mdiff2 = np.vstack([np.hstack([d2[2],d2[3]]), np.hstack([d2[0],d2[2]])])
+    elif readout_mode == '2AmpLeftRight':
+        mdiff1 = np.hstack([d1[0], d1[1]])
+        mdiff2 = np.hstack([d2[0], d2[1]])
+    elif readout_mode == '2AmpUpDown':
+        mdiff1 = np.vstack([d1[0], d1[1]])
+        mdiff2 = np.vstack([d2[0], d2[1]])
 
     assert mdiff1.shape == (ny,nx)
     assert mdiff2.shape == (ny,nx)