add --autocal-ff-solve-grad to override file and joint_proc

py/desispec/scripts/proc_joint_fit.py

@@ -356,6 +356,8 @@ def main(args=None, comm=None):
             cmd.append(f"-i")
             for flat in flats_for_arm[camera_arm]:
                 cmd.append(f"{flat}")
+            if args.autocal_ff_solve_grad:
+                cmd.append("--solve-gradient")
             num_cmd += 1
             cmdargs = cmd[1:]
 

py/desispec/scripts/proc_night.py

@@ -742,7 +742,12 @@ def submit_calibrations(cal_etable, ptable, cal_override, calibjobs, int_id,
         joint_prow, int_id = make_joint_prow(flat_prows, descriptor='nightlyflat',
                                              internal_id=int_id)
         ptable = set_calibrator_flag(flat_prows, ptable)
-        joint_prow, ptable = create_submit_add_and_save(joint_prow, ptable)
+        if 'nightlyflat' in cal_override:
+            extra_args = cal_override['nightlyflat']
+        else:
+            extra_args = None
+        joint_prow, ptable = create_submit_add_and_save(joint_prow, ptable,
+                                                        extra_job_args=extra_args)
         calibjobs[joint_prow['JOBDESC']] = joint_prow.copy()
         calibjobs['accounted_for']['fiberflatnight'] = True
 

py/desispec/test/test_proc_night.py

@@ -15,7 +15,8 @@
 
 from desispec.workflow.tableio import load_table, write_table
 from desispec.workflow.redshifts import get_ztile_script_pathname
-from desispec.workflow.desi_proc_funcs import get_desi_proc_tilenight_batch_file_pathname
+from desispec.workflow.desi_proc_funcs import \
+    get_desi_proc_tilenight_batch_file_pathname, get_desi_proc_batch_file_path
 from desispec.io import findfile
 from desispec.test.util import link_rawdata
 
@@ -329,6 +330,28 @@ def test_proc_night_linking_and_ccdcalib(self):
             self.assertIn(job, set(proctable['JOBDESC']))
         for job in ['nightlybias', 'ccdcalib']:
             self.assertNotIn(job, set(proctable['JOBDESC']))
+    def test_proc_night_override_flag_setting(self):
+        """Test if override file linking is working"""
+        ## Setup the basic dictionary for the override file
+        base_override_dict = {'calibration': {}}
+
+        ## Test basic case where we link everything
+        testdict = base_override_dict.copy()
+        flag = "--autocal-ff-solve-grad"
+        testdict['calibration']['nightlyflat'] = {'flags': [flag]}
+        proctable, unproctable = self._override_write_run_delete(testdict, dry_run_level=1)
+        for job in ['ccdcalib', 'psfnight', 'nightlyflat', 'tilenight']:
+            self.assertTrue(job in proctable['JOBDESC'])
+        for job in ['linkcal', 'nightlybias']:
+            self.assertTrue(job not in proctable['JOBDESC'])
+        scriptdir = get_desi_proc_batch_file_path(self.night, reduxdir=self.proddir)
+        script = glob.glob(os.path.join(scriptdir, 'nightlyflat*.slurm'))[0]
+        with open(script, 'r') as fil:
+            for line in fil.readlines():
+                log = get_logger()
+                log.error(line)
+                if 'desi_proc_joint_fit' in line:
+                    self.assertTrue(flag in line)
 
     @unittest.skipIf('SKIP_PROC_NIGHT_DAILY_TEST' in os.environ, 'Skipping test_proc_night_daily because $SKIP_PROC_NIGHT_DAILY_TEST is set')
     @unittest.skipUnless(os.path.isdir(_real_rawnight_dir), f'{_real_rawnight_dir} not available')

py/desispec/workflow/desi_proc_funcs.py

@@ -129,6 +129,9 @@ def add_desi_proc_joint_fit_terms(parser):
     #parser.add_argument("-n", "--nights", type=str, help="YEARMMDD nights")
     parser.add_argument("-e", "--expids", type=str, help="Exposure IDs")
     parser.add_argument("-i", "--inputs", type=str, help="input raw data files")
+    parser.add_argument("--autocal-ff-solve-grad", action="store_true",
+                        help="Perform a spatial gradient correction to the fiber flat"
+                             + " by running desi_autocalib_fiberflat with --solve-gradient")
     return parser
 
 def add_desi_proc_tilenight_terms(parser):
@@ -831,8 +834,8 @@ def create_desi_proc_batch_script(night, exp, cameras, jobdesc, queue,
             nightlycte = True
 
     ## nightlycte jobs add time to the job
-    ## hardcoding a runtime for nightlycte, as it matures this
-    ## should be moved into determine_resources()
+    ## hardcoding a runtime for nightlycte.
+    ## TODO should be moved into determine_resources()
     if nightlycte:
         cte_runtime = 5
         runtime += cte_runtime

py/desispec/workflow/processing.py

@@ -524,6 +524,11 @@ def create_batch_script(prow, queue='realtime', dry_run=0, joint=False,
             ## in the future we can eliminate the need for "joint"
             if joint or prow['JOBDESC'].lower() in ['psfnight', 'nightlyflat']:
                 cmd = desi_proc_joint_fit_command(prow, queue=queue)
+                ## For consistency with how we edit the other commands, do them
+                ## here, but future TODO would be to move these into the command
+                ## generation itself
+                if 'flags' in extra_job_args:
+                    cmd += ' ' + ' '.join(np.atleast_1d(extra_job_args['flags']))
             else:
                 cmd = desi_proc_command(prow, system_name, use_specter, queue=queue)
                 if nightlybias:

py/desispec/workflow/timing.py

@@ -45,9 +45,9 @@ def get_nightly_end_time():
     """
     month = time.localtime().tm_mon
     if np.abs(month - 6) > 2:
-        end_night = 8
+        end_night = 8 # 7:23 is latest sunrise
     else:
-        end_night = 7
+        end_night = 7 # 5:15 is earliest sunrise
     return end_night  # local Tucson time the following morning