Merge branch 'main' into tilenight-laststep

bin/plot_spectra

@@ -133,7 +133,7 @@ for tid in targetids :
     something_to_show = False
     for spec in spectra :
         jj=np.where(spec.fibermap["TARGETID"]==tid)[0]
-        if np.sum(jj)==0 :
+        if len(jj)==0 :
             log.warning("TARGETID {} not in spectra".format(tid))
             continue
         something_to_show = True

doc/changes.rst

@@ -5,7 +5,9 @@ desispec Change Log
 0.57.1 (unreleased)
 -------------------
 
-No changes yet.
+* Fix failing ``io.photo`` unit tests (PR `#2017`_).
+
+.. _`#2017`: https://github.com/desihub/desispec/pull/2017
 
 0.57.0 (2023-02-17)
 -------------------

doc/conf.py

@@ -136,7 +136,7 @@
 autodoc_mock_imports = []
 for missing in ('astropy', 'desimodel', 'desisim', 'desitarget', 'desiutil', 'fitsio', 'healpy',
                 'matplotlib', 'numba', 'numpy', 'pandas', 'PIL', 'psycopg2', 'quasarnp', 'redrock', 'requests',
-                'scipy', 'speclite', 'specter', 'specex', 'sqlalchemy', 'yaml'):
+                'scipy', 'speclite', 'specter', 'specex', 'sqlalchemy', 'yaml', 'pytz'):
     try:
         foo = import_module(missing)
     except ImportError:

py/desispec/coaddition.py

@@ -224,18 +224,28 @@ def coadd_fibermap(fibermap, onetile=False):
     #             xx = Column(np.arange(ntarget, dtype=np.int16))
     #             tfmap.add_column(xx,name='NUM_'+k)
 
-    tfmap.rename_column('FIBERSTATUS', 'COADD_FIBERSTATUS')
+    if 'FIBERSTATUS' in tfmap.dtype.names :
+        tfmap.rename_column('FIBERSTATUS', 'COADD_FIBERSTATUS')
+    if not  'COADD_FIBERSTATUS' in tfmap.dtype.names :
+        raise KeyError("no COADD_FIBERSTATUS column in tfmap")
+
+    if  'FIBERSTATUS' in fibermap.dtype.names :
+        fiberstatus_key='FIBERSTATUS'
+    elif  'COADD_FIBERSTATUS' in fibermap.dtype.names :
+         fiberstatus_key='COADD_FIBERSTATUS'
+    else :
+         raise KeyError("no FIBERSTATUS nor COADD_FIBERSTATUS column in fibermap")
 
     for i,tid in enumerate(targets) :
         jj = fibermap["TARGETID"]==tid
 
         #- coadded FIBERSTATUS = bitwise AND of input FIBERSTATUS
-        tfmap['COADD_FIBERSTATUS'][i] = np.bitwise_and.reduce(fibermap['FIBERSTATUS'][jj])
+        tfmap['COADD_FIBERSTATUS'][i] = np.bitwise_and.reduce(fibermap[fiberstatus_key][jj])
 
         #- Only FIBERSTATUS=0 were included in the coadd
         fiberstatus_nonamp_bits = get_all_nonamp_fiberbitmask_val()
         fiberstatus_amp_bits = get_justamps_fiberbitmask()
-        targ_fibstatuses = fibermap['FIBERSTATUS'][jj]
+        targ_fibstatuses = fibermap[fiberstatus_key][jj]
         nonamp_fiberstatus_flagged = ( (targ_fibstatuses & fiberstatus_nonamp_bits) > 0 )
         allamps_flagged = ( (targ_fibstatuses & fiberstatus_amp_bits) == fiberstatus_amp_bits )
         good_coadds = np.bitwise_not( nonamp_fiberstatus_flagged | allamps_flagged )

py/desispec/scripts/proc_tilenight.py

@@ -83,6 +83,11 @@ def main(args=None, comm=None):
     if args.laststeps is not None:
         keep &= np.isin(exptable['LASTSTEP'].astype(str), args.laststeps)
 
+    #- Camera superset from args.cameras; if None default to all
+    camera_superset = 'a0123456789'
+    if args.cameras is not None:
+        camera_superset = args.cameras
+
     exptable = exptable[keep]
 
     if len(exptable) == 0:
@@ -105,6 +110,7 @@ def main(args=None, comm=None):
             prestd_camwords[expids[i]] = difference_camwords(camword,badcamword,suppress_logging=True)
         else:
             prestd_camwords[expids[i]] = camword
+        prestd_camwords[expids[i]] = camword_intersection([prestd_camwords[expids[i]],camera_superset])
 
         laststep = str(exptable['LASTSTEP'][i]).lower()
         if laststep in prestd_laststeps:
@@ -114,10 +120,11 @@ def main(args=None, comm=None):
             poststdstar_expids.append(expid)
 
     joint_camwords = camword_union(list(prestd_camwords.values()), full_spectros_only=True)
+    joint_camwords = camword_intersection([joint_camwords, camera_superset])
 
     poststd_camwords = dict()
     for expid, camword in prestd_camwords.items():
-        poststd_camwords[expid] = camword_intersection([joint_camwords, camword])
+        poststd_camwords[expid] = camword_intersection([joint_camwords, camword, camera_superset])
 
     #-------------------------------------------------------------------------
     #- Create and submit a batch job if requested
@@ -133,6 +140,7 @@ def main(args=None, comm=None):
                                                    mpistdstars=args.mpistdstars,
                                                    use_specter=args.use_specter,
                                                    no_gpu=args.no_gpu,
+                                                   cameras=camera_superset
                                                    )
         err = 0
         if not args.nosubmit:

py/desispec/scripts/submit_night.py

@@ -239,9 +239,8 @@ def submit_night(night, proc_obstypes=None, z_submit_types=None, queue='realtime
         tiles_this_night = tiles_this_night[tiles_this_night>0]  # science tiles, not calibs
         allexp = read_minimal_exptables_columns(tileids=tiles_this_night)
         for tileid in tiles_this_night:
-            ii = (allexp['TILEID'] == tileid)
-            lastnight = np.max(allexp['NIGHT'][ii])
-            if lastnight == night:
+            nights_with_tile = allexp['NIGHT'][allexp['TILEID'] == tileid]
+            if len(nights_with_tile) > 0 and night == np.max(nights_with_tile):
                 tiles_cumulative.append(tileid)
 
         log.info(f'Submitting cumulative redshifts for {len(tiles_cumulative)}/{len(tiles_this_night)} tiles '

py/desispec/test/test_photo.py

@@ -21,42 +21,44 @@ def setUp(self):
 
         # for testing gather_targetphot and gather_tractorphot
 
-        # sv3-dark (secondary), sv1-bright (primary), main-dark (secondary), main-dark (primary, south), main-dark (primary, north)
+        # sv3-dark (secondary), sv1-bright (primary), main-dark (secondary), main-dark (primary, south), main-dark (primary, north), sv3-bright (TOO)
         input_cat = Table()
-        input_cat['TARGETID'] = [929413348720641, 39627061832191195, 2253225356951552, 39627565329026280, 39633086782113662]
-        input_cat['TARGET_RA'] = [216.615157366494, 59.237567466068484, 217.118753472706, 195.3373716438542, 230.3357972402121]
-        input_cat['TARGET_DEC'] = [1.61111164514945, -31.46536118661697, 1.40073953879623, -9.135959353230087, 40.614361185571525]
+        input_cat['TARGETID'] = [929413348720641, 39627061832191195, 2253225356951552, 39627565329026280, 39633086782113662, 43977491483722156]
+        input_cat['TARGET_RA'] = [216.615157366494, 59.237567466068484, 217.118753472706, 195.3373716438542, 230.3357972402121, 150.1306899]
+        input_cat['TARGET_DEC'] = [1.61111164514945, -31.46536118661697, 1.40073953879623, -9.135959353230087, 40.614361185571525, 1.505752]
+        input_cat['TILEID'] = [280, 80638, 1086, 1573, 1766, 19]
         self.input_cat = input_cat
-        self.tileids = [280, 80638, 1086, 1573, 1766]
 
         # targetphot results
         targetphot = Table() 
-        targetphot['FLUX_R'] = np.array([0.0, 22.768417, 0.0, 2.0220177, 0.3754208]).astype('f4')
-        targetphot['FLUX_W1'] = np.array([0.0, 38.42719, 0.0, 8.39509, 1.7937177]).astype('f4')
-        targetphot['FLUX_IVAR_W2'] = np.array([0.0, 0.72876793, 0.0, 0.53684264, 1.5837417]).astype('f4')
-        targetphot['NUMOBS_INIT'] = np.array([1,1,1,4,9]).astype(np.int64)
+        targetphot['FLUX_R'] = np.array([0.0, 22.768417, 0.0, 2.0220177, 0.3754208, 0.0]).astype('f4')
+        targetphot['FLUX_W1'] = np.array([0.0, 38.42719, 0.0, 8.39509, 1.7937177, 0.0]).astype('f4')
+        targetphot['FLUX_IVAR_W2'] = np.array([0.0, 0.72876793, 0.0, 0.53684264, 1.5837417, 0.0]).astype('f4')
+        targetphot['NUMOBS_INIT'] = np.array([1,1,1,4,9, 1]).astype(np.int64)
         self.targetphot = targetphot
 
         # tractorphot results
         tractorphot = Table() 
-        tractorphot['FLUX_R'] = np.array([0.0, 22.768417, 0.4996462, 2.0220177, 0.3754208]).astype('f4')
-        tractorphot['FLUX_IVAR_W1'] = np.array([0.0, 2.6306653, 2.2135038, 2.3442872, 6.2124352]).astype('f4')
-        tractorphot['LS_ID'] = np.array([0, 9906610122001627, 9906622040377206, 9906617989139688, 9907735053993854]).astype(np.int64)
+        tractorphot['FLUX_R'] = np.array([0.0, 22.768417, 0.4996462, 2.0220177, 0.3754208, 4.8209643]).astype('f4')
+        tractorphot['FLUX_IVAR_W1'] = np.array([0.0, 2.6306653, 2.2135038, 2.3442872, 6.2124352, 2.5682714]).astype('f4')
+        tractorphot['LS_ID'] = np.array([0, 9906610122001627, 9906622040377206, 9906617989139688, 9907735053993854, 9906622022814265]).astype(np.int64)
         self.tractorphot = tractorphot
 
     @unittest.skipUnless(standard_nersc_environment, "not at NERSC")
     def test_gather_targetdirs(self):
         """Test that we get the correct targeting directories given a tile."""
+        surveyops_dir = os.environ['DESI_SURVEYOPS']
         truedirs = {
             # sv1
             '80613': ['/global/cfs/cdirs/desi/target/catalogs/dr9/0.47.0/targets/sv1/resolve/bright/'],
             # sv3 including ToOs
-            '19': ['/global/cfs/cdirs/desi/survey/ops/surveyops/trunk/mtl/sv3/ToO/ToO.ecsv',
+            '19': [surveyops_dir+'/mtl/sv3/ToO/ToO.ecsv',
                    '/global/cfs/cdirs/desi/target/catalogs/dr9/0.57.0/targets/sv3/resolve/bright',
-                   '/global/cfs/cdirs/desi/target/catalogs/dr9/0.57.0/targets/sv3/secondary/bright'],
+                   '/global/cfs/cdirs/desi/target/catalogs/dr9/0.57.0/targets/sv3/secondary/bright/sv3targets-bright-secondary.fits'],
             # main
-            '2070': ['/global/cfs/cdirs/desi/target/catalogs/dr9/1.1.1/targets/main/resolve/dark',
-                     '/global/cfs/cdirs/desi/target/catalogs/dr9/1.1.1/targets/main/secondary/dark']
+            '2070': [surveyops_dir+'/mtl/main/ToO/ToO.ecsv',
+                     '/global/cfs/cdirs/desi/target/catalogs/dr9/1.1.1/targets/main/resolve/dark',
+                     '/global/cfs/cdirs/desi/target/catalogs/dr9/1.1.1/targets/main/secondary/dark/targets-dark-secondary.fits']
                    }
 
         for tileid in truedirs.keys():
@@ -68,7 +70,7 @@ def test_gather_targetdirs(self):
     def test_gather_targetphot(self):
         """Test that we get the correct targeting photometry for an input set of objects."""
 
-        targetphot = gather_targetphot(self.input_cat, tileids=self.tileids)
+        targetphot = gather_targetphot(self.input_cat)
         for col in self.targetphot.colnames:
             self.assertTrue(np.all(targetphot[col] == self.targetphot[col]))
 

py/desispec/workflow/desi_proc_funcs.py

@@ -880,7 +880,7 @@ def create_desi_proc_batch_script(night, exp, cameras, jobdesc, queue, runtime=N
 
 def create_desi_proc_tilenight_batch_script(night, exp, tileid, ncameras, queue, runtime=None, batch_opts=None,
                                   system_name=None, mpistdstars=True, use_specter=False,
-                                  no_gpu=False, laststeps=None
+                                  no_gpu=False, laststeps=None, cameras=None
                                   ):
     """
     Generate a SLURM batch script to be submitted to the slurm scheduler to run desi_proc.
@@ -891,7 +891,6 @@ def create_desi_proc_tilenight_batch_script(night, exp, tileid, ncameras, queue,
         tileid: str or int. The tile id for the data.
         ncameras: int. The number of cameras used for joint fitting.
         queue: str. Queue to be used.
-        laststeps (list of str, optional): A list of laststeps to pass as the laststeps argument to tilenight.
 
     Options:
         runtime: str. Timeout wall clock time.
@@ -900,6 +899,8 @@ def create_desi_proc_tilenight_batch_script(night, exp, tileid, ncameras, queue,
         mpistdstars: bool. Whether to use MPI for stdstar fitting.
         use_specter: bool. Use classic specter instead of gpu_specter for extractions
         no_gpu: bool. Do not use GPU even if available
+        laststeps: list of str. A list of laststeps to pass as the laststeps argument to tilenight
+        cameras: str, must be camword.
 
     Returns:
         scriptfile: the full path name for the script written.
@@ -977,13 +978,16 @@ def create_desi_proc_tilenight_batch_script(night, exp, tileid, ncameras, queue,
         cmd += f' -n {night}'
         cmd += f' -t {tileid}'
         cmd += f' --mpi'
+        if cameras is not None:
+            cmd += f' --cameras {cameras}'
+        else:
+            cmd += f' --cameras a0123456789'
         if mpistdstars:
             cmd += f' --mpistdstars'
         if no_gpu:
             cmd += f' --no-gpu'
         elif use_specter:
             cmd += f' --use-specter'
-
         if laststeps is not None:
             cmd += f' --laststeps="{",".join(laststeps)}"'