Merge f1d1166 into 1273d08

py/desispec/io/fibermap.py

@@ -24,7 +24,8 @@
 from desiutil.depend import add_dependencies, mergedep
 from desimodel.focalplane import get_tile_radius_deg
 from desimodel.io import load_focalplane
-from desispec.io.util import fitsheader, write_bintable, makepath, addkeys, parse_badamps
+from desispec.io.util import (fitsheader, write_bintable, makepath, addkeys,
+    parse_badamps, checkgzip)
 from desispec.io.meta import rawdata_root, findfile
 from . import iotime
 
@@ -410,32 +411,53 @@ def assemble_fibermap(night, expid, badamps=None, badfibers_filename=None,
     except KeyError:
         rawheader = fits.getheader(rawfile, 'SPS', disable_image_compression=True)
 
-    #- Find fiberassign file
-    rawfafile = fafile = find_fiberassign_file(night, expid)
-
-    #- Look for override fiberassign file in svn
-    try:
+    #- Look for fiberassign file, gzipped or not
+    if 'TILEID' in rawheader:
         tileid = rawheader['TILEID']
-    except KeyError:
-        #
-        # 20210114/00072405 is an example of where this happens.
-        # We use the fiberassign file that find_fiberassign_file() found
-        # by walking backwards through earlier exposures on the same night,
-        # *even if it is actually wrong*.
-        #
-        log.error("TILEID not in the header from %s!", rawfile)
-        rawfafile_expid = int(os.path.basename(os.path.dirname(rawfafile)))
-        if rawfafile_expid != expid:
-            log.error('Determining TILEID from an earlier exposure %08d/%08d!', night, rawfafile_expid)
+        rawfafile = findfile('fiberassign',night=night,expid=expid,tile=tileid)
+        try:
+            rawfafile = checkgzip(rawfafile)
+        except FileNotFoundError:
+            log.error("%s not found; looking in earlier exposures", rawfafile)
+            rawfafile = find_fiberassign_file(night, expid, tileid=tileid)
+
+    #- No TILEID in raw file, but don't give up yet
+    # 20210114/00072405 is example of looking back and finding wrong file
+    # 20210220/00077103 is example of looking back and finding right one
+    #   (albeit not with a useful coordinates file)
+    else:
+        log.error("TILEID not %s; looking for most recent fiberassign file", rawfile)
+        rawfafile = find_fiberassign_file(night, expid)
+        log.info("Found %s", rawfafile)
         tileid = int(os.path.basename(rawfafile).split('-')[1].split('.')[0])
+        log.info("Guessing TILEID=%d from filename; checking RA,dec", tileid)
+
+        #- Check for RA,DEC consistency
+        fahdr = fitsio.read_header(rawfafile, 'FIBERASSIGN')
+        if fahdr['TILERA'] != rawheader['TARGTRA']:
+            msg = 'fiberassign RA mismatch: {} {} vs. {} {}'.format(
+                    os.path.basename(rawfile), rawheader['TARGTRA'],
+                    os.path.basename(rawfafile), fahdr['TILERA'])
+            log.critical(msg)
+            raise RuntimeError(msg)
+
+        if fahdr['TILEDEC'] != rawheader['TARGTDEC']:
+            msg = 'fiberassign DEC mismatch: {} {} vs. {} {}'.format(
+                    os.path.basename(rawfile), rawheader['TARGTDEC'],
+                    os.path.basename(rawfafile), fahdr['TILEDEC'])
+            log.critical(msg)
+            raise RuntimeError(msg)
+
+    fafile = rawfafile
+    rawfafile_expid = int(os.path.basename(os.path.dirname(rawfafile)))
+    if rawfafile_expid != expid:
+        log.error('Using fiberassign from an earlier exposure %08d/%08d', night, rawfafile_expid)
+
+    #- Look for override fiberassign file in svn
     if allow_svn_override and ('DESI_TARGET' in os.environ):
         targdir = os.getenv('DESI_TARGET')
         testfile = f'{targdir}/fiberassign/tiles/trunk/{tileid//1000:03d}/fiberassign-{tileid:06d}.fits'
-        if os.path.exists(testfile+'.gz'):
-            fafile = testfile+'.gz'
-        elif os.path.exists(testfile):
-            fafile = testfile
-
+        fafile = checkgzip(testfile)
         if rawfafile != fafile:
             log.info(f'Overriding raw fiberassign file {rawfafile} with svn {fafile}')
         else:
@@ -741,6 +763,7 @@ def assemble_fibermap(night, expid, badamps=None, badfibers_filename=None,
         fibermap['DELTA_Y'] = 0.0
         fibermap['FIBER_RA'] = 0.0
         fibermap['FIBER_DEC'] = 0.0
+        fibermap['FIBERSTATUS'] |= fibermask.MISSINGPOSITION
         # Update data types to be consistent with updated value if coord file was used.
         for val in ['FIBER_X','FIBER_Y','DELTA_X','DELTA_Y']:
             old_col = fibermap[val]

py/desispec/io/meta.py

@@ -61,7 +61,7 @@ def findfile(filetype, night=None, expid=None, camera=None,
         #
         raw = '{rawdata_dir}/{night}/{expid:08d}/desi-{expid:08d}.fits.fz',
         coordinates = '{rawdata_dir}/{night}/{expid:08d}/coordinates-{expid:08d}.fits',
-        # fibermap = '{rawdata_dir}/{night}/{expid:08d}/fibermap-{expid:08d}.fits',
+        fiberassign = '{rawdata_dir}/{night}/{expid:08d}/fiberassign-{tile:06d}.fits.gz',
         etc = '{rawdata_dir}/{night}/{expid:08d}/etc-{expid:08d}.json',
         #
         # preproc/

py/desispec/io/util.py

@@ -15,6 +15,28 @@
 
 from ..util import healpix_degrade_fixed
 
+def checkgzip(filename):
+    """
+    Check for existence of filename, with or without .gz extension
+
+    Args:
+        filename (str): filename to check for
+
+    Returns path of existing file without or without .gz,
+    or raises FileNotFoundError if neither exists
+    """
+    if os.path.exists(filename):
+        return filename
+
+    if filename.endswith('.gz'):
+        altfilename = filename[0:-3]
+    else:
+        altfilename = filename + '.gz'
+
+    if os.path.exists(altfilename):
+        return altfilename
+    else:
+        raise FileNotFoundError(f'Neither {filename} nor {altfilename}')
 
 def iterfiles(root, prefix, suffix=None):
     '''

py/desispec/test/test_io.py

@@ -674,6 +674,36 @@ def test_native_endian(self):
             self.assertTrue(data2.dtype.isnative, dtype+' is not native endian')
             self.assertTrue(np.all(data1 == data2))
 
+    def test_checkzip(self):
+        """Test desispec.io.util.checkzip"""
+        from ..io.util import checkgzip
+
+        #- create test files
+        fitsfile = os.path.join(self.testDir, 'abc.fits')
+        gzfile = os.path.join(self.testDir, 'xyz.fits.gz')
+        fx = open(fitsfile, 'w'); fx.close()
+        fx = open(gzfile, 'w'); fx.close()
+
+        #- non-gzip file exists
+        fn = checkgzip(fitsfile)
+        self.assertEqual(fn, fitsfile)
+
+        #- looking for .gz but finding non-gz
+        fn = checkgzip(fitsfile+'.gz')
+        self.assertEqual(fn, fitsfile)
+
+        #- gzip file exists
+        fn = checkgzip(gzfile)
+        self.assertEqual(fn, gzfile)
+
+        #- looking for non-gzip file but finding gzip file
+        fn = checkgzip(gzfile[0:-3])
+        self.assertEqual(fn, gzfile)
+
+        #- Don't find what isn't there
+        with self.assertRaises(FileNotFoundError):
+            checkgzip(os.path.join(self.testDir, 'nope.fits'))
+
     def test_findfile(self):
         """Test desispec.io.meta.findfile and desispec.io.download.filepath2url.
         """