Merge pull request #152 from desihub/fixes

Improve logging of holding pen construction & check permissions

bin/afternoon_plan

@@ -2,13 +2,15 @@
 
 import desisurvey.scripts.afternoon_plan
 import sys
+import traceback
 
 if __name__ == '__main__':
     try:
         args = desisurvey.scripts.afternoon_plan.parse()
         retval = desisurvey.scripts.afternoon_plan.main(args)
     except Exception as e:
         print(e)
+        print(traceback.format_exc())
         sys.exit(-1)
     if retval is not None and retval != 0:
         sys.exit(retval)

doc/changes.rst

@@ -13,6 +13,9 @@ desisurvey change log
 * Add option for disabling network access during afternoon planning.
 * Do not force BACKUP program during day if another program is
   explicitly requested.
+* More carefully log things associated with holding pen construction.
+* Check that permissions of holding pen are correct following construction.
+* Print out tiles for which offline information is missing during AP.
 
 .. _`#151`: https://github.com/desihub/desisurvey/pull/151
 

py/desisurvey/forecast.py

@@ -199,7 +199,7 @@ def update(self):
 
 def forecast_plots(tmain=None, exps=None, surveyopsdir=None,
                    include_backup=False, cfgfile=None, ratio=False,
-                   nownight=None):
+                   nownight=None, airmasspower=1.25):
     from matplotlib import pyplot as p
     if surveyopsdir is None:
         surveyopsdir = os.environ['DESI_SURVEYOPS']
@@ -231,7 +231,8 @@ def forecast_plots(tmain=None, exps=None, surveyopsdir=None,
     cz = desisurvey.utils.cos_zenith(tmain['DESIGNHA']*u.deg,
                                      tmain['DEC']*u.deg)
     am = desisurvey.utils.cos_zenith_to_airmass(cz)
-    amfac = desisurvey.etc.airmass_exposure_factor(am)
+    # amfac = desisurvey.etc.airmass_exposure_factor(am)
+    amfac = am ** airmasspower
     dustfac = desisurvey.etc.dust_exposure_factor(tmain['EBV_MED'])
     cost = amfac*dustfac
     costetime = cost*desisurvey.tiles.get_nominal_program_times(

py/desisurvey/scripts/collect_etc.py

@@ -405,6 +405,25 @@ def update_donefrac_from_offline(exps, offlinefn):
         raise ValueError('weird duplicate EXPID in exps or offline')
     exps = exps.copy()
     exps['EFFTIME_SPEC'][me] = offline_eff_time[mo]
+    m = ((exps['TILEID'] > 0) & (exps['TILEID'] < 70000) &
+         (exps['EFFTIME_SPEC'] < 0) &
+         (exps['EFFTIME_ETC'] > 0))
+    # exposures where we're relying on the EFFTIME_ETC rather than
+    # EFFTIME_SPEC.
+    if np.any(m):
+        log.warning(f'Some ({np.sum(m)}) exposures are missing '
+                    'offline effective times.')
+        errmsg = 'List of nights with tiles with exposures with missing times:\n'
+        for night in np.unique(exps['NIGHT'][m]):
+            m2 = exps['NIGHT'] == night
+            errmsg += f'{night} ({np.sum(m & m2)}): '
+            for tileid in np.unique(exps['TILEID'][m & m2]):
+                m3 = exps['TILEID'] == tileid
+                tilestr = f'{tileid} (%s), ' % ' '.join(
+                    [str(x) for x in exps['EXPID'][m & m3]])
+                errmsg += tilestr
+            errmsg += '\n'
+        log.warning(errmsg)
     return exps
 
 

py/desisurvey/scripts/run_plan.py

@@ -10,9 +10,11 @@
 import desisurvey.utils
 import desisurvey.tiles
 import desisurvey.ephem
+import desisurvey.config
 from astropy.time import Time
 from astropy.coordinates import EarthLocation
 from astropy import units as u
+from functools import partial
 
 
 def mjd_to_azstr(mjd):
@@ -22,16 +24,24 @@ def mjd_to_azstr(mjd):
     return tt.astimezone(tz).strftime('%H:%M')
 
 
-def worktile(tileid):
+def worktile(tileid, logdir=None):
+    if logdir is None:
+        stdout = subprocess.DEVNULL
+    else:
+        stdout = open(os.path.join(logdir, f'log-{tileid}-main.log'), 'w')
     return subprocess.call(
         ['fba-main-onthefly.sh', str(tileid), 'n', 'manual'],
-        stdout=subprocess.DEVNULL)
+        stdout=stdout)
 
 
-def workqa(tileid):
+def workqa(tileid, logdir=None):
+    if logdir is None:
+        stdout = subprocess.DEVNULL
+    else:
+        stdout = open(os.path.join(logdir, f'log-{tileid}-qa.log'), 'w')
     return subprocess.call(
         ['fba-main-onthefly.sh', str(tileid), 'y', 'manual'],
-        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        stdout=stdout, stderr=stdout)
 
 
 def planplot(tileid, plan, title='Nightly plan'):
@@ -66,7 +76,7 @@ def planplot(tileid, plan, title='Nightly plan'):
     p.show()
 
 
-def make_tiles(tilelist, plan, nprocess=10):
+def make_tiles(tilelist, plan, nprocess=10, logdir=None):
     import glob
     hpdir = os.environ['FA_HOLDING_PEN']
     if len(hpdir) == 0:
@@ -89,13 +99,13 @@ def make_tiles(tilelist, plan, nprocess=10):
     tilestrings = np.array([str(t) for t in tilelist])
     print('Starting to write fiberassign tiles for ' +
           ' '.join(tilestrings))
-    retcode1 = pool.map(worktile, tilelist)
+    retcode1 = pool.map(partial(worktile, logdir=logdir), tilelist)
     retcode1 = np.array(retcode1)
     if np.any(retcode1 != 0):
         print('Weird return code for ' +
               ' '.join(tilestrings[retcode1 != 0]))
     print('Starting to write QA...')
-    retcode2 = pool.map(workqa, tilelist)
+    retcode2 = pool.map(partial(workqa, logdir=logdir), tilelist)
     retcode2 = np.array(retcode2)
     if np.any(retcode2 != 0):
         print('Weird return code for ' +
@@ -216,7 +226,34 @@ def run_plan(night=None, nts_dir=None, verbose=False, survey=None,
     planplot(tilelist, nts.planner,
              title='%4d%02d%02d' % (night.year, night.month, night.day))
     if makebackuptiles:
-        make_tiles(tilelist, nts.planner)
+        backuplogdir = os.path.join(
+            os.path.dirname(desisurvey.config.Configuration().file_name),
+            'backup-tile-logs')
+        if not os.path.exists(backuplogdir):
+            os.mkdir(backuplogdir)
+        make_tiles(tilelist, nts.planner, logdir=backuplogdir)
+        check_permissions()
+
+
+def check_permissions():
+    import stat
+    hpdir = os.environ.get('FA_HOLDING_PEN', None)
+    log = desiutil.log.get_logger()
+    bad = False
+    if hpdir is None:
+        log.warning('FA_HOLDING_PEN is not set!')
+    else:
+        dirs = os.listdir(hpdir)
+        for dirname in dirs:
+            tdirname = os.path.join(hpdir, dirname)
+            mode = stat.S_IMODE(
+                os.lstat(tdirname).st_mode)
+            if mode != 0o2775:
+                bad = True
+                log.error(f'{tdirname} has bad permissions!')
+    if bad:
+        log.error('Some files in the holding pen have bad permissions!'
+                  'Raise the alarm in #survey-ops!')
 
 
 def parse(options=None):

py/desisurvey/tileqa.py

@@ -459,13 +459,14 @@ def qa(desitiles, nside=1024, npts=1000, compare=False,
     p.savefig('onepass.pdf', dpi=200)
 
     p.figure('Full Sky')
-    setup_print((8, 5), scalefont=1.2)
-    p.subplots_adjust(left=0.1, bottom=0.06, top=0.97, right=0.97)
+    setup_print((8, 4.3), scalefont=1.2)
+    p.subplots_adjust(left=0.1, bottom=0.06, top=0.99, right=0.97)
     p.clf()
-    tim, xx, yy = heal2cart(ims['Tiles v3'], interp=False, return_pts=True)
-    p.imshow(tim, cmap='binary', origin='lower', extent=[360, 0, -90, 90],
+    tim, xx, yy = heal2cart(ims['Tiles v3'], interp=False, return_pts=True,
+                            startra=300)
+    p.imshow(tim, cmap='binary', origin='lower', extent=[300, -60, -90, 90],
              vmin=0, vmax=9)
-    p.gca().xaxis.set_ticks([360, 300, 240, 180, 120, 60, 0])
+    p.gca().xaxis.set_ticks([300, 300, 240, 180, 120, 60, 0, -60])
     p.gca().set_aspect('equal')
     p.xlabel(r'$\alpha$ ($\degree$)')
     p.ylabel(r'$\delta$ ($\degree$)')
@@ -872,13 +873,13 @@ def rotate2(l, b, l0, b0, phi0=0.):
     return rotate(l, b, phi0, b0, phi0=l0)
 
 
-def heal2cart(heal, interp=True, return_pts=False):
+def heal2cart(heal, interp=True, return_pts=False, startra=360):
     import healpy
     nside = healpy.get_nside(heal)  #*(2 if interp else 1)
     owidth = 8*nside
     oheight = 4*nside-1
     dm, rm = np.mgrid[0:oheight, 0:owidth]
-    rm = 360.-(rm+0.5) / float(owidth) * 360.
+    rm = startra-(rm+0.5) / float(owidth) * 360.
     dm = -90. + (dm+0.5) / float(oheight) * 180.
     t, p = lb2tp(rm.ravel(), dm.ravel())
     if interp: