From 7a103ea424c4e57b809b0949677d382353619053 Mon Sep 17 00:00:00 2001 From: Eddie Schlafly Date: Tue, 13 Jun 2023 09:13:40 -0700 Subject: [PATCH 1/2] Try to make afternoon planning more noisy about failures to set permissions on the holding pen. --- py/desisurvey/forecast.py | 5 +-- py/desisurvey/scripts/collect_etc.py | 19 ++++++++++ py/desisurvey/scripts/run_plan.py | 52 +++++++++++++++++++++++----- py/desisurvey/tileqa.py | 15 ++++---- 4 files changed, 74 insertions(+), 17 deletions(-) diff --git a/py/desisurvey/forecast.py b/py/desisurvey/forecast.py index 386e36e..ae07ed0 100644 --- a/py/desisurvey/forecast.py +++ b/py/desisurvey/forecast.py @@ -199,7 +199,7 @@ def update(self): def forecast_plots(tmain=None, exps=None, surveyopsdir=None, include_backup=False, cfgfile=None, ratio=False, - nownight=None): + nownight=None, airmasspower=1.25): from matplotlib import pyplot as p if surveyopsdir is None: surveyopsdir = os.environ['DESI_SURVEYOPS'] @@ -231,7 +231,8 @@ def forecast_plots(tmain=None, exps=None, surveyopsdir=None, cz = desisurvey.utils.cos_zenith(tmain['DESIGNHA']*u.deg, tmain['DEC']*u.deg) am = desisurvey.utils.cos_zenith_to_airmass(cz) - amfac = desisurvey.etc.airmass_exposure_factor(am) + # amfac = desisurvey.etc.airmass_exposure_factor(am) + amfac = am ** airmasspower dustfac = desisurvey.etc.dust_exposure_factor(tmain['EBV_MED']) cost = amfac*dustfac costetime = cost*desisurvey.tiles.get_nominal_program_times( diff --git a/py/desisurvey/scripts/collect_etc.py b/py/desisurvey/scripts/collect_etc.py index 532dc1b..7c7171a 100755 --- a/py/desisurvey/scripts/collect_etc.py +++ b/py/desisurvey/scripts/collect_etc.py @@ -405,6 +405,25 @@ def update_donefrac_from_offline(exps, offlinefn): raise ValueError('weird duplicate EXPID in exps or offline') exps = exps.copy() exps['EFFTIME_SPEC'][me] = offline_eff_time[mo] + m = ((exps['TILEID'] > 0) & (exps['TILEID'] < 70000) & + (exps['EFFTIME_SPEC'] < 0) & + (exps['EFFTIME_ETC'] > 0)) + # exposures where we're relying on the EFFTIME_ETC rather than + # EFFTIME_SPEC. + if np.any(m): + log.warning(f'Some ({np.sum(m)}) exposures are missing ' + 'offline effective times.') + errmsg = 'List of nights with tiles with exposures with missing times:\n' + for night in np.unique(exps['NIGHT'][m]): + m2 = exps['NIGHT'] == night + errmsg += f'{night} ({np.sum(m & m2)}): ' + for tileid in np.unique(exps['TILEID'][m & m2]): + m3 = exps['TILEID'] == tileid + tilestr = f'{tileid} (%s), ' % ' '.join( + [str(x) for x in exps['EXPID'][m & m3]]) + errmsg += tilestr + errmsg += '\n' + log.warning(errmsg) return exps diff --git a/py/desisurvey/scripts/run_plan.py b/py/desisurvey/scripts/run_plan.py index 85a4a05..69e8e87 100644 --- a/py/desisurvey/scripts/run_plan.py +++ b/py/desisurvey/scripts/run_plan.py @@ -13,6 +13,7 @@ from astropy.time import Time from astropy.coordinates import EarthLocation from astropy import units as u +from functools import partial def mjd_to_azstr(mjd): @@ -22,16 +23,24 @@ def mjd_to_azstr(mjd): return tt.astimezone(tz).strftime('%H:%M') -def worktile(tileid): +def worktile(tileid, logdir=None): + if logdir is None: + stdout = subprocess.DEVNULL + else: + stdout = open(os.path.join(logdir, f'log-{tileid}-main.log'), 'w') return subprocess.call( ['fba-main-onthefly.sh', str(tileid), 'n', 'manual'], - stdout=subprocess.DEVNULL) + stdout=stdout) -def workqa(tileid): +def workqa(tileid, logdir=None): + if logdir is None: + stdout = subprocess.DEVNULL + else: + stdout = open(os.path.join(logdir, f'log-{tileid}-qa.log'), 'w') return subprocess.call( ['fba-main-onthefly.sh', str(tileid), 'y', 'manual'], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + stdout=stdout, stderr=stdout) def planplot(tileid, plan, title='Nightly plan'): @@ -66,7 +75,7 @@ def planplot(tileid, plan, title='Nightly plan'): p.show() -def make_tiles(tilelist, plan, nprocess=10): +def make_tiles(tilelist, plan, nprocess=10, logdir=None): import glob hpdir = os.environ['FA_HOLDING_PEN'] if len(hpdir) == 0: @@ -89,13 +98,13 @@ def make_tiles(tilelist, plan, nprocess=10): tilestrings = np.array([str(t) for t in tilelist]) print('Starting to write fiberassign tiles for ' + ' '.join(tilestrings)) - retcode1 = pool.map(worktile, tilelist) + retcode1 = pool.map(partial(worktile, logdir=logdir), tilelist) retcode1 = np.array(retcode1) if np.any(retcode1 != 0): print('Weird return code for ' + ' '.join(tilestrings[retcode1 != 0])) print('Starting to write QA...') - retcode2 = pool.map(workqa, tilelist) + retcode2 = pool.map(partial(workqa, logdir=logdir), tilelist) retcode2 = np.array(retcode2) if np.any(retcode2 != 0): print('Weird return code for ' + @@ -216,7 +225,34 @@ def run_plan(night=None, nts_dir=None, verbose=False, survey=None, planplot(tilelist, nts.planner, title='%4d%02d%02d' % (night.year, night.month, night.day)) if makebackuptiles: - make_tiles(tilelist, nts.planner) + import desisurvey.config + backuplogdir = os.path.join( + os.path.dirname(desisurvey.config.Configuration.file_name), + 'backup-tile-logs') + os.mkdir(backuplogdir) + make_tiles(tilelist, nts.planner, logdir=backuplogdir) + check_permissions() + + +def check_permissions(): + from os import stat + hpdir = os.environ.get('FA_HOLDING_PEN', None) + log = desiutil.log.get_logger() + bad = False + if hpdir is None: + log.warning('FA_HOLDING_PEN is not set!') + else: + dirs = os.listdir(hpdir) + for dirname in dirs: + tdirname = os.path.join(hpdir, dirname) + mode = stat.S_IMODE( + os.lstat(tdirname).st_mode) + if mode != 0o2775: + bad = True + log.error(f'{tdirname} has bad permissions!') + if bad: + log.error('Some files in the holding pen have bad permissions!' + 'Raise the alarm in #survey-ops!') def parse(options=None): diff --git a/py/desisurvey/tileqa.py b/py/desisurvey/tileqa.py index c98291f..a12b3e3 100644 --- a/py/desisurvey/tileqa.py +++ b/py/desisurvey/tileqa.py @@ -459,13 +459,14 @@ def qa(desitiles, nside=1024, npts=1000, compare=False, p.savefig('onepass.pdf', dpi=200) p.figure('Full Sky') - setup_print((8, 5), scalefont=1.2) - p.subplots_adjust(left=0.1, bottom=0.06, top=0.97, right=0.97) + setup_print((8, 4.3), scalefont=1.2) + p.subplots_adjust(left=0.1, bottom=0.06, top=0.99, right=0.97) p.clf() - tim, xx, yy = heal2cart(ims['Tiles v3'], interp=False, return_pts=True) - p.imshow(tim, cmap='binary', origin='lower', extent=[360, 0, -90, 90], + tim, xx, yy = heal2cart(ims['Tiles v3'], interp=False, return_pts=True, + startra=300) + p.imshow(tim, cmap='binary', origin='lower', extent=[300, -60, -90, 90], vmin=0, vmax=9) - p.gca().xaxis.set_ticks([360, 300, 240, 180, 120, 60, 0]) + p.gca().xaxis.set_ticks([300, 300, 240, 180, 120, 60, 0, -60]) p.gca().set_aspect('equal') p.xlabel(r'$\alpha$ ($\degree$)') p.ylabel(r'$\delta$ ($\degree$)') @@ -872,13 +873,13 @@ def rotate2(l, b, l0, b0, phi0=0.): return rotate(l, b, phi0, b0, phi0=l0) -def heal2cart(heal, interp=True, return_pts=False): +def heal2cart(heal, interp=True, return_pts=False, startra=360): import healpy nside = healpy.get_nside(heal) #*(2 if interp else 1) owidth = 8*nside oheight = 4*nside-1 dm, rm = np.mgrid[0:oheight, 0:owidth] - rm = 360.-(rm+0.5) / float(owidth) * 360. + rm = startra-(rm+0.5) / float(owidth) * 360. dm = -90. + (dm+0.5) / float(oheight) * 180. t, p = lb2tp(rm.ravel(), dm.ravel()) if interp: From 53aa670c252abe9b433afaf26b7ca7aa87e0c825 Mon Sep 17 00:00:00 2001 From: Planner Date: Tue, 13 Jun 2023 11:25:16 -0700 Subject: [PATCH 2/2] Fix bugs. --- bin/afternoon_plan | 2 ++ doc/changes.rst | 3 +++ py/desisurvey/scripts/run_plan.py | 9 +++++---- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/bin/afternoon_plan b/bin/afternoon_plan index 607aecf..2316da8 100755 --- a/bin/afternoon_plan +++ b/bin/afternoon_plan @@ -2,6 +2,7 @@ import desisurvey.scripts.afternoon_plan import sys +import traceback if __name__ == '__main__': try: @@ -9,6 +10,7 @@ if __name__ == '__main__': retval = desisurvey.scripts.afternoon_plan.main(args) except Exception as e: print(e) + print(traceback.format_exc()) sys.exit(-1) if retval is not None and retval != 0: sys.exit(retval) diff --git a/doc/changes.rst b/doc/changes.rst index 226e973..b8c5610 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -13,6 +13,9 @@ desisurvey change log * Add option for disabling network access during afternoon planning. * Do not force BACKUP program during day if another program is explicitly requested. +* More carefully log things associated with holding pen construction. +* Check that permissions of holding pen are correct following construction. +* Print out tiles for which offline information is missing during AP. .. _`#151`: https://github.com/desihub/desisurvey/pull/151 diff --git a/py/desisurvey/scripts/run_plan.py b/py/desisurvey/scripts/run_plan.py index 69e8e87..568964c 100644 --- a/py/desisurvey/scripts/run_plan.py +++ b/py/desisurvey/scripts/run_plan.py @@ -10,6 +10,7 @@ import desisurvey.utils import desisurvey.tiles import desisurvey.ephem +import desisurvey.config from astropy.time import Time from astropy.coordinates import EarthLocation from astropy import units as u @@ -225,17 +226,17 @@ def run_plan(night=None, nts_dir=None, verbose=False, survey=None, planplot(tilelist, nts.planner, title='%4d%02d%02d' % (night.year, night.month, night.day)) if makebackuptiles: - import desisurvey.config backuplogdir = os.path.join( - os.path.dirname(desisurvey.config.Configuration.file_name), + os.path.dirname(desisurvey.config.Configuration().file_name), 'backup-tile-logs') - os.mkdir(backuplogdir) + if not os.path.exists(backuplogdir): + os.mkdir(backuplogdir) make_tiles(tilelist, nts.planner, logdir=backuplogdir) check_permissions() def check_permissions(): - from os import stat + import stat hpdir = os.environ.get('FA_HOLDING_PEN', None) log = desiutil.log.get_logger() bad = False