diff --git a/py/desisurvey/etc.py b/py/desisurvey/etc.py index 8e44139..d3b4317 100644 --- a/py/desisurvey/etc.py +++ b/py/desisurvey/etc.py @@ -302,17 +302,19 @@ def __init__(self, save_history=False): self.MIN_NEXP = config.min_exposures() self.TEXP_TOTAL = {} self.log = desiutil.log.get_logger() + unknownprograms = [] for program in desisurvey.tiles.Tiles.PROGRAMS: nomtime = getattr(config.nominal_exposure_time, program, None) if nomtime is None: - self.log.warning( - 'Unrecognized program {}, '.format(program) + - 'using default exposure time of 1000 s') + unknownprograms.append(program) nomtime = 1000/24/60/60 else: nomtime = nomtime().to(u.day).value - self.TEXP_TOTAL[program] = nomtime + if len(unknownprograms) > 0: + self.log.warning( + 'Unrecognized program {}, '.format(' '.join(unknownprograms)) + + 'using default exposure time of 1000 s') moon_up_factor = getattr(config, 'moon_up_factor', None) if moon_up_factor: for cond in ['DARK', 'GRAY', 'BRIGHT']: diff --git a/py/desisurvey/scripts/afternoon_plan.py b/py/desisurvey/scripts/afternoon_plan.py index 3357c72..21ece8b 100755 --- a/py/desisurvey/scripts/afternoon_plan.py +++ b/py/desisurvey/scripts/afternoon_plan.py @@ -155,8 +155,22 @@ def afternoon_plan(night=None, restore_etc_stats='most_recent', raise ValueError('Must pass spectra_dir to afternoon_plan or set ' 'DESI_SPECTRA_DIR.') - tiles, exps = collect_etc.scan_directory(spectra_dir, - start_from=restore_etc_stats) + if sv: + if os.environ.get('DESI_COLLAB_PASSWORD', None) is not None: + pwstr = '--password ' + os.environ['DESI_COLLAB_PASSWORD'] + ' ' + else: + print('Enter desi collaboration password to download status file.') + pwstr = '--ask-password ' + os.system('wget -q https://data.desi.lbl.gov/desi/users/raichoor/' + 'fiberassign-sv1/sv1-exposures.fits ' + pwstr + + '--user=desi -O ./sv1-exposures.fits') + offlinedepthfn = './sv1-exposures.fits' + else: + offlinedepthfn = None + + tiles, exps = collect_etc.scan_directory( + spectra_dir, start_from=restore_etc_stats, + offlinedepth=offlinedepthfn) collect_etc.write_tile_exp(tiles, exps, os.path.join( directory, 'etc-stats-{}.fits'.format(subdir))) @@ -168,8 +182,8 @@ def afternoon_plan(night=None, restore_etc_stats='most_recent', from desisurvey import svstats numcond = collect_etc.number_in_conditions(exps) donefraccond = svstats.donefrac_in_conditions(numcond) - nneeded = np.zeros(len(donefraccond)) - nobserved = np.zeros(len(donefraccond)) + nneeded = np.zeros(len(donefraccond), dtype='f4') + nobserved = np.zeros(len(donefraccond), dtype='f4') allcond = ['DARK', 'GRAY', 'BRIGHT'] for cond in allcond: nneeded += donefraccond['NNIGHT_NEEDED_'+cond] @@ -178,6 +192,7 @@ def afternoon_plan(night=None, restore_etc_stats='most_recent', tiles['TILEID'], return_indices=True) lastexp = np.zeros(len(donefraccond), dtype='i4')-1 lastexp[md] = tiles['LASTEXPID_ETC'][mt] + nneeded = nneeded + (nneeded == 0) planner.set_donefrac(donefraccond['TILEID'], nobserved / nneeded, lastexp) hdulist = fits.open(newtilefn) @@ -235,7 +250,7 @@ def parse(options=None): parser.add_argument('--night', type=str, help='night to plan, default: tonight', default=None) - parser.add_argument('--restore_etc_stats', type=str, + parser.add_argument('--restore-etc-stats', type=str, help=('etc_stats file to restore. Default: ' '"most_recent", search for most recent. ' '"fresh" to start fresh.'), diff --git a/py/desisurvey/scripts/collect_etc.py b/py/desisurvey/scripts/collect_etc.py index ea47a62..6471ae4 100755 --- a/py/desisurvey/scripts/collect_etc.py +++ b/py/desisurvey/scripts/collect_etc.py @@ -4,6 +4,9 @@ import numpy as np from astropy.io import fits import desiutil.log +import desisurvey.etc +from astropy import units as u + log = desiutil.log.get_logger() @@ -49,7 +52,8 @@ def cull_old_files(files, start_from): return [f for (f, e) in zip(files, expid) if e > maxexpid] -def scan_directory(dirname, simulate_donefrac=False, start_from=None): +def scan_directory(dirname, simulate_donefrac=False, start_from=None, + offlinedepth=None): """Scan directory for spectra with ETC statistics to collect. Parameters @@ -69,6 +73,9 @@ def scan_directory(dirname, simulate_donefrac=False, start_from=None): "fresh" or None indicates starting fresh. "most_recent" indicates searching DESISURVEY_OUTPUT for the most recent file to restore. + offlinedepth : str + offline depth file to use. Fills out donefracs according to + R_DEPTH in the file, plus config.nominal_exposure_time """ log.info('Scanning {} for desi exposures...'.format(dirname)) if start_from is None or (start_from == "fresh"): @@ -124,7 +131,7 @@ def scan_directory(dirname, simulate_donefrac=False, start_from=None): 'information...').format(len(files))) exps = np.zeros(len(files), dtype=[ ('EXPID', 'i4'), ('FILENAME', 'U200'), - ('TILEID', 'f4'), ('DONEFRAC_EXP_ETC', 'f4'), + ('TILEID', 'i4'), ('DONEFRAC_EXP_ETC', 'f4'), ('EXPTIME', 'f4'), ('MJD_OBS', 'f8'), ('FLAVOR', 'U80')]) for i, fn in enumerate(files): if (i % 1000) == 0: @@ -157,6 +164,8 @@ def scan_directory(dirname, simulate_donefrac=False, start_from=None): exps = exps[np.argsort(exps['EXPID'])] if start_exps is not None: exps = np.concatenate([start_exps, exps]) + if offlinedepth is not None: + exps = update_donefrac_from_offline(exps, offlinedepth) ntiles = len(np.unique(exps['TILEID'])) tiles = np.zeros(ntiles, dtype=[ ('TILEID', 'i4'), ('DONEFRAC_ETC', 'f4'), ('EXPTIME', 'f4'), @@ -233,8 +242,9 @@ def get_conditions(mjd): # a lot of exposures apparently taken during the day? # We should mark these as belonging to the "day" program? mday = nightind != nightind.astype('i4') - log.info('%d/%d exposures were taken during the day.' % - (np.sum(mday & okmjd), np.sum(okmjd))) + if np.sum(mday & okmjd) > 0: + log.info('%d/%d exposures were taken during the day.' % + (np.sum(mday & okmjd), np.sum(okmjd))) nightind = nightind.astype('i4') conditions = [] for mjd0, night in zip(mjd[~mday & okmjd], @@ -264,7 +274,7 @@ def get_conditions(mjd): return newout -def number_in_conditions(exps): +def number_in_conditions(exps, nightly_donefrac_requirement=0.5): import desisurvey.tiles tiles = desisurvey.tiles.get_tiles() m = exps['EXPTIME'] > 30 @@ -274,8 +284,9 @@ def number_in_conditions(exps): out = np.zeros(len(np.unique(exps['TILEID'])), dtype=[('TILEID', 'i4'), ('NEXP_BRIGHT', 'i4'), ('NEXP_GRAY', 'i4'), ('NEXP_DARK', 'i4'), - ('NNIGHT_BRIGHT', 'i4'), ('NNIGHT_GRAY', 'i4'), - ('NNIGHT_DARK', 'i4')]) + ('NNIGHT_BRIGHT', 'f4'), ('NNIGHT_GRAY', 'f4'), + ('NNIGHT_DARK', 'f4')]) + conddict = {ind: cond for cond, ind in tiles.OBSCONDITIONS.items()} for i, (f, l) in enumerate(subslices(exps['TILEID'][s])): ind = s[f:l] out['TILEID'][i] = exps['TILEID'][ind[0]] @@ -284,12 +295,65 @@ def number_in_conditions(exps): out['NEXP_'+cond][i] = np.sum(m) out['NNIGHT_'+cond][i] = len(np.unique( exps['MJD_OBS'][ind[m]].astype('i4'))) - # at Kitt Peak, a night never crosses an MJD boundary. - # So we count nights by counting the number of unique - # mjd integers. + # at Kitt Peak, a night never crosses an MJD boundary. + # So we count nights by counting the number of unique + # mjd integers. + nights = exps['MJD_OBS'][ind].astype('i4') + for night in np.unique(nights): + m = nights == night + totaldonefrac = np.sum(exps['DONEFRAC_EXP_ETC'][ind[m]]) + cond = conditions[ind[m]] + cond = cond[cond >= 0] + if len(cond) == 0: + # all images taken ~during the day, don't know what to do. + continue + cond = conddict[cond[0]] + if totaldonefrac > nightly_donefrac_requirement: + out['NNIGHT_'+cond][i] += 1 + else: + out['NNIGHT_'+cond][i] += 0.1 + # still gets credit for having been started return out +def update_donefrac_from_offline(exps, offlinefn): + offline = fits.getdata(offlinefn) + tiles = desisurvey.tiles.Tiles() + tileprogram = tiles.tileprogram + tileprograms = np.zeros(len(exps), dtype='U80') + mt, me = desisurvey.utils.match(tiles.tileID, exps['TILEID']) + tileprograms[:] = 'UNKNOWN' + tileprograms[me] = tileprogram[mt] + me, mo = desisurvey.utils.match(exps['EXPID'], offline['EXPID']) + offline_eff_time = offline['R_DEPTH'] + if ((len(np.unique(exps['EXPID'])) != len(exps)) or + (len(np.unique(offline['EXPID'])) != len(offline))): + raise ValueError('weird duplicate EXPID in exps or offline') + # offline has R_DEPTH in units of time + # now we need the goal exposure times + # these are just the nominal times + config = desisurvey.config.Configuration() + cfgnomtimes = config.nominal_exposure_time + nomtimes = [] + unknownprograms = [] + nunknown = 0 + for ei, oi in zip(me, mo): + nomprogramtime = getattr(cfgnomtimes, tileprograms[ei], 300) + if not isinstance(nomprogramtime, int): + nomprogramtime = nomprogramtime().to(u.s).value + else: + unknownprograms.append(tileprograms[ei]) + nunknown += 1 + nomtimes.append(nomprogramtime) + if nunknown > 0: + print('%d observations of unknown programs' % nunknown) + print('unknown programs: ', np.unique(unknownprograms)) + nomtimes = np.array(nomtimes) + exps = exps.copy() + exps['DONEFRAC_EXP_ETC'][me] = offline_eff_time[mo]/nomtimes + return exps + + def read_tile_exp(fn): """Read tile & exposure ETC statistics file from filename. @@ -323,6 +387,10 @@ def parse(options=None): help='etc_stats file to start from') parser.add_argument('--simulate_donefrac', action='store_true', help='use exptime/1000 instead of DONEFRAC') + parser.add_argument('--offlinedepth', type=str, + help='offline depth file to use') + parser.add_argument('--config', type=str, default=None, + help='configuration file to use') if options is None: args = parser.parse_args() else: @@ -332,8 +400,11 @@ def parse(options=None): def main(args): + import desisurvey.config + _ = desisurvey.config.Configuration(args.config) res = scan_directory(args.directory, start_from=args.start_from, - simulate_donefrac=args.simulate_donefrac) + simulate_donefrac=args.simulate_donefrac, + offlinedepth=args.offlinedepth) if res is not None: tiles, exps = res write_tile_exp(tiles, exps, args.outfile) diff --git a/py/desisurvey/scripts/run_plan.py b/py/desisurvey/scripts/run_plan.py index f859563..a3c939c 100644 --- a/py/desisurvey/scripts/run_plan.py +++ b/py/desisurvey/scripts/run_plan.py @@ -69,7 +69,9 @@ def run_plan(nts_dir=None): nsofar['NNIGHT_BRIGHT']] else: nsofar = [0, 0, 0] - print('%s %5.1f %6s %d %5.1f %5.1f %10s %3.1f %4d %6s %d/%d/%d' % ( + print( + ('%s %5.1f %6s %d %5.1f %5.1f %10s %3.1f ' + '%4d %6s %3.1f/%3.1f/%3.1f') % ( mjd_to_azstr(t0), lst, res['conditions'], res['fiberassign'], ra, dec, res['program'], res['exposure_factor'], res['esttime'].astype('i4'), diff --git a/py/desisurvey/svstats.py b/py/desisurvey/svstats.py index 0c6d49b..93dc7bc 100644 --- a/py/desisurvey/svstats.py +++ b/py/desisurvey/svstats.py @@ -6,7 +6,7 @@ def donefrac_in_conditions(condnexp, configfn=None): """Get donefrac corresponding to number of tiles observed in different conditions. - condnexp from desisurvey.scripts.collect_etc.nexp_in_conditions + condnexp from desisurvey.scripts.collect_etc.number_in_conditions """ import desisurvey.config config = desisurvey.config.Configuration(configfn) @@ -15,8 +15,8 @@ def donefrac_in_conditions(condnexp, configfn=None): ('TILEID', 'i4'), ('DONEFRAC_DARK', 'f4'), ('DONEFRAC_GRAY', 'f4'), ('DONEFRAC_BRIGHT', 'f4'), - ('NNIGHT_DARK', 'i4'), ('NNIGHT_GRAY', 'i4'), - ('NNIGHT_BRIGHT', 'i4'), + ('NNIGHT_DARK', 'f4'), ('NNIGHT_GRAY', 'f4'), + ('NNIGHT_BRIGHT', 'f4'), ('NNIGHT_NEEDED_DARK', 'i4'), ('NNIGHT_NEEDED_GRAY', 'i4'), ('NNIGHT_NEEDED_BRIGHT', 'i4'), ]) @@ -29,8 +29,10 @@ def donefrac_in_conditions(condnexp, configfn=None): out['TILEID'] = tiles.tileID for i, program in enumerate(tiles.tileprogram): for cond in CONDITIONS: - needed = getattr(getattr(config.completeness, program), cond, None) - needed = 0 if needed is None else needed() + needed = getattr(config.completeness, program, 0) + if not isinstance(needed, int): + needed = getattr(needed, cond, None) + needed = 0 if needed is None else needed() out['NNIGHT_NEEDED_'+cond][i] = needed for ic, it in zip(mc, mt): for cond in CONDITIONS: diff --git a/py/desisurvey/tileqa.py b/py/desisurvey/tileqa.py index 235b584..26285f1 100644 --- a/py/desisurvey/tileqa.py +++ b/py/desisurvey/tileqa.py @@ -917,12 +917,15 @@ def edgetiles(tiles, sel): def make_tiles_from_fiberassign(dirname, gaiadensitymapfile, tycho2file, covfile): - fn = glob.glob(os.path.join(dirname, '**/fiberassign*.fits.gz'), + fn = glob.glob(os.path.join(dirname, '**/fiberassign*.fits*'), recursive=True) fn = sorted(fn) tiles = numpy.zeros(len(fn), dtype=basetiledtype) for i, fn0 in enumerate(fn): h = fits.getheader(fn0) + if 'TILEID' not in h: + tiles['tileid'][i] = -1 + continue tiles['tileid'][i] = h['TILEID'] tiles['ra'][i] = h['TILERA'] tiles['dec'][i] = h['TILEDEC'] @@ -933,7 +936,7 @@ def make_tiles_from_fiberassign(dirname, gaiadensitymapfile, isdither = True except Exception: isdither = False - progstr = h['FAFLAVOR'].strip() + progstr = h.get('FAFLAVOR', 'unknown').strip() fa_surv = h['FA_SURV'].strip() if progstr[:len(fa_surv)] != fa_surv: progstr = fa_surv + '_' + progstr @@ -944,6 +947,7 @@ def make_tiles_from_fiberassign(dirname, gaiadensitymapfile, tiles['obsconditions'] = targetmask.obsconditions.mask(h['OBSCON']) else: tiles['obsconditions'] = 2**31-1 + tiles = tiles[tiles['tileid'] != -1] tiles['airmass'] = airmass( numpy.ones(len(tiles), dtype='f4')*15., tiles['dec'], 31.96) tiles_add = add_info_fields(tiles, gaiadensitymapfile, diff --git a/py/desisurvey/tiles.py b/py/desisurvey/tiles.py index c5a3991..5699544 100644 --- a/py/desisurvey/tiles.py +++ b/py/desisurvey/tiles.py @@ -323,7 +323,6 @@ def get_tiles(tiles_file=None, use_cache=True, write_cache=True): pinfo = [] for passnum in tiles.program_passes[pname]: pinfo.append('{}({})'.format(passnum, tiles.pass_ntiles[passnum])) - log.info('{:6s} passes(tiles): {}.'.format(pname, ', '.join(pinfo))) if write_cache: _cached_tiles[tiles_file] = tiles diff --git a/py/desisurvey/utils.py b/py/desisurvey/utils.py index b7612c1..d8a4fd0 100644 --- a/py/desisurvey/utils.py +++ b/py/desisurvey/utils.py @@ -419,3 +419,28 @@ def separation_matrix(ra1, dec1, ra2, dec2, max_separation=None): return havPHI <= threshold else: return np.rad2deg(np.arccos(np.clip(1 - 2 * havPHI, -1, +1))) + + +def match(a, b): + """Find matching elements of b in unique array a by index. + + Returns indices ma, mb such that a[ma] == b[mb] + + Parameters + ---------- + a : unique array + b : array + + Returns + ------- + ma, mb : indices such that a[ma] == b[mb] + """ + sa = np.argsort(a) + _, ua = np.unique(a[sa], return_index=True) + if len(ua) != len(a): + raise ValueError('All keys in a must be unique.') + ind = np.searchsorted(a[sa], b) + m = (ind >= 0) & (ind < len(a)) + matches = a[sa[ind[m]]] == b[m] + m[m] &= matches + return sa[ind[m]], np.flatnonzero(m)