Incorporate offline pipeline results into afternoon planning.

* Download offline pipeline results during afternoon planning.
* Update tile status with offline pipeline results.
* Count a visit as completed if the accumulated exptime on that night is >50% nominal.
* Reduce verbosity.

py/desisurvey/etc.py

@@ -302,17 +302,19 @@ def __init__(self, save_history=False):
         self.MIN_NEXP = config.min_exposures()
         self.TEXP_TOTAL = {}
         self.log = desiutil.log.get_logger()
+        unknownprograms = []
         for program in desisurvey.tiles.Tiles.PROGRAMS:
             nomtime = getattr(config.nominal_exposure_time, program, None)
             if nomtime is None:
-                self.log.warning(
-                    'Unrecognized program {}, '.format(program) +
-                    'using default exposure time of 1000 s')
+                unknownprograms.append(program)
                 nomtime = 1000/24/60/60
             else:
                 nomtime = nomtime().to(u.day).value
-
             self.TEXP_TOTAL[program] = nomtime
+        if len(unknownprograms) > 0:
+            self.log.warning(
+                'Unrecognized program {}, '.format(' '.join(unknownprograms)) +
+                'using default exposure time of 1000 s')
         moon_up_factor = getattr(config, 'moon_up_factor', None)
         if moon_up_factor:
             for cond in ['DARK', 'GRAY', 'BRIGHT']:

py/desisurvey/scripts/afternoon_plan.py

@@ -155,8 +155,22 @@ def afternoon_plan(night=None, restore_etc_stats='most_recent',
         raise ValueError('Must pass spectra_dir to afternoon_plan or set '
                          'DESI_SPECTRA_DIR.')
 
-    tiles, exps = collect_etc.scan_directory(spectra_dir,
-                                             start_from=restore_etc_stats)
+    if sv:
+        if os.environ.get('DESI_COLLAB_PASSWORD', None) is not None:
+            pwstr = '--password ' + os.environ['DESI_COLLAB_PASSWORD'] + ' '
+        else:
+            print('Enter desi collaboration password to download status file.')
+            pwstr = '--ask-password '
+        os.system('wget -q https://data.desi.lbl.gov/desi/users/raichoor/'
+                  'fiberassign-sv1/sv1-exposures.fits ' + pwstr +
+                  '--user=desi -O ./sv1-exposures.fits')
+        offlinedepthfn = './sv1-exposures.fits'
+    else:
+        offlinedepthfn = None
+
+    tiles, exps = collect_etc.scan_directory(
+        spectra_dir, start_from=restore_etc_stats,
+        offlinedepth=offlinedepthfn)
     collect_etc.write_tile_exp(tiles, exps, os.path.join(
         directory, 'etc-stats-{}.fits'.format(subdir)))
 
@@ -168,8 +182,8 @@ def afternoon_plan(night=None, restore_etc_stats='most_recent',
         from desisurvey import svstats
         numcond = collect_etc.number_in_conditions(exps)
         donefraccond = svstats.donefrac_in_conditions(numcond)
-        nneeded = np.zeros(len(donefraccond))
-        nobserved = np.zeros(len(donefraccond))
+        nneeded = np.zeros(len(donefraccond), dtype='f4')
+        nobserved = np.zeros(len(donefraccond), dtype='f4')
         allcond = ['DARK', 'GRAY', 'BRIGHT']
         for cond in allcond:
             nneeded += donefraccond['NNIGHT_NEEDED_'+cond]
@@ -178,6 +192,7 @@ def afternoon_plan(night=None, restore_etc_stats='most_recent',
                                    tiles['TILEID'], return_indices=True)
         lastexp = np.zeros(len(donefraccond), dtype='i4')-1
         lastexp[md] = tiles['LASTEXPID_ETC'][mt]
+        nneeded = nneeded + (nneeded == 0)
         planner.set_donefrac(donefraccond['TILEID'],
                              nobserved / nneeded, lastexp)
         hdulist = fits.open(newtilefn)
@@ -235,7 +250,7 @@ def parse(options=None):
     parser.add_argument('--night', type=str,
                         help='night to plan, default: tonight',
                         default=None)
-    parser.add_argument('--restore_etc_stats', type=str,
+    parser.add_argument('--restore-etc-stats', type=str,
                         help=('etc_stats file to restore. Default: '
                               '"most_recent", search for most recent.  '
                               '"fresh" to start fresh.'),

py/desisurvey/scripts/collect_etc.py

@@ -4,6 +4,9 @@
 import numpy as np
 from astropy.io import fits
 import desiutil.log
+import desisurvey.etc
+from astropy import units as u
+
 
 log = desiutil.log.get_logger()
 
@@ -49,7 +52,8 @@ def cull_old_files(files, start_from):
     return [f for (f, e) in zip(files, expid) if e > maxexpid]
 
 
-def scan_directory(dirname, simulate_donefrac=False, start_from=None):
+def scan_directory(dirname, simulate_donefrac=False, start_from=None,
+                   offlinedepth=None):
     """Scan directory for spectra with ETC statistics to collect.
 
     Parameters
@@ -69,6 +73,9 @@ def scan_directory(dirname, simulate_donefrac=False, start_from=None):
         "fresh" or None indicates starting fresh.
         "most_recent" indicates searching DESISURVEY_OUTPUT for the most recent
         file to restore.
+    offlinedepth : str
+        offline depth file to use.  Fills out donefracs according to
+        R_DEPTH in the file, plus config.nominal_exposure_time
     """
     log.info('Scanning {} for desi exposures...'.format(dirname))
     if start_from is None or (start_from == "fresh"):
@@ -124,7 +131,7 @@ def scan_directory(dirname, simulate_donefrac=False, start_from=None):
               'information...').format(len(files)))
     exps = np.zeros(len(files), dtype=[
         ('EXPID', 'i4'), ('FILENAME', 'U200'),
-        ('TILEID', 'f4'), ('DONEFRAC_EXP_ETC', 'f4'),
+        ('TILEID', 'i4'), ('DONEFRAC_EXP_ETC', 'f4'),
         ('EXPTIME', 'f4'), ('MJD_OBS', 'f8'), ('FLAVOR', 'U80')])
     for i, fn in enumerate(files):
         if (i % 1000) == 0:
@@ -157,6 +164,8 @@ def scan_directory(dirname, simulate_donefrac=False, start_from=None):
     exps = exps[np.argsort(exps['EXPID'])]
     if start_exps is not None:
         exps = np.concatenate([start_exps, exps])
+    if offlinedepth is not None:
+        exps = update_donefrac_from_offline(exps, offlinedepth)
     ntiles = len(np.unique(exps['TILEID']))
     tiles = np.zeros(ntiles, dtype=[
         ('TILEID', 'i4'), ('DONEFRAC_ETC', 'f4'), ('EXPTIME', 'f4'),
@@ -233,8 +242,9 @@ def get_conditions(mjd):
     # a lot of exposures apparently taken during the day?
     # We should mark these as belonging to the "day" program?
     mday = nightind != nightind.astype('i4')
-    log.info('%d/%d exposures were taken during the day.' %
-             (np.sum(mday & okmjd), np.sum(okmjd)))
+    if np.sum(mday & okmjd) > 0:
+        log.info('%d/%d exposures were taken during the day.' %
+                 (np.sum(mday & okmjd), np.sum(okmjd)))
     nightind = nightind.astype('i4')
     conditions = []
     for mjd0, night in zip(mjd[~mday & okmjd],
@@ -264,7 +274,7 @@ def get_conditions(mjd):
     return newout
 
 
-def number_in_conditions(exps):
+def number_in_conditions(exps, nightly_donefrac_requirement=0.5):
     import desisurvey.tiles
     tiles = desisurvey.tiles.get_tiles()
     m = exps['EXPTIME'] > 30
@@ -274,8 +284,9 @@ def number_in_conditions(exps):
     out = np.zeros(len(np.unique(exps['TILEID'])),
                    dtype=[('TILEID', 'i4'), ('NEXP_BRIGHT', 'i4'),
                           ('NEXP_GRAY', 'i4'), ('NEXP_DARK', 'i4'),
-                          ('NNIGHT_BRIGHT', 'i4'), ('NNIGHT_GRAY', 'i4'),
-                          ('NNIGHT_DARK', 'i4')])
+                          ('NNIGHT_BRIGHT', 'f4'), ('NNIGHT_GRAY', 'f4'),
+                          ('NNIGHT_DARK', 'f4')])
+    conddict = {ind: cond for cond, ind in tiles.OBSCONDITIONS.items()}
     for i, (f, l) in enumerate(subslices(exps['TILEID'][s])):
         ind = s[f:l]
         out['TILEID'][i] = exps['TILEID'][ind[0]]
@@ -284,12 +295,65 @@ def number_in_conditions(exps):
             out['NEXP_'+cond][i] = np.sum(m)
             out['NNIGHT_'+cond][i] = len(np.unique(
                 exps['MJD_OBS'][ind[m]].astype('i4')))
-            # at Kitt Peak, a night never crosses an MJD boundary.
-            # So we count nights by counting the number of unique
-            # mjd integers.
+        # at Kitt Peak, a night never crosses an MJD boundary.
+        # So we count nights by counting the number of unique
+        # mjd integers.
+        nights = exps['MJD_OBS'][ind].astype('i4')
+        for night in np.unique(nights):
+            m = nights == night
+            totaldonefrac = np.sum(exps['DONEFRAC_EXP_ETC'][ind[m]])
+            cond = conditions[ind[m]]
+            cond = cond[cond >= 0]
+            if len(cond) == 0:
+                # all images taken ~during the day, don't know what to do.
+                continue
+            cond = conddict[cond[0]]
+            if totaldonefrac > nightly_donefrac_requirement:
+                out['NNIGHT_'+cond][i] += 1
+            else:
+                out['NNIGHT_'+cond][i] += 0.1
+                # still gets credit for having been started
     return out
 
 
+def update_donefrac_from_offline(exps, offlinefn):
+    offline = fits.getdata(offlinefn)
+    tiles = desisurvey.tiles.Tiles()
+    tileprogram = tiles.tileprogram
+    tileprograms = np.zeros(len(exps), dtype='U80')
+    mt, me = desisurvey.utils.match(tiles.tileID, exps['TILEID'])
+    tileprograms[:] = 'UNKNOWN'
+    tileprograms[me] = tileprogram[mt]
+    me, mo = desisurvey.utils.match(exps['EXPID'], offline['EXPID'])
+    offline_eff_time = offline['R_DEPTH']
+    if ((len(np.unique(exps['EXPID'])) != len(exps)) or
+            (len(np.unique(offline['EXPID'])) != len(offline))):
+        raise ValueError('weird duplicate EXPID in exps or offline')
+    # offline has R_DEPTH in units of time
+    # now we need the goal exposure times
+    # these are just the nominal times
+    config = desisurvey.config.Configuration()
+    cfgnomtimes = config.nominal_exposure_time
+    nomtimes = []
+    unknownprograms = []
+    nunknown = 0
+    for ei, oi in zip(me, mo):
+        nomprogramtime = getattr(cfgnomtimes, tileprograms[ei], 300)
+        if not isinstance(nomprogramtime, int):
+            nomprogramtime = nomprogramtime().to(u.s).value
+        else:
+            unknownprograms.append(tileprograms[ei])
+            nunknown += 1
+        nomtimes.append(nomprogramtime)
+    if nunknown > 0:
+        print('%d observations of unknown programs' % nunknown)
+        print('unknown programs: ', np.unique(unknownprograms))
+    nomtimes = np.array(nomtimes)
+    exps = exps.copy()
+    exps['DONEFRAC_EXP_ETC'][me] = offline_eff_time[mo]/nomtimes
+    return exps
+
+
 def read_tile_exp(fn):
     """Read tile & exposure ETC statistics file from filename.
 
@@ -323,6 +387,10 @@ def parse(options=None):
                         help='etc_stats file to start from')
     parser.add_argument('--simulate_donefrac', action='store_true',
                         help='use exptime/1000 instead of DONEFRAC')
+    parser.add_argument('--offlinedepth', type=str,
+                        help='offline depth file to use')
+    parser.add_argument('--config', type=str, default=None,
+                        help='configuration file to use')
     if options is None:
         args = parser.parse_args()
     else:
@@ -332,8 +400,11 @@ def parse(options=None):
 
 
 def main(args):
+    import desisurvey.config
+    _ = desisurvey.config.Configuration(args.config)
     res = scan_directory(args.directory, start_from=args.start_from,
-                         simulate_donefrac=args.simulate_donefrac)
+                         simulate_donefrac=args.simulate_donefrac,
+                         offlinedepth=args.offlinedepth)
     if res is not None:
         tiles, exps = res
         write_tile_exp(tiles, exps, args.outfile)

py/desisurvey/scripts/run_plan.py

@@ -69,7 +69,9 @@ def run_plan(nts_dir=None):
                       nsofar['NNIGHT_BRIGHT']]
         else:
             nsofar = [0, 0, 0]
-        print('%s %5.1f %6s %d %5.1f %5.1f %10s %3.1f %4d %6s %d/%d/%d' % (
+        print(
+            ('%s %5.1f %6s %d %5.1f %5.1f %10s %3.1f '
+             '%4d %6s %3.1f/%3.1f/%3.1f') % (
             mjd_to_azstr(t0), lst, res['conditions'],
             res['fiberassign'], ra, dec, res['program'],
             res['exposure_factor'], res['esttime'].astype('i4'),

py/desisurvey/svstats.py

@@ -6,7 +6,7 @@ def donefrac_in_conditions(condnexp, configfn=None):
     """Get donefrac corresponding to number of tiles observed
     in different conditions.
 
-    condnexp from desisurvey.scripts.collect_etc.nexp_in_conditions
+    condnexp from desisurvey.scripts.collect_etc.number_in_conditions
     """
     import desisurvey.config
     config = desisurvey.config.Configuration(configfn)
@@ -15,8 +15,8 @@ def donefrac_in_conditions(condnexp, configfn=None):
         ('TILEID', 'i4'),
         ('DONEFRAC_DARK', 'f4'), ('DONEFRAC_GRAY', 'f4'),
         ('DONEFRAC_BRIGHT', 'f4'),
-        ('NNIGHT_DARK', 'i4'), ('NNIGHT_GRAY', 'i4'),
-        ('NNIGHT_BRIGHT', 'i4'),
+        ('NNIGHT_DARK', 'f4'), ('NNIGHT_GRAY', 'f4'),
+        ('NNIGHT_BRIGHT', 'f4'),
         ('NNIGHT_NEEDED_DARK', 'i4'), ('NNIGHT_NEEDED_GRAY', 'i4'),
         ('NNIGHT_NEEDED_BRIGHT', 'i4'),
     ])
@@ -29,8 +29,10 @@ def donefrac_in_conditions(condnexp, configfn=None):
     out['TILEID'] = tiles.tileID
     for i, program in enumerate(tiles.tileprogram):
         for cond in CONDITIONS:
-            needed = getattr(getattr(config.completeness, program), cond, None)
-            needed = 0 if needed is None else needed()
+            needed = getattr(config.completeness, program, 0)
+            if not isinstance(needed, int):
+                needed = getattr(needed, cond, None)
+                needed = 0 if needed is None else needed()
             out['NNIGHT_NEEDED_'+cond][i] = needed
     for ic, it in zip(mc, mt):
         for cond in CONDITIONS:

py/desisurvey/tileqa.py

@@ -917,12 +917,15 @@ def edgetiles(tiles, sel):
 
 def make_tiles_from_fiberassign(dirname, gaiadensitymapfile,
                                 tycho2file, covfile):
-    fn = glob.glob(os.path.join(dirname, '**/fiberassign*.fits.gz'),
+    fn = glob.glob(os.path.join(dirname, '**/fiberassign*.fits*'),
                    recursive=True)
     fn = sorted(fn)
     tiles = numpy.zeros(len(fn), dtype=basetiledtype)
     for i, fn0 in enumerate(fn):
         h = fits.getheader(fn0)
+        if 'TILEID' not in h:
+            tiles['tileid'][i] = -1
+            continue
         tiles['tileid'][i] = h['TILEID']
         tiles['ra'][i] = h['TILERA']
         tiles['dec'][i] = h['TILEDEC']
@@ -933,7 +936,7 @@ def make_tiles_from_fiberassign(dirname, gaiadensitymapfile,
                 isdither = True
         except Exception:
             isdither = False
-        progstr = h['FAFLAVOR'].strip()
+        progstr = h.get('FAFLAVOR', 'unknown').strip()
         fa_surv = h['FA_SURV'].strip()
         if progstr[:len(fa_surv)] != fa_surv:
             progstr = fa_surv + '_' + progstr
@@ -944,6 +947,7 @@ def make_tiles_from_fiberassign(dirname, gaiadensitymapfile,
             tiles['obsconditions'] = targetmask.obsconditions.mask(h['OBSCON'])
         else:
             tiles['obsconditions'] = 2**31-1
+    tiles = tiles[tiles['tileid'] != -1]
     tiles['airmass'] = airmass(
         numpy.ones(len(tiles), dtype='f4')*15., tiles['dec'], 31.96)
     tiles_add = add_info_fields(tiles, gaiadensitymapfile,

py/desisurvey/tiles.py

@@ -323,7 +323,6 @@ def get_tiles(tiles_file=None, use_cache=True, write_cache=True):
             pinfo = []
             for passnum in tiles.program_passes[pname]:
                 pinfo.append('{}({})'.format(passnum, tiles.pass_ntiles[passnum]))
-            log.info('{:6s} passes(tiles): {}.'.format(pname, ', '.join(pinfo)))
 
     if write_cache:
         _cached_tiles[tiles_file] = tiles

py/desisurvey/utils.py

@@ -419,3 +419,28 @@ def separation_matrix(ra1, dec1, ra2, dec2, max_separation=None):
         return havPHI <= threshold
     else:
         return np.rad2deg(np.arccos(np.clip(1 - 2 * havPHI, -1, +1)))
+
+
+def match(a, b):
+    """Find matching elements of b in unique array a by index.
+
+    Returns indices ma, mb such that a[ma] == b[mb]
+
+    Parameters
+    ----------
+    a : unique array
+    b : array
+
+    Returns
+    -------
+    ma, mb : indices such that a[ma] == b[mb]
+    """
+    sa = np.argsort(a)
+    _, ua = np.unique(a[sa], return_index=True)
+    if len(ua) != len(a):
+        raise ValueError('All keys in a must be unique.')
+    ind = np.searchsorted(a[sa], b)
+    m = (ind >= 0) & (ind < len(a))
+    matches = a[sa[ind[m]]] == b[m]
+    m[m] &= matches
+    return sa[ind[m]], np.flatnonzero(m)