Merge 31ab816 into ff869bf

bin/fba-main-onthefly.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# this script is a wrapper for a fba_launch call for one main tile
+# there is no "svn up" here, no SVN checks
+# the code will exit if a TILEID already existing in the official SVN folder exists
+
+TILEID=$1 # e.g. 1000
+QA=$2   # y or n
+
+OUTDIR=$FA_HOLDING_PEN
+TILESFN=$SURVEYOPS/ops/tiles-main.ecsv
+DTCATVER=1.1.1
+
+# setting the proper environment
+# https://desi.lbl.gov/trac/wiki/SurveyOps/FiberAssignAtKPNO version #18
+if [ -z $NERSC_HOST ]; then
+    # at KPNO
+    export DESI_PRODUCT_ROOT=/software/datasystems/desiconda/20200924
+    export DESI_ROOT=/data/datasystems
+    export DESI_TARGET=$DESI_ROOT/target
+    export DESI_SURVEYOPS=$DESI_ROOT/survey/ops/surveyops/trunk
+    module use $DESI_PRODUCT_ROOT/modulefiles
+    module load desiconda
+    module load desimodules/21.5
+    module swap desitarget/1.2.2
+    module swap fiberassign/5.1.1
+    module swap desimeter/0.6.7
+    module swap desimodel/master
+else
+    echo Fiberassign on the fly should only be run for testing purposes at NERSC!
+    # source /global/cfs/cdirs/desi/software/desi_environment.sh 21.5
+fi
+
+if [ -z $FIBER_ASSIGN_DIR ]; then
+    SVNTILEDIR=$DESI_TARGET/fiberassign/tiles/trunk
+else
+    SVNTILEDIR=$FIBER_ASSIGN_DIR
+fi
+
+# grabbing the RA, DEC, PROGRAM, STATUS, HA for TILEID
+LINE=`awk '{if ($1 == '$TILEID') print $0}' $TILESFN`
+RA=`echo $LINE | awk '{print $3}'`
+DEC=`echo $LINE | awk '{print $4}'`
+PROGRAM=`echo $LINE | awk '{print $5}'`
+STATUS=`echo $LINE | awk '{print $8}'`
+HA=`echo $LINE | awk '{print $10}'`
+
+# small sanity check
+if [[ "$STATUS" != "unobs" ]]
+then
+    echo "TILEID=$TILEID has already been observed; exiting"
+    exit
+fi
+
+# calling fba_launch
+if [[ "$QA" == "y" ]]
+then
+    CMD="fba_launch --outdir $OUTDIR --tileid $TILEID --tilera $RA --tiledec $DEC --survey main --program $PROGRAM --ha $HA --dtver $DTCATVER --steps qa --log-stdout --doclean y --svntiledir $SVNTILEDIR --worldreadable"
+else
+    CMD="fba_launch --outdir $OUTDIR --tileid $TILEID --tilera $RA --tiledec $DEC --survey main --program $PROGRAM --ha $HA --dtver $DTCATVER --nosteps qa --doclean n --svntiledir $SVNTILEDIR --worldreadable"
+fi
+echo $CMD
+eval $CMD

doc/changes.rst

@@ -23,11 +23,13 @@ desisurvey change log
   `#141`_).
 * Let ICS move tiles into place.  Make tile selection only use speed
   (PR `#142`_).
+* Add fiberassign-on-the-fly capability (PR `#143`_)
 
 .. _`#139`: https://github.com/desihub/desisurvey/pull/139
 .. _`#140`: https://github.com/desihub/desisurvey/pull/140
 .. _`#141`: https://github.com/desihub/desisurvey/pull/141
 .. _`#142`: https://github.com/desihub/desisurvey/pull/142
+.. _`#143`: https://github.com/desihub/desisurvey/pull/143
 
 0.17.0 (2021-04-23)
 -------------------

py/desisurvey/NTS.py

@@ -51,6 +51,7 @@
 from astropy import units as u
 from astropy import time
 import numpy as np
+import subprocess
 
 try:
     import DOSlib.logger as Log
@@ -177,6 +178,15 @@ def azinrange(az, low, high):
     return (az >= low) & (az <= high)
 
 
+def get_fiberassign_dir():
+    fadir = os.environ.get('DOS_DESI_TILES', None)
+    if fadir is None:
+        fadir = os.environ.get('FIBER_ASSIGN_DIR', None)
+    if fadir is None:
+        logob.error('DOS_DESI_TILES and FIBER_ASSIGN_DIR not set!')
+    return fadir
+
+
 def move_tile_into_place(tileid, speculative=False):
     """Move fiberassign file into place if not already there.
 
@@ -192,11 +202,8 @@ def move_tile_into_place(tileid, speculative=False):
     speculative : bool
         if True, do not perform the actual copy; just check existence.
     """
-    fadir = os.environ.get('DOS_DESI_TILES', None)
+    fadir = get_fiberassign_dir()
     if fadir is None:
-        fadir = os.environ.get('FIBER_ASSIGN_DIR', None)
-    if fadir is None:
-        logob.error('DOS_DESI_TILES and FIBER_ASSIGN_DIR not set!')
         return False
     tileidstr = '%06d' % tileid
     fabasefn = os.path.join(tileidstr[0:3], 'fiberassign-%s' % tileidstr)
@@ -236,6 +243,30 @@ def move_tile_into_place(tileid, speculative=False):
     return True
 
 
+def design_tile_on_fly(tileid, speculative=False):
+    # don't actually design the tile, but say we did
+    # not sure what else we might want to check here.
+    if speculative:
+        return True
+    fadir = get_fiberassign_dir()
+    if fadir is None:
+        return False
+    tileidpad = '%06d' % tileid
+    subdir = tileidpad[:3]
+    outfnnogz = os.path.join(fadir, subdir, 'fiberassign-%s.fits' % tileidpad)
+    outfn = os.path.join(fadir, subdir, 'fiberassign-%s.fits.gz' % tileidpad)
+    if os.path.exists(outfnnogz) or os.path.exists(outfn):
+        return True
+    logob.info('Designing tile %d on fly.' % tileid)
+    subprocess.call(['fba-main-onthefly.sh', str(tileid), 'n'],
+                    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    subprocess.Popen(['fba-main-onthefly.sh', str(tileid), 'y'],
+                     stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    if os.path.exists(outfn):
+        return True
+    return False
+
+
 class NTS():
     def __init__(self, obsplan=None, defaults={}, night=None,
                  nts_survey=None):
@@ -287,6 +318,8 @@ def __init__(self, obsplan=None, defaults={}, night=None,
             raise ValueError('Could not find obsplan configuration!')
         desisurvey.config.Configuration.reset()
         config = desisurvey.config.Configuration(obsplan)
+        self.log.info('Loading obsplan from {}, desisurvey version {}'.format(
+            obsplan, desisurvey.__version__))
         _ = desisurvey.tiles.get_tiles(use_cache=False, write_cache=True)
 
         self.default_seeing = defaults.get('seeing', 1.1)
@@ -296,6 +329,11 @@ def __init__(self, obsplan=None, defaults={}, night=None,
         self.rules = desisurvey.rules.Rules(
             config.get_path(config.rules_file()))
         self.config = config
+        fa_on_the_fly = getattr(config, 'fa_on_the_fly', None)
+        if fa_on_the_fly is not None:
+            self.fa_on_the_fly = fa_on_the_fly()
+        else:
+            self.fa_on_the_fly = False
         try:
             self.planner = desisurvey.plan.Planner(
                 self.rules, restore=config.tiles_file(),
@@ -468,10 +506,16 @@ def next_tile(self, conditions=None, exposure=None, constraints=None,
             self.requestlog.logresponse(badtile)
             return badtile
 
-        if not move_tile_into_place(tileid, speculative=speculative):
-            self.log.error('Could not find tile {}!'.format(tileid))
-            self.requestlog.logresponse(badtile)
-            return badtile
+        if self.fa_on_the_fly and not constraints.get('static_fa_only', False):
+            if not design_tile_on_fly(tileid, speculative=speculative):
+                self.log.error('fa-on-the-fly failed!')
+                self.requestlog.logresponse(badtile)
+                return badtile
+        else:
+            if not move_tile_into_place(tileid, speculative=speculative):
+                self.log.error('Could not find tile {}!'.format(tileid))
+                self.requestlog.logresponse(badtile)
+                return badtile
 
         self.scheduler.plan.add_pending_tile(tileid)
 

py/desisurvey/holdingpen.py

@@ -61,10 +61,13 @@ def tileid_to_clean(faholddir, fadir, mtldone):
                 len(existing_fafiles))
     mtltime = [fits.getheader(fn).get('MTLTIME', 'None')
                for fn in existing_fafiles]
-    m = np.array([mtltime0 is not None for mtltime0 in mtltime])
+    m = np.array([mtltime0 is not None for mtltime0 in mtltime],
+                 dtype='bool')
     if np.any(~m):
         logger.warning('MTLTIME not found for tiles {}!'.format(
             ' '.join([x for x in existing_fafiles[~m]])))
+    if np.sum(m) == 0:
+        return np.zeros(0, dtype='i4')
     existing_tileids = existing_tileids[m]
     existing_fafiles = existing_fafiles[m]
     mtltime = np.array(mtltime)[m]
@@ -158,7 +161,7 @@ def missing_tileid(fadir, faholddir):
         if not os.path.exists(os.path.join(
                 fadir, expidstr[:3],
                 'fiberassign-{}.fits.gz'.format(expidstr))):
-            logger.error('TILEID %d should be checked into and is not!' %
+            logger.error('TILEID %d should be checked into svn and is not!' %
                          tileid0)
         else:
             count += 1

py/desisurvey/plan.py

@@ -169,6 +169,11 @@ def __init__(self, rules=None, restore=None, simulate=False, log=None):
             nogray = nogray()
         self.nogray = nogray
 
+        fa_on_the_fly = getattr(config, 'fa_on_the_fly', False)
+        if not isinstance(fa_on_the_fly, bool):
+            fa_on_the_fly = fa_on_the_fly()
+        self.fa_on_the_fly = fa_on_the_fly
+
         self.tiles = desisurvey.tiles.get_tiles()
         self.ephem = desisurvey.ephem.get_ephem()
         if restore is not None:
@@ -501,23 +506,24 @@ def afternoon_plan(self, night):
                         (self.tile_available == 0))
         self.tile_priority[zeropriority] = 0
         # tiles with priority > 0 may be designed.
-        if not self.simulate:
+        if not self.simulate and not self.fa_on_the_fly:
             m = self.tile_available & ~filesavailable
+            self.tile_available[~filesavailable] = 0
             for prog, progmask in self.tiles.program_mask.items():
                 ma = m & progmask
                 nundesigned = np.sum(ma)
-                if nundesigned > 0:
-                    if nundesigned < 100:
-                        self.log.info(
-                            'Newly available {} tiles that '.format(prog) +
-                            'have not yet been designed:  ' +
-                            ' '.join([str(x) for x in self.tiles.tileID[ma]]))
-                    else:
-                        self.log.info(
-                            'There are many ({}) available {} tiles that have '
-                            'not been designed.  Suppressing full '
-                            'list.'.format(nundesigned, prog))
-            self.tile_available[~filesavailable] = 0
+                if nundesigned == 0:
+                    continue
+                elif nundesigned < 200:
+                    self.log.info(
+                        'Newly available {} tiles that '.format(prog) +
+                        'have not yet been designed:  ' +
+                        ' '.join([str(x) for x in self.tiles.tileID[ma]]))
+                else:
+                    self.log.info(
+                        'There are many ({}) available {} tiles that have '
+                        'not been designed.  Suppressing full '
+                        'list.'.format(nundesigned, prog))
         newlycompleted = ((self.tile_status == 'done') &
                           (oldstatus != 'done'))
         return newlystarted, newlycompleted

py/desisurvey/scripts/afternoon_plan.py

@@ -106,6 +106,17 @@ def afternoon_plan(night=None, exposures=None,
         log.info('SURVEYOPS directory not found; not performing '
                  'surveyops updates.')
 
+    # compulsively make sure that the main SURVEYOPS repository is up to date
+    # this ensures that tiles designed on the fly will use recent MTLs.
+    # note, this occurs after the "private" surveyops/mtl/mtl-done-tiles.ecsv
+    # update.  In the event that a check in landed between the two svn updates,
+    # AP would not release updated tiles for observation that night.  That's
+    # fine.  In the opposite order, AP would release tiles, but they would use
+    # out of date MTLs, a disaster.
+    mainsurveyopsdir = os.path.join(
+        os.environ['DESI_ROOT'], 'survey', 'ops', 'surveyops', 'trunk')
+    subprocess.run(['svn', 'up', mainsurveyopsdir])
+
     fbadir = os.environ['FIBER_ASSIGN_DIR']
     # update FIBER_ASSIGN_DIR
     subprocess.run(['svn', 'up', fbadir])
@@ -206,24 +217,8 @@ def afternoon_plan(night=None, exposures=None,
             offlinepipelinefiles[i] = None
 
     if surveyopsdir is None:
-        os.system(
-            'wget -q '
-            'https://data.desi.lbl.gov/desi/survey/ops/surveyops/trunk/mtl/'
-            'mtl-done-tiles.ecsv -O ./mtl-done-tiles.new.ecsv')
-        try:
-            filelen = os.stat('mtl-done-tiles.new.ecsv').st_size
-        except Exception:
-            filelen = 0
-        if filelen > 0:
-            os.rename('mtl-done-tiles.new.ecsv', 'mtl-done-tiles.ecsv')
-            mtldonefn = './mtl-done-tiles.ecsv'
-        else:
-            log.warning('Updating mtl-done-tiles failed!')
-            if os.path.exists('./mtl-done-tiles.ecsv'):
-                mtldonefn = './mtl-done-tiles.ecsv'
-            else:
-                mtldonefn = None
-        badexpfn = None
+        raise ValueError('SURVEYOPS is not set; cannot do MTL updates; '
+                         'failing!')
     else:
         mtldonefn = os.path.join(surveyopsdir, 'mtl', 'mtl-done-tiles.ecsv')
         badexpfn = os.path.join(surveyopsdir, 'ops', 'bad_exp_list.csv')
@@ -275,8 +270,8 @@ def afternoon_plan(night=None, exposures=None,
         desisurvey.holdingpen.maintain_holding_pen_and_svn(
             fbadir, faholddir, mtldonefn)
     else:
-        logger.error('FA_HOLDING_PEN is None, skipping holding pen '
-                     'maintenance!')
+        log.error('FA_HOLDING_PEN is None, skipping holding pen '
+                  'maintenance!')
     # pick up any changes to available tiles.
     planner.afternoon_plan(night)
     planner.save(os.path.join(subdir, os.path.basename(newtilefn)))