Merge pull request #108 from desihub/statusfile

Migrate to single status file; rudimentary afternoon planning; allow tile & status files to not match.

py/desisurvey/NTS.py

@@ -49,51 +49,129 @@
 import desisurvey.scheduler
 import desisurvey.etc
 import desisurvey.utils
-import datetime
+import desisurvey.config
+import desiutil.log
+from astropy.io import ascii
+from astropy import coordinates
+from astropy import units as u
+
+class QueuedList():
+    """Simple class to manage list of exposures already observed in a night.
+    
+    Parameters
+    ----------
+    fn : str
+        file name where QueuedList is backed to disk.
+    """
+    def __init__(self, fn):
+        self.fn = fn
+        self.log = desiutil.log.get_logger()
+        self.restore()
+
+    def restore(self):
+        if os.path.exists(self.fn):
+            try:
+                self.queued = ascii.read(self.fn, comment='#',
+                                         names=['tileid'], format='no_header')
+            except OSError:
+                self.log.error('Could not read in queued file; '
+                               'record of past exposures lost!')
+            self.queued = list(self.queued['tileid'])
+        else:
+            self.queued = []
+
+    def add(self, tileid):
+        self.queued.append(tileid)
+        try:
+            open(self.fn, 'a').write(str(tileid)+'\n')
+            # could work harder to make this atomic.
+        except OSError:
+            self.log.error('Could not write out queued file; '
+                           'record of last exposure lost!')
+
+
+def azinrange(az, low, high):
+    """Return whether azimuth is between low and high, trying to respect the
+    360 deg boundary.
+
+    We transform high so that it is in the range [low, low+360].  We then
+    transform az likewise, so that the test can be done as low <= az <= high.
+    In this scheme, azinrange(0, 2, 1) = True, since low, high = [2, 1] is
+    interpreted as all angles between 2 and 361 degrees.
+
+    Parameters
+    ----------
+    az: azimuth (deg)
+    low: lower bound on azimuth (deg)
+    high: upper bound on azimuth (deg)
+
+    Returns
+    -------
+    Array of same shape as az, indicating if az is between low and high.
+    """
+
+    if low > high:
+        high = ((high - low) % 360) + low
+    az = ((az - low) % 360) + low
+    return (az >= low) & (az <= high)
 
 
 class NTS():
-    def __init__(self, obsplan, fiber_assign_dir, defaults={}, night=None):
+    def __init__(self, obsplan='config.yaml', defaults={}, night=None):
         """Initialize a new instance of the Next Tile Selector.
 
         Parameters
         ----------
-        obsplan : not currently used; planner initialized from default
-            scheduler directory.
-
-        fiber_assign_dir : directory where fiber assign files are located
+        obsplan : config.yaml to load
 
         defaults : dictionary giving default values of 'seeing',
             'transparency', 'sky_level', and 'program', for next tile
             selection.
 
-        night : night to plan, ISO 8601.
+        night : night for which to assign tiles, YYYMMDD, default tonight.
 
         Returns
         -------
         NTS object. Tiles can be generated via next_tile(...)
         """
         self.obsplan = obsplan
-        self.fiber_assign_dir = fiber_assign_dir
+        self.log = desiutil.log.get_logger()
+        # making a new NTS; clear out old configuration / tile information
+        if night is None:
+            self.night = desisurvey.utils.get_current_date()
+            self.log.info('No night selected, '
+                          'using current date: {}.'.format(self.night))
+        else:
+            self.night = night
+        nightstr = desisurvey.utils.night_to_str(self.night)
+        if not os.path.exists(obsplan):
+            obsplannew = os.path.join(os.environ['DESISURVEY_OUTPUT'],
+                                      nightstr, obsplan)
+            if not os.path.exists(obsplannew):
+                self.log.error('Could not find obsplan configuration '
+                               '{}!'.format(obsplan))
+                raise ValueError('Could not find obsplan configuration!')
+            else:
+                obsplan = obsplannew
+        desisurvey.config.Configuration.reset()
+        config = desisurvey.config.Configuration(obsplan)
+        _ = desisurvey.tiles.get_tiles(use_cache=False, write_cache=True)
+
         self.default_seeing = defaults.get('seeing', 1.0)
         self.default_transparency = defaults.get('transparency', 0.9)
         self.default_skylevel = defaults.get('skylevel', 1000.0)
         self.default_program = defaults.get('program', 'DESI DARK')
-        if night is None:
-            self.night = datetime.date.today()
-            print('Warning: no night selected, using current date!',
-                  self.night)
-        else:
-            self.night = night
-        self.rules = desisurvey.rules.Rules()
-        # should look for rules file in obsplan dir?
+        self.rules = desisurvey.rules.Rules(
+            config.get_path(config.rules_file()))
+        self.config = config
         try:
-            nightstr = self.night.isoformat()
             self.planner = desisurvey.plan.Planner(
                 self.rules,
-                restore='planner_afternoon_{}.fits'.format(nightstr))
+                restore='{}/desi-status-{}.fits'.format(nightstr, nightstr))
             self.scheduler = desisurvey.scheduler.Scheduler(
-                restore='scheduler_{}.fits'.format(nightstr))
+                restore='{}/desi-status-{}.fits'.format(nightstr, nightstr))
+            self.queuedlist = QueuedList(
+                config.get_path('{}/queued-{}.dat'.format(nightstr, nightstr)))
         except:
             raise ValueError('Error restoring scheduler & planner files; '
                              'has afternoon planning been performed?')
@@ -104,7 +182,7 @@ def __init__(self, obsplan, fiber_assign_dir, defaults={}, night=None):
 
     def next_tile(self, mjd=None, skylevel=None, transparency=None,
                   seeing=None, program=None, lastexp=None, fiber_assign=None,
-                  previoustiles=None):
+                  previoustiles=[], azrange=None):
         """
         Select the next tile.
 
@@ -137,6 +215,7 @@ def next_tile(self, mjd=None, skylevel=None, transparency=None,
         maxtime : float, do not observe for longer than maxtime (seconds)
         fiber_assign : str, file name of fiber_assign file
         foundtile : bool, a valid tile was found
+        azrange : [lowaz, highaz], azimuth of tile must be in this range
         """
 
         if fiber_assign is not None:
@@ -147,22 +226,34 @@ def next_tile(self, mjd=None, skylevel=None, transparency=None,
 
         if mjd is None:
             from astropy import time
-            mjd = time.Time.now().mjd
-            print('Warning: no time specified, using current time, MJD: %f' %
-                  mjd)
+            now = time.Time.now()
+            mjd = now.mjd
+            self.log.info('No time specified, using current time, MJD: %f' %
+                          mjd)
         seeing = self.default_seeing if seeing is None else seeing
         skylevel = self.default_skylevel if skylevel is None else skylevel
         transparency = (self.default_transparency if transparency is None
                         else transparency)
 
-        if previoustiles is not None:
-            ind, mask = self.scheduler.tiles.index(previoustiles,
-                                                   return_mask=True)
-            self.scheduler.in_night_pool[ind[mask]] = False
+        self.queuedlist.restore()
+        previoustiles = previoustiles + self.queuedlist.queued
+        ind, mask = self.scheduler.tiles.index(previoustiles,
+                                               return_mask=True)
+        save_in_night_pool = self.scheduler.in_night_pool[ind[mask]].copy()
+        self.scheduler.in_night_pool[ind[mask]] = False
         # remove previous tiles from possible tiles to schedule
-        # note: this will be remembered until NTS is restarted!  EFS
+        if azrange is not None:
+            tra = self.scheduler.tiles.tileRA
+            tdec = self.scheduler.tiles.tileDEC
+            altazframe = desisurvey.utils.get_observer(now)
+            coordrd = coordinates.ICRS(ra=tra*u.deg, dec=tdec*u.deg)
+            coordaz = coordrd.transform_to(altazframe)
+            az = coordaz.az.to(u.deg).value
+            self.scheduler.in_night_pool &= azinrange(az, azrange)
+
         result = self.scheduler.next_tile(
             mjd, self.ETC, seeing, transparency, skylevel, program=program)
+        self.scheduler.in_night_pool[ind[mask]] = save_in_night_pool
         (tileid, passnum, snr2frac_start, exposure_factor, airmass,
          sched_program, mjd_program_end) = result
         if tileid is None:
@@ -182,81 +273,20 @@ def next_tile(self, mjd=None, skylevel=None, transparency=None,
             self.ETC.TEXP_TOTAL[sched_program]*exposure_factor)  # EFS hack
         exptime = texp_remaining
         maxtime = self.ETC.MAX_EXPTIME
-        self.scheduler.update_snr(
-            tileid, snr2frac_start + min([exptime, maxtime])/texp_tot)
-        self.scheduler.save('scheduler_{}.fits'.format(self.night.isoformat()))
         if program is None:
             maxtime = min([maxtime, mjd_program_end-maxtime])
 
-        fiber_assign = os.path.join(self.fiber_assign_dir,
-                                    'tile_%d.fits' % tileid)
+        tileidstr = '{:06d}'.format(tileid)
+        fiber_assign = os.path.join(self.config.fiber_assign_dir(),
+                                    tileidstr[:3],
+                                    'fiberassign-%s.fits' % tileidstr)
         days_to_seconds = 60*60*24
 
         selection = {'tileid': tileid, 's2n': s2n,
                      'esttime': exptime*days_to_seconds,
                      'maxtime': maxtime*days_to_seconds,
                      'fiber_assign': fiber_assign,
                      'foundtile': True}
+        self.queuedlist.add(tileid)
 
         return selection
-
-
-def afternoon_plan(night=None, lastnight=None):
-    """
-    Perform daily afternoon planning.
-
-    Afternoon planning identifies tiles available for observation and assigns
-    priorities.  It must be performed before the NTS can identify new tiles to
-    observe.
-
-    Params
-    ------
-    night : str, ISO 8601.  The night to plan.  Default tonight.
-
-    lastnight : str, ISO 8601.  The previous planned night.  Used for restoring
-        the previous completion status of all tiles.  Defaults to not
-        restoring status, i.e., all previous tile completion information is
-        ignored!
-    """
-    if night is None:
-        night = datetime.date.today().isoformat()
-    rules = desisurvey.rules.Rules()
-    # should look for rules file in obsplan dir?
-    if lastnight is not None:
-        planner = desisurvey.plan.Planner(
-            rules, restore='planner_end_{}.fits' % lastnight)
-        scheduler = desisurvey.scheduler.Scheduler(
-            restore='scheduler_end_{}.fits')
-    else:
-        planner = desisurvey.plan.Planner(rules)
-        scheduler = desisurvey.scheduler.Scheduler()
-    # restore: maybe check directory, and restore if file present?  EFS
-    # planner.save(), scheduler.save()
-    # planner.restore(), scheduler.restore()
-    planner.afternoon_plan(desisurvey.utils.get_date(night),
-                           scheduler.completed)
-    # currently afternoon planning checks to see what tiles have been marked
-    # as done, and what new tiles may now be fiberassigned.
-    # currently moves tiles closer to fiber assignment each time it's called
-    # (countdon decreases), depending on fiber_assign_cadence, etc.
-    # eventually we want this to be ~totally different, so while this isn't
-    # really the behavior we'd want on the mountain, I'm leaving it until
-    # we have something much different.
-    planner.save('planner_afternoon_{}.fits'.format(night))
-    scheduler.save('scheduler_{}.fits'.format(night))
-
-
-if __name__ == "__main__":
-    import argparse
-    parser = argparse.ArgumentParser(
-        description='Perform afternoon planning.',
-        epilog='EXAMPLE: %(prog)s --night 2020-01-01')
-    parser.add_argument('--night', type=str,
-                        help='night to plan, default: tonight',
-                        default=None)
-    parser.add_argument('--lastnight', type=str,
-                        help='night to restore, default: start fresh.',
-                        default=None)
-
-    parser.parse_args()
-    afternoon_plan(parser.night, parser.lastnight)

py/desisurvey/config.py

@@ -137,6 +137,17 @@ def reset():
         Configuration.__instance = None
 
 
+    @staticmethod
+    def _get_full_path(file_name):
+        # Locate the config file in our pkg data/ directory if no path is given.
+        if os.path.split(file_name)[0] == '':
+            full_path = astropy.utils.data._find_pkg_data_path(
+                os.path.join('data', file_name))
+        else:
+            full_path = file_name
+        return full_path
+
+
     def __new__(cls, file_name=None):
         """Implement a singleton access pattern.
         """
@@ -175,12 +186,7 @@ def _initialize(self, file_name=None):
         # Remember the file name since it is not allowed to change.
         self.file_name = file_name
 
-        # Locate the config file in our pkg data/ directory if no path is given.
-        if os.path.split(file_name)[0] == '':
-            full_path = astropy.utils.data._find_pkg_data_path(
-                os.path.join('data', file_name))
-        else:
-            full_path = file_name
+        full_path = self._get_full_path(file_name)
 
         # Validate that all mapping keys are valid python identifiers
         # and that there are no embedded sequences.

py/desisurvey/data/config-cmx.yaml

@@ -105,6 +105,7 @@ nominal_exposure_time:
     CATASTROPHE_MEDSTARS: 300 s
     EVEN_ODD: 300 s
     SKY_WITH_STAR: 300 s
+    CMX_DITHERING: 30 s
 
 nominal_conditions:
     # Moon below the horizon
@@ -166,7 +167,7 @@ tile_radius: 1.63 deg
 # - Pass numbers are arbitrary integers and do not need to be consecutive
 #   or dense. However use of non-standard values will generally require
 #   an update to fiber_assignment_order, above.
-tiles_file: '{DESISURVEY_OUTPUT}/ALL_CMX_tiles2.fits'
+tiles_file: SV0_612tiles_March2020_uniquepassprogram.fits
 
 commissioning: True
 # tile file is a commissioning tile file. This disables checks related
@@ -176,3 +177,9 @@ commissioning: True
 # writing files managed by this package. The pattern {...} will be expanded
 # using environment variables.
 output_path: '{DESISURVEY_OUTPUT}'
+
+rules_file: rules-cmx.yaml
+
+fiber_assign_dir: /global/cfs/cdirs/desi/target/fiberassign/tiles/trunk/
+
+spectra_dir: /global/cfs/cdirs/desi/spectro/data

py/desisurvey/data/config.yaml

@@ -160,3 +160,5 @@ tiles_file: desi-tiles.fits
 # writing files managed by this package. The pattern {...} will be expanded
 # using environment variables.
 output_path: '{DESISURVEY_OUTPUT}'
+
+rules_file: rules.yaml

py/desisurvey/data/rules-cmx.yaml

@@ -6,9 +6,28 @@
 # Implement a dummy rules file for commissioning.
 
 A:
-    passes: 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19
+    passes: 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20
     dec_min: -90
     dec_order: +0.2
     rules:
         A(0): { START: 1.0 }
-        # looks to me like this is further assumed for all passes
+        A(1): { START: 1.0 }
+        A(2): { START: 1.0 }
+        A(3): { START: 1.0 }
+        A(4): { START: 1.0 }
+        A(5): { START: 1.0 }
+        A(6): { START: 1.0 }
+        A(7): { START: 1.0 }
+        A(8): { START: 1.0 }
+        A(9): { START: 1.0 }
+        A(10): { START: 1.0 }
+        A(11): { START: 1.0 }
+        A(12): { START: 1.0 }
+        A(13): { START: 1.0 }
+        A(14): { START: 1.0 }
+        A(15): { START: 1.0 }
+        A(16): { START: 1.0 }
+        A(17): { START: 1.0 }
+        A(18): { START: 1.0 }
+        A(19): { START: 1.0 }
+        A(20): { START: 1.0 }