[WIP] Updates for Release 19.9

.readthedocs.yml

@@ -0,0 +1,18 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the doc/ directory with Sphinx
+sphinx:
+    configuration: doc/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+    version: 3
+    # system_packages: true

.travis.yml

@@ -36,7 +36,7 @@ env:
         # overidden underneath. They are defined here in order to save having
         # to repeat them for all configurations.
         - MAIN_CMD='python setup.py'
-        - DESIUTIL_VERSION=1.9.15
+        - DESIUTIL_VERSION=2.0.1
         - SPHINX_VERSION=1.8.5
     matrix:
         - SETUP_CMD='egg_info'

LICENSE.rst

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2017, DESI Collaboration <desi-data@desi.lbl.gov>
+Copyright (c) 2014-2019, DESI Collaboration <desi-data@desi.lbl.gov>
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

bin/check_minitest.sh

@@ -15,22 +15,17 @@ export PIXPROD=mini
 export SPECPROD=mini
 export DESI_SPECTRO_DATA=${DESI_SPECTRO_SIM}/${PIXPROD}
 #
-# Clean up log files.
-#
-[[ -f ${SCRATCH}/DESI_SPECTRO_DATA_${run}.log ]] && /bin/rm -f ${SCRATCH}/DESI_SPECTRO_DATA_${run}.log
-[[ -f ${SCRATCH}/DESI_SPECTRO_REDUX_${run}.log ]] && /bin/rm -f ${SCRATCH}/DESI_SPECTRO_REDUX_${run}.log
-[[ -f ${SCRATCH}/DESI_SPECTRO_SIM_${run}.log ]] && /bin/rm -f ${SCRATCH}/DESI_SPECTRO_SIM_${run}.log
-[[ -f ${SCRATCH}/DESI_TARGET_${run}.log ]] && /bin/rm -f ${SCRATCH}/DESI_TARGET_${run}.log && touch ${SCRATCH}/DESI_TARGET_${run}.log
-#
 # Run the tests.
 #
-check_model DESI_SPECTRO_DATA ${DESI_SPECTRO_DATA} > ${SCRATCH}/DESI_SPECTRO_DATA_${run}.log 2>&1
-check_model DESI_SPECTRO_REDUX ${DESI_SPECTRO_REDUX} > ${SCRATCH}/DESI_SPECTRO_REDUX_${run}.log 2>&1
-check_model DESI_SPECTRO_SIM ${DESI_SPECTRO_SIM} > ${SCRATCH}/DESI_SPECTRO_SIM_${run}.log 2>&1
-check_model -F DESI_TARGET/mtl.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/mtl.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
-check_model -F DESI_TARGET/sky.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/sky.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
-check_model -F DESI_TARGET/stdstar.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/standards-dark.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
-check_model -F DESI_TARGET/stdstar.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/standards-bright.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
-check_model -F DESI_TARGET/targets.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/targets.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
-check_model -F DESI_TARGET/truth.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/truth.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
-check_model -F DESI_TARGET/fiberassign/tile.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/fiberassign/tile_11108.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
+for d in DESI_SPECTRO_DATA DESI_SPECTRO_REDUX DESI_SPECTRO_SIM DESISURVEY_OUTPUT; do
+    [[ -f ${SCRATCH}/${d}_${run}.log ]] && /bin/rm -f ${SCRATCH}/${d}_${run}.log
+    check_model ${d} ${!d} > ${SCRATCH}/${d}_${run}.log 2>&1
+done
+[[ -f ${SCRATCH}/DESI_TARGET_${run}.log ]] && /bin/rm -f ${SCRATCH}/DESI_TARGET_${run}.log && touch ${SCRATCH}/DESI_TARGET_${run}.log
+check_model -F DESI_TARGET/mtl.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/mtl-dark.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
+check_model -F DESI_TARGET/skies.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/sky.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
+# check_model -F DESI_TARGET/stdstar.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/standards-dark.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
+# check_model -F DESI_TARGET/stdstar.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/standards-bright.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
+check_model -F DESI_TARGET/targets.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/targets-dark.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
+check_model -F DESI_TARGET/truth.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/targets/truth-dark.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1
+check_model -F DESI_TARGET/fiberassign/tile-TILEID-FIELDNUM.rst /global/projecta/projectdirs/desi/datachallenge/reference_runs/${run}/fiberassign/tile-011108.fits >> ${SCRATCH}/DESI_TARGET_${run}.log 2>&1

doc/DESISURVEY_OUTPUT/exposures.rst

@@ -3,8 +3,8 @@ exposures
 =========
 
 :Summary: Record of simulated exposures.
-:Naming Convention: ``exposures.fits``
-:Regex: ``exposures\.fits``
+:Naming Convention: ``exposures_surveysim.fits``
+:Regex: ``exposures_surveysim\.fits``
 :File Type: FITS, 2 MB  (scales with the number of exposures)
 
 *Note*: currently this is only an output from surveysim, but it may become an
@@ -40,9 +40,7 @@ KEY     Example Value     Type Comment
 ======= ================= ==== =======
 TILES   desi--tiles.fits  str  Name of the tiles file specified in desisurvey config.
 NEXP    41875             int  Number of exposures recorded so far.
-COMMENT Baseline (seed=1) str  Comment describing this simulation.
 INITIAL 2019-12-01        str  YEAR-MM-DD of initial night used for integer offsets (or blank before any observing).
-EXTNAME META              str  Extension name.
 ======= ================= ==== =======
 
 Empty HDU.
@@ -62,7 +60,6 @@ KEY     Example Value Type Comment
 ======= ============= ==== =====================
 NAXIS1  41875         int  length of dimension 1
 NAXIS2  8             int  length of dimension 2
-EXTNAME EXPOSURES     str  extension name
 ======= ============= ==== =====================
 
 Required Data Table Columns
@@ -76,7 +73,7 @@ EXPTIME  float32 s      Length of time shutter was open.
 TILEID   int32          ID of the observed tile.
 SNR2FRAC float32        Fractional SNR2 accumulated on this tile during this exposure.
 AIRMASS  float32        Average airmass during this exposure.
-SEEING   float32 arcsec Average FWHM atmospheric seeing during this exposure. 
+SEEING   float32 arcsec Average FWHM atmospheric seeing during this exposure.
 TRANSP   float32        Average atmospheric transparency during this exposure.
 SKY      float32        Average sky background level during this exposure.
 ======== ======= ====== ===========
@@ -96,7 +93,6 @@ KEY     Example Value Type Comment
 ======= ============= ==== =====================
 NAXIS1  20            int  length of dimension 1
 NAXIS2  10            int  length of dimension 2
-EXTNAME TILEDATA      str  extension name
 ======= ============= ==== =====================
 
 Required Data Table Columns
@@ -112,7 +108,7 @@ SNR2FRAC float32       Total fractional SNR2 accumulated on this tile.
 NEXP     int32         Total number of exposures of this tile.
 ======== ======= ===== ===========
 
-There is one table row per tile, indexed to match `desisurvey.tiles.Tiles 
+There is one table row per tile, indexed to match `desisurvey.tiles.Tiles
 <https://desisurvey.readthedocs.io/en/latest/api.html#desisurvey.tiles.Tiles>`__.
 
 The integer ``AVAIL`` and ``PLANNED`` values are nights since the date specified
@@ -135,11 +131,11 @@ object records exposures during simulation::
 
     import surveysim.exposures
     explist = surveysim.exposures.ExposureList()
-    
+
 Its internal state after a simulation (or each night) can be saved using, for example::
 
     explist.save('exposures.fits', comment='Baseline (seed=1)')
-    
+
 This state can then later be restored using::
 
     explist = surveysim.exposures.ExposureList(restore='exposures.fits')

doc/DESISURVEY_OUTPUT/stats.rst

@@ -3,8 +3,8 @@ stats
 =====
 
 :Summary: Nightly summary statistics, recorded by pass, from a surveysim run.
-:Naming Convention: ``stats.fits``
-:Regex: ``stats\.fits``
+:Naming Convention: ``stats_surveysim.fits``
+:Regex: ``stats_surveysim\.fits``
 :File Type: FITS, 810 KB
 
 *Note*: currently this is an output of surveysim, but in the future may become
@@ -16,7 +16,7 @@ Contents
 ====== ======= ======== ===================
 Number EXTNAME Type     Contents
 ====== ======= ======== ===================
-HDU0_  META    IMAGE    Blank.
+HDU0_          IMAGE    Blank.
 HDU1_  STATS   BINTABLE Nighly summary statistics by pass.
 ====== ======= ======== ===================
 
@@ -26,7 +26,7 @@ FITS Header Units
 HDU0
 ----
 
-EXTNAME = (None)
+EXTNAME = PRIMARY
 
 Empty HDU.
 
@@ -48,8 +48,6 @@ NAXIS2  13                int  length of dimension 2
 TILES   desi-tiles.fits   str  Name of the tiles file specified in desisurvey config.
 START   2020-03-15        str  YEAR-MM-YY of the first night of the survey.
 STOP    2020-04-15        str  YEAR-MM-YY of the most recent night with statistics recorded.
-COMMENT Baseline (seed=1) str  Comment describing this simulation.
-EXTNAME STATS             str  extension name
 ======= ================= ==== =====================
 
 Required Data Table Columns
@@ -59,7 +57,7 @@ Required Data Table Columns
 Name         Type       Units Description
 ============ ========== ===== ===========
 MJD          float64          MJD of local noon before this night.
-tsched       float64    d     Total scheduled time for all programs during this night.      
+tsched       float64    d     Total scheduled time for all programs during this night.
 topen        float64[3] d     Actual time dome was open during this night, per program.
 tdead        float64[3] d     Deadtime during this night (dome open, but idling), per program.
 tscience     float64[8] d     Time spent with the shutter open for science exposures, per pass.
@@ -96,11 +94,11 @@ object tracks survey statistics during simulation::
 
     import surveysim.stats
     stats = surveysim.stats.SurveyStatistics()
-    
+
 Its internal state after a simulation (or each night) can be saved using, for example::
 
     stats.save('stats.fits', comment='Baseline (seed=1)')
-    
+
 This state can then later be restored using::
 
     stats = surveysim.stats.SurveyStatistics(restore='stats.fits')

doc/DESISURVEY_OUTPUT/surveyinit.rst

@@ -43,7 +43,6 @@ YEARS    2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017 str  List of yea
 START    2019-12-01                                             str  Nominal survey start date used for hour-angle optimization.
 STOP     2024-11-30                                             str  Nominal survey stop date used for hour-angle optimization.
 TWILIGHT F                                                      bool Was twilight included in the BRIGHT program schedule for optimzation?
-EXTNAME  WEATHER                                                str   extension name
 ======== ====================================================== ==== ==============
 
 Data: FITS image [float64, 2556]
@@ -72,7 +71,6 @@ KEY     Example Value Type  Comment
 NAXIS1  24            int   length of dimension 1
 NAXIS2  192           int   length of dimension 2
 ORIGIN  -60.0         float Low edge of first LST histogram bin in degrees.
-EXTNAME DARK          str   extension name
 ======= ============= ===== =====================
 
 Required Data Table Columns
@@ -104,7 +102,6 @@ KEY     Example Value Type  Comment
 NAXIS1  24            int   length of dimension 1
 NAXIS2  192           int   length of dimension 2
 ORIGIN  -60.0         float Low edge of first LST histogram bin in degrees.
-EXTNAME GRAY          str   extension name
 ======= ============= ===== =====================
 
 Required Data Table Columns
@@ -136,7 +133,6 @@ KEY     Example Value Type  Comment
 NAXIS1  24            int   length of dimension 1
 NAXIS2  192           int   length of dimension 2
 ORIGIN  -60.0         float Low edge of first LST histogram bin in degrees.
-EXTNAME BRIGHT        str   extension name
 ======= ============= ===== =====================
 
 Required Data Table Columns
@@ -166,7 +162,7 @@ Required Header Keywords
 KEY     Example Value Type Comment
 ======= ============= ==== =====================
 NAXIS1  16071         int  length of dimension 1
-EXTNAME DESIGN        str  extension name
+NAXIS2  10            int  length of dimension 2
 ======= ============= ==== =====================
 
 Required Data Table Columns
@@ -189,7 +185,7 @@ the ``HA`` column as a 1D array.
 Notes and Examples
 ==================
 
-The histograms of available LST in each program are calculated by `get_available_lst 
+The histograms of available LST in each program are calculated by `get_available_lst
 <https://desisurvey.readthedocs.io/en/latest/api.html#desisurvey.ephem.Ephemerides.get_available_lst>`__.
 
 Hour angle optimization is performed by `desisurvey.optimize.Optimizer

doc/DESI_SPECTRO_REDUX/SPECPROD/exposures/NIGHT/EXPID/calib-CAMERA-EXPID.rst

@@ -54,20 +54,21 @@ MOONFRAC 0.4083473802955095         float Moon illumination fraction 0-1; 1=full
 MOONALT  -79.39563600071901         float Moon altitude [degrees]
 MOONSEP  131.2947533254612          float Moon:tile separation angle [degrees]
 DATE-OBS 2020-03-17T02:42:47.160    str   Start of exposure
+MJD      58924.37858796309          float
+SNR2FRAC 0.501188337802887          float
+TRANSP   0.9980747699737549         float
+SKY      1.0                        float
+RA       150.73                     float
+DEC      30.52                      float
 PASS     1                          int
-RA       150.87                     float
-DEC      31.23                      float
-EBMV     0.0189034678041935         float
-MJD      58925.11304582807          float
-TRANSPAR 0.7192993760108948         float
 DOSVER   SIM                        str
 FEEVER   SIM                        str
-BUNIT    10**+17 cm2 count s / erg  str   i.e. (electron/Angstrom) / (1e-17 erg/(s cm2 Angstrom))
+BUNIT    10**+17 cm2 count s / erg  str   *i.e.* (elec/A) / (1e-17 erg/s/cm2/A)
 AIRORVAC vac                        str   Vacuum wavelengths
-CAMERA   r3                         str
-FIBERMIN 1557                       int
-CHECKSUM AXAlBW3iAWAiAW3i           str   HDU checksum updated 2018-03-01T15:08:08
-DATASUM  3423741220                 str   data unit checksum updated 2018-03-01T15:08:08
+CAMERA   z1                         str
+FIBERMIN 500                        int
+CHECKSUM 1HdW1GZU1GbU1GZU           str   HDU checksum updated 2018-03-01T15:04:16
+DATASUM  4250999040                 str   data unit checksum updated 2018-03-01T15:04:16
 ======== ========================== ===== ==============================================
 
 Data: FITS image [float32, 2645x500]

doc/DESI_SPECTRO_REDUX/SPECPROD/exposures/NIGHT/EXPID/cframe-CAMERA-EXPID.rst

@@ -59,20 +59,21 @@ MOONFRAC 0.4083473802955095           float Moon illumination fraction 0-1; 1=fu
 MOONALT  -79.39563600071901           float Moon altitude [degrees]
 MOONSEP  131.2947533254612            float Moon:tile separation angle [degrees]
 DATE-OBS 2020-03-17T02:42:47.160      str   Start of exposure
+MJD      58924.37858796309            float
+SNR2FRAC 0.501188337802887            float
+TRANSP   0.9980747699737549           float
+SKY      1.0                          float
+RA       150.73                       float
+DEC      30.52                        float
 PASS     1                            int
-RA       150.87                       float
-DEC      31.23                        float
-EBMV     0.0189034678041935           float
-MJD      58925.11304582807            float
-TRANSPAR 0.7192993760108948           float
 DOSVER   SIM                          str
 FEEVER   SIM                          str
 BUNIT    10**-17 erg/(s cm2 Angstrom) str
 AIRORVAC vac                          str   Vacuum wavelengths
-CAMERA   z7                           str
-FIBERMIN 3500                         int
-CHECKSUM 9GKaD9HU9EHZC9HZ             str   HDU checksum updated 2018-03-01T15:08:14
-DATASUM  1412099935                   str   data unit checksum updated 2018-03-01T15:08:14
+CAMERA   z1                           str
+FIBERMIN 500                          int
+CHECKSUM 1HdW1GZU1GbU1GZU             str   HDU checksum updated 2018-03-01T15:04:16
+DATASUM  4250999040                   str   data unit checksum updated 2018-03-01T15:04:16
 ======== ============================ ===== ==============================================
 
 Data: FITS image [float32, 2999x500]
@@ -193,16 +194,16 @@ DATASUM  4154192770       str  data unit checksum updated 2018-03-01T15:08:15
 Required Data Table Columns
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-================= ======= ===== ===========
+================= ======= ===== ============================================
 Name              Type    Units Description
-================= ======= ===== ===========
-TARGETID          int64
-DESI_TARGET       int64
-BGS_TARGET        int64
-MWS_TARGET        int64
+================= ======= ===== ============================================
+TARGETID          int64         Unique target ID
+DESI_TARGET       int64         DESI dark+calib targeting bit mask
+BGS_TARGET        int64         DESI Bright Galaxy Survey targeting bit mask
+MWS_TARGET        int64         DESI Milky Way Survey targeting bit mask
 SECONDARY_TARGET  int64
-TARGET_RA         float64
-TARGET_DEC        float64
+TARGET_RA         float64       Target right ascension [degrees]
+TARGET_DEC        float64       Target declination [degrees]
 TARGET_RA_IVAR    float64
 TARGET_DEC_IVAR   float64
 BRICKID           int64
@@ -213,8 +214,10 @@ SUBPRIORITY       float64
 REF_ID            int64
 PMRA              float32
 PMDEC             float32
+REF_EPOCH         float32
 PMRA_IVAR         float32
 PMDEC_IVAR        float32
+RELEASE           int16
 FLUX_G            float32
 FLUX_R            float32
 FLUX_Z            float32
@@ -240,31 +243,38 @@ MW_TRANSMISSION_R float32
 MW_TRANSMISSION_Z float32
 EBV               float32
 PHOTSYS           char[1]
-FIBER             int32
+OBSCONDITIONS     int32
+NUMOBS_INIT       int64
+PRIORITY_INIT     int64
+NUMOBS_MORE       int32
+HPXPIXEL          int64
+FIBER             int32         Fiber ID [0-4999]
 PETAL_LOC         int32
 DEVICE_LOC        int32
 LOCATION          int32
 FIBERSTATUS       int32
 OBJTYPE           char[3]
-LAMBDA_REF        float32
-DESIGN_X          float32
-DESIGN_Y          float32
-DESIGN_Q          float32
-DESIGN_S          float32
+LAMBDA_REF        float32       Reference wavelength at which to align fiber
+FIBERASSIGN_X     float32
+FIBERASSIGN_Y     float32
+FA_TARGET         int64
+FA_TYPE           byte
 NUMTARGET         int16
 FIBER_RA          float64
 FIBER_DEC         float64
 FIBER_RA_IVAR     float32
 FIBER_DEC_IVAR    float32
-DELTA_X           float32
-DELTA_Y           float32
-DELTA_X_IVAR      float32
-DELTA_Y_IVAR      float32
+PLATEMAKER_X      float32
+PLATEMAKER_Y      float32
+PLATEMAKER_RA     float32
+PLATEMAKER_DEC    float32
 NUM_ITER          int32
 SPECTROID         int32
-BRICKNAME         char[8]
+BRICKNAME         char[8]       Brickname from target imaging
 LAMBDAREF         float64
-================= ======= ===== ===========
+DELTA_X           float32
+DELTA_Y           float32
+================= ======= ===== ============================================
 
 HDU6
 ----