updated documentation for v2.0 Fujilupe and v1.0 Iron VACs

bin/fastspecfit-setup.sh

@@ -33,7 +33,7 @@ elif [[ $1 == "env" ]]; then
     export DESI_ROOT='/global/cfs/cdirs/desi'
     export DUST_DIR='/global/cfs/cdirs/cosmo/data/dust/v0_1'
     export DR9_DIR='/global/cfs/cdirs/desi/external/legacysurvey/dr9'
-    export FTEMPLATES_DIR='/global/cfs/cdirs/desi/science/gqp/templates/fastspecfit'
+    export FTEMPLATES_DIR='/global/cfs/cdirs/desi/external/templates/fastspecfit'
 
     export OMP_NUM_THREADS=1
     export MKL_NUM_THREADS=1

bin/mpi-fastspecfit.sh

@@ -10,25 +10,25 @@
 #salloc -N 4 -C cpu -A desi -L cfs -t 04:00:00 --qos interactive
 
 # Testing--
-#srun -n 4 -c 128 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec-test fuji 128 > /global/cfs/cdirs/desi/spectro/fastspecfit/fuji/logs/fastspec-test.log.1 2>&1 &
+#srun -n 4 -c 128 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec-test iron 128 > /global/cfs/cdirs/desi/spectro/fastspecfit/iron/logs/fastspec-test.log.1 2>&1 &
 
 # Perlmutter w/ shifter
 #salloc -N 4 -C cpu -A desi -L cfs -t 04:00:00 --qos interactive --image=docker:desihub/fastspecfit:2.1.1
-#srun -n 4 -c 128 --kill-on-bad-exit=0 --no-kill shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec fuji 128 healpix sv1 > /global/cfs/cdirs/desi/spectro/fastspecfit/fuji/logs/fastspec-fuji-sv1.log.1 2>&1 &
+#srun -n 4 -c 128 shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec iron 128 healpix cmx,sv1,sv2,sv3,special > /global/cfs/cdirs/desi/spectro/fastspecfit/iron/logs/fastspec-iron-sv1.log.1 2>&1 &
 
-#srun -n 16 -c 32 --kill-on-bad-exit=0 --no-kill shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec 32 sv2 dark > /global/cfs/cdirs/desi/spectro/fastspecfit/fuji/logs/fastspec-fuji-sv2-dark.log.1 2>&1 &
-#srun -n 16 -c 32 --kill-on-bad-exit=0 --no-kill shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastphot 32 sv2 dark > /global/cfs/cdirs/desi/spectro/fastspecfit/fuji/logs/fastphot-fuji-sv2-dark.log.1 2>&1 &
-#srun -n 16 -c 32 --kill-on-bad-exit=0 --no-kill shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh qafastspec 32 > qafastspec-fuji.log.1 2>&1 &
-#srun -n 16 -c 32 --kill-on-bad-exit=0 --no-kill shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh qafastphot 32 > qafastphot-fuji.log.1 2>&1 &
+#srun -n 16 -c 32 shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec 32 sv2 dark > /global/cfs/cdirs/desi/spectro/fastspecfit/iron/logs/fastspec-iron-sv2-dark.log.1 2>&1 &
+#srun -n 16 -c 32 shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastphot 32 sv2 dark > /global/cfs/cdirs/desi/spectro/fastspecfit/iron/logs/fastphot-iron-sv2-dark.log.1 2>&1 &
+#srun -n 16 -c 32 shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh qafastspec 32 > qafastspec-iron.log.1 2>&1 &
+#srun -n 16 -c 32 shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh qafastphot 32 > qafastphot-iron.log.1 2>&1 &
 
 # Cori
 #salloc -N 32 -C haswell -A desi -L cfs -t 04:00:00 --qos interactive --image=docker:desihub/fastspecfit:2.1.1
-#srun -n 32 -c 32 --kill-on-bad-exit=0 --no-kill shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec fuji 32 healpix sv1 > /global/cfs/cdirs/desi/spectro/fastspecfit/fuji/logs/fastspec-fuji-sv2sv3.log.1 2>&1 &
+#srun -n 32 -c 32 shifter --module=mpich /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec iron 32 healpix sv1 > /global/cfs/cdirs/desi/spectro/fastspecfit/iron/logs/fastspec-iron-sv2sv3.log.1 2>&1 &
 
-#srun -n 16 -c 32 --kill-on-bad-exit=0 --no-kill shifter --module=mpich-cle6 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec fuji 32 - sv1 > /global/cfs/cdirs/desi/spectro/fastspecfit/fuji/logs/fastspec-fuji-sv1.log.1 2>&1 &
-#srun -n 16 -c 32 --kill-on-bad-exit=0 --no-kill shifter --module=mpich-cle6 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastphot 32 sv2 dark > /global/cfs/cdirs/desi/spectro/fastspecfit/fuji/logs/fastphot-fuji-sv2-dark.log.1 2>&1 &
-#srun -n 16 -c 32 --kill-on-bad-exit=0 --no-kill shifter --module=mpich-cle6 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh qafastspec 32 > qafastspec-fuji.log.1 2>&1 &
-#srun -n 16 -c 32 --kill-on-bad-exit=0 --no-kill shifter --module=mpich-cle6 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh qafastphot 32 > qafastphot-fuji.log.1 2>&1 &
+#srun -n 16 -c 32 shifter --module=mpich-cle6 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec iron 32 - sv1 > /global/cfs/cdirs/desi/spectro/fastspecfit/iron/logs/fastspec-iron-sv1.log.1 2>&1 &
+#srun -n 16 -c 32 shifter --module=mpich-cle6 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastphot 32 sv2 dark > /global/cfs/cdirs/desi/spectro/fastspecfit/iron/logs/fastphot-iron-sv2-dark.log.1 2>&1 &
+#srun -n 16 -c 32 shifter --module=mpich-cle6 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh qafastspec 32 > qafastspec-iron.log.1 2>&1 &
+#srun -n 16 -c 32 shifter --module=mpich-cle6 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh qafastphot 32 > qafastphot-iron.log.1 2>&1 &
 
 codedir=/global/homes/i/ioannis/code/desihub
 #codedir=/usr/local/bin
@@ -51,7 +51,7 @@ outdir_html=/global/cfs/cdirs/desi/users/ioannis/fastspecfit
 export DESI_ROOT='/global/cfs/cdirs/desi'
 export DUST_DIR='/global/cfs/cdirs/cosmo/data/dust/v0_1'
 export DR9_DIR='/global/cfs/cdirs/desi/external/legacysurvey/dr9'
-export FTEMPLATES_DIR='/global/cfs/cdirs/desi/science/gqp/templates/fastspecfit'
+export FTEMPLATES_DIR='/global/cfs/cdirs/desi/external/templates/fastspecfit'
 
 export TMPCACHE=$(mktemp -d)
 export MPLCONFIGDIR=$TMPCACHE/matplotlib

bin/mpi-fastspecfit.slurm

@@ -11,5 +11,5 @@
 
 # sbatch /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.slurm
 
-time srun -n 32 -c 128 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh makeqa fuji 128 healpix sv3
-#time srun -n 64 -c 128 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec fuji 128 healpix
+time srun -n 32 -c 128 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh makeqa fuji 128 healpix
+#time srun -n 32 -c 128 /global/homes/i/ioannis/code/desihub/fastspecfit/bin/mpi-fastspecfit.sh fastspec iron 128 healpix main bright

bin/validate-production

@@ -77,6 +77,7 @@ def check_catalogs(specprod_dir, fastspecdir, specprod, mp=1):
     fast = fitsio.read(fastfile, ext='METADATA', columns=['Z', 'Z_RR'])
     zdiff = fast['Z'] != fast['Z_RR']
     fastfile = os.path.basename(fastfile)
+    #fastfile = fastfile.replace('fastspec-', '{fastspec,fastphot}-')
     fastfile = fastfile.replace('fastspec-', '{fastspec}-')
 
     fastfiles.append(fastfile)
@@ -169,7 +170,40 @@ def fileinfo(filelist, sortbyrows=False):
 
     return widths, lines, headers
 
-def build_table(specwidths, speclines, photwidths, photlines, specheaders, photheaders):
+def build_table(widths, lines, headers):
+
+    for ii, width in enumerate(widths):
+        if ii == len(widths)-1:
+            print(''.join(np.repeat('=', width)))
+        else:
+            print(''.join(np.repeat('=', width))+' ', end='')            
+
+    for ii, (width, dd) in enumerate(zip(widths, headers)):
+        if ii == len(widths)-1:
+            print(str.ljust(dd, width))
+        else:
+            print(str.ljust(dd, width)+' ', end='')
+
+    for ii, width in enumerate(widths):
+        if ii == len(widths)-1:
+            print(''.join(np.repeat('=', width)))
+        else:
+            print(''.join(np.repeat('=', width))+' ', end='')
+
+    for line in lines:
+        for ii, (width, dd) in enumerate(zip(widths, line)):
+            if ii == len(widths)-1:
+                print(str.ljust(dd, width))
+            else:
+                print(str.ljust(dd, width)+' ', end='')
+
+    for ii, width in enumerate(widths):
+        if ii == len(widths)-1:
+            print(''.join(np.repeat('=', width)))
+        else:
+            print(''.join(np.repeat('=', width))+' ', end='')            
+
+def build_table_withphot(specwidths, speclines, photwidths, photlines, specheaders, photheaders):
 
     headers = np.hstack((specheaders, photheaders))
     widths = np.hstack((specwidths, photwidths))
@@ -290,29 +324,28 @@ def main():
     fastspecdir = '/global/cfs/cdirs/desi/spectro/fastspecfit/'+args.specprod
     specprod_dir = os.path.join(desi_root, 'spectro', 'redux', args.specprod)
 
-    if args.specprod == 'guadalupe':
-        sortbyrows = True
-    else:
+    if args.specprod == 'fuji':
         sortbyrows = False
+    else:
+        sortbyrows = True
 
     # Do statistics.
     if args.statistics:
         # by survey
         fastspecfiles = np.hstack((sorted(glob(os.path.join(fastspecdir, 'catalogs', 'fastspec-{}-*.fits'.format(args.specprod)))),
                                    os.path.join(fastspecdir, 'catalogs', 'fastspec-{}.fits'.format(args.specprod))))
-        fastphotfiles = np.hstack((sorted(glob(os.path.join(fastspecdir, 'catalogs', 'fastphot-{}-*.fits'.format(args.specprod)))),
-                                   os.path.join(fastspecdir, 'catalogs', 'fastphot-{}.fits'.format(args.specprod))))
-        #fastspecfiles = np.array(sorted(glob(os.path.join(fastspecdir, 'catalogs', 'fastspec-{}-*.fits'.format(args.specprod)))))
-        #fastphotfiles = np.array(sorted(glob(os.path.join(fastspecdir, 'catalogs', 'fastphot-{}-*.fits'.format(args.specprod)))))
         specwidths, speclines, specheaders = fileinfo(fastspecfiles, sortbyrows=sortbyrows)
-        photwidths, photlines, photheaders = fileinfo(fastphotfiles, sortbyrows=sortbyrows)
-        build_table(specwidths, speclines, photwidths, photlines, specheaders, photheaders)
+        build_table(specwidths, speclines, specheaders)
+        #fastphotfiles = np.hstack((sorted(glob(os.path.join(fastspecdir, 'catalogs', 'fastphot-{}-*.fits'.format(args.specprod)))),
+        #                           os.path.join(fastspecdir, 'catalogs', 'fastphot-{}.fits'.format(args.specprod))))
+        #photwidths, photlines, photheaders = fileinfo(fastphotfiles, sortbyrows=sortbyrows)
+        #build_table_withphot(specwidths, speclines, photwidths, photlines, specheaders, photheaders)
 
     # Validate files.
     if args.validate:
         # check for infinity and other crap
         check_infinity(fastspecdir, specprod_dir, args.specprod, mp=args.mp)
-        check_infinity(fastspecdir, specprod_dir, args.specprod, mp=args.mp, fastphot=True)
+        #check_infinity(fastspecdir, specprod_dir, args.specprod, mp=args.mp, fastphot=True)
 
         # verify that we have all the targets we expect and also summarize the
         # number of targets with corrected redshifts

doc/algorithms.rst

@@ -3,6 +3,10 @@
 Algorithms
 ==========
 
+.. note::  
+
+  This page is still under construction!
+
 .. contents:: Contents
     :depth: 3
 
@@ -11,33 +15,22 @@ Overview
 
 ``FastSpecFit`` leverages our understanding of galaxies and quasars to build a
 physical model of their observed-frame optical spectra and UV/optical/IR
-spectral energy distributions (SEDs). Specifically, ``FastSpecFit`` uses 
-
-
+spectral energy distributions (SEDs). 
 
 Simple Stellar Population Templates
 -----------------------------------
 
-Document the SSPs.
-
+Document the templates, physical properties, etc.
 
 Modeling the Stellar Continuum
 ------------------------------
 
 * Initial line-sigma estimation.
 * Finding significant lines; masking.  
-* Fast NNLS.
-* Reddening. If CONTINUUM_AV_IVAR is zero it means that fitted for the
-  (intrinsic) dust extinction failed.
 * Velocity dispersion (with criteria).
-* Smooth continuum correction.  
-
-
-Modeling the Emission Lines
----------------------------
-
-* Line-fitting (broad, narrow).
-* Continuum, EWs, upper limits, etc.
+* Aperture correction.  
+* Smooth continuum correction.
+* No-photometry and photometry-only modes.
 
 K-Corrections and Rest-Frame Photometry
 ---------------------------------------
@@ -49,12 +42,15 @@ If the inverse variance on a given absolutely magnitude is zero it means that
 the absolute magnitude was derived from *synthesized* photometry based on the
 best-fitting model (i.e., use with care).
 
+Modeling the Emission Lines
+---------------------------
+
+* Line-fitting (broad, narrow).
+* Continuum, EWs, upper limits, etc.
+
 .. _`planned improvements`:
 
 Planned Improvements
 --------------------
 
-* Joint fitting of the broadband photometry and spectroscopy.
-* Adding dust emission to the templates.
 * Additional QSO templates.
-* Broadband photometry is missing emission lines.

doc/changes.rst

@@ -8,10 +8,12 @@ Change Log
 * Web-app updates needed for Fuji/v2.0 database load [`PR #107`_].
 * Get target cutouts using image coadds on-disk [`PR #108`_].
 * Initial hooks to fit stacked spectra [`PR #113`_].
+* Updated documentation for v2.0 Fujilupe and v1.0 Iron VACs [`PR #114`_]. 
 
 .. _`PR #107`: https://github.com/desihub/fastspecfit/pull/107
 .. _`PR #108`: https://github.com/desihub/fastspecfit/pull/108
 .. _`PR #113`: https://github.com/desihub/fastspecfit/pull/113
+.. _`PR #114`: https://github.com/desihub/fastspecfit/pull/114
 
 2.1.1 (2023-02-22)
 ------------------

doc/conf.py

@@ -73,7 +73,7 @@
 
 # General information about the project.
 project = u'fastspecfit'
-copyright = u'2019-2022, John Moustakas & the DESI Collaboration'
+copyright = u'2019-2023, John Moustakas & the DESI Collaboration'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -315,3 +315,4 @@
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
+

doc/fastphot.rst

@@ -1,7 +1,7 @@
 .. _fastphot datamodel:
 
 ===================
-fastphot Data Model
+Fastphot Data Model
 ===================
 
 :Summary: Broadband photometric fitting results.

doc/fastspec.rst

@@ -1,7 +1,7 @@
 .. _fastspec datamodel:
 
 ===================
-fastspec Data Model
+Fastspec Data Model
 ===================
 
 :Summary: Spectroscopic fitting results.
@@ -113,7 +113,6 @@ Name                         Type         Units                         Descript
                   DN4000_OBS      float32                               Narrow 4000-A break index measured from the observed spectroscopic data.
                  DN4000_IVAR      float32                               Inverse variance of DN4000_OBS or DN4000.
                 DN4000_MODEL      float32                               Narrow 4000-A break index measured from the best-fitting continuum model.
-                DN4000_MODEL      float32                               Narrow 4000-A break index (from Balogh et al. 1999) measured from the best-fitting continuum model.
                 FLUX_SYNTH_G      float32                          nmgy g-band flux synthesized from the data.
                 FLUX_SYNTH_R      float32                          nmgy r-band flux synthesized from the data.
                 FLUX_SYNTH_Z      float32                          nmgy z-band flux synthesized from the data.
@@ -912,7 +911,7 @@ Name                   Type        Units      Description
              TSNR2_ELG float32                Template signal-to-noise ratio squared for ELG targets.
              TSNR2_QSO float32                Template signal-to-noise ratio squared for QSO targets.
              TSNR2_LYA float32                Template signal-to-noise ratio squared for LYA targets.
-               PHOTSYS  bytes1                Photometric system (*N* or *S*).
+               PHOTSYS    str1                Photometric system (*N* or *S*).
                LS_ID     int64                Unique Legacy Surveys identification number.
            FIBERFLUX_G float32           nmgy Fiber g-band flux corrected for Galactic extinction.
            FIBERFLUX_R float32           nmgy Fiber r-band flux corrected for Galactic extinction.