Scripts n doc

bin/run_specview_cmx_targets

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#
+# See top-level LICENSE.rst file for Copyright information
+#
+
+"""
+Write static html files from list of targets, based on cmx coadds
+"""
+
+import prospect.scripts.specview_cmx_targets as specview_cmx_targets
+
+if __name__ == '__main__':
+    args = specview_cmx_targets.parse()
+    specview_cmx_targets.main(args)
+
+

doc/nb/Prospect_targetid.ipynb

@@ -82,7 +82,8 @@
     "# Create \"homemade mini-db\"\n",
     "tiledir   = '/global/cfs/cdirs/desi/spectro/redux/andes/tiles/' # EDIT THIS\n",
     "tiles = ['68000','68001','68002'] # EDIT THIS\n",
-    "obs_db = utils_specviewer.make_targetdict(tiledir, tiles=tiles) # petals, tiles = optional arguments\n",
+    "nights = ['20200315','20200315'] # EDIT THIS\n",
+    "obs_db = utils_specviewer.make_targetdict(tiledir, tiles=tiles, nights=nights) # petals, tiles, nights = optional arguments\n",
     "\n",
     "## Enter list of targets here. Targetids must be int64.\n",
     "targets = [ 35191281135124504, 35191295232180275]\n",
@@ -113,7 +114,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.6"
+   "version": "3.8.3"
   }
  },
  "nbformat": 4,

js/select_model.js

@@ -79,10 +79,10 @@ if (model_select.value == 'Best fit') {
     var spectype =  fit_results['SPECTYPE'][ifiberslider.value][i_fit]
     var subtype =  fit_results['SUBTYPE'][ifiberslider.value][i_fit]
     spec_z = fit_results['Z'][ifiberslider.value][i_fit]
-
+    
     var template_wave = fit_templates["wave_"+spectype+"_"+subtype]
     var template_flux = fit_templates["flux_"+spectype+"_"+subtype]
-
+    
     var model_wave = template_wave.slice()
     var model_flux = template_wave.slice()
     for (var j=0; j<model_wave.length; j++) {

py/prospect/data/absorption_lines.txt

@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 # Comma-separated entries
 # List of absorption lines for prospect viewer
-# Sources: SDSS spZline files ($IDLSPEC2D_DIR/etc/emlines.par, wavelengths in air for
-#   lambda > 2000, vacuum for lambda < 2000); A. Edge list (sent by email to T. Davis,
-#   07/04/2020); few others.
+# Sources: 
+#   - SDSS spZline files ($IDLSPEC2D_DIR/etc/emlines.par, wavelengths in air for lambda > 2000, vacuum for lambda < 2000);
+#   - A. Edge list (sent by email to T. Davis, 07/04/2020);
+#   - desisim recombination/forbidden line lists (J. Moustakas; selected those whose quoted line-ratio wrt Hbeta is > 0.1); 
+#   - few others.
 # name,longname,lambda,vacuum,major
 Si II,Si II 1260,1260.4221,True,False
 O I,O I 1302,1302.1685,True,False

py/prospect/data/emission_lines.txt

@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 # Comma-separated entries
 # List of emission lines for prospect viewer
-# Sources: SDSS spZline files ($IDLSPEC2D_DIR/etc/emlines.par, wavelengths in air for
-#   lambda > 2000, vacuum for lambda < 2000); A. Edge list (sent by email to T. Davis,
-#   07/04/2020); desisim recombination/forbidden line lists (J. Moustakas); few others.
+# Sources: 
+#   - SDSS spZline files ($IDLSPEC2D_DIR/etc/emlines.par, wavelengths in air for lambda > 2000, vacuum for lambda < 2000); 
+#   - A. Edge list (sent by email to T. Davis, 07/04/2020); 
+#   - desisim recombination/forbidden line lists (J. Moustakas; selected those whose quoted line-ratio wrt Hbeta is > 0.1); 
+#   - few others.
 # name,longname,lambda,vacuum,major
 Lyβ,Lyman β,1025.18,True,False
 Lyα,Lyman α,1215.67,True,True

py/prospect/myspecselect.py

@@ -5,7 +5,7 @@
 
 import desispec.spectra
 
-def myspecselect(thespec, nights=None, bands=None, targets=None, fibers=None, expids=None, indices=None, invert=False, remove_scores=False, clean_fiberstatus=False):
+def myspecselect(thespec, nights=None, bands=None, targets=None, fibers=None, expids=None, indices=None, invert=False, remove_scores=False, clean_fiberstatus=False, output_indices=False):
     """
     Select a subset of the data.
     This filters the data based on a logical AND of the different
@@ -19,6 +19,7 @@ def myspecselect(thespec, nights=None, bands=None, targets=None, fibers=None, ex
         ADDED =>indices (list) : list of raw (arbitrary) indices in the Spectra object to select. 
         invert (bool): after combining all criteria, invert selection.
         remove_scores (bool): probably tmp trick, TODO
+        output_indices (bool): if True, also returns indices of kept spectra
     Returns (Spectra):
         a new Spectra object containing the selected data.
     """
@@ -127,4 +128,7 @@ def myspecselect(thespec, nights=None, bands=None, targets=None, fibers=None, ex
         fibermap=thespec.fibermap[keep], meta=thespec.meta, extra=keep_extra,
         single=thespec._single, scores=keep_scores)
 
+    if output_indices :
+        return (ret, keep)
+
     return ret

py/prospect/plotframes.py

@@ -25,7 +25,7 @@
 from bokeh.models.widgets import (
     Slider, Button, Div, CheckboxGroup, CheckboxButtonGroup, RadioButtonGroup, 
     TextInput, Select, DataTable, TableColumn, Spinner)
-from bokeh.layouts import widgetbox, Spacer, gridplot
+from bokeh.layouts import Column, Spacer, gridplot
 import bokeh.events
 # from bokeh.layouts import row, column
 
@@ -1195,7 +1195,7 @@ def plotspectra(spectra, nspec=None, startspec=None, zcatalog=None, redrock_cat=
     z_tovi_button.js_on_event('button_click', z_tovi_callback)
 
     #- Optional VI information on spectral type
-    vi_spectypes = [" "] + utils_specviewer._vi_spectypes
+    vi_spectypes = [''] + utils_specviewer._vi_spectypes
     vi_category_select = Select(value=" ", title="VI spectype:", options=vi_spectypes)
     with open(os.path.join(js_dir,"CSVtoArray.js"), 'r') as f : vi_category_code = f.read()
     with open(os.path.join(js_dir,"save_vi.js"), 'r') as f : vi_category_code += f.read()
@@ -1411,92 +1411,92 @@ def plotspectra(spectra, nspec=None, startspec=None, zcatalog=None, redrock_cat=
 
     slider_width = plot_width - 2*navigation_button_width
     navigator = bk.Row(
-        widgetbox(prev_button, width=navigation_button_width+15),
-        widgetbox(next_button, width=navigation_button_width+20),
-        widgetbox(ifiberslider, width=plot_width+(plot_height//2)-(60*len(vi_class_labels)+2*navigation_button_width+35))
+        Column(prev_button, width=navigation_button_width+15),
+        Column(next_button, width=navigation_button_width+20),
+        Column(ifiberslider, width=plot_width+(plot_height//2)-(60*len(vi_class_labels)+2*navigation_button_width+35))
     )
     if with_vi_widgets :
-        navigator.children.insert(1, widgetbox(vi_class_input, width=60*len(vi_class_labels)) )
+        navigator.children.insert(1, Column(vi_class_input, width=60*len(vi_class_labels)) )
         vi_widget_set = bk.Column(
-            widgetbox( Div(text="VI optional indications :"), width=300 ),
+            Column( Div(text="VI optional indications :"), width=300 ),
             bk.Row(
-                widgetbox(vi_issue_input, width=150),
-                widgetbox(vi_z_input, width=150),
-                widgetbox(vi_category_select, width=150)
+                Column(vi_issue_input, width=150),
+                Column(vi_z_input, width=150),
+                Column(vi_category_select, width=150)
             ),
             bk.Row(
-                widgetbox(vi_comment_input, width=300),
-                widgetbox(vi_std_comment_select, width=200),
+                Column(vi_comment_input, width=300),
+                Column(vi_std_comment_select, width=200),
             ),
             bk.Row(
-                widgetbox(vi_name_input, width=200),
-                widgetbox(vi_filename_input, width=300)
+                Column(vi_name_input, width=200),
+                Column(vi_filename_input, width=300)
             ),
-            widgetbox(save_vi_button, width=100),
-            widgetbox(vi_table),        
+            Column(save_vi_button, width=100),
+            Column(vi_table),        
             bk.Row(
-                widgetbox(recover_vi_button, width=150),
-                widgetbox(clear_vi_button, width=150)
+                Column(recover_vi_button, width=150),
+                Column(clear_vi_button, width=150)
             ),
             background='#f5f5f0'
         )
     plot_widget_set = bk.Column(
-        widgetbox( Div(text="Pipeline fit: ") ),
-        widgetbox(zcat_display, width=plot_widget_width),
+        Column( Div(text="Pipeline fit: ") ),
+        Column(zcat_display, width=plot_widget_width),
         bk.Row(
             bk.Column(
                 bk.Row(
-                    widgetbox(z_minus_button, width=z_button_width+15),
-                    widgetbox(zslider, width=plot_widget_width-2*z_button_width-135),
-                    widgetbox(z_plus_button, width=z_button_width)
+                    Column(z_minus_button, width=z_button_width+15),
+                    Column(zslider, width=plot_widget_width-2*z_button_width-135),
+                    Column(z_plus_button, width=z_button_width)
                 ),
                 bk.Row(
-                    widgetbox(dzslider, width=plot_widget_width-235),
-                    widgetbox(Spacer(width=20)),
-                    widgetbox(zreset_button, width=100)
+                    Column(dzslider, width=plot_widget_width-235),
+                    Column(Spacer(width=20)),
+                    Column(zreset_button, width=100)
                 )
             ),
-            widgetbox(Spacer(width=15)),
+            Column(Spacer(width=15)),
             bk.Column(
-                widgetbox(z_input, width=100),
-                widgetbox(z_tovi_button, width=100)
+                Column(z_input, width=100),
+                Column(z_tovi_button, width=100)
             ),
             background='#fff7e6'
         ),
-        widgetbox(smootherslider, width=plot_widget_width),
+        Column(smootherslider, width=plot_widget_width),
 #        widgetbox(display_options_group,width=120),
         bk.Row(
-            widgetbox(coaddcam_buttons, width=200),
-            widgetbox(Spacer(width=30)),
-            widgetbox(waveframe_buttons, width=120)
+            Column(coaddcam_buttons, width=200),
+            Column(Spacer(width=30)),
+            Column(waveframe_buttons, width=120)
         ),
         bk.Row(
-            widgetbox(lines_button_group, width=200),
-            widgetbox(Spacer(width=30)),
-            widgetbox(majorline_checkbox, width=120)
+            Column(lines_button_group, width=200),
+            Column(Spacer(width=30)),
+            Column(majorline_checkbox, width=120)
         )
     )
     if model_select is not None :
-        plot_widget_set.children.insert(3, widgetbox(model_select, width=200))
+        plot_widget_set.children.insert(3, Column(model_select, width=200))
     if with_vi_widgets :
-        plot_widget_set.children.append( widgetbox(Spacer(height=30)) )
-        plot_widget_set.children.append( widgetbox(vi_guideline_div, width=plot_widget_width) )
+        plot_widget_set.children.append( Column(Spacer(height=30)) )
+        plot_widget_set.children.append( Column(vi_guideline_div, width=plot_widget_width) )
         full_widget_set = bk.Row(
             vi_widget_set,
-            widgetbox(Spacer(width=40)),
+            Column(Spacer(width=40)),
             plot_widget_set
         )
     else : full_widget_set = plot_widget_set
 
     main_bokehsetup = bk.Column(
         bk.Row(fig, bk.Column(imfig, zoomfig), Spacer(width=20), sizing_mode='stretch_width'),
         bk.Row(
-            widgetbox(targ_display, width=600), # plot_width - 200
-            widgetbox(Spacer(width=20)),
-            widgetbox(reset_plotrange_button, width = 120),
-            widgetbox(Spacer(width=80)),
-            widgetbox(oii_zoom_button, width=80),
-            widgetbox(oii_undo_button, width=50),
+            Column(targ_display, width=600), # plot_width - 200
+            Column(Spacer(width=20)),
+            Column(reset_plotrange_button, width = 120),
+            Column(Spacer(width=80)),
+            Column(oii_zoom_button, width=80),
+            Column(oii_undo_button, width=50),
         ),
         navigator,
         full_widget_set,
@@ -1539,11 +1539,11 @@ def plotspectra(spectra, nspec=None, startspec=None, zcatalog=None, redrock_cat=
         miniplot_width = ( plot_width + (plot_height//2) ) // ncols_grid
         thumb_grid = grid_thumbs(spectra, miniplot_width, x_range=(xmin,xmax), ncols_grid=ncols_grid, titles=titles)
         thumb_viewer = bk.Column(
-            widgetbox( Div(text=
+            Column( Div(text=
                            " <h3> Thumbnail gallery for DESI spectra in "+title+" </h3>" +
                            " <p> Click <a href='specviewer_"+title+".html'>here</a> to access the spectral viewer corresponding to these spectra. </p>"
                           ), width=plot_width ),
-            widgetbox( thumb_grid )
+            Column( thumb_grid )
         )
         bk.save(thumb_viewer)
 

py/prospect/scripts/specview_cmx_targets.py

@@ -0,0 +1,88 @@
+"""
+prospect.scripts.specview_cmx_targets
+===================================
+
+Write static html files from a set of targets,
+ using tile-based coadds in CMX data
+"""
+
+import os
+import argparse
+import numpy as np
+
+import desispec.io
+from desiutil.log import get_logger
+
+from prospect import plotframes
+from prospect import utils_specviewer, myspecselect
+
+def parse() :
+
+    parser = argparse.ArgumentParser(description='Create static html pages from a set of targets, using CMX tile-based coadds')
+    parser.add_argument('--specprod_dir', help='Location of directory tree (data in specprod_dir/tiles/)', type=str)
+    parser.add_argument('--target_list', help='ASCII file providing the list of targetids', type=str)
+    parser.add_argument('--tiles', help='Name of tile[s] to be processed (avoids to scan all tiles)', nargs='+', type=str, default=None)
+    parser.add_argument('--nights', help='Name of night[s] to be processed (avoids to scan all nights)', nargs='+', type=str, default=None)
+    parser.add_argument('--nspecperfile', help='Number of spectra in each html page', type=int, default=50)
+    parser.add_argument('--webdir', help='Base directory for webpages', type=str)
+    parser.add_argument('--titlepage_prefix', help='Prefix for webpage title', type=str, default='targetlist')
+    parser.add_argument('--with_multiple_models', help='Display several models (requires full redrock outputs)', action='store_true')
+    parser.add_argument('--template_dir', help='Redrock template directory', type=str, default=None)
+
+    args = parser.parse_args()
+    return args
+
+
+def main(args) :
+
+    log = get_logger()
+
+    tile_dir = os.path.join(args.specprod_dir,'tiles')
+    if args.tiles :
+        tile_list = args.tiles
+    else :
+        tile_list = os.listdir(tile_dir)
+    obs_db = utils_specviewer.make_targetdict(tile_dir, tiles=tile_list, nights=args.nights)
+
+    targetids = np.loadtxt(args.target_list, dtype='int64', comments='#')
+    log.info(str(len(targetids))+" targets provided.")
+
+    if args.with_multiple_models :
+        spectra, zcat, rrtable = utils_specviewer.load_spectra_zcat_from_targets(targetids, tile_dir, obs_db, with_redrock=True)
+    else :
+        spectra, zcat = utils_specviewer.load_spectra_zcat_from_targets(targetids, tile_dir, obs_db, with_redrock=False)
+
+    # TODO? this may be put in a standalone fct, avoid code duplicate with other script
+    # Create several html pages : sort by targetid
+    nspec = spectra.num_spectra()
+    log.info(str(nspec)+" spectra obtained.")
+    sort_indices = np.argsort(spectra.fibermap["TARGETID"])
+    nbpages = int(np.ceil((nspec/args.nspecperfile)))
+    for i_page in range(1,1+nbpages) :
+
+        log.info(" * Page "+str(i_page)+" / "+str(nbpages))
+        the_indices = sort_indices[(i_page-1)*args.nspecperfile:i_page*args.nspecperfile]            
+        thespec, kept_ind = myspecselect.myspecselect(spectra, indices=the_indices, remove_scores=True, output_indices=True)
+        the_zcat = zcat[kept_ind]
+        if not np.array_equal(the_zcat['TARGETID'], thespec.fibermap['TARGETID']) :
+            raise RuntimeError("targetids do not match between spec and zcat")
+        #the_zcat, kk = utils_specviewer.match_zcat_to_spectra(zcat, thespec)
+        if args.with_multiple_models :
+            the_rrtable = rrtable[kept_ind]
+            #the_rrtable, kk = utils_specviewer.match_zcat_to_spectra(rrtable, thespec)
+            num_approx_fits = 4 # TODO settle option
+            with_full_2ndfit = True # TODO settle option
+        else :
+            the_rrtable = None
+            num_approx_fits = None
+            with_full_2ndfit = False
+
+        titlepage = args.titlepage_prefix+"_"+str(i_page)
+        plotframes.plotspectra(thespec, with_noise=True, is_coadded=True, zcatalog=the_zcat,
+                    title=titlepage, html_dir=args.webdir, mask_type='CMX_TARGET', with_thumb_only_page=True,
+                    template_dir=args.template_dir, redrock_cat=the_rrtable, num_approx_fits=num_approx_fits,
+                    with_full_2ndfit=with_full_2ndfit)
+
+    log.info("End of specview_cmx_targets script.")
+    return 0
+