diff --git a/.gitignore b/.gitignore index dfaaafd6a..1c8d2c110 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ utils/config.json build/ dist/ -jwql.egg-info/ \ No newline at end of file +jwql.egg-info/ +docs/build/ +docs/source/api/ \ No newline at end of file diff --git a/README.md b/README.md index 11c7d78d6..9212a5b56 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ The following is a bare bones example of a best work flow for contributing to th 10. Delete your local copy of your branch. -## Git Ignore +## `gitignore` The `jwql` repository also contains a file named `.gitignore` that indicates specific directories, files or file types that should not be commited to the repository. Feel free to add additional lines to this file if you want to avoid committing anything. Some examples may include `.fits` files, `.jpeg` files, or `.ipynb_checkpoints/`. @@ -86,7 +86,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Matthew Bourque (INS) - Lauren Chambers (INS) - Misty Cracraft (INS) -- Joseph Filippazo (INS) +- Joe Filippazzo (INS) - Bryan Hilbert (INS) - Graham Kanarek (INS) - Catherine Martlin (INS) @@ -109,6 +109,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Phil Grant (ITSD) - Dean Hines (INS) - Sherrie Holfeltz (INS) +- Joe Hunkeler (OED) - Catherine Kaleida (OED) - Mark Kyprianou (OED) - Karen Levay (OED) @@ -125,4 +126,5 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Linda Smith (INS) - Dave Unger (ITSD) - Jeff Valenti (INS) +- Lara Wilkinson (OPO) - Joe Zahn (ITSD) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..f88da7e2b --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = jwql +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/source/api_docs.rst b/docs/source/api_docs.rst new file mode 100644 index 000000000..73d9da6dc --- /dev/null +++ b/docs/source/api_docs.rst @@ -0,0 +1,7 @@ +Preview Image Test +================== +.. automodapi:: jwql.preview_image.preview_image + +Utils Test +========== +.. automodapi:: jwql.utils.utils diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 000000000..c4c75e108 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# jwql documentation build configuration file, created by +# sphinx-quickstart on Wed Apr 4 10:30:20 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) +import stsci_rtd_theme + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx_automodapi.automodapi', + 'sphinx_automodapi.automodsumm', + 'numpydoc', + 'sphinx.ext.autodoc', + 'sphinx.ext.mathjax', + 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# Numpy doc setting, right now this supresses some warnings, not exactly sure why? +numpydoc_show_class_members = False + +# General information about the project. +project = 'jwql' +copyright = '2018, STScI' +author = 'STScI' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.4.1' +# The full version, including alpha/beta/rc tags. +release = '0.4.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'stsci_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] +html_theme_path = [stsci_rtd_theme.get_html_theme_path()] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +#html_sidebars = {} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'jwqldoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'jwql.tex', 'jwql Documentation', + 'STScI', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'jwql', 'jwql Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'jwql', 'jwql Documentation', + author, 'jwql', 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 000000000..4c3820628 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,30 @@ +.. jwql documentation master file, created by + sphinx-quickstart on Wed Apr 4 10:30:20 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to jwql's documentation! +================================ + +General JWQL description here. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +API documentation +================= + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + api_docs.rst + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/environment.yml b/environment.yml index 1985e77b5..3fa1223b9 100644 --- a/environment.yml +++ b/environment.yml @@ -4,14 +4,19 @@ channels: - http://ssb.stsci.edu/astroconda-dev dependencies: - astropy=3.0 +- astroquery=0.3.8 +- bokeh=0.12.5 - django=1.11.8 - jwst=0.7.8rc9 - matplotlib=2.1.1 - numpy=1.14.0 +- numpydoc=0.8.0 - postgresql=9.6.6 - python=3.6.4 - python-dateutil=2.6.1 - sphinx=1.6.6 +- sphinx_rtd_theme=0.1.9 - sqlalchemy=1.2.0 +- stsci_rtd_theme=0.0.1 - pip: - sphinx-automodapi==0.7 \ No newline at end of file diff --git a/jwql/__init__.py b/jwql/__init__.py index e69de29bb..6d9900a73 100644 --- a/jwql/__init__.py +++ b/jwql/__init__.py @@ -0,0 +1 @@ +from . import dbmonitor \ No newline at end of file diff --git a/jwql/dbmonitor/__init__.py b/jwql/dbmonitor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py new file mode 100644 index 000000000..5ed0a9654 --- /dev/null +++ b/jwql/dbmonitor/dbmonitor.py @@ -0,0 +1,204 @@ +"""This module is home to a suite of MAST queries that gather bulk properties +of available JWST data for JWQL + +Authors +------- + + Joe Filippazzo + +Use +--- + + To get an inventory of all JWST files do: + :: + + from jwql.dbmonitor import dbmonitor + inventory, keywords = dbmonitor.jwst_inventory() +""" + +import os + +from astroquery.mast import Mast +from bokeh.charts import Donut, save, output_file +import pandas as pd + +from ..permissions.permissions import set_permissions +from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS + + +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters=None, add_requests=None, + caom=False, return_data=False): + """Get the counts for a given instrument and data product + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + add_filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. add_filters = {'filter':'GR150R'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} + caom: bool + Query CAOM service + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int, dict + The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` + """ + filters = [] + + # Make sure the dataproduct is a list + if isinstance(dataproduct, str): + dataproduct = [dataproduct] + + # Make sure the instrument is supported + if instrument.lower() not in [ins.lower() for ins in JWST_INSTRUMENTS]: + raise TypeError('Supported instruments include:', JWST_INSTRUMENTS) + + # CAOM service + if caom: + + # Declare the service + service = 'Mast.Caom.Filtered' + + # Set the filters + filters += [{'paramName': 'obs_collection', 'values': ['JWST']}, + {'paramName': 'instrument_name', 'values': [instrument]}, + {'paramName': 'dataproduct_type', 'values': dataproduct}] + + # Instruent filtered service + else: + + # Declare the service + service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) + + # Include additonal filters + if isinstance(add_filters, dict): + filters += [{"paramName": name, "values": [val]} + for name, val in add_filters.items()] + + # Assemble the request + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns': True} + + # Just get the counts + if return_data: + params['columns'] = '*' + + # Add requests + if isinstance(add_requests, dict): + params.update(add_requests) + + response = Mast.service_request_async(service, params) + result = response[0].json() + + # Return all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + + +def instrument_keywords(instrument, caom=False): + """Get the keywords for a given instrument service + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + caom: bool + Query CAOM service + + Returns + ------- + pd.DataFrame + A DataFrame of the keywords + """ + # Retrieve one dataset to get header keywords + sample = instrument_inventory(instrument, return_data=True, caom=caom, + add_requests={'pagesize': 1, 'page': 1}) + data = [[i['name'], i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + return keywords + + +def jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + caom=False, plot=False): + """Gather a full inventory of all JWST data in each instrument + service by instrument/dtype + + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + caom: bool + Query CAOM service + plot: bool + Return a pie chart of the data + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Iterate through instruments + inventory, keywords = [], {} + for instrument in instruments: + ins = [instrument] + for dp in dataproducts: + count = instrument_inventory(instrument, dataproduct=dp, caom=caom) + ins.append(count) + + # Get the total + ins.append(sum(ins[-3:])) + + # Add it to the list + inventory.append(ins) + + # Add the keywords to the dict + keywords[instrument] = instrument_keywords(instrument, caom=caom) + + # Make the table + all_cols = ['instrument']+dataproducts+['total'] + table = pd.DataFrame(inventory, columns=all_cols) + + # Melt the table + table = pd.melt(table, id_vars=['instrument'], + value_vars=dataproducts, + value_name='files', var_name='dataproduct') + + # Plot it + if plot: + + # Make the plot + plt = Donut(table, label=['instrument', 'dataproduct'], values='files', + text_font_size='12pt', hover_text='files', + name="JWST Inventory", plot_width=600, plot_height=600) + + # Save the plot + if caom: + output_filename = 'database_monitor_caom.html' + else: + output_filename = 'database_monitor_jwst.html' + outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename) + output_file(outfile) + save(plt) + set_permissions(outfile, verbose=False) + + return table, keywords diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py new file mode 100755 index 000000000..516b0d118 --- /dev/null +++ b/jwql/tests/test_dbmonitor.py @@ -0,0 +1,65 @@ +#! /usr/bin/env python + +"""Tests for the dbmonitor module. + +Authors +------- + + Joe Filippazzo + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s test_dbmonitor.py +""" + +from ..dbmonitor import dbmonitor as db +from ..utils.utils import JWST_INSTRUMENTS + + +def test_caom_instrument_keywords(): + """Test to see that the CAOM keywords are the same for all + instruments""" + kw = [] + for ins in JWST_INSTRUMENTS: + kw.append(db.instrument_keywords(ins, caom=True)['keyword'].tolist()) + + assert kw[0] == kw[1] == kw[2] == kw[3] == kw[4] + + +def test_filtered_instrument_keywords(): + """Test to see that the instrument specific service keywords are + different for all instruments""" + kw = [] + for ins in JWST_INSTRUMENTS: + kw.append(db.instrument_keywords(ins, caom=False)['keyword'].tolist()) + + assert kw[0] != kw[1] != kw[2] != kw[3] != kw[4] + + +def test_instrument_inventory_filtering(): + """Test to see that the instrument inventory can be filtered""" + filt = 'GR150R' + data = db.instrument_inventory('niriss', + add_filters={'filter': filt}, + return_data=True) + + filters = [row['filter'] for row in data['data']] + + assert all([i == filt for i in filters]) + + +def test_instrument_dataproduct_filtering(): + """Test to see that the instrument inventory can be filtered + by data product""" + dp = 'spectrum' + data = db.instrument_inventory('nirspec', dataproduct=dp, caom=True, + return_data=True) + + dps = [row['dataproduct_type'] for row in data['data']] + + assert all([i == dp for i in dps]) diff --git a/jwql/tests/test_permissions.py b/jwql/tests/test_permissions.py old mode 100644 new mode 100755 diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py new file mode 100644 index 000000000..2a79c83e7 --- /dev/null +++ b/jwql/tests/test_utils.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +"""Tests for the ``utils`` module. + +Authors +------- + + - Lauren Chambers + +Use +--- + + These tests can be run via the command line (omit the -s to + suppress verbose output to stdout): + + :: + + pytest -s test_utils.py +""" + +import pytest + +from jwql.utils.utils import get_config, filename_parser + + +def test_get_config(): + '''Assert that the ``get_config`` function successfuly creates a + dictionary. + ''' + settings = get_config() + assert isinstance(settings, dict) + + +def test_filename_parser_filename(): + '''Generate a dictionary with parameters from a JWST filename. + Assert that the dictionary matches what is expected. + ''' + filename = 'jw00327001001_02101_00002_nrca1_rate.fits' + filename_dict = filename_parser(filename) + + correct_dict = {'activity': '01', + 'detector': 'nrca1', + 'exposure_id': '00002', + 'observation': '001', + 'parallel_seq_id': '1', + 'program_id': '00327', + 'suffix': 'rate', + 'visit': '001', + 'visit_group': '02'} + + assert filename_dict == correct_dict + + +def test_filename_parser_filepath(): + '''Generate a dictionary with parameters from a JWST filepath + (not just the basename). Assert that the dictionary matches what + is expected. + ''' + filepath = '/test/dir/to/the/file/jw90002/jw90002001001_02102_00001_nis_rateints.fits' + filename_dict = filename_parser(filepath) + + correct_dict = {'activity': '02', + 'detector': 'nis', + 'exposure_id': '00001', + 'observation': '001', + 'parallel_seq_id': '1', + 'program_id': '90002', + 'suffix': 'rateints', + 'visit': '001', + 'visit_group': '02'} + + assert filename_dict == correct_dict + + +def test_filename_parser_nonJWST(): + '''Attempt to generate a file parameter dictionary from a file + that is not formatted in the JWST naming convention. Ensure the + appropriate error is raised. + ''' + with pytest.raises(ValueError, + match=r'Provided file .+ does not follow JWST naming conventions \(jw____\.fits\)'): + filename = 'not_a_jwst_file.fits' + filename_parser(filename) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index a0f122afc..f907612f2 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -3,7 +3,8 @@ Authors ------- - Matthew Bourque + - Matthew Bourque + - Lauren Chambers Use --- @@ -12,10 +13,23 @@ >>> import utils settings = get_config() + +References +---------- + + Filename parser modifed from Joe Hunkeler: + https://gist.github.com/jhunkeler/f08783ca2da7bfd1f8e9ee1d207da5ff """ import json +import os +import re +__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) + +JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', + 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] def get_config(): """Return a dictionary that holds the contents of the jwql config @@ -27,7 +41,50 @@ def get_config(): A dictionary that holds the contents of the config file. """ - with open('config.json', 'r') as config_file: + with open(os.path.join(__location__, 'config.json'), 'r') as config_file: settings = json.load(config_file) return settings + + +def filename_parser(filename): + """Return a dictionary that contains the properties of a given + JWST file (e.g. program ID, visit number, detector, etc.) + + Parameters + ---------- + filename : str + Path or name of JWST file to parse + + Returns + ------- + filename_dict : dict + Collection of file properties + + Raises + ------ + ValueError + When the provided file does not follow naming conventions + """ + filename = os.path.basename(filename) + + elements = \ + re.compile(r"[a-z]+" + "(?P\d{5})" + "(?P\d{3})" + "(?P\d{3})" + "_(?P\d{2})" + "(?P\d{1})" + "(?P\d{2})" + "_(?P\d+)" + "_(?P\w+)" + "_(?P\w+).*") + + jwst_file = elements.match(filename) + + if jwst_file is not None: + filename_dict = jwst_file.groupdict() + else: + raise ValueError('Provided file {} does not follow JWST naming conventions (jw____.fits)'.format(filename)) + + return filename_dict diff --git a/setup.py b/setup.py index af40a2fea..245dac2de 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ AUTHORS = 'Matthew Bourque, Sara Ogaz, Joe Filippazzo, Bryan Hilbert, Misty Cracraft, Graham Kanarek' AUTHORS += 'Johannes Sahlmann, Lauren Chambers, Catherine Martlin' -REQUIRES = ['astropy', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] +REQUIRES = ['astropy', 'astroquery', 'bokeh', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] setup( name = 'jwql',