Merge pull request #52 from hover2pi/db_monitor

Db monitor

README.md

@@ -86,7 +86,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@
 - Matthew Bourque (INS)
 - Lauren Chambers (INS)
 - Misty Cracraft (INS)
-- Joseph Filippazo (INS)
+- Joe Filippazzo (INS)
 - Bryan Hilbert (INS)
 - Graham Kanarek (INS)
 - Catherine Martlin (INS)

environment.yml

@@ -4,6 +4,8 @@ channels:
 - http://ssb.stsci.edu/astroconda-dev
 dependencies:
 - astropy=3.0
+- astroquery=0.3.8
+- bokeh=0.12.5
 - django=1.11.8
 - jwst=0.7.8rc9
 - matplotlib=2.1.1

jwql/__init__.py

@@ -0,0 +1 @@
+from . import dbmonitor

jwql/dbmonitor/dbmonitor.py

@@ -0,0 +1,204 @@
+"""This module is home to a suite of MAST queries that gather bulk properties
+of available JWST data for JWQL
+
+Authors
+-------
+
+    Joe Filippazzo
+
+Use
+---
+
+    To get an inventory of all JWST files do:
+    ::
+
+        from jwql.dbmonitor import dbmonitor
+        inventory, keywords = dbmonitor.jwst_inventory()
+"""
+
+import os
+
+from astroquery.mast import Mast
+from bokeh.charts import Donut, save, output_file
+import pandas as pd
+
+from ..permissions.permissions import set_permissions
+from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS
+
+
+def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
+                         add_filters=None, add_requests=None,
+                         caom=False, return_data=False):
+    """Get the counts for a given instrument and data product
+
+    Parameters
+    ----------
+    instrument: str
+        The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS']
+    dataproduct: sequence, str
+        The type of data product to search
+    add_filters: dict
+        The ('paramName':'values') pairs to include in the 'filters' argument
+        of the request e.g. add_filters = {'filter':'GR150R'}
+    add_requests: dict
+        The ('request':'value') pairs to include in the request
+        e.g. add_requests = {'pagesize':1, 'page':1}
+    caom: bool
+        Query CAOM service
+    return_data: bool
+        Return the actual data instead of counts only
+
+    Returns
+    -------
+    int, dict
+        The number of database records that satisfy the search criteria
+        or a dictionary of the data if `return_data=True`
+    """
+    filters = []
+
+    # Make sure the dataproduct is a list
+    if isinstance(dataproduct, str):
+        dataproduct = [dataproduct]
+
+    # Make sure the instrument is supported
+    if instrument.lower() not in [ins.lower() for ins in JWST_INSTRUMENTS]:
+        raise TypeError('Supported instruments include:', JWST_INSTRUMENTS)
+
+    # CAOM service
+    if caom:
+
+        # Declare the service
+        service = 'Mast.Caom.Filtered'
+
+        # Set the filters
+        filters += [{'paramName': 'obs_collection', 'values': ['JWST']},
+                    {'paramName': 'instrument_name', 'values': [instrument]},
+                    {'paramName': 'dataproduct_type', 'values': dataproduct}]
+
+    # Instruent filtered service
+    else:
+
+        # Declare the service
+        service = 'Mast.Jwst.Filtered.{}'.format(instrument.title())
+
+    # Include additonal filters
+    if isinstance(add_filters, dict):
+        filters += [{"paramName": name, "values": [val]}
+                    for name, val in add_filters.items()]
+
+    # Assemble the request
+    params = {'columns': 'COUNT_BIG(*)',
+              'filters': filters,
+              'removenullcolumns': True}
+
+    # Just get the counts
+    if return_data:
+        params['columns'] = '*'
+
+    # Add requests
+    if isinstance(add_requests, dict):
+        params.update(add_requests)
+
+    response = Mast.service_request_async(service, params)
+    result = response[0].json()
+
+    # Return all the data
+    if return_data:
+        return result
+
+    # Or just the counts
+    else:
+        return result['data'][0]['Column1']
+
+
+def instrument_keywords(instrument, caom=False):
+    """Get the keywords for a given instrument service
+
+    Parameters
+    ----------
+    instrument: str
+        The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS']
+    caom: bool
+        Query CAOM service
+
+    Returns
+    -------
+    pd.DataFrame
+        A DataFrame of the keywords
+    """
+    # Retrieve one dataset to get header keywords
+    sample = instrument_inventory(instrument, return_data=True, caom=caom,
+                                  add_requests={'pagesize': 1, 'page': 1})
+    data = [[i['name'], i['type']] for i in sample['fields']]
+    keywords = pd.DataFrame(data, columns=('keyword', 'dtype'))
+
+    return keywords
+
+
+def jwst_inventory(instruments=JWST_INSTRUMENTS,
+                   dataproducts=['image', 'spectrum', 'cube'],
+                   caom=False, plot=False):
+    """Gather a full inventory of all JWST data in each instrument
+    service by instrument/dtype
+
+    Parameters
+    ----------
+    instruments: sequence
+        The list of instruments to count
+    dataproducts: sequence
+        The types of dataproducts to count
+    caom: bool
+        Query CAOM service
+    plot: bool
+        Return a pie chart of the data
+
+    Returns
+    -------
+    astropy.table.table.Table
+        The table of record counts for each instrument and mode
+    """
+    # Iterate through instruments
+    inventory, keywords = [], {}
+    for instrument in instruments:
+        ins = [instrument]
+        for dp in dataproducts:
+            count = instrument_inventory(instrument, dataproduct=dp, caom=caom)
+            ins.append(count)
+
+        # Get the total
+        ins.append(sum(ins[-3:]))
+
+        # Add it to the list
+        inventory.append(ins)
+
+        # Add the keywords to the dict
+        keywords[instrument] = instrument_keywords(instrument, caom=caom)
+
+    # Make the table
+    all_cols = ['instrument']+dataproducts+['total']
+    table = pd.DataFrame(inventory, columns=all_cols)
+
+    # Melt the table
+    table = pd.melt(table, id_vars=['instrument'],
+                    value_vars=dataproducts,
+                    value_name='files', var_name='dataproduct')
+
+    # Plot it
+    if plot:
+
+        # Make the plot
+        plt = Donut(table, label=['instrument', 'dataproduct'], values='files',
+                    text_font_size='12pt', hover_text='files',
+                    name="JWST Inventory", plot_width=600, plot_height=600)
+
+        # Save the plot
+        if caom:
+            output_filename = 'database_monitor_caom.html'
+        else:
+            output_filename = 'database_monitor_jwst.html'
+        outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename)
+        output_file(outfile)
+        save(plt)
+        set_permissions(outfile, verbose=False)
+
+    return table, keywords

jwql/tests/test_dbmonitor.py

@@ -0,0 +1,65 @@
+#! /usr/bin/env python
+
+"""Tests for the dbmonitor module.
+
+Authors
+-------
+
+    Joe Filippazzo
+
+Use
+---
+
+    These tests can be run via the command line (omit the ``-s`` to
+    suppress verbose output to stdout):
+    ::
+
+        pytest -s test_dbmonitor.py
+"""
+
+from ..dbmonitor import dbmonitor as db
+from ..utils.utils import JWST_INSTRUMENTS
+
+
+def test_caom_instrument_keywords():
+    """Test to see that the CAOM keywords are the same for all
+    instruments"""
+    kw = []
+    for ins in JWST_INSTRUMENTS:
+        kw.append(db.instrument_keywords(ins, caom=True)['keyword'].tolist())
+
+    assert kw[0] == kw[1] == kw[2] == kw[3] == kw[4]
+
+
+def test_filtered_instrument_keywords():
+    """Test to see that the instrument specific service keywords are
+    different for all instruments"""
+    kw = []
+    for ins in JWST_INSTRUMENTS:
+        kw.append(db.instrument_keywords(ins, caom=False)['keyword'].tolist())
+
+    assert kw[0] != kw[1] != kw[2] != kw[3] != kw[4]
+
+
+def test_instrument_inventory_filtering():
+    """Test to see that the instrument inventory can be filtered"""
+    filt = 'GR150R'
+    data = db.instrument_inventory('niriss',
+                                   add_filters={'filter': filt},
+                                   return_data=True)
+
+    filters = [row['filter'] for row in data['data']]
+
+    assert all([i == filt for i in filters])
+
+
+def test_instrument_dataproduct_filtering():
+    """Test to see that the instrument inventory can be filtered
+    by data product"""
+    dp = 'spectrum'
+    data = db.instrument_inventory('nirspec', dataproduct=dp, caom=True,
+                                   return_data=True)
+
+    dps = [row['dataproduct_type'] for row in data['data']]
+
+    assert all([i == dp for i in dps])

jwql/utils/utils.py

@@ -27,6 +27,9 @@
 
 __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
 
+JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS']
+JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY',
+                     'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL']
 
 def get_config():
     """Return a dictionary that holds the contents of the ``jwql``

setup.py

@@ -7,7 +7,7 @@
 AUTHORS = 'Matthew Bourque, Sara Ogaz, Joe Filippazzo, Bryan Hilbert, Misty Cracraft, Graham Kanarek'
 AUTHORS += 'Johannes Sahlmann, Lauren Chambers, Catherine Martlin'
 
-REQUIRES = ['astropy', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy']
+REQUIRES = ['astropy', 'astroquery', 'bokeh', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy']
 
 setup(
     name = 'jwql',