diff --git a/README.md b/README.md index bd56169e7..eb374eb11 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Matthew Bourque (INS) - Lauren Chambers (INS) - Misty Cracraft (INS) -- Joseph Filippazo (INS) +- Joe Filippazzo (INS) - Bryan Hilbert (INS) - Graham Kanarek (INS) - Catherine Martlin (INS) diff --git a/environment.yml b/environment.yml index ff3c2a99a..3fa1223b9 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,8 @@ channels: - http://ssb.stsci.edu/astroconda-dev dependencies: - astropy=3.0 +- astroquery=0.3.8 +- bokeh=0.12.5 - django=1.11.8 - jwst=0.7.8rc9 - matplotlib=2.1.1 diff --git a/jwql/__init__.py b/jwql/__init__.py index e69de29bb..6d9900a73 100644 --- a/jwql/__init__.py +++ b/jwql/__init__.py @@ -0,0 +1 @@ +from . import dbmonitor \ No newline at end of file diff --git a/jwql/dbmonitor/__init__.py b/jwql/dbmonitor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py new file mode 100644 index 000000000..5ed0a9654 --- /dev/null +++ b/jwql/dbmonitor/dbmonitor.py @@ -0,0 +1,204 @@ +"""This module is home to a suite of MAST queries that gather bulk properties +of available JWST data for JWQL + +Authors +------- + + Joe Filippazzo + +Use +--- + + To get an inventory of all JWST files do: + :: + + from jwql.dbmonitor import dbmonitor + inventory, keywords = dbmonitor.jwst_inventory() +""" + +import os + +from astroquery.mast import Mast +from bokeh.charts import Donut, save, output_file +import pandas as pd + +from ..permissions.permissions import set_permissions +from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS + + +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters=None, add_requests=None, + caom=False, return_data=False): + """Get the counts for a given instrument and data product + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + add_filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. add_filters = {'filter':'GR150R'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} + caom: bool + Query CAOM service + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int, dict + The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` + """ + filters = [] + + # Make sure the dataproduct is a list + if isinstance(dataproduct, str): + dataproduct = [dataproduct] + + # Make sure the instrument is supported + if instrument.lower() not in [ins.lower() for ins in JWST_INSTRUMENTS]: + raise TypeError('Supported instruments include:', JWST_INSTRUMENTS) + + # CAOM service + if caom: + + # Declare the service + service = 'Mast.Caom.Filtered' + + # Set the filters + filters += [{'paramName': 'obs_collection', 'values': ['JWST']}, + {'paramName': 'instrument_name', 'values': [instrument]}, + {'paramName': 'dataproduct_type', 'values': dataproduct}] + + # Instruent filtered service + else: + + # Declare the service + service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) + + # Include additonal filters + if isinstance(add_filters, dict): + filters += [{"paramName": name, "values": [val]} + for name, val in add_filters.items()] + + # Assemble the request + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns': True} + + # Just get the counts + if return_data: + params['columns'] = '*' + + # Add requests + if isinstance(add_requests, dict): + params.update(add_requests) + + response = Mast.service_request_async(service, params) + result = response[0].json() + + # Return all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + + +def instrument_keywords(instrument, caom=False): + """Get the keywords for a given instrument service + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + caom: bool + Query CAOM service + + Returns + ------- + pd.DataFrame + A DataFrame of the keywords + """ + # Retrieve one dataset to get header keywords + sample = instrument_inventory(instrument, return_data=True, caom=caom, + add_requests={'pagesize': 1, 'page': 1}) + data = [[i['name'], i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + return keywords + + +def jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + caom=False, plot=False): + """Gather a full inventory of all JWST data in each instrument + service by instrument/dtype + + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + caom: bool + Query CAOM service + plot: bool + Return a pie chart of the data + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Iterate through instruments + inventory, keywords = [], {} + for instrument in instruments: + ins = [instrument] + for dp in dataproducts: + count = instrument_inventory(instrument, dataproduct=dp, caom=caom) + ins.append(count) + + # Get the total + ins.append(sum(ins[-3:])) + + # Add it to the list + inventory.append(ins) + + # Add the keywords to the dict + keywords[instrument] = instrument_keywords(instrument, caom=caom) + + # Make the table + all_cols = ['instrument']+dataproducts+['total'] + table = pd.DataFrame(inventory, columns=all_cols) + + # Melt the table + table = pd.melt(table, id_vars=['instrument'], + value_vars=dataproducts, + value_name='files', var_name='dataproduct') + + # Plot it + if plot: + + # Make the plot + plt = Donut(table, label=['instrument', 'dataproduct'], values='files', + text_font_size='12pt', hover_text='files', + name="JWST Inventory", plot_width=600, plot_height=600) + + # Save the plot + if caom: + output_filename = 'database_monitor_caom.html' + else: + output_filename = 'database_monitor_jwst.html' + outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename) + output_file(outfile) + save(plt) + set_permissions(outfile, verbose=False) + + return table, keywords diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py new file mode 100755 index 000000000..516b0d118 --- /dev/null +++ b/jwql/tests/test_dbmonitor.py @@ -0,0 +1,65 @@ +#! /usr/bin/env python + +"""Tests for the dbmonitor module. + +Authors +------- + + Joe Filippazzo + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s test_dbmonitor.py +""" + +from ..dbmonitor import dbmonitor as db +from ..utils.utils import JWST_INSTRUMENTS + + +def test_caom_instrument_keywords(): + """Test to see that the CAOM keywords are the same for all + instruments""" + kw = [] + for ins in JWST_INSTRUMENTS: + kw.append(db.instrument_keywords(ins, caom=True)['keyword'].tolist()) + + assert kw[0] == kw[1] == kw[2] == kw[3] == kw[4] + + +def test_filtered_instrument_keywords(): + """Test to see that the instrument specific service keywords are + different for all instruments""" + kw = [] + for ins in JWST_INSTRUMENTS: + kw.append(db.instrument_keywords(ins, caom=False)['keyword'].tolist()) + + assert kw[0] != kw[1] != kw[2] != kw[3] != kw[4] + + +def test_instrument_inventory_filtering(): + """Test to see that the instrument inventory can be filtered""" + filt = 'GR150R' + data = db.instrument_inventory('niriss', + add_filters={'filter': filt}, + return_data=True) + + filters = [row['filter'] for row in data['data']] + + assert all([i == filt for i in filters]) + + +def test_instrument_dataproduct_filtering(): + """Test to see that the instrument inventory can be filtered + by data product""" + dp = 'spectrum' + data = db.instrument_inventory('nirspec', dataproduct=dp, caom=True, + return_data=True) + + dps = [row['dataproduct_type'] for row in data['data']] + + assert all([i == dp for i in dps]) diff --git a/jwql/tests/test_permissions.py b/jwql/tests/test_permissions.py old mode 100644 new mode 100755 diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 20fe14f4d..2249951f3 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -27,6 +27,9 @@ __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) +JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', + 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] def get_config(): """Return a dictionary that holds the contents of the ``jwql`` diff --git a/setup.py b/setup.py index af40a2fea..245dac2de 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ AUTHORS = 'Matthew Bourque, Sara Ogaz, Joe Filippazzo, Bryan Hilbert, Misty Cracraft, Graham Kanarek' AUTHORS += 'Johannes Sahlmann, Lauren Chambers, Catherine Martlin' -REQUIRES = ['astropy', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] +REQUIRES = ['astropy', 'astroquery', 'bokeh', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] setup( name = 'jwql',