From 72a877ddd51142cdabca9e3d9003a166e52fcdda Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 11:01:08 -0400 Subject: [PATCH 01/24] Added functions to query MAST for record counts and data. --- jwql/__init__.py | 1 + jwql/dbmonitor/__init__.py | 0 jwql/dbmonitor/dbmonitor.py | 149 ++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 jwql/dbmonitor/__init__.py create mode 100644 jwql/dbmonitor/dbmonitor.py diff --git a/jwql/__init__.py b/jwql/__init__.py index e69de29bb..6d9900a73 100644 --- a/jwql/__init__.py +++ b/jwql/__init__.py @@ -0,0 +1 @@ +from . import dbmonitor \ No newline at end of file diff --git a/jwql/dbmonitor/__init__.py b/jwql/dbmonitor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py new file mode 100644 index 000000000..b613e94b5 --- /dev/null +++ b/jwql/dbmonitor/dbmonitor.py @@ -0,0 +1,149 @@ +"""This module is home to a suite of MAST queries that gather bulk properties of available JWST data for JWQL + +Authors +------- + + Joe Filippazzo + +Use +--- + + This module can be imported as such: + + >>> from jwql.dbmonitor import dbmonitor +""" +import sys +import os +import time +import re +import json +from urllib.parse import quote as urlencode +from urllib.request import urlretrieve +import http.client as httplib +from astroquery.mast import Mast, Observations + +JWST_INSTRUMENTS = ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] + +def listMissions(): + """Small function to test sevice_request and make sure JWST is in the list of services + + Returns + ------- + astropy.table.table.Table + The table of all supported services + """ + service = 'Mast.Missions.List' + params = {} + request = Mast.service_request(service, params) + + return request + +def mastQuery(request): + """A small MAST API wrapper + + Parameters + ---------- + request: dict + The dictionary of 'service', 'format' and 'params' for the request + + Returns + ------- + dict + A dictionary of the 'data', 'fields', 'paging', and 'status' of the request + """ + # Define the server + server='mast.stsci.edu' + + # Grab Python Version + version = ".".join(map(str, sys.version_info[:3])) + + # Create Http Header Variables + headers = {"Content-type": "application/x-www-form-urlencoded", + "Accept": "text/plain", + "User-agent":"python-requests/"+version} + + # Encoding the request as a json string + requestString = json.dumps(request) + requestString = urlencode(requestString) + + # opening the https connection + conn = httplib.HTTPSConnection(server) + + # Making the query + conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) + + # Getting the response + resp = conn.getresponse() + head = resp.getheaders() + content = resp.read().decode('utf-8') + + # Close the https connection + conn.close() + + return head,content + +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data=False): + """Get the counts for a given instrument and data product + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int + The number of database records that satisfy the search criteria + """ + # Make sure the dataproduct is a list + if isinstance(dataproduct,str): + dataproduct = [dataproduct] + + # Assemble the request + request = {'service':'Mast.Caom.Filtered', + 'format':'json', + 'params':{ + 'columns':'COUNT_BIG(*)', + 'filters':[ + {'paramName':'obs_collection', 'values':['JWST']}, + {'paramName':'instrument_name', 'values':[instrument]}, + {'paramName':'dataproduct_type', 'values':dataproduct} + ]}} + + # Just get the counts + if return_data: + request['params']['columns'] = '*' + + # Perform the query + headers, outString = mastQuery(request) + result = json.loads(outString) + + # REturn all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + +def jwst_inventory(): + """Gather a full inventory of all JWST data on MAST binned by instrument and mode + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Make master table + inventory = at.Table(names=('instrument','dataproduct','count')) + + # Iterate through instruments + for instrument in JWST_INSTRUMENTS: + for dataproduct in ['cube','image','spectrum']: + count = instrument_inventory(instrument, dataproduct=dataproduct) + \ No newline at end of file From b52a0c5337f248a068447ffd39a434598b72e3c7 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 11:40:11 -0400 Subject: [PATCH 02/24] Generates table of result counts --- jwql/dbmonitor/dbmonitor.py | 41 +++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index b613e94b5..c08420211 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -17,10 +17,12 @@ import time import re import json +import numpy as np from urllib.parse import quote as urlencode from urllib.request import urlretrieve import http.client as httplib from astroquery.mast import Mast, Observations +import astropy.table as at JWST_INSTRUMENTS = ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] @@ -83,7 +85,7 @@ def mastQuery(request): return head,content -def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data=False): +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters={}, return_data=False): """Get the counts for a given instrument and data product Parameters @@ -92,6 +94,9 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data= The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search + filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument of the request + e.g. filters = {'target_classification':'moving'} return_data: bool Return the actual data instead of counts only @@ -104,16 +109,21 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data= if isinstance(dataproduct,str): dataproduct = [dataproduct] + # Set default filters + filters = [{'paramName':'obs_collection', 'values':['JWST']}, + {'paramName':'instrument_name', 'values':[instrument]}, + {'paramName':'dataproduct_type', 'values':dataproduct}] + + # Include additonal filters + for k,v in add_filters.items(): + filters.append({'paramName':k, 'values':[v] if isinstance(v,str) else v}) + # Assemble the request request = {'service':'Mast.Caom.Filtered', 'format':'json', 'params':{ 'columns':'COUNT_BIG(*)', - 'filters':[ - {'paramName':'obs_collection', 'values':['JWST']}, - {'paramName':'instrument_name', 'values':[instrument]}, - {'paramName':'dataproduct_type', 'values':dataproduct} - ]}} + 'filters':filters}} # Just get the counts if return_data: @@ -131,19 +141,28 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data= else: return result['data'][0]['Column1'] -def jwst_inventory(): +def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image','spectrum','cube']): """Gather a full inventory of all JWST data on MAST binned by instrument and mode + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + Returns ------- astropy.table.table.Table The table of record counts for each instrument and mode """ # Make master table - inventory = at.Table(names=('instrument','dataproduct','count')) + inventory = at.Table(names=('instrument','dataproduct','count'), dtype=('S8','S12',int)) # Iterate through instruments - for instrument in JWST_INSTRUMENTS: - for dataproduct in ['cube','image','spectrum']: + for instrument in instruments: + for dataproduct in dataproducts: count = instrument_inventory(instrument, dataproduct=dataproduct) - \ No newline at end of file + inventory.add_row([instrument, dataproduct, count]) + + return inventory \ No newline at end of file From 9a363a5ecd1b14012c6f9817a9555f4880a25b37 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 11:53:58 -0400 Subject: [PATCH 03/24] PEP8 compliance. --- jwql/dbmonitor/dbmonitor.py | 137 +++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 64 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index c08420211..89e25985a 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -1,4 +1,5 @@ -"""This module is home to a suite of MAST queries that gather bulk properties of available JWST data for JWQL +"""This module is home to a suite of MAST queries that gather bulk properties +of available JWST data for JWQL Authors ------- @@ -8,28 +9,28 @@ Use --- - This module can be imported as such: + To get an inventory of all JWST files do: >>> from jwql.dbmonitor import dbmonitor + >>> results = dbmonitor.jwst_inventory() + """ import sys -import os -import time -import re import json -import numpy as np from urllib.parse import quote as urlencode -from urllib.request import urlretrieve -import http.client as httplib -from astroquery.mast import Mast, Observations +import http.client as httplib +from astroquery.mast import Mast import astropy.table as at -JWST_INSTRUMENTS = ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] -JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] +JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', + 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] + def listMissions(): - """Small function to test sevice_request and make sure JWST is in the list of services - + """Small function to test sevice_request and make sure JWST is + in the list of services + Returns ------- astropy.table.table.Table @@ -38,56 +39,60 @@ def listMissions(): service = 'Mast.Missions.List' params = {} request = Mast.service_request(service, params) - + return request - + + def mastQuery(request): """A small MAST API wrapper - + Parameters ---------- request: dict The dictionary of 'service', 'format' and 'params' for the request - + Returns ------- dict - A dictionary of the 'data', 'fields', 'paging', and 'status' of the request + A dictionary of the 'data', 'fields', 'paging', and 'status' + of the request """ # Define the server - server='mast.stsci.edu' - - # Grab Python Version + server = 'mast.stsci.edu' + + # Grab Python Version version = ".".join(map(str, sys.version_info[:3])) - + # Create Http Header Variables headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain", - "User-agent":"python-requests/"+version} - + "User-agent": "python-requests/"+version} + # Encoding the request as a json string requestString = json.dumps(request) requestString = urlencode(requestString) - + # opening the https connection conn = httplib.HTTPSConnection(server) - + # Making the query conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) - + # Getting the response resp = conn.getresponse() head = resp.getheaders() content = resp.read().decode('utf-8') - + # Close the https connection conn.close() - - return head,content - -def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters={}, return_data=False): + + return head, content + + +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters={}, return_data=False): """Get the counts for a given instrument and data product - + Parameters ---------- instrument: str @@ -95,74 +100,78 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters= dataproduct: sequence, str The type of data product to search filters: dict - The ('paramName':'values') pairs to include in the 'filters' argument of the request - e.g. filters = {'target_classification':'moving'} + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. filters = {'target_classification':'moving'} return_data: bool Return the actual data instead of counts only - + Returns ------- int The number of database records that satisfy the search criteria """ # Make sure the dataproduct is a list - if isinstance(dataproduct,str): + if isinstance(dataproduct, str): dataproduct = [dataproduct] - + # Set default filters - filters = [{'paramName':'obs_collection', 'values':['JWST']}, - {'paramName':'instrument_name', 'values':[instrument]}, - {'paramName':'dataproduct_type', 'values':dataproduct}] - + filters = [{'paramName': 'obs_collection', 'values': ['JWST']}, + {'paramName': 'instrument_name', 'values': [instrument]}, + {'paramName': 'dataproduct_type', 'values': dataproduct}] + # Include additonal filters - for k,v in add_filters.items(): - filters.append({'paramName':k, 'values':[v] if isinstance(v,str) else v}) - + for k, v in add_filters.items(): + filters.append({'paramName': k, 'values': [v] + if isinstance(v, str) else v}) + # Assemble the request - request = {'service':'Mast.Caom.Filtered', - 'format':'json', - 'params':{ - 'columns':'COUNT_BIG(*)', - 'filters':filters}} - + request = {'service': 'Mast.Caom.Filtered', + 'format': 'json', + 'params': { + 'columns': 'COUNT_BIG(*)', + 'filters': filters}} + # Just get the counts if return_data: request['params']['columns'] = '*' - + # Perform the query headers, outString = mastQuery(request) result = json.loads(outString) - - # REturn all the data + + # Return all the data if return_data: return result - + # Or just the counts else: return result['data'][0]['Column1'] - -def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image','spectrum','cube']): - """Gather a full inventory of all JWST data on MAST binned by instrument and mode - + + +def jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube']): + """Gather a full inventory of all JWST data on MAST by instrument/dtype + Parameters ---------- instruments: sequence The list of instruments to count dataproducts: sequence The types of dataproducts to count - + Returns ------- astropy.table.table.Table The table of record counts for each instrument and mode """ # Make master table - inventory = at.Table(names=('instrument','dataproduct','count'), dtype=('S8','S12',int)) - + inventory = at.Table(names=('instrument', 'dataproduct', 'count'), + dtype=('S8', 'S12', int)) + # Iterate through instruments for instrument in instruments: for dataproduct in dataproducts: count = instrument_inventory(instrument, dataproduct=dataproduct) inventory.add_row([instrument, dataproduct, count]) - - return inventory \ No newline at end of file + + return inventory From 39a945d2bacbf2d60f6b90be392895d1c33b8275 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 12:06:55 -0400 Subject: [PATCH 04/24] Added total db counts --- jwql/dbmonitor/dbmonitor.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 89e25985a..34a3dadbd 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -95,7 +95,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, Parameters ---------- - instrument: str + instrument: sequence, str The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search @@ -114,9 +114,13 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(dataproduct, str): dataproduct = [dataproduct] + # Make sure the instrument is a list + if isinstance(instrument, str): + instrument = [instrument] + # Set default filters filters = [{'paramName': 'obs_collection', 'values': ['JWST']}, - {'paramName': 'instrument_name', 'values': [instrument]}, + {'paramName': 'instrument_name', 'values': instrument}, {'paramName': 'dataproduct_type', 'values': dataproduct}] # Include additonal filters @@ -174,4 +178,12 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, count = instrument_inventory(instrument, dataproduct=dataproduct) inventory.add_row([instrument, dataproduct, count]) + # Get number of files for all datatypes for this instrument + count = instrument_inventory(instrument, dataproduct=dataproducts) + inventory.add_row([instrument, '*', count]) + + # Get number of files for all datatypes for this instrument + count = instrument_inventory(instruments, dataproduct=dataproducts) + inventory.add_row(['*', '*', count]) + return inventory From 7ad6e741f0d4496890dc26214db9a236ed9b2db2 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 12:09:52 -0400 Subject: [PATCH 05/24] Added total db counts comment --- jwql/dbmonitor/dbmonitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 34a3dadbd..f53cfd65b 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -182,7 +182,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, count = instrument_inventory(instrument, dataproduct=dataproducts) inventory.add_row([instrument, '*', count]) - # Get number of files for all datatypes for this instrument + # Get total number of files count = instrument_inventory(instruments, dataproduct=dataproducts) inventory.add_row(['*', '*', count]) From 61f16f9a9b2c7d5229570343ac386369944d5776 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 12:57:20 -0400 Subject: [PATCH 06/24] Added keyword count to jwst_inventory() --- jwql/dbmonitor/dbmonitor.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index f53cfd65b..5f337734d 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -12,7 +12,7 @@ To get an inventory of all JWST files do: >>> from jwql.dbmonitor import dbmonitor - >>> results = dbmonitor.jwst_inventory() + >>> inventory, keywords = dbmonitor.jwst_inventory() """ import sys @@ -90,7 +90,7 @@ def mastQuery(request): def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters={}, return_data=False): + add_filters={}, add_requests={}, return_data=False): """Get the counts for a given instrument and data product Parameters @@ -99,9 +99,12 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search - filters: dict + add_filters: dict The ('paramName':'values') pairs to include in the 'filters' argument - of the request e.g. filters = {'target_classification':'moving'} + of the request e.g. add_filters = {'target_classification':'moving'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} return_data: bool Return the actual data instead of counts only @@ -139,6 +142,10 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, if return_data: request['params']['columns'] = '*' + # Add requests + if add_requests: + request.update(add_requests) + # Perform the query headers, outString = mastQuery(request) result = json.loads(outString) @@ -186,4 +193,12 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, count = instrument_inventory(instruments, dataproduct=dataproducts) inventory.add_row(['*', '*', count]) - return inventory + # Retrieve one dataset to get header keywords + sample = instrument_inventory(instruments, dataproduct=dataproducts, + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + names = [i['name'] for i in sample['fields']] + types = [i['type'] for i in sample['fields']] + keywords = at.Table([names, types], names=('keyword', 'dtype')) + + return inventory, keywords From 8f914ab530ff4318aba2398a7c1da26ed28d73cd Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 25 Apr 2018 12:27:57 -0400 Subject: [PATCH 07/24] Moved global lists to utils.py and removed POST calls. --- jwql/dbmonitor/dbmonitor.py | 160 ++++++++++++++++++------------------ jwql/utils/utils.py | 4 + 2 files changed, 82 insertions(+), 82 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 5f337734d..b39130e8c 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -15,78 +15,16 @@ >>> inventory, keywords = dbmonitor.jwst_inventory() """ -import sys + +import http.client as httplib import json +import sys from urllib.parse import quote as urlencode -import http.client as httplib + from astroquery.mast import Mast import astropy.table as at -JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] -JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', - 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] - - -def listMissions(): - """Small function to test sevice_request and make sure JWST is - in the list of services - - Returns - ------- - astropy.table.table.Table - The table of all supported services - """ - service = 'Mast.Missions.List' - params = {} - request = Mast.service_request(service, params) - - return request - - -def mastQuery(request): - """A small MAST API wrapper - - Parameters - ---------- - request: dict - The dictionary of 'service', 'format' and 'params' for the request - - Returns - ------- - dict - A dictionary of the 'data', 'fields', 'paging', and 'status' - of the request - """ - # Define the server - server = 'mast.stsci.edu' - - # Grab Python Version - version = ".".join(map(str, sys.version_info[:3])) - - # Create Http Header Variables - headers = {"Content-type": "application/x-www-form-urlencoded", - "Accept": "text/plain", - "User-agent": "python-requests/"+version} - - # Encoding the request as a json string - requestString = json.dumps(request) - requestString = urlencode(requestString) - - # opening the https connection - conn = httplib.HTTPSConnection(server) - - # Making the query - conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) - - # Getting the response - resp = conn.getresponse() - head = resp.getheaders() - content = resp.read().decode('utf-8') - - # Close the https connection - conn.close() - - return head, content +from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, @@ -110,8 +48,9 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, Returns ------- - int + int, dict The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` """ # Make sure the dataproduct is a list if isinstance(dataproduct, str): @@ -127,28 +66,23 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, {'paramName': 'dataproduct_type', 'values': dataproduct}] # Include additonal filters - for k, v in add_filters.items(): - filters.append({'paramName': k, 'values': [v] - if isinstance(v, str) else v}) + for param, value in add_filters.items(): + filters.append({'paramName': param, 'values': [value] + if isinstance(value, str) else value}) # Assemble the request - request = {'service': 'Mast.Caom.Filtered', - 'format': 'json', - 'params': { - 'columns': 'COUNT_BIG(*)', - 'filters': filters}} + params = {'columns': 'COUNT_BIG(*)', 'filters': filters,} # Just get the counts if return_data: - request['params']['columns'] = '*' + params['columns'] = '*' # Add requests if add_requests: - request.update(add_requests) - - # Perform the query - headers, outString = mastQuery(request) - result = json.loads(outString) + params.update(add_requests) + + response = Mast.service_request_async('Mast.Caom.Filtered', params) + result = response[0].json() # Return all the data if return_data: @@ -202,3 +136,65 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, keywords = at.Table([names, types], names=('keyword', 'dtype')) return inventory, keywords + + +def listMissions(): + """Small function to test sevice_request and make sure JWST is + in the list of services + + Returns + ------- + astropy.table.table.Table + The table of all supported services + """ + service = 'Mast.Missions.List' + params = {} + request = Mast.service_request(service, params) + + return request + + +def mastQuery(request): + """A small MAST API wrapper + + Parameters + ---------- + request: dict + The dictionary of 'service', 'format' and 'params' for the request + + Returns + ------- + dict + A dictionary of the 'data', 'fields', 'paging', and 'status' + of the request + """ + # Define the server + server = 'mast.stsci.edu' + + # Grab Python Version + version = ".".join(map(str, sys.version_info[:3])) + + # Create Http Header Variables + headers = {"Content-type": "application/x-www-form-urlencoded", + "Accept": "text/plain", + "User-agent": "python-requests/"+version} + + # Encoding the request as a json string + requestString = json.dumps(request) + requestString = urlencode(requestString) + + # opening the https connection + conn = httplib.HTTPSConnection(server) + + # Making the query + conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) + + # Getting the response + resp = conn.getresponse() + head = resp.getheaders() + content = resp.read().decode('utf-8') + + # Close the https connection + conn.close() + + return head, content diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index a0f122afc..79b82c53e 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -17,6 +17,10 @@ import json +JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', + 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] + def get_config(): """Return a dictionary that holds the contents of the jwql config file. From 087fde480c7bc22d03da1f4511aab39d6431d1d8 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 25 Apr 2018 14:18:33 -0400 Subject: [PATCH 08/24] Added CAOM functions and non-CAOM functions. --- jwql/dbmonitor/dbmonitor.py | 117 ++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 4 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index b39130e8c..28196d327 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -31,6 +31,70 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters={}, add_requests={}, return_data=False): """Get the counts for a given instrument and data product + Parameters + ---------- + instrument: sequence, str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + add_filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. add_filters = {'target_classification':'moving'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int, dict + The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` + """ + # Declare the service + service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) + + # Make sure the dataproduct is a list + if isinstance(dataproduct, str): + dataproduct = [dataproduct] + + # Make sure the instrument is a list + if isinstance(instrument, str): + instrument = [instrument] + + # Set default filters + filters = [] + + # Assemble the request + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns':True,} + + # Just get the counts + if return_data: + params['columns'] = '*' + + # Add requests + if add_requests: + params.update(add_requests) + + response = Mast.service_request_async(service, params) + result = response[0].json() + + # Return all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + + +def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters={}, add_requests={}, return_data=False): + """Get the counts for a given instrument and data product + Parameters ---------- instrument: sequence, str @@ -116,19 +180,64 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Iterate through instruments for instrument in instruments: for dataproduct in dataproducts: - count = instrument_inventory(instrument, dataproduct=dataproduct) + count = instrument_inventory_caom(instrument, dataproduct=dataproduct) + inventory.add_row([instrument, dataproduct, count]) + + # Get number of files for all datatypes for this instrument + count = instrument_inventory_caom(instrument, dataproduct=dataproducts) + inventory.add_row([instrument, '*', count]) + + # Get total number of files + count = instrument_inventory_caom(instruments, dataproduct=dataproducts) + inventory.add_row(['*', '*', count]) + + # Retrieve one dataset to get header keywords + sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + names = [i['name'] for i in sample['fields']] + types = [i['type'] for i in sample['fields']] + keywords = at.Table([names, types], names=('keyword', 'dtype')) + + return inventory, keywords + + +def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube']): + """Gather a full inventory of all JWST data on MAST by instrument/dtype + + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Make master table + inventory = at.Table(names=('instrument', 'dataproduct', 'count'), + dtype=('S8', 'S12', int)) + + # Iterate through instruments + for instrument in instruments: + for dataproduct in dataproducts: + count = instrument_inventory_caom(instrument, dataproduct=dataproduct) inventory.add_row([instrument, dataproduct, count]) # Get number of files for all datatypes for this instrument - count = instrument_inventory(instrument, dataproduct=dataproducts) + count = instrument_inventory_caom(instrument, dataproduct=dataproducts) inventory.add_row([instrument, '*', count]) # Get total number of files - count = instrument_inventory(instruments, dataproduct=dataproducts) + count = instrument_inventory_caom(instruments, dataproduct=dataproducts) inventory.add_row(['*', '*', count]) # Retrieve one dataset to get header keywords - sample = instrument_inventory(instruments, dataproduct=dataproducts, + sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, add_requests={'pagesize': 1, 'page': 1}, return_data=True) names = [i['name'] for i in sample['fields']] From 72d8e7b2c4fc539a0a048b1c21adedde44acc6fd Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 25 Apr 2018 16:17:55 -0400 Subject: [PATCH 09/24] Added donut plot. --- jwql/dbmonitor/dbmonitor.py | 327 ++++++++++++++++++++++++++++++++++++ jwql/utils/utils.py | 3 + 2 files changed, 330 insertions(+) create mode 100644 jwql/dbmonitor/dbmonitor.py diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py new file mode 100644 index 000000000..69a990374 --- /dev/null +++ b/jwql/dbmonitor/dbmonitor.py @@ -0,0 +1,327 @@ +"""This module is home to a suite of MAST queries that gather bulk properties +of available JWST data for JWQL + +Authors +------- + + Joe Filippazzo + +Use +--- + + To get an inventory of all JWST files do: + + >>> from jwql.dbmonitor import dbmonitor + >>> inventory, keywords = dbmonitor.jwst_inventory() + +""" + +import http.client as httplib +import json +import sys +from urllib.parse import quote as urlencode + +from astroquery.mast import Mast +from bokeh.charts import Donut, show, output_file +import pandas as pd + +from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS + + +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters={}, add_requests={}, return_data=False): + """Get the counts for a given instrument and data product + + Parameters + ---------- + instrument: sequence, str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + add_filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. add_filters = {'target_classification':'moving'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int, dict + The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` + """ + # Declare the service + service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) + + # Make sure the dataproduct is a list + if isinstance(dataproduct, str): + dataproduct = [dataproduct] + + # Make sure the instrument is a list + if isinstance(instrument, str): + instrument = [instrument] + + # Set default filters + filters = [] + + # Assemble the request + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns':True,} + + # Just get the counts + if return_data: + params['columns'] = '*' + + # Add requests + if add_requests: + params.update(add_requests) + + response = Mast.service_request_async(service, params) + result = response[0].json() + + # Return all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + + +def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters={}, add_requests={}, return_data=False): + """Get the counts for a given instrument and data product + + Parameters + ---------- + instrument: sequence, str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + add_filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. add_filters = {'target_classification':'moving'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int, dict + The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` + """ + # Make sure the dataproduct is a list + if isinstance(dataproduct, str): + dataproduct = [dataproduct] + + # Make sure the instrument is a list + if isinstance(instrument, str): + instrument = [instrument] + + # Set default filters + filters = [{'paramName': 'obs_collection', 'values': ['JWST']}, + {'paramName': 'instrument_name', 'values': instrument}, + {'paramName': 'dataproduct_type', 'values': dataproduct}] + + # Include additonal filters + for param, value in add_filters.items(): + filters.append({'paramName': param, 'values': [value] + if isinstance(value, str) else value}) + + # Assemble the request + params = {'columns': 'COUNT_BIG(*)', 'filters': filters,} + + # Just get the counts + if return_data: + params['columns'] = '*' + + # Add requests + if add_requests: + params.update(add_requests) + + response = Mast.service_request_async('Mast.Caom.Filtered', params) + result = response[0].json() + + # Return all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + + +# def jwst_inventory(instruments=JWST_INSTRUMENTS, +# dataproducts=['image', 'spectrum', 'cube']): +# """Gather a full inventory of all JWST data on MAST by instrument/dtype +# +# Parameters +# ---------- +# instruments: sequence +# The list of instruments to count +# dataproducts: sequence +# The types of dataproducts to count +# +# Returns +# ------- +# astropy.table.table.Table +# The table of record counts for each instrument and mode +# """ +# # Make master table +# inventory = at.Table(names=('instrument', 'dataproduct', 'count'), +# dtype=('S8', 'S12', int)) +# +# # Iterate through instruments +# for instrument in instruments: +# for dataproduct in dataproducts: +# count = instrument_inventory_caom(instrument, dataproduct=dataproduct) +# inventory.add_row([instrument, dataproduct, count]) +# +# # Get number of files for all datatypes for this instrument +# count = instrument_inventory_caom(instrument, dataproduct=dataproducts) +# inventory.add_row([instrument, '*', count]) +# +# # Get total number of files +# count = instrument_inventory_caom(instruments, dataproduct=dataproducts) +# inventory.add_row(['*', '*', count]) +# +# # Retrieve one dataset to get header keywords +# sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, +# add_requests={'pagesize': 1, 'page': 1}, +# return_data=True) +# names = [i['name'] for i in sample['fields']] +# types = [i['type'] for i in sample['fields']] +# keywords = at.Table([names, types], names=('keyword', 'dtype')) +# +# return inventory, keywords + + +def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + plot=False): + """Gather a full inventory of all JWST data on MAST by instrument/dtype + + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + plot: bool + Return a pie chart of the data + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Iterate through instruments + inventory = [] + for instrument in instruments: + ins = [instrument] + for dp in dataproducts: + count = instrument_inventory_caom(instrument, dataproduct=dp) + ins.append(count) + + # Get the total + ins.append(sum(ins[-3:])) + + # Add it to the list + inventory.append(ins) + + # Make the table + table = pd.DataFrame(inventory, columns=['instrument']+dataproducts+['total']) + + # Melt the table + table = pd.melt(table, id_vars=['instrument'], + value_vars=dataproducts, + value_name='files', var_name='dataproduct') + + # Retrieve one dataset to get header keywords + sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + data = [[i['name'],i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + # Plot it + if plot: + + # Make the plot + plt = Donut(table, label=['instrument', 'dataproduct'], values='files', + text_font_size='12pt', hover_text='files', + name="JWST Inventory", plot_width=600, plot_height=600) + + show(plt) + + return table, keywords + + +def listMissions(): + """Small function to test sevice_request and make sure JWST is + in the list of services + + Returns + ------- + astropy.table.table.Table + The table of all supported services + """ + service = 'Mast.Missions.List' + params = {} + request = Mast.service_request(service, params) + + return request + + +def mastQuery(request): + """A small MAST API wrapper + + Parameters + ---------- + request: dict + The dictionary of 'service', 'format' and 'params' for the request + + Returns + ------- + dict + A dictionary of the 'data', 'fields', 'paging', and 'status' + of the request + """ + # Define the server + server = 'mast.stsci.edu' + + # Grab Python Version + version = ".".join(map(str, sys.version_info[:3])) + + # Create Http Header Variables + headers = {"Content-type": "application/x-www-form-urlencoded", + "Accept": "text/plain", + "User-agent": "python-requests/"+version} + + # Encoding the request as a json string + requestString = json.dumps(request) + requestString = urlencode(requestString) + + # opening the https connection + conn = httplib.HTTPSConnection(server) + + # Making the query + conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) + + # Getting the response + resp = conn.getresponse() + head = resp.getheaders() + content = resp.read().decode('utf-8') + + # Close the https connection + conn.close() + + return head, content + diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 024b2f7e1..f907612f2 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -27,6 +27,9 @@ __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) +JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', + 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] def get_config(): """Return a dictionary that holds the contents of the jwql config From e5f1282cf48a56770406afea98714ff3d52c00bd Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 2 May 2018 11:54:45 -0400 Subject: [PATCH 10/24] Tidying up the inventory functions. --- jwql/dbmonitor/dbmonitor.py | 110 +++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 46 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 69a990374..a06efd868 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -22,7 +22,7 @@ from urllib.parse import quote as urlencode from astroquery.mast import Mast -from bokeh.charts import Donut, show, output_file +from bkcharts import Donut, show, output_file import pandas as pd from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS @@ -70,7 +70,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, # Assemble the request params = {'columns': 'COUNT_BIG(*)', 'filters': filters, - 'removenullcolumns':True,} + 'removenullcolumns': True,} # Just get the counts if return_data: @@ -136,7 +136,9 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(value, str) else value}) # Assemble the request - params = {'columns': 'COUNT_BIG(*)', 'filters': filters,} + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns': True,} # Just get the counts if return_data: @@ -158,50 +160,66 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, return result['data'][0]['Column1'] -# def jwst_inventory(instruments=JWST_INSTRUMENTS, -# dataproducts=['image', 'spectrum', 'cube']): -# """Gather a full inventory of all JWST data on MAST by instrument/dtype -# -# Parameters -# ---------- -# instruments: sequence -# The list of instruments to count -# dataproducts: sequence -# The types of dataproducts to count -# -# Returns -# ------- -# astropy.table.table.Table -# The table of record counts for each instrument and mode -# """ -# # Make master table -# inventory = at.Table(names=('instrument', 'dataproduct', 'count'), -# dtype=('S8', 'S12', int)) -# -# # Iterate through instruments -# for instrument in instruments: -# for dataproduct in dataproducts: -# count = instrument_inventory_caom(instrument, dataproduct=dataproduct) -# inventory.add_row([instrument, dataproduct, count]) -# -# # Get number of files for all datatypes for this instrument -# count = instrument_inventory_caom(instrument, dataproduct=dataproducts) -# inventory.add_row([instrument, '*', count]) -# -# # Get total number of files -# count = instrument_inventory_caom(instruments, dataproduct=dataproducts) -# inventory.add_row(['*', '*', count]) -# -# # Retrieve one dataset to get header keywords -# sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, -# add_requests={'pagesize': 1, 'page': 1}, -# return_data=True) -# names = [i['name'] for i in sample['fields']] -# types = [i['type'] for i in sample['fields']] -# keywords = at.Table([names, types], names=('keyword', 'dtype')) -# -# return inventory, keywords +def jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + plot=False): + """Gather a full inventory of all JWST data on MAST by instrument/dtype + + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + plot: bool + Return a pie chart of the data + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Iterate through instruments + inventory = [] + for instrument in instruments: + ins = [instrument] + for dp in dataproducts: + count = instrument_inventory(instrument, dataproduct=dp) + ins.append(count) + + # Get the total + ins.append(sum(ins[-3:])) + + # Add it to the list + inventory.append(ins) + + # Make the table + table = pd.DataFrame(inventory, columns=['instrument']+dataproducts+['total']) + + # Melt the table + table = pd.melt(table, id_vars=['instrument'], + value_vars=dataproducts, + value_name='files', var_name='dataproduct') + + # Retrieve one dataset to get header keywords + sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + data = [[i['name'],i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + # Plot it + if plot: + + # Make the plot + plt = Donut(table, label=['instrument', 'dataproduct'], values='files', + text_font_size='12pt', hover_text='files', + name="JWST Inventory", plot_width=600, plot_height=600) + + show(plt) + return table, keywords + def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], From 4b6f6e8d276a6bfb000dae8560e4b05fe7ba9770 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 2 May 2018 12:08:56 -0400 Subject: [PATCH 11/24] PEP8 compliance. --- jwql/dbmonitor/dbmonitor.py | 66 +++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index a06efd868..576c1b66d 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -22,7 +22,7 @@ from urllib.parse import quote as urlencode from astroquery.mast import Mast -from bkcharts import Donut, show, output_file +from bkcharts import Donut, show import pandas as pd from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS @@ -55,7 +55,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, """ # Declare the service service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) - + # Make sure the dataproduct is a list if isinstance(dataproduct, str): dataproduct = [dataproduct] @@ -68,9 +68,9 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, filters = [] # Assemble the request - params = {'columns': 'COUNT_BIG(*)', - 'filters': filters, - 'removenullcolumns': True,} + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns': True} # Just get the counts if return_data: @@ -79,7 +79,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, # Add requests if add_requests: params.update(add_requests) - + response = Mast.service_request_async(service, params) result = response[0].json() @@ -93,7 +93,8 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters={}, add_requests={}, return_data=False): + add_filters={}, add_requests={}, + return_data=False): """Get the counts for a given instrument and data product Parameters @@ -136,9 +137,9 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(value, str) else value}) # Assemble the request - params = {'columns': 'COUNT_BIG(*)', + params = {'columns': 'COUNT_BIG(*)', 'filters': filters, - 'removenullcolumns': True,} + 'removenullcolumns': True} # Just get the counts if return_data: @@ -147,7 +148,7 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, # Add requests if add_requests: params.update(add_requests) - + response = Mast.service_request_async('Mast.Caom.Filtered', params) result = response[0].json() @@ -161,8 +162,8 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, def jwst_inventory(instruments=JWST_INSTRUMENTS, - dataproducts=['image', 'spectrum', 'cube'], - plot=False): + dataproducts=['image', 'spectrum', 'cube'], + plot=False): """Gather a full inventory of all JWST data on MAST by instrument/dtype Parameters @@ -186,26 +187,27 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, for dp in dataproducts: count = instrument_inventory(instrument, dataproduct=dp) ins.append(count) - + # Get the total ins.append(sum(ins[-3:])) - + # Add it to the list inventory.append(ins) - + # Make the table - table = pd.DataFrame(inventory, columns=['instrument']+dataproducts+['total']) - + all_cols = ['instrument']+dataproducts+['total'] + table = pd.DataFrame(inventory, columns=all_cols) + # Melt the table table = pd.melt(table, id_vars=['instrument'], value_vars=dataproducts, value_name='files', var_name='dataproduct') - + # Retrieve one dataset to get header keywords sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, - add_requests={'pagesize': 1, 'page': 1}, - return_data=True) - data = [[i['name'],i['type']] for i in sample['fields']] + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + data = [[i['name'], i['type']] for i in sample['fields']] keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) # Plot it @@ -219,7 +221,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, show(plt) return table, keywords - + def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], @@ -247,26 +249,27 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, for dp in dataproducts: count = instrument_inventory_caom(instrument, dataproduct=dp) ins.append(count) - + # Get the total ins.append(sum(ins[-3:])) - + # Add it to the list inventory.append(ins) - + # Make the table - table = pd.DataFrame(inventory, columns=['instrument']+dataproducts+['total']) - + all_cols = ['instrument']+dataproducts+['total'] + table = pd.DataFrame(inventory, columns=all_cols) + # Melt the table table = pd.melt(table, id_vars=['instrument'], value_vars=dataproducts, value_name='files', var_name='dataproduct') - + # Retrieve one dataset to get header keywords sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, - add_requests={'pagesize': 1, 'page': 1}, - return_data=True) - data = [[i['name'],i['type']] for i in sample['fields']] + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + data = [[i['name'], i['type']] for i in sample['fields']] keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) # Plot it @@ -342,4 +345,3 @@ def mastQuery(request): conn.close() return head, content - From b00f7c53dd870a3a99700bb25a852d137bab971b Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 4 May 2018 14:43:22 -0400 Subject: [PATCH 12/24] Updated the module docstring to render more nicely in the API docs --- jwql/dbmonitor/dbmonitor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 576c1b66d..3a5935d79 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -10,10 +10,10 @@ --- To get an inventory of all JWST files do: - - >>> from jwql.dbmonitor import dbmonitor - >>> inventory, keywords = dbmonitor.jwst_inventory() - + :: + + from jwql.dbmonitor import dbmonitor + inventory, keywords = dbmonitor.jwst_inventory() """ import http.client as httplib From 92761dc571518bce9a1c3fd8ee21a202870cdb42 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Fri, 11 May 2018 15:47:42 -0400 Subject: [PATCH 13/24] Fixed dbmonitor to accept filters and broke out keyword queries. --- jwql/dbmonitor/dbmonitor.py | 174 ++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 98 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 576c1b66d..5e22f02b7 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -16,11 +16,6 @@ """ -import http.client as httplib -import json -import sys -from urllib.parse import quote as urlencode - from astroquery.mast import Mast from bkcharts import Donut, show import pandas as pd @@ -29,18 +24,19 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters={}, add_requests={}, return_data=False): + add_filters=None, add_requests=None, + return_data=False): """Get the counts for a given instrument and data product Parameters ---------- - instrument: sequence, str + instrument: str The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search add_filters: dict The ('paramName':'values') pairs to include in the 'filters' argument - of the request e.g. add_filters = {'target_classification':'moving'} + of the request e.g. add_filters = {'filter':'GR150R'} add_requests: dict The ('request':'value') pairs to include in the request e.g. add_requests = {'pagesize':1, 'page':1} @@ -64,8 +60,12 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(instrument, str): instrument = [instrument] - # Set default filters - filters = [] + # Include filters + if isinstance(add_filters, dict): + filters = [{"paramName": name, "values": [val]} + for name, val in add_filters.items()] + else: + filters = [] # Assemble the request params = {'columns': 'COUNT_BIG(*)', @@ -77,7 +77,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, params['columns'] = '*' # Add requests - if add_requests: + if isinstance(add_requests, dict): params.update(add_requests) response = Mast.service_request_async(service, params) @@ -93,13 +93,14 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters={}, add_requests={}, + add_filters=None, add_requests=None, return_data=False): - """Get the counts for a given instrument and data product + """Get the counts for a given instrument and data product in the + CAOM service Parameters ---------- - instrument: sequence, str + instrument: str The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search @@ -132,9 +133,10 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, {'paramName': 'dataproduct_type', 'values': dataproduct}] # Include additonal filters - for param, value in add_filters.items(): - filters.append({'paramName': param, 'values': [value] - if isinstance(value, str) else value}) + if isinstance(add_filters, dict): + for param, value in add_filters.items(): + filters.append({'paramName': param, 'values': [value] + if isinstance(value, str) else value}) # Assemble the request params = {'columns': 'COUNT_BIG(*)', @@ -146,7 +148,7 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, params['columns'] = '*' # Add requests - if add_requests: + if isinstance(add_requests, dict): params.update(add_requests) response = Mast.service_request_async('Mast.Caom.Filtered', params) @@ -161,10 +163,55 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, return result['data'][0]['Column1'] +def instrument_keywords(instrument): + """Get the keywords for a given instrument service + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + + Returns + ------- + pd.DataFrame + A DataFrame of the keywords + """ + # Retrieve one dataset to get header keywords + sample = instrument_inventory(instrument, return_data=True, + add_requests={'pagesize': 1, 'page': 1}) + data = [[i['name'], i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + return keywords + + +def instrument_keywords_caom(instrument): + """Get the keywords for a given instrument in the CAOM service + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + + Returns + ------- + pd.DataFrame + A DataFrame of the keywords + """ + # Retrieve one dataset to get header keywords + sample = instrument_inventory_caom(instrument, return_data=True, + add_requests={'pagesize': 1, 'page': 1}) + data = [[i['name'], i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + return keywords + + def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], plot=False): - """Gather a full inventory of all JWST data on MAST by instrument/dtype + """Gather a full inventory of all JWST data in each instrument + service by instrument/dtype Parameters ---------- @@ -181,7 +228,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, The table of record counts for each instrument and mode """ # Iterate through instruments - inventory = [] + inventory, keywords = [], {} for instrument in instruments: ins = [instrument] for dp in dataproducts: @@ -194,6 +241,9 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Add it to the list inventory.append(ins) + # Add the keywords to the dict + keywords[instrument] = instrument_keywords(instrument) + # Make the table all_cols = ['instrument']+dataproducts+['total'] table = pd.DataFrame(inventory, columns=all_cols) @@ -203,13 +253,6 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, value_vars=dataproducts, value_name='files', var_name='dataproduct') - # Retrieve one dataset to get header keywords - sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, - add_requests={'pagesize': 1, 'page': 1}, - return_data=True) - data = [[i['name'], i['type']] for i in sample['fields']] - keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) - # Plot it if plot: @@ -226,7 +269,8 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], plot=False): - """Gather a full inventory of all JWST data on MAST by instrument/dtype + """Gather a full inventory of all JWST data in the CAOM service + by instrument/dtype Parameters ---------- @@ -243,7 +287,7 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, The table of record counts for each instrument and mode """ # Iterate through instruments - inventory = [] + inventory, keywords = [], {} for instrument in instruments: ins = [instrument] for dp in dataproducts: @@ -256,6 +300,9 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, # Add it to the list inventory.append(ins) + # Add the keywords to the dict + keywords[instrument] = instrument_keywords_caom(instrument) + # Make the table all_cols = ['instrument']+dataproducts+['total'] table = pd.DataFrame(inventory, columns=all_cols) @@ -265,13 +312,6 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, value_vars=dataproducts, value_name='files', var_name='dataproduct') - # Retrieve one dataset to get header keywords - sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, - add_requests={'pagesize': 1, 'page': 1}, - return_data=True) - data = [[i['name'], i['type']] for i in sample['fields']] - keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) - # Plot it if plot: @@ -283,65 +323,3 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, show(plt) return table, keywords - - -def listMissions(): - """Small function to test sevice_request and make sure JWST is - in the list of services - - Returns - ------- - astropy.table.table.Table - The table of all supported services - """ - service = 'Mast.Missions.List' - params = {} - request = Mast.service_request(service, params) - - return request - - -def mastQuery(request): - """A small MAST API wrapper - - Parameters - ---------- - request: dict - The dictionary of 'service', 'format' and 'params' for the request - - Returns - ------- - dict - A dictionary of the 'data', 'fields', 'paging', and 'status' - of the request - """ - # Define the server - server = 'mast.stsci.edu' - - # Grab Python Version - version = ".".join(map(str, sys.version_info[:3])) - - # Create Http Header Variables - headers = {"Content-type": "application/x-www-form-urlencoded", - "Accept": "text/plain", - "User-agent": "python-requests/"+version} - - # Encoding the request as a json string - requestString = json.dumps(request) - requestString = urlencode(requestString) - - # opening the https connection - conn = httplib.HTTPSConnection(server) - - # Making the query - conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) - - # Getting the response - resp = conn.getresponse() - head = resp.getheaders() - content = resp.read().decode('utf-8') - - # Close the https connection - conn.close() - - return head, content From c899d1e4d8e87ac8846469f005af0820178d691f Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 11:48:59 -0400 Subject: [PATCH 14/24] Combined CAOM and filtered service queries. --- jwql/dbmonitor/dbmonitor.py | 192 ++++++------------------------------ 1 file changed, 30 insertions(+), 162 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index fe8a25bfb..d13ffbc8c 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -25,7 +25,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters=None, add_requests=None, - return_data=False): + caom=False, return_data=False): """Get the counts for a given instrument and data product Parameters @@ -40,6 +40,8 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_requests: dict The ('request':'value') pairs to include in the request e.g. add_requests = {'pagesize':1, 'page':1} + caom: bool + Query CAOM service return_data: bool Return the actual data instead of counts only @@ -49,94 +51,37 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, The number of database records that satisfy the search criteria or a dictionary of the data if `return_data=True` """ - # Declare the service - service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) + filters = [] # Make sure the dataproduct is a list if isinstance(dataproduct, str): dataproduct = [dataproduct] - # Make sure the instrument is a list - if isinstance(instrument, str): - instrument = [instrument] + # Make sure the instrument is supported + if instrument.lower() not in [ins.lower() for ins in JWST_INSTRUMENTS]: + raise TypeError('Supported instruments include:', JWST_INSTRUMENTS) - # Include filters - if isinstance(add_filters, dict): - filters = [{"paramName": name, "values": [val]} - for name, val in add_filters.items()] - else: - filters = [] - - # Assemble the request - params = {'columns': 'COUNT_BIG(*)', - 'filters': filters, - 'removenullcolumns': True} - - # Just get the counts - if return_data: - params['columns'] = '*' - - # Add requests - if isinstance(add_requests, dict): - params.update(add_requests) + # CAOM service + if caom: - response = Mast.service_request_async(service, params) - result = response[0].json() + # Declare the service + service = 'Mast.Caom.Filtered' - # Return all the data - if return_data: - return result + # Set the filters + filters += [{'paramName': 'obs_collection', 'values': ['JWST']}, + {'paramName': 'instrument_name', 'values': [instrument]}, + {'paramName': 'dataproduct_type', 'values': dataproduct}] - # Or just the counts + # Instruent filtered service else: - return result['data'][0]['Column1'] - - -def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters=None, add_requests=None, - return_data=False): - """Get the counts for a given instrument and data product in the - CAOM service - - Parameters - ---------- - instrument: str - The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] - dataproduct: sequence, str - The type of data product to search - add_filters: dict - The ('paramName':'values') pairs to include in the 'filters' argument - of the request e.g. add_filters = {'target_classification':'moving'} - add_requests: dict - The ('request':'value') pairs to include in the request - e.g. add_requests = {'pagesize':1, 'page':1} - return_data: bool - Return the actual data instead of counts only - - Returns - ------- - int, dict - The number of database records that satisfy the search criteria - or a dictionary of the data if `return_data=True` - """ - # Make sure the dataproduct is a list - if isinstance(dataproduct, str): - dataproduct = [dataproduct] - - # Make sure the instrument is a list - if isinstance(instrument, str): - instrument = [instrument] - # Set default filters - filters = [{'paramName': 'obs_collection', 'values': ['JWST']}, - {'paramName': 'instrument_name', 'values': instrument}, - {'paramName': 'dataproduct_type', 'values': dataproduct}] + # Declare the service + service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) # Include additonal filters if isinstance(add_filters, dict): - for param, value in add_filters.items(): - filters.append({'paramName': param, 'values': [value] - if isinstance(value, str) else value}) + filters += [{"paramName": name, "values": [val]} + for name, val in add_filters.items()] # Assemble the request params = {'columns': 'COUNT_BIG(*)', @@ -151,7 +96,7 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(add_requests, dict): params.update(add_requests) - response = Mast.service_request_async('Mast.Caom.Filtered', params) + response = Mast.service_request_async(service, params) result = response[0].json() # Return all the data @@ -163,13 +108,15 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, return result['data'][0]['Column1'] -def instrument_keywords(instrument): +def instrument_keywords(instrument, caom=False): """Get the keywords for a given instrument service Parameters ---------- instrument: str The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + caom: bool + Query CAOM service Returns ------- @@ -177,7 +124,7 @@ def instrument_keywords(instrument): A DataFrame of the keywords """ # Retrieve one dataset to get header keywords - sample = instrument_inventory(instrument, return_data=True, + sample = instrument_inventory(instrument, return_data=True, caom=caom, add_requests={'pagesize': 1, 'page': 1}) data = [[i['name'], i['type']] for i in sample['fields']] keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) @@ -185,31 +132,9 @@ def instrument_keywords(instrument): return keywords -def instrument_keywords_caom(instrument): - """Get the keywords for a given instrument in the CAOM service - - Parameters - ---------- - instrument: str - The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] - - Returns - ------- - pd.DataFrame - A DataFrame of the keywords - """ - # Retrieve one dataset to get header keywords - sample = instrument_inventory_caom(instrument, return_data=True, - add_requests={'pagesize': 1, 'page': 1}) - data = [[i['name'], i['type']] for i in sample['fields']] - keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) - - return keywords - - def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], - plot=False): + caom=False, plot=False): """Gather a full inventory of all JWST data in each instrument service by instrument/dtype @@ -219,6 +144,8 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, The list of instruments to count dataproducts: sequence The types of dataproducts to count + caom: bool + Query CAOM service plot: bool Return a pie chart of the data @@ -232,66 +159,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, for instrument in instruments: ins = [instrument] for dp in dataproducts: - count = instrument_inventory(instrument, dataproduct=dp) - ins.append(count) - - # Get the total - ins.append(sum(ins[-3:])) - - # Add it to the list - inventory.append(ins) - - # Add the keywords to the dict - keywords[instrument] = instrument_keywords(instrument) - - # Make the table - all_cols = ['instrument']+dataproducts+['total'] - table = pd.DataFrame(inventory, columns=all_cols) - - # Melt the table - table = pd.melt(table, id_vars=['instrument'], - value_vars=dataproducts, - value_name='files', var_name='dataproduct') - - # Plot it - if plot: - - # Make the plot - plt = Donut(table, label=['instrument', 'dataproduct'], values='files', - text_font_size='12pt', hover_text='files', - name="JWST Inventory", plot_width=600, plot_height=600) - - show(plt) - - return table, keywords - - -def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, - dataproducts=['image', 'spectrum', 'cube'], - plot=False): - """Gather a full inventory of all JWST data in the CAOM service - by instrument/dtype - - Parameters - ---------- - instruments: sequence - The list of instruments to count - dataproducts: sequence - The types of dataproducts to count - plot: bool - Return a pie chart of the data - - Returns - ------- - astropy.table.table.Table - The table of record counts for each instrument and mode - """ - # Iterate through instruments - inventory, keywords = [], {} - for instrument in instruments: - ins = [instrument] - for dp in dataproducts: - count = instrument_inventory_caom(instrument, dataproduct=dp) + count = instrument_inventory(instrument, dataproduct=dp, caom=caom) ins.append(count) # Get the total @@ -301,7 +169,7 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, inventory.append(ins) # Add the keywords to the dict - keywords[instrument] = instrument_keywords_caom(instrument) + keywords[instrument] = instrument_keywords(instrument, caom=caom) # Make the table all_cols = ['instrument']+dataproducts+['total'] From 59ef757af31a57d5073e340ae848ac8ca46473c7 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 12:27:37 -0400 Subject: [PATCH 15/24] Removed duplicate global variables --- jwql/utils/utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 08b07967a..f907612f2 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -27,10 +27,6 @@ __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) -JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] -JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', - 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] - JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] From b247a38a34bb273c0b1a25a6f3304fcdf2b4d074 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 14:12:43 -0400 Subject: [PATCH 16/24] PEP 8 --- jwql/dbmonitor/dbmonitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index d13ffbc8c..8591475cc 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -11,7 +11,7 @@ To get an inventory of all JWST files do: :: - + from jwql.dbmonitor import dbmonitor inventory, keywords = dbmonitor.jwst_inventory() """ From e0e735e95ac0218e6865e75bc69391a8aebad812 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 16:12:55 -0400 Subject: [PATCH 17/24] Started adding tests. --- jwql/tests/test_dbmonitor.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 jwql/tests/test_dbmonitor.py diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py new file mode 100755 index 000000000..424bd5590 --- /dev/null +++ b/jwql/tests/test_dbmonitor.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +"""Tests for the dbmonitor module. + +Authors +------- + + - Joe Filippazzo + + +Use +--- + + These tests can be run via the command line (omit the -s to suppress verbose output to stdout): + + :: + pytest -s test_dbmonitor.py + +""" + +from ..dbmonitor import dbmonitor as db +from ..utils.utils import JWST_INSTRUMENTS + + +def test_caom_keywords(): + """Test to see that the CAOM keywords are the same for all instruments""" + kw = [] + for ins in JWST_INSTRUMENTS: + kw.append(db.instrument_keywords(ins, caom=True)['keyword'].tolist()) + + assert kw[0] == kw[1] == kw[2] == kw[3] == kw[4] From 6ce1f2df7d0b50ed09c855fdd5cd5577244030c0 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 16:16:32 -0400 Subject: [PATCH 18/24] Formatting docstring. --- jwql/tests/test_dbmonitor.py | 3 +-- jwql/tests/test_permissions.py | 0 2 files changed, 1 insertion(+), 2 deletions(-) mode change 100644 => 100755 jwql/tests/test_permissions.py diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py index 424bd5590..faacb17ef 100755 --- a/jwql/tests/test_dbmonitor.py +++ b/jwql/tests/test_dbmonitor.py @@ -4,8 +4,7 @@ Authors ------- - - Joe Filippazzo - + Joe Filippazzo Use --- diff --git a/jwql/tests/test_permissions.py b/jwql/tests/test_permissions.py old mode 100644 new mode 100755 From bcfe728dce01cb6abfdf08f04f2e43c4e8d47356 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 29 May 2018 15:09:13 -0400 Subject: [PATCH 19/24] Added a few unittests. --- jwql/tests/test_dbmonitor.py | 41 +++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py index faacb17ef..7ac0cb2fc 100755 --- a/jwql/tests/test_dbmonitor.py +++ b/jwql/tests/test_dbmonitor.py @@ -7,9 +7,10 @@ Joe Filippazzo Use ---- +- - These tests can be run via the command line (omit the -s to suppress verbose output to stdout): + These tests can be run via the command line (omit the -s to suppress +verbose output to stdout): :: pytest -s test_dbmonitor.py @@ -20,10 +21,44 @@ from ..utils.utils import JWST_INSTRUMENTS -def test_caom_keywords(): +def test_caom_instrument_keywords(): """Test to see that the CAOM keywords are the same for all instruments""" kw = [] for ins in JWST_INSTRUMENTS: kw.append(db.instrument_keywords(ins, caom=True)['keyword'].tolist()) assert kw[0] == kw[1] == kw[2] == kw[3] == kw[4] + + +def test_filtered_instrument_keywords(): + """Test to see that the instrument specific service keywords are + different for all instruments""" + kw = [] + for ins in JWST_INSTRUMENTS: + kw.append(db.instrument_keywords(ins, caom=False)['keyword'].tolist()) + + assert kw[0] != kw[1] != kw[2] != kw[3] != kw[4] + + +def test_instrument_inventory_filtering(): + """Test to see that the instrument inventory can be filtered""" + filt = 'GR150R' + data = db.instrument_inventory('niriss', + add_filters={'filter': filt}, + return_data=True) + + filters = [row['filter'] for row in data['data']] + + assert all([i == filt for i in filters]) + + +def test_instrument_dataproduct_filtering(): + """Test to see that the instrument inventory can be filtered + by data product""" + dp = 'spectrum' + data = db.instrument_inventory('nirspec', dataproduct=dp, caom=True, + return_data=True) + + dps = [row['dataproduct_type'] for row in data['data']] + + assert all([i == dp for i in dps]) From 1b87971906d8a10130ec370fda0f4381607801f2 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 29 May 2018 15:12:38 -0400 Subject: [PATCH 20/24] Fixed my name in the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f06decc7b..9212a5b56 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Matthew Bourque (INS) - Lauren Chambers (INS) - Misty Cracraft (INS) -- Joseph Filippazo (INS) +- Joe Filippazzo (INS) - Bryan Hilbert (INS) - Graham Kanarek (INS) - Catherine Martlin (INS) From 8ce3800a14e50a2786be9d34eb136b1cc88b2965 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 31 May 2018 11:14:07 -0400 Subject: [PATCH 21/24] Saving the plot to the jwql central storage area instead of just showing it. Using bokeh.charts instead of bkcharts. --- environment.yml | 1 + jwql/dbmonitor/dbmonitor.py | 15 ++++++++++++--- jwql/tests/test_dbmonitor.py | 15 ++++++++------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/environment.yml b/environment.yml index ff3c2a99a..cd9105095 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,7 @@ channels: - http://ssb.stsci.edu/astroconda-dev dependencies: - astropy=3.0 +- bokeh=0.12.5 - django=1.11.8 - jwst=0.7.8rc9 - matplotlib=2.1.1 diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 8591475cc..2a25b2f7f 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -16,11 +16,13 @@ inventory, keywords = dbmonitor.jwst_inventory() """ +import os + from astroquery.mast import Mast -from bkcharts import Donut, show +from bokeh.charts import Donut, save, output_file import pandas as pd -from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS +from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, @@ -188,6 +190,13 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, text_font_size='12pt', hover_text='files', name="JWST Inventory", plot_width=600, plot_height=600) - show(plt) + # Save the plot + if caom: + output_filename = 'database_monitor_caom.html' + else: + output_filename = 'database_monitor_jwst.html' + outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename) + output_file(outfile) + save(plt) return table, keywords diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py index 7ac0cb2fc..516b0d118 100755 --- a/jwql/tests/test_dbmonitor.py +++ b/jwql/tests/test_dbmonitor.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#! /usr/bin/env python + """Tests for the dbmonitor module. Authors @@ -7,14 +8,13 @@ Joe Filippazzo Use -- - - These tests can be run via the command line (omit the -s to suppress -verbose output to stdout): +--- + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): :: - pytest -s test_dbmonitor.py + pytest -s test_dbmonitor.py """ from ..dbmonitor import dbmonitor as db @@ -22,7 +22,8 @@ def test_caom_instrument_keywords(): - """Test to see that the CAOM keywords are the same for all instruments""" + """Test to see that the CAOM keywords are the same for all + instruments""" kw = [] for ins in JWST_INSTRUMENTS: kw.append(db.instrument_keywords(ins, caom=True)['keyword'].tolist()) From c109a5797a51ade615d992953f4db4e2a55b13bf Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 31 May 2018 11:21:42 -0400 Subject: [PATCH 22/24] Added call to set_permissions for setting the permissions for the output plots. --- jwql/dbmonitor/dbmonitor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 2a25b2f7f..5ed0a9654 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -22,6 +22,7 @@ from bokeh.charts import Donut, save, output_file import pandas as pd +from ..permissions.permissions import set_permissions from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS @@ -198,5 +199,6 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename) output_file(outfile) save(plt) + set_permissions(outfile, verbose=False) return table, keywords From d2f633788c9cfb4d5314282dec0b1327d310ea28 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 31 May 2018 11:30:51 -0400 Subject: [PATCH 23/24] Added bokeh to list of requires. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index af40a2fea..681a0789e 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ AUTHORS = 'Matthew Bourque, Sara Ogaz, Joe Filippazzo, Bryan Hilbert, Misty Cracraft, Graham Kanarek' AUTHORS += 'Johannes Sahlmann, Lauren Chambers, Catherine Martlin' -REQUIRES = ['astropy', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] +REQUIRES = ['astropy', 'bokeh', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] setup( name = 'jwql', From 0cf6175afc29fcd3391f661f3881900562d26ffc Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 31 May 2018 11:42:12 -0400 Subject: [PATCH 24/24] Added astroquery dependency --- environment.yml | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index cd9105095..3fa1223b9 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,7 @@ channels: - http://ssb.stsci.edu/astroconda-dev dependencies: - astropy=3.0 +- astroquery=0.3.8 - bokeh=0.12.5 - django=1.11.8 - jwst=0.7.8rc9 diff --git a/setup.py b/setup.py index 681a0789e..245dac2de 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ AUTHORS = 'Matthew Bourque, Sara Ogaz, Joe Filippazzo, Bryan Hilbert, Misty Cracraft, Graham Kanarek' AUTHORS += 'Johannes Sahlmann, Lauren Chambers, Catherine Martlin' -REQUIRES = ['astropy', 'bokeh', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] +REQUIRES = ['astropy', 'astroquery', 'bokeh', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] setup( name = 'jwql',