From 72a877ddd51142cdabca9e3d9003a166e52fcdda Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 11:01:08 -0400 Subject: [PATCH 01/35] Added functions to query MAST for record counts and data. --- jwql/__init__.py | 1 + jwql/dbmonitor/__init__.py | 0 jwql/dbmonitor/dbmonitor.py | 149 ++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 jwql/dbmonitor/__init__.py create mode 100644 jwql/dbmonitor/dbmonitor.py diff --git a/jwql/__init__.py b/jwql/__init__.py index e69de29bb..6d9900a73 100644 --- a/jwql/__init__.py +++ b/jwql/__init__.py @@ -0,0 +1 @@ +from . import dbmonitor \ No newline at end of file diff --git a/jwql/dbmonitor/__init__.py b/jwql/dbmonitor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py new file mode 100644 index 000000000..b613e94b5 --- /dev/null +++ b/jwql/dbmonitor/dbmonitor.py @@ -0,0 +1,149 @@ +"""This module is home to a suite of MAST queries that gather bulk properties of available JWST data for JWQL + +Authors +------- + + Joe Filippazzo + +Use +--- + + This module can be imported as such: + + >>> from jwql.dbmonitor import dbmonitor +""" +import sys +import os +import time +import re +import json +from urllib.parse import quote as urlencode +from urllib.request import urlretrieve +import http.client as httplib +from astroquery.mast import Mast, Observations + +JWST_INSTRUMENTS = ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] + +def listMissions(): + """Small function to test sevice_request and make sure JWST is in the list of services + + Returns + ------- + astropy.table.table.Table + The table of all supported services + """ + service = 'Mast.Missions.List' + params = {} + request = Mast.service_request(service, params) + + return request + +def mastQuery(request): + """A small MAST API wrapper + + Parameters + ---------- + request: dict + The dictionary of 'service', 'format' and 'params' for the request + + Returns + ------- + dict + A dictionary of the 'data', 'fields', 'paging', and 'status' of the request + """ + # Define the server + server='mast.stsci.edu' + + # Grab Python Version + version = ".".join(map(str, sys.version_info[:3])) + + # Create Http Header Variables + headers = {"Content-type": "application/x-www-form-urlencoded", + "Accept": "text/plain", + "User-agent":"python-requests/"+version} + + # Encoding the request as a json string + requestString = json.dumps(request) + requestString = urlencode(requestString) + + # opening the https connection + conn = httplib.HTTPSConnection(server) + + # Making the query + conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) + + # Getting the response + resp = conn.getresponse() + head = resp.getheaders() + content = resp.read().decode('utf-8') + + # Close the https connection + conn.close() + + return head,content + +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data=False): + """Get the counts for a given instrument and data product + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int + The number of database records that satisfy the search criteria + """ + # Make sure the dataproduct is a list + if isinstance(dataproduct,str): + dataproduct = [dataproduct] + + # Assemble the request + request = {'service':'Mast.Caom.Filtered', + 'format':'json', + 'params':{ + 'columns':'COUNT_BIG(*)', + 'filters':[ + {'paramName':'obs_collection', 'values':['JWST']}, + {'paramName':'instrument_name', 'values':[instrument]}, + {'paramName':'dataproduct_type', 'values':dataproduct} + ]}} + + # Just get the counts + if return_data: + request['params']['columns'] = '*' + + # Perform the query + headers, outString = mastQuery(request) + result = json.loads(outString) + + # REturn all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + +def jwst_inventory(): + """Gather a full inventory of all JWST data on MAST binned by instrument and mode + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Make master table + inventory = at.Table(names=('instrument','dataproduct','count')) + + # Iterate through instruments + for instrument in JWST_INSTRUMENTS: + for dataproduct in ['cube','image','spectrum']: + count = instrument_inventory(instrument, dataproduct=dataproduct) + \ No newline at end of file From b52a0c5337f248a068447ffd39a434598b72e3c7 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 11:40:11 -0400 Subject: [PATCH 02/35] Generates table of result counts --- jwql/dbmonitor/dbmonitor.py | 41 +++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index b613e94b5..c08420211 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -17,10 +17,12 @@ import time import re import json +import numpy as np from urllib.parse import quote as urlencode from urllib.request import urlretrieve import http.client as httplib from astroquery.mast import Mast, Observations +import astropy.table as at JWST_INSTRUMENTS = ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] @@ -83,7 +85,7 @@ def mastQuery(request): return head,content -def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data=False): +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters={}, return_data=False): """Get the counts for a given instrument and data product Parameters @@ -92,6 +94,9 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data= The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search + filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument of the request + e.g. filters = {'target_classification':'moving'} return_data: bool Return the actual data instead of counts only @@ -104,16 +109,21 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data= if isinstance(dataproduct,str): dataproduct = [dataproduct] + # Set default filters + filters = [{'paramName':'obs_collection', 'values':['JWST']}, + {'paramName':'instrument_name', 'values':[instrument]}, + {'paramName':'dataproduct_type', 'values':dataproduct}] + + # Include additonal filters + for k,v in add_filters.items(): + filters.append({'paramName':k, 'values':[v] if isinstance(v,str) else v}) + # Assemble the request request = {'service':'Mast.Caom.Filtered', 'format':'json', 'params':{ 'columns':'COUNT_BIG(*)', - 'filters':[ - {'paramName':'obs_collection', 'values':['JWST']}, - {'paramName':'instrument_name', 'values':[instrument]}, - {'paramName':'dataproduct_type', 'values':dataproduct} - ]}} + 'filters':filters}} # Just get the counts if return_data: @@ -131,19 +141,28 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, return_data= else: return result['data'][0]['Column1'] -def jwst_inventory(): +def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image','spectrum','cube']): """Gather a full inventory of all JWST data on MAST binned by instrument and mode + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + Returns ------- astropy.table.table.Table The table of record counts for each instrument and mode """ # Make master table - inventory = at.Table(names=('instrument','dataproduct','count')) + inventory = at.Table(names=('instrument','dataproduct','count'), dtype=('S8','S12',int)) # Iterate through instruments - for instrument in JWST_INSTRUMENTS: - for dataproduct in ['cube','image','spectrum']: + for instrument in instruments: + for dataproduct in dataproducts: count = instrument_inventory(instrument, dataproduct=dataproduct) - \ No newline at end of file + inventory.add_row([instrument, dataproduct, count]) + + return inventory \ No newline at end of file From 9a363a5ecd1b14012c6f9817a9555f4880a25b37 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 11:53:58 -0400 Subject: [PATCH 03/35] PEP8 compliance. --- jwql/dbmonitor/dbmonitor.py | 137 +++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 64 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index c08420211..89e25985a 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -1,4 +1,5 @@ -"""This module is home to a suite of MAST queries that gather bulk properties of available JWST data for JWQL +"""This module is home to a suite of MAST queries that gather bulk properties +of available JWST data for JWQL Authors ------- @@ -8,28 +9,28 @@ Use --- - This module can be imported as such: + To get an inventory of all JWST files do: >>> from jwql.dbmonitor import dbmonitor + >>> results = dbmonitor.jwst_inventory() + """ import sys -import os -import time -import re import json -import numpy as np from urllib.parse import quote as urlencode -from urllib.request import urlretrieve -import http.client as httplib -from astroquery.mast import Mast, Observations +import http.client as httplib +from astroquery.mast import Mast import astropy.table as at -JWST_INSTRUMENTS = ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] -JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] +JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', + 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] + def listMissions(): - """Small function to test sevice_request and make sure JWST is in the list of services - + """Small function to test sevice_request and make sure JWST is + in the list of services + Returns ------- astropy.table.table.Table @@ -38,56 +39,60 @@ def listMissions(): service = 'Mast.Missions.List' params = {} request = Mast.service_request(service, params) - + return request - + + def mastQuery(request): """A small MAST API wrapper - + Parameters ---------- request: dict The dictionary of 'service', 'format' and 'params' for the request - + Returns ------- dict - A dictionary of the 'data', 'fields', 'paging', and 'status' of the request + A dictionary of the 'data', 'fields', 'paging', and 'status' + of the request """ # Define the server - server='mast.stsci.edu' - - # Grab Python Version + server = 'mast.stsci.edu' + + # Grab Python Version version = ".".join(map(str, sys.version_info[:3])) - + # Create Http Header Variables headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain", - "User-agent":"python-requests/"+version} - + "User-agent": "python-requests/"+version} + # Encoding the request as a json string requestString = json.dumps(request) requestString = urlencode(requestString) - + # opening the https connection conn = httplib.HTTPSConnection(server) - + # Making the query conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) - + # Getting the response resp = conn.getresponse() head = resp.getheaders() content = resp.read().decode('utf-8') - + # Close the https connection conn.close() - - return head,content - -def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters={}, return_data=False): + + return head, content + + +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters={}, return_data=False): """Get the counts for a given instrument and data product - + Parameters ---------- instrument: str @@ -95,74 +100,78 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters= dataproduct: sequence, str The type of data product to search filters: dict - The ('paramName':'values') pairs to include in the 'filters' argument of the request - e.g. filters = {'target_classification':'moving'} + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. filters = {'target_classification':'moving'} return_data: bool Return the actual data instead of counts only - + Returns ------- int The number of database records that satisfy the search criteria """ # Make sure the dataproduct is a list - if isinstance(dataproduct,str): + if isinstance(dataproduct, str): dataproduct = [dataproduct] - + # Set default filters - filters = [{'paramName':'obs_collection', 'values':['JWST']}, - {'paramName':'instrument_name', 'values':[instrument]}, - {'paramName':'dataproduct_type', 'values':dataproduct}] - + filters = [{'paramName': 'obs_collection', 'values': ['JWST']}, + {'paramName': 'instrument_name', 'values': [instrument]}, + {'paramName': 'dataproduct_type', 'values': dataproduct}] + # Include additonal filters - for k,v in add_filters.items(): - filters.append({'paramName':k, 'values':[v] if isinstance(v,str) else v}) - + for k, v in add_filters.items(): + filters.append({'paramName': k, 'values': [v] + if isinstance(v, str) else v}) + # Assemble the request - request = {'service':'Mast.Caom.Filtered', - 'format':'json', - 'params':{ - 'columns':'COUNT_BIG(*)', - 'filters':filters}} - + request = {'service': 'Mast.Caom.Filtered', + 'format': 'json', + 'params': { + 'columns': 'COUNT_BIG(*)', + 'filters': filters}} + # Just get the counts if return_data: request['params']['columns'] = '*' - + # Perform the query headers, outString = mastQuery(request) result = json.loads(outString) - - # REturn all the data + + # Return all the data if return_data: return result - + # Or just the counts else: return result['data'][0]['Column1'] - -def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image','spectrum','cube']): - """Gather a full inventory of all JWST data on MAST binned by instrument and mode - + + +def jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube']): + """Gather a full inventory of all JWST data on MAST by instrument/dtype + Parameters ---------- instruments: sequence The list of instruments to count dataproducts: sequence The types of dataproducts to count - + Returns ------- astropy.table.table.Table The table of record counts for each instrument and mode """ # Make master table - inventory = at.Table(names=('instrument','dataproduct','count'), dtype=('S8','S12',int)) - + inventory = at.Table(names=('instrument', 'dataproduct', 'count'), + dtype=('S8', 'S12', int)) + # Iterate through instruments for instrument in instruments: for dataproduct in dataproducts: count = instrument_inventory(instrument, dataproduct=dataproduct) inventory.add_row([instrument, dataproduct, count]) - - return inventory \ No newline at end of file + + return inventory From 39a945d2bacbf2d60f6b90be392895d1c33b8275 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 12:06:55 -0400 Subject: [PATCH 04/35] Added total db counts --- jwql/dbmonitor/dbmonitor.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 89e25985a..34a3dadbd 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -95,7 +95,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, Parameters ---------- - instrument: str + instrument: sequence, str The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search @@ -114,9 +114,13 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(dataproduct, str): dataproduct = [dataproduct] + # Make sure the instrument is a list + if isinstance(instrument, str): + instrument = [instrument] + # Set default filters filters = [{'paramName': 'obs_collection', 'values': ['JWST']}, - {'paramName': 'instrument_name', 'values': [instrument]}, + {'paramName': 'instrument_name', 'values': instrument}, {'paramName': 'dataproduct_type', 'values': dataproduct}] # Include additonal filters @@ -174,4 +178,12 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, count = instrument_inventory(instrument, dataproduct=dataproduct) inventory.add_row([instrument, dataproduct, count]) + # Get number of files for all datatypes for this instrument + count = instrument_inventory(instrument, dataproduct=dataproducts) + inventory.add_row([instrument, '*', count]) + + # Get number of files for all datatypes for this instrument + count = instrument_inventory(instruments, dataproduct=dataproducts) + inventory.add_row(['*', '*', count]) + return inventory From 7ad6e741f0d4496890dc26214db9a236ed9b2db2 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 12:09:52 -0400 Subject: [PATCH 05/35] Added total db counts comment --- jwql/dbmonitor/dbmonitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 34a3dadbd..f53cfd65b 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -182,7 +182,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, count = instrument_inventory(instrument, dataproduct=dataproducts) inventory.add_row([instrument, '*', count]) - # Get number of files for all datatypes for this instrument + # Get total number of files count = instrument_inventory(instruments, dataproduct=dataproducts) inventory.add_row(['*', '*', count]) From 61f16f9a9b2c7d5229570343ac386369944d5776 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 2 Apr 2018 12:57:20 -0400 Subject: [PATCH 06/35] Added keyword count to jwst_inventory() --- jwql/dbmonitor/dbmonitor.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index f53cfd65b..5f337734d 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -12,7 +12,7 @@ To get an inventory of all JWST files do: >>> from jwql.dbmonitor import dbmonitor - >>> results = dbmonitor.jwst_inventory() + >>> inventory, keywords = dbmonitor.jwst_inventory() """ import sys @@ -90,7 +90,7 @@ def mastQuery(request): def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters={}, return_data=False): + add_filters={}, add_requests={}, return_data=False): """Get the counts for a given instrument and data product Parameters @@ -99,9 +99,12 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search - filters: dict + add_filters: dict The ('paramName':'values') pairs to include in the 'filters' argument - of the request e.g. filters = {'target_classification':'moving'} + of the request e.g. add_filters = {'target_classification':'moving'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} return_data: bool Return the actual data instead of counts only @@ -139,6 +142,10 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, if return_data: request['params']['columns'] = '*' + # Add requests + if add_requests: + request.update(add_requests) + # Perform the query headers, outString = mastQuery(request) result = json.loads(outString) @@ -186,4 +193,12 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, count = instrument_inventory(instruments, dataproduct=dataproducts) inventory.add_row(['*', '*', count]) - return inventory + # Retrieve one dataset to get header keywords + sample = instrument_inventory(instruments, dataproduct=dataproducts, + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + names = [i['name'] for i in sample['fields']] + types = [i['type'] for i in sample['fields']] + keywords = at.Table([names, types], names=('keyword', 'dtype')) + + return inventory, keywords From c4324a1984d7e2b95cc71153deeb219e0b08a459 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 16 Apr 2018 10:36:40 -0400 Subject: [PATCH 07/35] Added some more acknowledgements --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 11c7d78d6..f06decc7b 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ The following is a bare bones example of a best work flow for contributing to th 10. Delete your local copy of your branch. -## Git Ignore +## `gitignore` The `jwql` repository also contains a file named `.gitignore` that indicates specific directories, files or file types that should not be commited to the repository. Feel free to add additional lines to this file if you want to avoid committing anything. Some examples may include `.fits` files, `.jpeg` files, or `.ipynb_checkpoints/`. @@ -109,6 +109,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Phil Grant (ITSD) - Dean Hines (INS) - Sherrie Holfeltz (INS) +- Joe Hunkeler (OED) - Catherine Kaleida (OED) - Mark Kyprianou (OED) - Karen Levay (OED) @@ -125,4 +126,5 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Linda Smith (INS) - Dave Unger (ITSD) - Jeff Valenti (INS) +- Lara Wilkinson (OPO) - Joe Zahn (ITSD) From ef7c148ba2a8ad76309ba383db052e858b6cab1b Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Mon, 16 Apr 2018 10:40:40 -0400 Subject: [PATCH 08/35] Add Joe Hunkeler's filename parser to utils.py with minor modifications --- jwql/utils/utils.py | 46 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index a0f122afc..e8901648a 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -4,6 +4,8 @@ ------- Matthew Bourque + Joe Hunkeler (filename_parser) + Lauren Chambers Use --- @@ -15,6 +17,8 @@ """ import json +import re +import os def get_config(): @@ -31,3 +35,45 @@ def get_config(): settings = json.load(config_file) return settings + +def filename_parser(filename): + """Return a dictionary that contains the properties of a given + JWST file (e.g. program ID, visit number, detector, etc.) + + Parameters + ---------- + filename : str + Path or name of JWST file to parse + + Returns + ------- + dict + Collection of file properties + + Raises + ------ + ValueError + When the provided file does not follow naming conventions + """ + filename = os.path.basename(filename) + + elements = \ + re.compile(r"[a-z]+" + "(?P\d{5})" + "(?P\d{3})" + "(?P\d{3})" + "_(?P\d{2})" + "(?P\d{1})" + "(?P\d{2})" + "_(?P\d+)" + "_(?P\w+)" + "_(?P\w+).*") + + jwst_file = elements.match(filename) + + if jwst_file is not None: + filename_dict = jwst_file.groupdict() + else: + raise ValueError('Provided file {} does not follow JWST naming conventions (jw____ .fits)') + + return filename_dict From f55f25f3a0ff3c217206d583fd7ffe6b92a4cfe1 Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Mon, 16 Apr 2018 10:41:47 -0400 Subject: [PATCH 09/35] Update references/authors --- jwql/utils/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index e8901648a..166e66185 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -4,7 +4,6 @@ ------- Matthew Bourque - Joe Hunkeler (filename_parser) Lauren Chambers Use @@ -14,6 +13,12 @@ >>> import utils settings = get_config() + +References +---------- + + Filename parser modifed from Joe Hunkeler: + https://gist.github.com/jhunkeler/f08783ca2da7bfd1f8e9ee1d207da5ff """ import json From 86cd3888b4c2f436270abb832f4b31b1f1d50a5e Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Wed, 4 Apr 2018 11:04:54 -0400 Subject: [PATCH 10/35] first pass at getting sphinx and automodapi running, not storing automodapi generated docs for now. #51 --- .gitignore | 4 +- docs/Makefile | 20 +++++ docs/source/api_docs.rst | 7 ++ docs/source/conf.py | 170 +++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 30 +++++++ 5 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 docs/Makefile create mode 100644 docs/source/api_docs.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst diff --git a/.gitignore b/.gitignore index dfaaafd6a..1c8d2c110 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ utils/config.json build/ dist/ -jwql.egg-info/ \ No newline at end of file +jwql.egg-info/ +docs/build/ +docs/source/api/ \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..f88da7e2b --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = jwql +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/source/api_docs.rst b/docs/source/api_docs.rst new file mode 100644 index 000000000..73d9da6dc --- /dev/null +++ b/docs/source/api_docs.rst @@ -0,0 +1,7 @@ +Preview Image Test +================== +.. automodapi:: jwql.preview_image.preview_image + +Utils Test +========== +.. automodapi:: jwql.utils.utils diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 000000000..6427bd81c --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# jwql documentation build configuration file, created by +# sphinx-quickstart on Wed Apr 4 10:30:20 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) +import stsci_rtd_theme + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx_automodapi.automodapi', + 'sphinx_automodapi.automodsumm', + 'numpydoc', + 'sphinx.ext.autodoc', + 'sphinx.ext.mathjax', + 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'jwql' +copyright = '2018, STScI' +author = 'STScI' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.4.1' +# The full version, including alpha/beta/rc tags. +release = '0.4.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'stsci_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] +html_theme_path = [stsci_rtd_theme.get_html_theme_path()] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +#html_sidebars = {} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'jwqldoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'jwql.tex', 'jwql Documentation', + 'STScI', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'jwql', 'jwql Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'jwql', 'jwql Documentation', + author, 'jwql', 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 000000000..4c3820628 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,30 @@ +.. jwql documentation master file, created by + sphinx-quickstart on Wed Apr 4 10:30:20 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to jwql's documentation! +================================ + +General JWQL description here. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +API documentation +================= + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + api_docs.rst + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` From b700d07858f346075c469ffc7185cd66cef3de1f Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Wed, 4 Apr 2018 11:14:06 -0400 Subject: [PATCH 11/35] Adding readthedocs files, issue #51 --- docs/environment.yml | 14 ++++++++++++++ readthedocs.yml | 5 +++++ 2 files changed, 19 insertions(+) create mode 100644 docs/environment.yml create mode 100644 readthedocs.yml diff --git a/docs/environment.yml b/docs/environment.yml new file mode 100644 index 000000000..a72b38e5d --- /dev/null +++ b/docs/environment.yml @@ -0,0 +1,14 @@ +name: stak-notebooks +channels: + - http://ssb.stsci.edu/astroconda + - conda-forge +dependencies: + - numpy + - scipy + - astropy + - numpydoc + - sphinx>=1.5 + - sphinx_rtd_theme + - stsci_rtd_theme + - pip: + - sphinx-automodapi diff --git a/readthedocs.yml b/readthedocs.yml new file mode 100644 index 000000000..ddcd2ca67 --- /dev/null +++ b/readthedocs.yml @@ -0,0 +1,5 @@ +type: sphinx +python: + version: 3 +conda: + file: docs/environment.yml \ No newline at end of file From cf5c37a58cff30d6f934d77695abac089bf1494e Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Wed, 4 Apr 2018 11:31:30 -0400 Subject: [PATCH 12/35] deleting these rtd files for now, rtd will not run on a private repo --- docs/environment.yml | 14 -------------- readthedocs.yml | 5 ----- 2 files changed, 19 deletions(-) delete mode 100644 docs/environment.yml delete mode 100644 readthedocs.yml diff --git a/docs/environment.yml b/docs/environment.yml deleted file mode 100644 index a72b38e5d..000000000 --- a/docs/environment.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: stak-notebooks -channels: - - http://ssb.stsci.edu/astroconda - - conda-forge -dependencies: - - numpy - - scipy - - astropy - - numpydoc - - sphinx>=1.5 - - sphinx_rtd_theme - - stsci_rtd_theme - - pip: - - sphinx-automodapi diff --git a/readthedocs.yml b/readthedocs.yml deleted file mode 100644 index ddcd2ca67..000000000 --- a/readthedocs.yml +++ /dev/null @@ -1,5 +0,0 @@ -type: sphinx -python: - version: 3 -conda: - file: docs/environment.yml \ No newline at end of file From aa88a2321f30422d4b92ec156f94d37d1d13f03c Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Wed, 4 Apr 2018 11:51:58 -0400 Subject: [PATCH 13/35] Updated environment file and a small conf tweak. --- docs/source/conf.py | 3 +++ environment.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 6427bd81c..c4c75e108 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -50,6 +50,9 @@ # The master toctree document. master_doc = 'index' +# Numpy doc setting, right now this supresses some warnings, not exactly sure why? +numpydoc_show_class_members = False + # General information about the project. project = 'jwql' copyright = '2018, STScI' diff --git a/environment.yml b/environment.yml index 1985e77b5..ff3c2a99a 100644 --- a/environment.yml +++ b/environment.yml @@ -8,10 +8,13 @@ dependencies: - jwst=0.7.8rc9 - matplotlib=2.1.1 - numpy=1.14.0 +- numpydoc=0.8.0 - postgresql=9.6.6 - python=3.6.4 - python-dateutil=2.6.1 - sphinx=1.6.6 +- sphinx_rtd_theme=0.1.9 - sqlalchemy=1.2.0 +- stsci_rtd_theme=0.0.1 - pip: - sphinx-automodapi==0.7 \ No newline at end of file From b31cdf7a18d28c1f332989e58335f6d2f0129aae Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Wed, 18 Apr 2018 10:00:05 -0400 Subject: [PATCH 14/35] Create simply pytests for utils.get_config and utils.filename_parser --- jwql/tests/test_utils.py | 83 ++++++++++++++++++++++++++++++++++++++++ jwql/utils/utils.py | 6 ++- 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 jwql/tests/test_utils.py diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py new file mode 100644 index 000000000..6dd4cda98 --- /dev/null +++ b/jwql/tests/test_utils.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +"""Tests for the utils module. + +Authors +------- + + - Lauren Chambers + + +Use +--- + + These tests can be run via the command line (omit the -s to + suppress verbose output to stdout): + + :: + pytest -s test_utils.py + +""" + +import pytest + +from jwql.utils.utils import get_config, filename_parser + + +def test_get_config(): + '''Assert that the get_config function successfuly creates a + dictionary. + ''' + settings = get_config() + assert isinstance(settings, dict) + + +def test_filename_parser_filename(): + '''Generate a dictionary with parameters from a JWST filename. + Assert that the dictionary matches what is expected. + ''' + filename = 'jw00327001001_02101_00002_nrca1_rate.fits' + filename_dict = filename_parser(filename) + + correct_dict = {'activity': '01', + 'detector': 'nrca1', + 'exposure_id': '00002', + 'observation': '001', + 'parallel_seq_id': '1', + 'program_id': '00327', + 'suffix': 'rate', + 'visit': '001', + 'visit_group': '02'} + + assert filename_dict == correct_dict + + +def test_filename_parser_filepath(): + '''Generate a dictionary with parameters from a JWST filepath + (not just the basename). Assert that the dictionary matches what + is expected. + ''' + filepath = '/test/dir/to/the/file/jw90002/jw90002001001_02102_00001_nis_rateints.fits' + filename_dict = filename_parser(filepath) + + correct_dict = {'activity': '02', + 'detector': 'nis', + 'exposure_id': '00001', + 'observation': '001', + 'parallel_seq_id': '1', + 'program_id': '90002', + 'suffix': 'rateints', + 'visit': '001', + 'visit_group': '02'} + + assert filename_dict == correct_dict + + +def test_filename_parser_nonJWST(): + '''Attempt to generate a file parameter dictionary from a file + that is not formatted in the JWST naming convention. Ensure the + appropriate error is raised. + ''' + with pytest.raises(ValueError, + match=r'Provided file .+ does not follow JWST naming conventions \(jw____\.fits\)'): + filename = 'not_a_jwst_file.fits' + filename_parser(filename) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 166e66185..f6dea7c4b 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -25,6 +25,8 @@ import re import os +__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) + def get_config(): """Return a dictionary that holds the contents of the jwql config @@ -36,7 +38,7 @@ def get_config(): A dictionary that holds the contents of the config file. """ - with open('config.json', 'r') as config_file: + with open(os.path.join(__location__, 'config.json'), 'r') as config_file: settings = json.load(config_file) return settings @@ -79,6 +81,6 @@ def filename_parser(filename): if jwst_file is not None: filename_dict = jwst_file.groupdict() else: - raise ValueError('Provided file {} does not follow JWST naming conventions (jw____ .fits)') + raise ValueError('Provided file {} does not follow JWST naming conventions (jw____.fits)'.format(filename)) return filename_dict From f865a2eb4f67905345e2c69777ea251b91339b66 Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Wed, 18 Apr 2018 17:07:11 -0400 Subject: [PATCH 15/35] Updating docstrings, etc. --- jwql/tests/test_utils.py | 5 +++-- jwql/utils/utils.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index 6dd4cda98..d79aa1f39 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""Tests for the utils module. +"""Tests for the ``utils`` module. Authors ------- @@ -14,6 +14,7 @@ suppress verbose output to stdout): :: + pytest -s test_utils.py """ @@ -24,7 +25,7 @@ def test_get_config(): - '''Assert that the get_config function successfuly creates a + '''Assert that the ``get_config`` function successfuly creates a dictionary. ''' settings = get_config() diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index f6dea7c4b..1bc0401a1 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -22,8 +22,8 @@ """ import json -import re import os +import re __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) @@ -54,7 +54,7 @@ def filename_parser(filename): Returns ------- - dict + filename_dict : dict Collection of file properties Raises From 72513a35e2081da2c666ca289380f157363ecdfa Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 19 Apr 2018 14:04:45 -0400 Subject: [PATCH 16/35] Removed some extra lines --- jwql/tests/test_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index d79aa1f39..2a79c83e7 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -6,7 +6,6 @@ - Lauren Chambers - Use --- @@ -16,7 +15,6 @@ :: pytest -s test_utils.py - """ import pytest From 4130d96d09897a355b8ad2c0266063f4f683104a Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 19 Apr 2018 14:05:35 -0400 Subject: [PATCH 17/35] Minor formatting changes --- jwql/utils/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 1bc0401a1..024b2f7e1 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -3,8 +3,8 @@ Authors ------- - Matthew Bourque - Lauren Chambers + - Matthew Bourque + - Lauren Chambers Use --- @@ -43,6 +43,7 @@ def get_config(): return settings + def filename_parser(filename): """Return a dictionary that contains the properties of a given JWST file (e.g. program ID, visit number, detector, etc.) From 8f914ab530ff4318aba2398a7c1da26ed28d73cd Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 25 Apr 2018 12:27:57 -0400 Subject: [PATCH 18/35] Moved global lists to utils.py and removed POST calls. --- jwql/dbmonitor/dbmonitor.py | 160 ++++++++++++++++++------------------ jwql/utils/utils.py | 4 + 2 files changed, 82 insertions(+), 82 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 5f337734d..b39130e8c 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -15,78 +15,16 @@ >>> inventory, keywords = dbmonitor.jwst_inventory() """ -import sys + +import http.client as httplib import json +import sys from urllib.parse import quote as urlencode -import http.client as httplib + from astroquery.mast import Mast import astropy.table as at -JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] -JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', - 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] - - -def listMissions(): - """Small function to test sevice_request and make sure JWST is - in the list of services - - Returns - ------- - astropy.table.table.Table - The table of all supported services - """ - service = 'Mast.Missions.List' - params = {} - request = Mast.service_request(service, params) - - return request - - -def mastQuery(request): - """A small MAST API wrapper - - Parameters - ---------- - request: dict - The dictionary of 'service', 'format' and 'params' for the request - - Returns - ------- - dict - A dictionary of the 'data', 'fields', 'paging', and 'status' - of the request - """ - # Define the server - server = 'mast.stsci.edu' - - # Grab Python Version - version = ".".join(map(str, sys.version_info[:3])) - - # Create Http Header Variables - headers = {"Content-type": "application/x-www-form-urlencoded", - "Accept": "text/plain", - "User-agent": "python-requests/"+version} - - # Encoding the request as a json string - requestString = json.dumps(request) - requestString = urlencode(requestString) - - # opening the https connection - conn = httplib.HTTPSConnection(server) - - # Making the query - conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) - - # Getting the response - resp = conn.getresponse() - head = resp.getheaders() - content = resp.read().decode('utf-8') - - # Close the https connection - conn.close() - - return head, content +from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, @@ -110,8 +48,9 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, Returns ------- - int + int, dict The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` """ # Make sure the dataproduct is a list if isinstance(dataproduct, str): @@ -127,28 +66,23 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, {'paramName': 'dataproduct_type', 'values': dataproduct}] # Include additonal filters - for k, v in add_filters.items(): - filters.append({'paramName': k, 'values': [v] - if isinstance(v, str) else v}) + for param, value in add_filters.items(): + filters.append({'paramName': param, 'values': [value] + if isinstance(value, str) else value}) # Assemble the request - request = {'service': 'Mast.Caom.Filtered', - 'format': 'json', - 'params': { - 'columns': 'COUNT_BIG(*)', - 'filters': filters}} + params = {'columns': 'COUNT_BIG(*)', 'filters': filters,} # Just get the counts if return_data: - request['params']['columns'] = '*' + params['columns'] = '*' # Add requests if add_requests: - request.update(add_requests) - - # Perform the query - headers, outString = mastQuery(request) - result = json.loads(outString) + params.update(add_requests) + + response = Mast.service_request_async('Mast.Caom.Filtered', params) + result = response[0].json() # Return all the data if return_data: @@ -202,3 +136,65 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, keywords = at.Table([names, types], names=('keyword', 'dtype')) return inventory, keywords + + +def listMissions(): + """Small function to test sevice_request and make sure JWST is + in the list of services + + Returns + ------- + astropy.table.table.Table + The table of all supported services + """ + service = 'Mast.Missions.List' + params = {} + request = Mast.service_request(service, params) + + return request + + +def mastQuery(request): + """A small MAST API wrapper + + Parameters + ---------- + request: dict + The dictionary of 'service', 'format' and 'params' for the request + + Returns + ------- + dict + A dictionary of the 'data', 'fields', 'paging', and 'status' + of the request + """ + # Define the server + server = 'mast.stsci.edu' + + # Grab Python Version + version = ".".join(map(str, sys.version_info[:3])) + + # Create Http Header Variables + headers = {"Content-type": "application/x-www-form-urlencoded", + "Accept": "text/plain", + "User-agent": "python-requests/"+version} + + # Encoding the request as a json string + requestString = json.dumps(request) + requestString = urlencode(requestString) + + # opening the https connection + conn = httplib.HTTPSConnection(server) + + # Making the query + conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) + + # Getting the response + resp = conn.getresponse() + head = resp.getheaders() + content = resp.read().decode('utf-8') + + # Close the https connection + conn.close() + + return head, content diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index a0f122afc..79b82c53e 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -17,6 +17,10 @@ import json +JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', + 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] + def get_config(): """Return a dictionary that holds the contents of the jwql config file. From 087fde480c7bc22d03da1f4511aab39d6431d1d8 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 25 Apr 2018 14:18:33 -0400 Subject: [PATCH 19/35] Added CAOM functions and non-CAOM functions. --- jwql/dbmonitor/dbmonitor.py | 117 ++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 4 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index b39130e8c..28196d327 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -31,6 +31,70 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters={}, add_requests={}, return_data=False): """Get the counts for a given instrument and data product + Parameters + ---------- + instrument: sequence, str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + add_filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. add_filters = {'target_classification':'moving'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int, dict + The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` + """ + # Declare the service + service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) + + # Make sure the dataproduct is a list + if isinstance(dataproduct, str): + dataproduct = [dataproduct] + + # Make sure the instrument is a list + if isinstance(instrument, str): + instrument = [instrument] + + # Set default filters + filters = [] + + # Assemble the request + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns':True,} + + # Just get the counts + if return_data: + params['columns'] = '*' + + # Add requests + if add_requests: + params.update(add_requests) + + response = Mast.service_request_async(service, params) + result = response[0].json() + + # Return all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + + +def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters={}, add_requests={}, return_data=False): + """Get the counts for a given instrument and data product + Parameters ---------- instrument: sequence, str @@ -116,19 +180,64 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Iterate through instruments for instrument in instruments: for dataproduct in dataproducts: - count = instrument_inventory(instrument, dataproduct=dataproduct) + count = instrument_inventory_caom(instrument, dataproduct=dataproduct) + inventory.add_row([instrument, dataproduct, count]) + + # Get number of files for all datatypes for this instrument + count = instrument_inventory_caom(instrument, dataproduct=dataproducts) + inventory.add_row([instrument, '*', count]) + + # Get total number of files + count = instrument_inventory_caom(instruments, dataproduct=dataproducts) + inventory.add_row(['*', '*', count]) + + # Retrieve one dataset to get header keywords + sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + names = [i['name'] for i in sample['fields']] + types = [i['type'] for i in sample['fields']] + keywords = at.Table([names, types], names=('keyword', 'dtype')) + + return inventory, keywords + + +def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube']): + """Gather a full inventory of all JWST data on MAST by instrument/dtype + + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Make master table + inventory = at.Table(names=('instrument', 'dataproduct', 'count'), + dtype=('S8', 'S12', int)) + + # Iterate through instruments + for instrument in instruments: + for dataproduct in dataproducts: + count = instrument_inventory_caom(instrument, dataproduct=dataproduct) inventory.add_row([instrument, dataproduct, count]) # Get number of files for all datatypes for this instrument - count = instrument_inventory(instrument, dataproduct=dataproducts) + count = instrument_inventory_caom(instrument, dataproduct=dataproducts) inventory.add_row([instrument, '*', count]) # Get total number of files - count = instrument_inventory(instruments, dataproduct=dataproducts) + count = instrument_inventory_caom(instruments, dataproduct=dataproducts) inventory.add_row(['*', '*', count]) # Retrieve one dataset to get header keywords - sample = instrument_inventory(instruments, dataproduct=dataproducts, + sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, add_requests={'pagesize': 1, 'page': 1}, return_data=True) names = [i['name'] for i in sample['fields']] From 72d8e7b2c4fc539a0a048b1c21adedde44acc6fd Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 25 Apr 2018 16:17:55 -0400 Subject: [PATCH 20/35] Added donut plot. --- jwql/dbmonitor/dbmonitor.py | 327 ++++++++++++++++++++++++++++++++++++ jwql/utils/utils.py | 3 + 2 files changed, 330 insertions(+) create mode 100644 jwql/dbmonitor/dbmonitor.py diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py new file mode 100644 index 000000000..69a990374 --- /dev/null +++ b/jwql/dbmonitor/dbmonitor.py @@ -0,0 +1,327 @@ +"""This module is home to a suite of MAST queries that gather bulk properties +of available JWST data for JWQL + +Authors +------- + + Joe Filippazzo + +Use +--- + + To get an inventory of all JWST files do: + + >>> from jwql.dbmonitor import dbmonitor + >>> inventory, keywords = dbmonitor.jwst_inventory() + +""" + +import http.client as httplib +import json +import sys +from urllib.parse import quote as urlencode + +from astroquery.mast import Mast +from bokeh.charts import Donut, show, output_file +import pandas as pd + +from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS + + +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters={}, add_requests={}, return_data=False): + """Get the counts for a given instrument and data product + + Parameters + ---------- + instrument: sequence, str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + add_filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. add_filters = {'target_classification':'moving'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int, dict + The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` + """ + # Declare the service + service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) + + # Make sure the dataproduct is a list + if isinstance(dataproduct, str): + dataproduct = [dataproduct] + + # Make sure the instrument is a list + if isinstance(instrument, str): + instrument = [instrument] + + # Set default filters + filters = [] + + # Assemble the request + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns':True,} + + # Just get the counts + if return_data: + params['columns'] = '*' + + # Add requests + if add_requests: + params.update(add_requests) + + response = Mast.service_request_async(service, params) + result = response[0].json() + + # Return all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + + +def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters={}, add_requests={}, return_data=False): + """Get the counts for a given instrument and data product + + Parameters + ---------- + instrument: sequence, str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + dataproduct: sequence, str + The type of data product to search + add_filters: dict + The ('paramName':'values') pairs to include in the 'filters' argument + of the request e.g. add_filters = {'target_classification':'moving'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int, dict + The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` + """ + # Make sure the dataproduct is a list + if isinstance(dataproduct, str): + dataproduct = [dataproduct] + + # Make sure the instrument is a list + if isinstance(instrument, str): + instrument = [instrument] + + # Set default filters + filters = [{'paramName': 'obs_collection', 'values': ['JWST']}, + {'paramName': 'instrument_name', 'values': instrument}, + {'paramName': 'dataproduct_type', 'values': dataproduct}] + + # Include additonal filters + for param, value in add_filters.items(): + filters.append({'paramName': param, 'values': [value] + if isinstance(value, str) else value}) + + # Assemble the request + params = {'columns': 'COUNT_BIG(*)', 'filters': filters,} + + # Just get the counts + if return_data: + params['columns'] = '*' + + # Add requests + if add_requests: + params.update(add_requests) + + response = Mast.service_request_async('Mast.Caom.Filtered', params) + result = response[0].json() + + # Return all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + + +# def jwst_inventory(instruments=JWST_INSTRUMENTS, +# dataproducts=['image', 'spectrum', 'cube']): +# """Gather a full inventory of all JWST data on MAST by instrument/dtype +# +# Parameters +# ---------- +# instruments: sequence +# The list of instruments to count +# dataproducts: sequence +# The types of dataproducts to count +# +# Returns +# ------- +# astropy.table.table.Table +# The table of record counts for each instrument and mode +# """ +# # Make master table +# inventory = at.Table(names=('instrument', 'dataproduct', 'count'), +# dtype=('S8', 'S12', int)) +# +# # Iterate through instruments +# for instrument in instruments: +# for dataproduct in dataproducts: +# count = instrument_inventory_caom(instrument, dataproduct=dataproduct) +# inventory.add_row([instrument, dataproduct, count]) +# +# # Get number of files for all datatypes for this instrument +# count = instrument_inventory_caom(instrument, dataproduct=dataproducts) +# inventory.add_row([instrument, '*', count]) +# +# # Get total number of files +# count = instrument_inventory_caom(instruments, dataproduct=dataproducts) +# inventory.add_row(['*', '*', count]) +# +# # Retrieve one dataset to get header keywords +# sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, +# add_requests={'pagesize': 1, 'page': 1}, +# return_data=True) +# names = [i['name'] for i in sample['fields']] +# types = [i['type'] for i in sample['fields']] +# keywords = at.Table([names, types], names=('keyword', 'dtype')) +# +# return inventory, keywords + + +def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + plot=False): + """Gather a full inventory of all JWST data on MAST by instrument/dtype + + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + plot: bool + Return a pie chart of the data + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Iterate through instruments + inventory = [] + for instrument in instruments: + ins = [instrument] + for dp in dataproducts: + count = instrument_inventory_caom(instrument, dataproduct=dp) + ins.append(count) + + # Get the total + ins.append(sum(ins[-3:])) + + # Add it to the list + inventory.append(ins) + + # Make the table + table = pd.DataFrame(inventory, columns=['instrument']+dataproducts+['total']) + + # Melt the table + table = pd.melt(table, id_vars=['instrument'], + value_vars=dataproducts, + value_name='files', var_name='dataproduct') + + # Retrieve one dataset to get header keywords + sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + data = [[i['name'],i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + # Plot it + if plot: + + # Make the plot + plt = Donut(table, label=['instrument', 'dataproduct'], values='files', + text_font_size='12pt', hover_text='files', + name="JWST Inventory", plot_width=600, plot_height=600) + + show(plt) + + return table, keywords + + +def listMissions(): + """Small function to test sevice_request and make sure JWST is + in the list of services + + Returns + ------- + astropy.table.table.Table + The table of all supported services + """ + service = 'Mast.Missions.List' + params = {} + request = Mast.service_request(service, params) + + return request + + +def mastQuery(request): + """A small MAST API wrapper + + Parameters + ---------- + request: dict + The dictionary of 'service', 'format' and 'params' for the request + + Returns + ------- + dict + A dictionary of the 'data', 'fields', 'paging', and 'status' + of the request + """ + # Define the server + server = 'mast.stsci.edu' + + # Grab Python Version + version = ".".join(map(str, sys.version_info[:3])) + + # Create Http Header Variables + headers = {"Content-type": "application/x-www-form-urlencoded", + "Accept": "text/plain", + "User-agent": "python-requests/"+version} + + # Encoding the request as a json string + requestString = json.dumps(request) + requestString = urlencode(requestString) + + # opening the https connection + conn = httplib.HTTPSConnection(server) + + # Making the query + conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) + + # Getting the response + resp = conn.getresponse() + head = resp.getheaders() + content = resp.read().decode('utf-8') + + # Close the https connection + conn.close() + + return head, content + diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 024b2f7e1..f907612f2 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -27,6 +27,9 @@ __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) +JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] +JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', + 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] def get_config(): """Return a dictionary that holds the contents of the jwql config From e5f1282cf48a56770406afea98714ff3d52c00bd Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 2 May 2018 11:54:45 -0400 Subject: [PATCH 21/35] Tidying up the inventory functions. --- jwql/dbmonitor/dbmonitor.py | 110 +++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 46 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 69a990374..a06efd868 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -22,7 +22,7 @@ from urllib.parse import quote as urlencode from astroquery.mast import Mast -from bokeh.charts import Donut, show, output_file +from bkcharts import Donut, show, output_file import pandas as pd from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS @@ -70,7 +70,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, # Assemble the request params = {'columns': 'COUNT_BIG(*)', 'filters': filters, - 'removenullcolumns':True,} + 'removenullcolumns': True,} # Just get the counts if return_data: @@ -136,7 +136,9 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(value, str) else value}) # Assemble the request - params = {'columns': 'COUNT_BIG(*)', 'filters': filters,} + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns': True,} # Just get the counts if return_data: @@ -158,50 +160,66 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, return result['data'][0]['Column1'] -# def jwst_inventory(instruments=JWST_INSTRUMENTS, -# dataproducts=['image', 'spectrum', 'cube']): -# """Gather a full inventory of all JWST data on MAST by instrument/dtype -# -# Parameters -# ---------- -# instruments: sequence -# The list of instruments to count -# dataproducts: sequence -# The types of dataproducts to count -# -# Returns -# ------- -# astropy.table.table.Table -# The table of record counts for each instrument and mode -# """ -# # Make master table -# inventory = at.Table(names=('instrument', 'dataproduct', 'count'), -# dtype=('S8', 'S12', int)) -# -# # Iterate through instruments -# for instrument in instruments: -# for dataproduct in dataproducts: -# count = instrument_inventory_caom(instrument, dataproduct=dataproduct) -# inventory.add_row([instrument, dataproduct, count]) -# -# # Get number of files for all datatypes for this instrument -# count = instrument_inventory_caom(instrument, dataproduct=dataproducts) -# inventory.add_row([instrument, '*', count]) -# -# # Get total number of files -# count = instrument_inventory_caom(instruments, dataproduct=dataproducts) -# inventory.add_row(['*', '*', count]) -# -# # Retrieve one dataset to get header keywords -# sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, -# add_requests={'pagesize': 1, 'page': 1}, -# return_data=True) -# names = [i['name'] for i in sample['fields']] -# types = [i['type'] for i in sample['fields']] -# keywords = at.Table([names, types], names=('keyword', 'dtype')) -# -# return inventory, keywords +def jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + plot=False): + """Gather a full inventory of all JWST data on MAST by instrument/dtype + + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + plot: bool + Return a pie chart of the data + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + # Iterate through instruments + inventory = [] + for instrument in instruments: + ins = [instrument] + for dp in dataproducts: + count = instrument_inventory(instrument, dataproduct=dp) + ins.append(count) + + # Get the total + ins.append(sum(ins[-3:])) + + # Add it to the list + inventory.append(ins) + + # Make the table + table = pd.DataFrame(inventory, columns=['instrument']+dataproducts+['total']) + + # Melt the table + table = pd.melt(table, id_vars=['instrument'], + value_vars=dataproducts, + value_name='files', var_name='dataproduct') + + # Retrieve one dataset to get header keywords + sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + data = [[i['name'],i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + # Plot it + if plot: + + # Make the plot + plt = Donut(table, label=['instrument', 'dataproduct'], values='files', + text_font_size='12pt', hover_text='files', + name="JWST Inventory", plot_width=600, plot_height=600) + + show(plt) + return table, keywords + def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], From 4b6f6e8d276a6bfb000dae8560e4b05fe7ba9770 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 2 May 2018 12:08:56 -0400 Subject: [PATCH 22/35] PEP8 compliance. --- jwql/dbmonitor/dbmonitor.py | 66 +++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index a06efd868..576c1b66d 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -22,7 +22,7 @@ from urllib.parse import quote as urlencode from astroquery.mast import Mast -from bkcharts import Donut, show, output_file +from bkcharts import Donut, show import pandas as pd from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS @@ -55,7 +55,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, """ # Declare the service service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) - + # Make sure the dataproduct is a list if isinstance(dataproduct, str): dataproduct = [dataproduct] @@ -68,9 +68,9 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, filters = [] # Assemble the request - params = {'columns': 'COUNT_BIG(*)', - 'filters': filters, - 'removenullcolumns': True,} + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns': True} # Just get the counts if return_data: @@ -79,7 +79,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, # Add requests if add_requests: params.update(add_requests) - + response = Mast.service_request_async(service, params) result = response[0].json() @@ -93,7 +93,8 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters={}, add_requests={}, return_data=False): + add_filters={}, add_requests={}, + return_data=False): """Get the counts for a given instrument and data product Parameters @@ -136,9 +137,9 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(value, str) else value}) # Assemble the request - params = {'columns': 'COUNT_BIG(*)', + params = {'columns': 'COUNT_BIG(*)', 'filters': filters, - 'removenullcolumns': True,} + 'removenullcolumns': True} # Just get the counts if return_data: @@ -147,7 +148,7 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, # Add requests if add_requests: params.update(add_requests) - + response = Mast.service_request_async('Mast.Caom.Filtered', params) result = response[0].json() @@ -161,8 +162,8 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, def jwst_inventory(instruments=JWST_INSTRUMENTS, - dataproducts=['image', 'spectrum', 'cube'], - plot=False): + dataproducts=['image', 'spectrum', 'cube'], + plot=False): """Gather a full inventory of all JWST data on MAST by instrument/dtype Parameters @@ -186,26 +187,27 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, for dp in dataproducts: count = instrument_inventory(instrument, dataproduct=dp) ins.append(count) - + # Get the total ins.append(sum(ins[-3:])) - + # Add it to the list inventory.append(ins) - + # Make the table - table = pd.DataFrame(inventory, columns=['instrument']+dataproducts+['total']) - + all_cols = ['instrument']+dataproducts+['total'] + table = pd.DataFrame(inventory, columns=all_cols) + # Melt the table table = pd.melt(table, id_vars=['instrument'], value_vars=dataproducts, value_name='files', var_name='dataproduct') - + # Retrieve one dataset to get header keywords sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, - add_requests={'pagesize': 1, 'page': 1}, - return_data=True) - data = [[i['name'],i['type']] for i in sample['fields']] + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + data = [[i['name'], i['type']] for i in sample['fields']] keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) # Plot it @@ -219,7 +221,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, show(plt) return table, keywords - + def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], @@ -247,26 +249,27 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, for dp in dataproducts: count = instrument_inventory_caom(instrument, dataproduct=dp) ins.append(count) - + # Get the total ins.append(sum(ins[-3:])) - + # Add it to the list inventory.append(ins) - + # Make the table - table = pd.DataFrame(inventory, columns=['instrument']+dataproducts+['total']) - + all_cols = ['instrument']+dataproducts+['total'] + table = pd.DataFrame(inventory, columns=all_cols) + # Melt the table table = pd.melt(table, id_vars=['instrument'], value_vars=dataproducts, value_name='files', var_name='dataproduct') - + # Retrieve one dataset to get header keywords sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, - add_requests={'pagesize': 1, 'page': 1}, - return_data=True) - data = [[i['name'],i['type']] for i in sample['fields']] + add_requests={'pagesize': 1, 'page': 1}, + return_data=True) + data = [[i['name'], i['type']] for i in sample['fields']] keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) # Plot it @@ -342,4 +345,3 @@ def mastQuery(request): conn.close() return head, content - From b00f7c53dd870a3a99700bb25a852d137bab971b Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 4 May 2018 14:43:22 -0400 Subject: [PATCH 23/35] Updated the module docstring to render more nicely in the API docs --- jwql/dbmonitor/dbmonitor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 576c1b66d..3a5935d79 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -10,10 +10,10 @@ --- To get an inventory of all JWST files do: - - >>> from jwql.dbmonitor import dbmonitor - >>> inventory, keywords = dbmonitor.jwst_inventory() - + :: + + from jwql.dbmonitor import dbmonitor + inventory, keywords = dbmonitor.jwst_inventory() """ import http.client as httplib From 92761dc571518bce9a1c3fd8ee21a202870cdb42 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Fri, 11 May 2018 15:47:42 -0400 Subject: [PATCH 24/35] Fixed dbmonitor to accept filters and broke out keyword queries. --- jwql/dbmonitor/dbmonitor.py | 174 ++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 98 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 576c1b66d..5e22f02b7 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -16,11 +16,6 @@ """ -import http.client as httplib -import json -import sys -from urllib.parse import quote as urlencode - from astroquery.mast import Mast from bkcharts import Donut, show import pandas as pd @@ -29,18 +24,19 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters={}, add_requests={}, return_data=False): + add_filters=None, add_requests=None, + return_data=False): """Get the counts for a given instrument and data product Parameters ---------- - instrument: sequence, str + instrument: str The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search add_filters: dict The ('paramName':'values') pairs to include in the 'filters' argument - of the request e.g. add_filters = {'target_classification':'moving'} + of the request e.g. add_filters = {'filter':'GR150R'} add_requests: dict The ('request':'value') pairs to include in the request e.g. add_requests = {'pagesize':1, 'page':1} @@ -64,8 +60,12 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(instrument, str): instrument = [instrument] - # Set default filters - filters = [] + # Include filters + if isinstance(add_filters, dict): + filters = [{"paramName": name, "values": [val]} + for name, val in add_filters.items()] + else: + filters = [] # Assemble the request params = {'columns': 'COUNT_BIG(*)', @@ -77,7 +77,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, params['columns'] = '*' # Add requests - if add_requests: + if isinstance(add_requests, dict): params.update(add_requests) response = Mast.service_request_async(service, params) @@ -93,13 +93,14 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters={}, add_requests={}, + add_filters=None, add_requests=None, return_data=False): - """Get the counts for a given instrument and data product + """Get the counts for a given instrument and data product in the + CAOM service Parameters ---------- - instrument: sequence, str + instrument: str The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] dataproduct: sequence, str The type of data product to search @@ -132,9 +133,10 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, {'paramName': 'dataproduct_type', 'values': dataproduct}] # Include additonal filters - for param, value in add_filters.items(): - filters.append({'paramName': param, 'values': [value] - if isinstance(value, str) else value}) + if isinstance(add_filters, dict): + for param, value in add_filters.items(): + filters.append({'paramName': param, 'values': [value] + if isinstance(value, str) else value}) # Assemble the request params = {'columns': 'COUNT_BIG(*)', @@ -146,7 +148,7 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, params['columns'] = '*' # Add requests - if add_requests: + if isinstance(add_requests, dict): params.update(add_requests) response = Mast.service_request_async('Mast.Caom.Filtered', params) @@ -161,10 +163,55 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, return result['data'][0]['Column1'] +def instrument_keywords(instrument): + """Get the keywords for a given instrument service + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + + Returns + ------- + pd.DataFrame + A DataFrame of the keywords + """ + # Retrieve one dataset to get header keywords + sample = instrument_inventory(instrument, return_data=True, + add_requests={'pagesize': 1, 'page': 1}) + data = [[i['name'], i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + return keywords + + +def instrument_keywords_caom(instrument): + """Get the keywords for a given instrument in the CAOM service + + Parameters + ---------- + instrument: str + The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + + Returns + ------- + pd.DataFrame + A DataFrame of the keywords + """ + # Retrieve one dataset to get header keywords + sample = instrument_inventory_caom(instrument, return_data=True, + add_requests={'pagesize': 1, 'page': 1}) + data = [[i['name'], i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + return keywords + + def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], plot=False): - """Gather a full inventory of all JWST data on MAST by instrument/dtype + """Gather a full inventory of all JWST data in each instrument + service by instrument/dtype Parameters ---------- @@ -181,7 +228,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, The table of record counts for each instrument and mode """ # Iterate through instruments - inventory = [] + inventory, keywords = [], {} for instrument in instruments: ins = [instrument] for dp in dataproducts: @@ -194,6 +241,9 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Add it to the list inventory.append(ins) + # Add the keywords to the dict + keywords[instrument] = instrument_keywords(instrument) + # Make the table all_cols = ['instrument']+dataproducts+['total'] table = pd.DataFrame(inventory, columns=all_cols) @@ -203,13 +253,6 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, value_vars=dataproducts, value_name='files', var_name='dataproduct') - # Retrieve one dataset to get header keywords - sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, - add_requests={'pagesize': 1, 'page': 1}, - return_data=True) - data = [[i['name'], i['type']] for i in sample['fields']] - keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) - # Plot it if plot: @@ -226,7 +269,8 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], plot=False): - """Gather a full inventory of all JWST data on MAST by instrument/dtype + """Gather a full inventory of all JWST data in the CAOM service + by instrument/dtype Parameters ---------- @@ -243,7 +287,7 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, The table of record counts for each instrument and mode """ # Iterate through instruments - inventory = [] + inventory, keywords = [], {} for instrument in instruments: ins = [instrument] for dp in dataproducts: @@ -256,6 +300,9 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, # Add it to the list inventory.append(ins) + # Add the keywords to the dict + keywords[instrument] = instrument_keywords_caom(instrument) + # Make the table all_cols = ['instrument']+dataproducts+['total'] table = pd.DataFrame(inventory, columns=all_cols) @@ -265,13 +312,6 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, value_vars=dataproducts, value_name='files', var_name='dataproduct') - # Retrieve one dataset to get header keywords - sample = instrument_inventory_caom(instruments, dataproduct=dataproducts, - add_requests={'pagesize': 1, 'page': 1}, - return_data=True) - data = [[i['name'], i['type']] for i in sample['fields']] - keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) - # Plot it if plot: @@ -283,65 +323,3 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, show(plt) return table, keywords - - -def listMissions(): - """Small function to test sevice_request and make sure JWST is - in the list of services - - Returns - ------- - astropy.table.table.Table - The table of all supported services - """ - service = 'Mast.Missions.List' - params = {} - request = Mast.service_request(service, params) - - return request - - -def mastQuery(request): - """A small MAST API wrapper - - Parameters - ---------- - request: dict - The dictionary of 'service', 'format' and 'params' for the request - - Returns - ------- - dict - A dictionary of the 'data', 'fields', 'paging', and 'status' - of the request - """ - # Define the server - server = 'mast.stsci.edu' - - # Grab Python Version - version = ".".join(map(str, sys.version_info[:3])) - - # Create Http Header Variables - headers = {"Content-type": "application/x-www-form-urlencoded", - "Accept": "text/plain", - "User-agent": "python-requests/"+version} - - # Encoding the request as a json string - requestString = json.dumps(request) - requestString = urlencode(requestString) - - # opening the https connection - conn = httplib.HTTPSConnection(server) - - # Making the query - conn.request("POST", "/api/v0/invoke", "request="+requestString, headers) - - # Getting the response - resp = conn.getresponse() - head = resp.getheaders() - content = resp.read().decode('utf-8') - - # Close the https connection - conn.close() - - return head, content From c899d1e4d8e87ac8846469f005af0820178d691f Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 11:48:59 -0400 Subject: [PATCH 25/35] Combined CAOM and filtered service queries. --- jwql/dbmonitor/dbmonitor.py | 192 ++++++------------------------------ 1 file changed, 30 insertions(+), 162 deletions(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index fe8a25bfb..d13ffbc8c 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -25,7 +25,7 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters=None, add_requests=None, - return_data=False): + caom=False, return_data=False): """Get the counts for a given instrument and data product Parameters @@ -40,6 +40,8 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_requests: dict The ('request':'value') pairs to include in the request e.g. add_requests = {'pagesize':1, 'page':1} + caom: bool + Query CAOM service return_data: bool Return the actual data instead of counts only @@ -49,94 +51,37 @@ def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, The number of database records that satisfy the search criteria or a dictionary of the data if `return_data=True` """ - # Declare the service - service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) + filters = [] # Make sure the dataproduct is a list if isinstance(dataproduct, str): dataproduct = [dataproduct] - # Make sure the instrument is a list - if isinstance(instrument, str): - instrument = [instrument] + # Make sure the instrument is supported + if instrument.lower() not in [ins.lower() for ins in JWST_INSTRUMENTS]: + raise TypeError('Supported instruments include:', JWST_INSTRUMENTS) - # Include filters - if isinstance(add_filters, dict): - filters = [{"paramName": name, "values": [val]} - for name, val in add_filters.items()] - else: - filters = [] - - # Assemble the request - params = {'columns': 'COUNT_BIG(*)', - 'filters': filters, - 'removenullcolumns': True} - - # Just get the counts - if return_data: - params['columns'] = '*' - - # Add requests - if isinstance(add_requests, dict): - params.update(add_requests) + # CAOM service + if caom: - response = Mast.service_request_async(service, params) - result = response[0].json() + # Declare the service + service = 'Mast.Caom.Filtered' - # Return all the data - if return_data: - return result + # Set the filters + filters += [{'paramName': 'obs_collection', 'values': ['JWST']}, + {'paramName': 'instrument_name', 'values': [instrument]}, + {'paramName': 'dataproduct_type', 'values': dataproduct}] - # Or just the counts + # Instruent filtered service else: - return result['data'][0]['Column1'] - - -def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters=None, add_requests=None, - return_data=False): - """Get the counts for a given instrument and data product in the - CAOM service - - Parameters - ---------- - instrument: str - The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] - dataproduct: sequence, str - The type of data product to search - add_filters: dict - The ('paramName':'values') pairs to include in the 'filters' argument - of the request e.g. add_filters = {'target_classification':'moving'} - add_requests: dict - The ('request':'value') pairs to include in the request - e.g. add_requests = {'pagesize':1, 'page':1} - return_data: bool - Return the actual data instead of counts only - - Returns - ------- - int, dict - The number of database records that satisfy the search criteria - or a dictionary of the data if `return_data=True` - """ - # Make sure the dataproduct is a list - if isinstance(dataproduct, str): - dataproduct = [dataproduct] - - # Make sure the instrument is a list - if isinstance(instrument, str): - instrument = [instrument] - # Set default filters - filters = [{'paramName': 'obs_collection', 'values': ['JWST']}, - {'paramName': 'instrument_name', 'values': instrument}, - {'paramName': 'dataproduct_type', 'values': dataproduct}] + # Declare the service + service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) # Include additonal filters if isinstance(add_filters, dict): - for param, value in add_filters.items(): - filters.append({'paramName': param, 'values': [value] - if isinstance(value, str) else value}) + filters += [{"paramName": name, "values": [val]} + for name, val in add_filters.items()] # Assemble the request params = {'columns': 'COUNT_BIG(*)', @@ -151,7 +96,7 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, if isinstance(add_requests, dict): params.update(add_requests) - response = Mast.service_request_async('Mast.Caom.Filtered', params) + response = Mast.service_request_async(service, params) result = response[0].json() # Return all the data @@ -163,13 +108,15 @@ def instrument_inventory_caom(instrument, dataproduct=JWST_DATAPRODUCTS, return result['data'][0]['Column1'] -def instrument_keywords(instrument): +def instrument_keywords(instrument, caom=False): """Get the keywords for a given instrument service Parameters ---------- instrument: str The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] + caom: bool + Query CAOM service Returns ------- @@ -177,7 +124,7 @@ def instrument_keywords(instrument): A DataFrame of the keywords """ # Retrieve one dataset to get header keywords - sample = instrument_inventory(instrument, return_data=True, + sample = instrument_inventory(instrument, return_data=True, caom=caom, add_requests={'pagesize': 1, 'page': 1}) data = [[i['name'], i['type']] for i in sample['fields']] keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) @@ -185,31 +132,9 @@ def instrument_keywords(instrument): return keywords -def instrument_keywords_caom(instrument): - """Get the keywords for a given instrument in the CAOM service - - Parameters - ---------- - instrument: str - The instrument name, i.e. ['NIRISS','NIRCam','NIRSpec','MIRI','FGS'] - - Returns - ------- - pd.DataFrame - A DataFrame of the keywords - """ - # Retrieve one dataset to get header keywords - sample = instrument_inventory_caom(instrument, return_data=True, - add_requests={'pagesize': 1, 'page': 1}) - data = [[i['name'], i['type']] for i in sample['fields']] - keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) - - return keywords - - def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], - plot=False): + caom=False, plot=False): """Gather a full inventory of all JWST data in each instrument service by instrument/dtype @@ -219,6 +144,8 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, The list of instruments to count dataproducts: sequence The types of dataproducts to count + caom: bool + Query CAOM service plot: bool Return a pie chart of the data @@ -232,66 +159,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, for instrument in instruments: ins = [instrument] for dp in dataproducts: - count = instrument_inventory(instrument, dataproduct=dp) - ins.append(count) - - # Get the total - ins.append(sum(ins[-3:])) - - # Add it to the list - inventory.append(ins) - - # Add the keywords to the dict - keywords[instrument] = instrument_keywords(instrument) - - # Make the table - all_cols = ['instrument']+dataproducts+['total'] - table = pd.DataFrame(inventory, columns=all_cols) - - # Melt the table - table = pd.melt(table, id_vars=['instrument'], - value_vars=dataproducts, - value_name='files', var_name='dataproduct') - - # Plot it - if plot: - - # Make the plot - plt = Donut(table, label=['instrument', 'dataproduct'], values='files', - text_font_size='12pt', hover_text='files', - name="JWST Inventory", plot_width=600, plot_height=600) - - show(plt) - - return table, keywords - - -def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, - dataproducts=['image', 'spectrum', 'cube'], - plot=False): - """Gather a full inventory of all JWST data in the CAOM service - by instrument/dtype - - Parameters - ---------- - instruments: sequence - The list of instruments to count - dataproducts: sequence - The types of dataproducts to count - plot: bool - Return a pie chart of the data - - Returns - ------- - astropy.table.table.Table - The table of record counts for each instrument and mode - """ - # Iterate through instruments - inventory, keywords = [], {} - for instrument in instruments: - ins = [instrument] - for dp in dataproducts: - count = instrument_inventory_caom(instrument, dataproduct=dp) + count = instrument_inventory(instrument, dataproduct=dp, caom=caom) ins.append(count) # Get the total @@ -301,7 +169,7 @@ def jwst_inventory_caom(instruments=JWST_INSTRUMENTS, inventory.append(ins) # Add the keywords to the dict - keywords[instrument] = instrument_keywords_caom(instrument) + keywords[instrument] = instrument_keywords(instrument, caom=caom) # Make the table all_cols = ['instrument']+dataproducts+['total'] From 59ef757af31a57d5073e340ae848ac8ca46473c7 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 12:27:37 -0400 Subject: [PATCH 26/35] Removed duplicate global variables --- jwql/utils/utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 08b07967a..f907612f2 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -27,10 +27,6 @@ __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) -JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] -JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', - 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] - JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] From b247a38a34bb273c0b1a25a6f3304fcdf2b4d074 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 14:12:43 -0400 Subject: [PATCH 27/35] PEP 8 --- jwql/dbmonitor/dbmonitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index d13ffbc8c..8591475cc 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -11,7 +11,7 @@ To get an inventory of all JWST files do: :: - + from jwql.dbmonitor import dbmonitor inventory, keywords = dbmonitor.jwst_inventory() """ From e0e735e95ac0218e6865e75bc69391a8aebad812 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 16:12:55 -0400 Subject: [PATCH 28/35] Started adding tests. --- jwql/tests/test_dbmonitor.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 jwql/tests/test_dbmonitor.py diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py new file mode 100755 index 000000000..424bd5590 --- /dev/null +++ b/jwql/tests/test_dbmonitor.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +"""Tests for the dbmonitor module. + +Authors +------- + + - Joe Filippazzo + + +Use +--- + + These tests can be run via the command line (omit the -s to suppress verbose output to stdout): + + :: + pytest -s test_dbmonitor.py + +""" + +from ..dbmonitor import dbmonitor as db +from ..utils.utils import JWST_INSTRUMENTS + + +def test_caom_keywords(): + """Test to see that the CAOM keywords are the same for all instruments""" + kw = [] + for ins in JWST_INSTRUMENTS: + kw.append(db.instrument_keywords(ins, caom=True)['keyword'].tolist()) + + assert kw[0] == kw[1] == kw[2] == kw[3] == kw[4] From 6ce1f2df7d0b50ed09c855fdd5cd5577244030c0 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 23 May 2018 16:16:32 -0400 Subject: [PATCH 29/35] Formatting docstring. --- jwql/tests/test_dbmonitor.py | 3 +-- jwql/tests/test_permissions.py | 0 2 files changed, 1 insertion(+), 2 deletions(-) mode change 100644 => 100755 jwql/tests/test_permissions.py diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py index 424bd5590..faacb17ef 100755 --- a/jwql/tests/test_dbmonitor.py +++ b/jwql/tests/test_dbmonitor.py @@ -4,8 +4,7 @@ Authors ------- - - Joe Filippazzo - + Joe Filippazzo Use --- diff --git a/jwql/tests/test_permissions.py b/jwql/tests/test_permissions.py old mode 100644 new mode 100755 From bcfe728dce01cb6abfdf08f04f2e43c4e8d47356 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 29 May 2018 15:09:13 -0400 Subject: [PATCH 30/35] Added a few unittests. --- jwql/tests/test_dbmonitor.py | 41 +++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py index faacb17ef..7ac0cb2fc 100755 --- a/jwql/tests/test_dbmonitor.py +++ b/jwql/tests/test_dbmonitor.py @@ -7,9 +7,10 @@ Joe Filippazzo Use ---- +- - These tests can be run via the command line (omit the -s to suppress verbose output to stdout): + These tests can be run via the command line (omit the -s to suppress +verbose output to stdout): :: pytest -s test_dbmonitor.py @@ -20,10 +21,44 @@ from ..utils.utils import JWST_INSTRUMENTS -def test_caom_keywords(): +def test_caom_instrument_keywords(): """Test to see that the CAOM keywords are the same for all instruments""" kw = [] for ins in JWST_INSTRUMENTS: kw.append(db.instrument_keywords(ins, caom=True)['keyword'].tolist()) assert kw[0] == kw[1] == kw[2] == kw[3] == kw[4] + + +def test_filtered_instrument_keywords(): + """Test to see that the instrument specific service keywords are + different for all instruments""" + kw = [] + for ins in JWST_INSTRUMENTS: + kw.append(db.instrument_keywords(ins, caom=False)['keyword'].tolist()) + + assert kw[0] != kw[1] != kw[2] != kw[3] != kw[4] + + +def test_instrument_inventory_filtering(): + """Test to see that the instrument inventory can be filtered""" + filt = 'GR150R' + data = db.instrument_inventory('niriss', + add_filters={'filter': filt}, + return_data=True) + + filters = [row['filter'] for row in data['data']] + + assert all([i == filt for i in filters]) + + +def test_instrument_dataproduct_filtering(): + """Test to see that the instrument inventory can be filtered + by data product""" + dp = 'spectrum' + data = db.instrument_inventory('nirspec', dataproduct=dp, caom=True, + return_data=True) + + dps = [row['dataproduct_type'] for row in data['data']] + + assert all([i == dp for i in dps]) From 1b87971906d8a10130ec370fda0f4381607801f2 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 29 May 2018 15:12:38 -0400 Subject: [PATCH 31/35] Fixed my name in the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f06decc7b..9212a5b56 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Matthew Bourque (INS) - Lauren Chambers (INS) - Misty Cracraft (INS) -- Joseph Filippazo (INS) +- Joe Filippazzo (INS) - Bryan Hilbert (INS) - Graham Kanarek (INS) - Catherine Martlin (INS) From 8ce3800a14e50a2786be9d34eb136b1cc88b2965 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 31 May 2018 11:14:07 -0400 Subject: [PATCH 32/35] Saving the plot to the jwql central storage area instead of just showing it. Using bokeh.charts instead of bkcharts. --- environment.yml | 1 + jwql/dbmonitor/dbmonitor.py | 15 ++++++++++++--- jwql/tests/test_dbmonitor.py | 15 ++++++++------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/environment.yml b/environment.yml index ff3c2a99a..cd9105095 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,7 @@ channels: - http://ssb.stsci.edu/astroconda-dev dependencies: - astropy=3.0 +- bokeh=0.12.5 - django=1.11.8 - jwst=0.7.8rc9 - matplotlib=2.1.1 diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 8591475cc..2a25b2f7f 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -16,11 +16,13 @@ inventory, keywords = dbmonitor.jwst_inventory() """ +import os + from astroquery.mast import Mast -from bkcharts import Donut, show +from bokeh.charts import Donut, save, output_file import pandas as pd -from ..utils.utils import JWST_DATAPRODUCTS, JWST_INSTRUMENTS +from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, @@ -188,6 +190,13 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, text_font_size='12pt', hover_text='files', name="JWST Inventory", plot_width=600, plot_height=600) - show(plt) + # Save the plot + if caom: + output_filename = 'database_monitor_caom.html' + else: + output_filename = 'database_monitor_jwst.html' + outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename) + output_file(outfile) + save(plt) return table, keywords diff --git a/jwql/tests/test_dbmonitor.py b/jwql/tests/test_dbmonitor.py index 7ac0cb2fc..516b0d118 100755 --- a/jwql/tests/test_dbmonitor.py +++ b/jwql/tests/test_dbmonitor.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#! /usr/bin/env python + """Tests for the dbmonitor module. Authors @@ -7,14 +8,13 @@ Joe Filippazzo Use -- - - These tests can be run via the command line (omit the -s to suppress -verbose output to stdout): +--- + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): :: - pytest -s test_dbmonitor.py + pytest -s test_dbmonitor.py """ from ..dbmonitor import dbmonitor as db @@ -22,7 +22,8 @@ def test_caom_instrument_keywords(): - """Test to see that the CAOM keywords are the same for all instruments""" + """Test to see that the CAOM keywords are the same for all + instruments""" kw = [] for ins in JWST_INSTRUMENTS: kw.append(db.instrument_keywords(ins, caom=True)['keyword'].tolist()) From c109a5797a51ade615d992953f4db4e2a55b13bf Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 31 May 2018 11:21:42 -0400 Subject: [PATCH 33/35] Added call to set_permissions for setting the permissions for the output plots. --- jwql/dbmonitor/dbmonitor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jwql/dbmonitor/dbmonitor.py b/jwql/dbmonitor/dbmonitor.py index 2a25b2f7f..5ed0a9654 100644 --- a/jwql/dbmonitor/dbmonitor.py +++ b/jwql/dbmonitor/dbmonitor.py @@ -22,6 +22,7 @@ from bokeh.charts import Donut, save, output_file import pandas as pd +from ..permissions.permissions import set_permissions from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS @@ -198,5 +199,6 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename) output_file(outfile) save(plt) + set_permissions(outfile, verbose=False) return table, keywords From d2f633788c9cfb4d5314282dec0b1327d310ea28 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 31 May 2018 11:30:51 -0400 Subject: [PATCH 34/35] Added bokeh to list of requires. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index af40a2fea..681a0789e 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ AUTHORS = 'Matthew Bourque, Sara Ogaz, Joe Filippazzo, Bryan Hilbert, Misty Cracraft, Graham Kanarek' AUTHORS += 'Johannes Sahlmann, Lauren Chambers, Catherine Martlin' -REQUIRES = ['astropy', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] +REQUIRES = ['astropy', 'bokeh', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] setup( name = 'jwql', From 0cf6175afc29fcd3391f661f3881900562d26ffc Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 31 May 2018 11:42:12 -0400 Subject: [PATCH 35/35] Added astroquery dependency --- environment.yml | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index cd9105095..3fa1223b9 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,7 @@ channels: - http://ssb.stsci.edu/astroconda-dev dependencies: - astropy=3.0 +- astroquery=0.3.8 - bokeh=0.12.5 - django=1.11.8 - jwst=0.7.8rc9 diff --git a/setup.py b/setup.py index 681a0789e..245dac2de 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ AUTHORS = 'Matthew Bourque, Sara Ogaz, Joe Filippazzo, Bryan Hilbert, Misty Cracraft, Graham Kanarek' AUTHORS += 'Johannes Sahlmann, Lauren Chambers, Catherine Martlin' -REQUIRES = ['astropy', 'bokeh', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] +REQUIRES = ['astropy', 'astroquery', 'bokeh', 'django', 'matplotlib', 'numpy', 'python-dateutil', 'sphinx', 'sphinx-automodapi', 'sqlalchemy'] setup( name = 'jwql',