From 9fde3778c4bb1517b2ab5e0ea8af848b5f45698c Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Fri, 22 Aug 2014 09:42:48 -0400 Subject: [PATCH 01/42] Updated some of the documentation for cosmosim. --- astroquery/cosmosim/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 8aa485416b..153d0a0c3b 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -417,6 +417,7 @@ def abort_job(self,jobid=None): self.check_all_jobs() + def delete_all_jobs(self,phase=None,regex=None): """ A public function which deletes any/all jobs from the server in any phase @@ -429,7 +430,7 @@ def delete_all_jobs(self,phase=None,regex=None): regex : string A regular expression to match all tablenames to. Matching table names will be deleted. """ - + self.check_all_jobs() if regex: From de1ed5849268446ef58b199c51fa1659ca42221b Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Sat, 23 Aug 2014 22:23:37 -0400 Subject: [PATCH 02/42] Added votable storage to download() in cosmosim/core.py. Also, began the creation of the Millennium Database Simulation Query Tool. --- astroquery/cosmosim/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 153d0a0c3b..85adaaffb7 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -629,6 +629,7 @@ def download(self,jobid=None,filename=None,format=None): print("Data written to file: {}.xml".format(filename)) elif format in ['FITS','fits']: print("Need to implement...") + else: if not filename: return headers, data From 5c8d70368a56ebcd7b641c4402a407d43cd2ef25 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Mon, 8 Sep 2014 22:15:36 -0400 Subject: [PATCH 03/42] Undid a git rm -r --- astroquery/millennium/__init__.py | 21 ------------------- astroquery/millennium/core.py | 0 .../tests/test_millennium_remote.py | 0 3 files changed, 21 deletions(-) delete mode 100644 astroquery/millennium/__init__.py delete mode 100644 astroquery/millennium/core.py delete mode 100644 astroquery/millennium/tests/test_millennium_remote.py diff --git a/astroquery/millennium/__init__.py b/astroquery/millennium/__init__.py deleted file mode 100644 index e1a940a15e..0000000000 --- a/astroquery/millennium/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -Millennium Database Query Tool ------------------------------ -.. topic:: Revision History - - Access to all cosmological simulations stored in the Millennium database, via the ??? service. - - http://www. - - :Author: Austen M. Groener -""" - -from astropy.config import ConfigurationItem - -MILLENNIUM_SERVER = ConfigurationItem('millennium_server',["http://www."],'Name of the Millennium mirror to use.') - -MILLENNIUM_TIMEOUT = ConfigurationItem('timeout', 60, 'time limit for connecting to Millennium server') - -from .core import Millennium - -__all__ = ['Millennium'] diff --git a/astroquery/millennium/core.py b/astroquery/millennium/core.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/astroquery/millennium/tests/test_millennium_remote.py b/astroquery/millennium/tests/test_millennium_remote.py deleted file mode 100644 index e69de29bb2..0000000000 From 00215b75c0cd3b086a2e2a533829431bb3b2c506 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Mon, 8 Sep 2014 22:22:44 -0400 Subject: [PATCH 04/42] Began the addition of the Millennium DB tool. --- astroquery/millennium/__init__.py | 0 astroquery/millennium/core.py | 0 astroquery/millennium/tests/__init__.py | 0 astroquery/millennium/tests/test_cosmosim_remote.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 astroquery/millennium/__init__.py create mode 100644 astroquery/millennium/core.py create mode 100644 astroquery/millennium/tests/__init__.py create mode 100644 astroquery/millennium/tests/test_cosmosim_remote.py diff --git a/astroquery/millennium/__init__.py b/astroquery/millennium/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/astroquery/millennium/core.py b/astroquery/millennium/core.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/astroquery/millennium/tests/__init__.py b/astroquery/millennium/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/astroquery/millennium/tests/test_cosmosim_remote.py b/astroquery/millennium/tests/test_cosmosim_remote.py new file mode 100644 index 0000000000..e69de29bb2 From 69410da072a51ee93618ef146ce4c419b156c715 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Mon, 8 Sep 2014 22:34:57 -0400 Subject: [PATCH 05/42] Added a test_cosmosim.py to cosmosim/tests. --- astroquery/cosmosim/tests/test_cosmosim.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 astroquery/cosmosim/tests/test_cosmosim.py diff --git a/astroquery/cosmosim/tests/test_cosmosim.py b/astroquery/cosmosim/tests/test_cosmosim.py new file mode 100644 index 0000000000..e69de29bb2 From 9181315dc8297ce225b0d3cde8968b81e0fc1a68 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Thu, 11 Sep 2014 09:35:30 -0400 Subject: [PATCH 06/42] Fixed up the login logic. Should work nicely with both scripts and interactive sessions. --- astroquery/cosmosim/core.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 85adaaffb7..06a5dc9268 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -41,24 +41,20 @@ def __init__(self): def _login(self, username, password=None): - self.session = requests.session() + if not hasattr(self,'session'): + self.session = requests.session() self.username = username # Get password from keyring or prompt password_from_keyring = keyring.get_password("astroquery:www.cosmosim.org", self.username) if not password_from_keyring: logging.warning("No password was found in the keychain for the provided username.") - - # Check if running from scipt or interactive python session - import __main__ as main - # For script - if hasattr(main,'__file__'): - assert password, "No password provided." + if password: self.password = password - # For interactive session else: self.password = getpass.getpass("{0}, enter your CosmoSim password:\n".format(self.username)) else: + logging.warning("Using the password found in the keychain for the provided username.") self.password = password_from_keyring # Authenticate @@ -79,7 +75,8 @@ def _login(self, username, password=None): # Delete job soup = BeautifulSoup(authenticated.content) - self.delete_job(jobid="{}".format(soup.find("uws:jobid").string),squash=True) + if authenticated.status_code == 200: + self.delete_job(jobid="{}".format(soup.find("uws:jobid").string),squash=True) return authenticated From 2deb9bbe91543498d2546f724f1e8cc16f9434da Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Thu, 11 Sep 2014 09:45:43 -0400 Subject: [PATCH 07/42] Separated out Millennium DB tool stuff into a different branch, entitled: millennium --- astroquery/millennium/__init__.py | 0 astroquery/millennium/core.py | 0 astroquery/millennium/tests/__init__.py | 0 astroquery/millennium/tests/test_cosmosim_remote.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 astroquery/millennium/__init__.py delete mode 100644 astroquery/millennium/core.py delete mode 100644 astroquery/millennium/tests/__init__.py delete mode 100644 astroquery/millennium/tests/test_cosmosim_remote.py diff --git a/astroquery/millennium/__init__.py b/astroquery/millennium/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/astroquery/millennium/core.py b/astroquery/millennium/core.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/astroquery/millennium/tests/__init__.py b/astroquery/millennium/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/astroquery/millennium/tests/test_cosmosim_remote.py b/astroquery/millennium/tests/test_cosmosim_remote.py deleted file mode 100644 index e69de29bb2..0000000000 From 7b077cc7b8c12e24805a7336504f74d46971575a Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Thu, 11 Sep 2014 10:20:16 -0400 Subject: [PATCH 08/42] Added login() option store_password. Also changed default behavior of deletepw (to False) for logout(), so password security is frontloaded. --- astroquery/cosmosim/core.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 06a5dc9268..fb6d6ae539 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -39,7 +39,7 @@ def __init__(self): super(CosmoSim, self).__init__() self.session = requests.session() - def _login(self, username, password=None): + def _login(self, username, password=None, store_password=False): if not hasattr(self,'session'): self.session = requests.session() @@ -70,7 +70,7 @@ def _login(self, username, password=None): # Generating dictionary of existing tables self._existing_tables() - if authenticated.status_code == 200 and password_from_keyring is None: + if authenticated.status_code == 200 and password_from_keyring is None and store_password: keyring.set_password("astroquery:www.cosmosim.org", self.username, self.password) # Delete job @@ -80,7 +80,7 @@ def _login(self, username, password=None): return authenticated - def logout(self,deletepw=True): + def logout(self,deletepw=False): """ Public function which allows the user to logout of their cosmosim credentials. @@ -94,8 +94,12 @@ def logout(self,deletepw=True): if hasattr(self,'username') and hasattr(self,'password') and hasattr(self,'session'): if deletepw is True: - keyring.delete_password("astroquery:www.cosmosim.org", self.username) - print("Removed password for {} in the keychain.".format(self.username)) + try: + keyring.delete_password("astroquery:www.cosmosim.org", self.username) + print("Removed password for {} in the keychain.".format(self.username)) + except: + print("Password for {} was never stored in the keychain.".format(self.username)) + del self.session del self.username del self.password From c0971efd4e233d7356f54deb39a4a9a7e4706286 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 9 Sep 2014 17:47:48 +0200 Subject: [PATCH 09/42] fix print_help and add an exception for HARPS From fd8a9130cc3d3d9b2f4e5a8840b3685934e6f4b0 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 10 Sep 2014 09:55:45 +0200 Subject: [PATCH 10/42] need to check for ipynb more intelligently --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index bcc7924a3b..a93c9bd3fa 100644 --- a/CHANGES +++ b/CHANGES @@ -29,7 +29,6 @@ - Added new tool: ALMA archive query tool http://astroquery.readthedocs.org/en/latest/alma/alma.html (#411) - - setup script and installation fixes 0.2 (2014-08-17) From 3e8b7386da1185dc07350a394cf7f883c16336ba Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 9 Sep 2014 14:08:04 +0200 Subject: [PATCH 11/42] add ALMA query tool [WIP] --- astroquery/alma/core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index 95e0878438..33759c0602 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -341,7 +341,6 @@ def retrieve_data_from_uid(self, uids, cache=True): log.info("Downloading files of size {0}...".format(totalsize.to(u.GB))) # TODO: Add cache=cache keyword here. Currently would have no effect. downloaded_files = self.download_files(file_urls) - return downloaded_files def _parse_result(self, response, verbose=False): @@ -642,7 +641,6 @@ def _validate_payload(self, payload): "by the ALMA query service:" " {0}".format(invalid_params)) - Alma = AlmaClass() def clean_uid(uid): From f09d04c9dca0ca98f9899288a93fc6452f9b4570 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 9 Sep 2014 14:12:49 +0200 Subject: [PATCH 12/42] remove __session reference (it was wrong) --- astroquery/alma/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index 33759c0602..558b1e015b 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -205,7 +205,6 @@ def stage_data(self, uids, cache=False): self._staging_log['request_id'] = request_id log.debug("Request ID: {0}".format(request_id)) - # Submit a request for the specific request ID identified above submission_url = os.path.join(self.dataarchive_url, 'rh', 'submission', request_id) From 13a3c3188dc54c1193b1e03aefdfe172e2168ac5 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 9 Sep 2014 17:11:09 +0200 Subject: [PATCH 13/42] docstrings --- astroquery/alma/core.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index 558b1e015b..8d519731b7 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -1,4 +1,4 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst +tmp_downloadurl = urls[formatlist.index('CSV')]# Licensed under a 3-clause BSD style license - see LICENSE.rst from __future__ import print_function import time import sys @@ -150,7 +150,6 @@ def _get_dataarchive_url(self): self.dataarchive_url = self.archive_url return self.dataarchive_url - def stage_data(self, uids, cache=False): """ Stage ALMA data @@ -247,7 +246,6 @@ def stage_data(self, uids, cache=False): errormessage = root.find('div', id='errorContent').string.strip() raise RemoteServiceError(errormessage) - data_table = root.findAll('table', class_='list', id='report')[0] columns = {'uid':[], 'URL':[], 'size':[]} for tr in data_table.findAll('tr'): @@ -274,7 +272,6 @@ def _HEADER_data_size(self, files): """ Given a list of file URLs, return the data size. This is useful for assessing how much data you might be downloading! - (This is discouraged by the ALMA archive, as it puts unnecessary load on their system) """ From b766a5cd39a1023ec8cd01ab88fa75ba5bc6b00b Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 9 Sep 2014 19:34:15 +0200 Subject: [PATCH 14/42] add tests --- astroquery/alma/tests/test_alma.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astroquery/alma/tests/test_alma.py b/astroquery/alma/tests/test_alma.py index 3d3c5c123a..65b54bd92d 100644 --- a/astroquery/alma/tests/test_alma.py +++ b/astroquery/alma/tests/test_alma.py @@ -71,7 +71,6 @@ def test_SgrAstar(monkeypatch): monkeypatch.setattr(Alma, '_get_dataarchive_url', _get_dataarchive_url) alma = Alma() monkeypatch.setattr(alma, '_get_dataarchive_url', _get_dataarchive_url) - # monkeypatch instructions from https://pytest.org/latest/monkeypatch.html monkeypatch.setattr(alma, '_request', alma_request) # set up local cache path to prevent remote query From 1b7f43b66d2d0f5ad24940ba68295c6ec970112a Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 9 Sep 2014 19:35:49 +0200 Subject: [PATCH 15/42] remove row limit From 266ef92133044af9dea1e55bbff5e941c201396e Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 9 Sep 2014 19:43:47 +0200 Subject: [PATCH 16/42] add remote tests (minimal) --- astroquery/alma/tests/test_alma_remote.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astroquery/alma/tests/test_alma_remote.py b/astroquery/alma/tests/test_alma_remote.py index 5e46882b2e..d0f1b61bdb 100644 --- a/astroquery/alma/tests/test_alma_remote.py +++ b/astroquery/alma/tests/test_alma_remote.py @@ -37,7 +37,6 @@ def test_stage_data(self, temp_dir): result_s = alma.query_object('Sgr A*') assert b'2011.0.00217.S' in result_s['Project_code'] - uid = result_s['Asdm_uid'][0] alma.stage_data([uid]) From 2f708ffca05a44f1a6d714c071efa74e2aae3df7 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 10 Sep 2014 07:50:23 +0200 Subject: [PATCH 17/42] docs! and update API to match the way I used it in the docs --- astroquery/alma/core.py | 1 - astroquery/alma/tests/test_alma_remote.py | 1 - astroquery/query.py | 1 - 3 files changed, 3 deletions(-) diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index 8d519731b7..8c1ccd2c9b 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -333,7 +333,6 @@ def retrieve_data_from_uid(self, uids, cache=True): #log.info("Determining download size for {0} files...".format(len(files))) #each_size,totalsize = self.data_size(files) - log.info("Downloading files of size {0}...".format(totalsize.to(u.GB))) # TODO: Add cache=cache keyword here. Currently would have no effect. downloaded_files = self.download_files(file_urls) diff --git a/astroquery/alma/tests/test_alma_remote.py b/astroquery/alma/tests/test_alma_remote.py index d0f1b61bdb..7724e31064 100644 --- a/astroquery/alma/tests/test_alma_remote.py +++ b/astroquery/alma/tests/test_alma_remote.py @@ -44,7 +44,6 @@ def test_stage_data(self, temp_dir): def test_doc_example(self, temp_dir): alma = Alma() alma.cache_location = temp_dir - m83_data = alma.query_object('M83') assert m83_data.colnames == ['Project_code', 'Source_name', 'RA', 'Dec', 'Band', 'Frequency_resolution', diff --git a/astroquery/query.py b/astroquery/query.py index bd0d8fe7b1..de9ef2d7f4 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -22,7 +22,6 @@ def to_cache(response, cache_file): with open(cache_file, "wb") as f: pickle.dump(response, f) - class AstroQuery(object): def __init__(self, method, url, params=None, data=None, headers=None, From 2734e1008370b25866d10ac1498c9cb0412f5d41 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 10 Sep 2014 07:53:22 +0200 Subject: [PATCH 18/42] py3 fix From c6afcf01237fcc5aa585ae66a4b327fdba103a42 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 10 Sep 2014 09:57:13 +0200 Subject: [PATCH 19/42] use system_tools.in_ipynb() instead of __IPYTHON__ From bd183812887ff79edde10938e37be8f3dba999b6 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 10 Sep 2014 09:35:31 +0200 Subject: [PATCH 20/42] setup.py -> v0.2.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b9d4b178fb..edbc02b6df 100755 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ builtins._ASTROPY_PACKAGE_NAME_ = PACKAGENAME # VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) -VERSION = '0.3.dev' +VERSION = '0.2.2' # Indicates if this version is a release version RELEASE = 'dev' not in VERSION From 9389978d57fb4dff90156a415bec754808e0a401 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 10 Sep 2014 09:36:09 +0200 Subject: [PATCH 21/42] setup.py back to v0.3.dev --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index edbc02b6df..b9d4b178fb 100755 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ builtins._ASTROPY_PACKAGE_NAME_ = PACKAGENAME # VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) -VERSION = '0.2.2' +VERSION = '0.3.dev' # Indicates if this version is a release version RELEASE = 'dev' not in VERSION From 0ea18a1897f73d8f58a799c9e44b7154575c2e8e Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 10 Sep 2014 11:12:50 +0200 Subject: [PATCH 22/42] LAMDA remote tests updated to respect new API From bd134cdb9b797131a96377455bf8bba0b5fdacdd Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 10 Sep 2014 11:24:58 +0200 Subject: [PATCH 23/42] resolves Skyview Issue #414 From 07a406e1f736102a89d8f05a4749310fc0a289a3 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Thu, 11 Sep 2014 16:11:33 +0200 Subject: [PATCH 24/42] use the Query _request scheme --- astroquery/cosmosim/core.py | 115 +++++++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 28 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index fb6d6ae539..0e6a77b777 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -29,20 +29,21 @@ __all__ = ['CosmoSim'] -class CosmoSim(QueryWithLogin): +class CosmoSimClass(QueryWithLogin): QUERY_URL = conf.query_url SCHEMA_URL = conf.schema_url TIMEOUT = conf.timeout def __init__(self): - super(CosmoSim, self).__init__() - self.session = requests.session() + super(CosmoSimClass, self).__init__() + self.session = self._BaseQuery__session def _login(self, username, password=None, store_password=False): if not hasattr(self,'session'): self.session = requests.session() + self._BaseQuery__session = self.session # NOTE FROM AG: I hope this works... self.username = username # Get password from keyring or prompt @@ -59,7 +60,9 @@ def _login(self, username, password=None, store_password=False): # Authenticate print("Authenticating {0} on www.cosmosim.org...".format(self.username)) - authenticated = self.session.post(CosmoSim.QUERY_URL,auth=(self.username,self.password)) + authenticated = self._request('POST', CosmoSim.QUERY_URL, + auth=(self.username,self.password), + cache=False) if authenticated.status_code == 200: print("Authentication successful!") elif authenticated.status_code == 401: @@ -101,6 +104,7 @@ def logout(self,deletepw=False): print("Password for {} was never stored in the keychain.".format(self.username)) del self.session + del self._BaseQuery__session del self.username del self.password else: @@ -112,7 +116,9 @@ def check_login_status(self): """ if hasattr(self,'username') and hasattr(self,'password') and hasattr(self,'session'): - authenticated = self.session.post(CosmoSim.QUERY_URL,auth=(self.username,self.password)) + authenticated = self._request('POST', CosmoSim.QUERY_URL, + auth=(self.username,self.password), + cache=False) if authenticated.status_code == 200: print("Status: You are logged in as {}.".format(self.username)) else: @@ -126,7 +132,8 @@ def check_login_status(self): self.delete_job(jobid="{}".format(soup.find("uws:jobid").string),squash=True) - def run_sql_query(self, query_string,tablename=None,queue=None,mail=None,text=None): + def run_sql_query(self, query_string, tablename=None, queue=None, + mail=None, text=None, cache=True): """ Public function which sends a POST request containing the sql query string. @@ -142,6 +149,8 @@ def run_sql_query(self, query_string,tablename=None,queue=None,mail=None,text=No The user's email address for receiving job completion alerts. text : string The user's cell phone number for receiving job completion alerts. + cache : bool + Whether to cache the query locally Returns ------- @@ -155,15 +164,26 @@ def run_sql_query(self, query_string,tablename=None,queue=None,mail=None,text=No queue = 'short' if tablename in self.table_dict.values(): - result = self.session.post(CosmoSim.QUERY_URL,auth=(self.username,self.password),data={'query':query_string,'phase':'run','queue':queue}) + result = self._request('POST', + CosmoSim.QUERY_URL, + auth=(self.username,self.password), + data={'query':query_string,'phase':'run','queue':queue}, + cache=cache) soup = BeautifulSoup(result.content) gen_tablename = str(soup.find(id="table").string) logging.warning("Table name {} is already taken.".format(tablename)) print("Generated table name: {}".format(gen_tablename)) elif tablename is None: - result = self.session.post(CosmoSim.QUERY_URL,auth=(self.username,self.password),data={'query':query_string,'phase':'run','queue':queue}) + result = self._request('POST', CosmoSim.QUERY_URL, + auth=(self.username, self.password), + data={'query':query_string, 'phase':'run', + 'queue':queue}) else: - result = self.session.post(CosmoSim.QUERY_URL,auth=(self.username,self.password),data={'query':query_string,'table':'{}'.format(tablename),'phase':'run','queue':queue}) + result = self._request('POST', CosmoSim.QUERY_URL, + auth=(self.username, self.password), + data={'query':query_string, + 'table':'{}'.format(tablename), + 'phase':'run', 'queue':queue}) self._existing_tables() soup = BeautifulSoup(result.content) @@ -202,7 +222,8 @@ def check_job_status(self,jobid=None): Parameters ---------- jobid : string - The jobid of the sql query. If no jobid is given, it attemps to use the most recent job (if it exists in this session). + The jobid of the sql query. If no jobid is given, it attempts to + use the most recent job (if it exists in this session). Returns ------- @@ -219,11 +240,14 @@ def check_job_status(self,jobid=None): except: raise AttributeError - response = self.session.get(CosmoSim.QUERY_URL+'/{}'.format(jobid)+'/phase',auth=(self.username,self.password),data={'print':'b'}) - print("Job {}: {}".format(jobid,response.content)) + response = self._request('GET', + CosmoSim.QUERY_URL+'/{}'.format(jobid)+'/phase', + auth=(self.username, self.password), + data={'print':'b'}) + logging.info("Job {}: {}".format(jobid,response.content)) return response.content - def check_all_jobs(self,phase=None,regex=None): + def check_all_jobs(self, phase=None, regex=None): """ Public function which builds a dictionary whose keys are each jobid for a given set of user credentials and whose values are the phase status (e.g. - @@ -242,7 +266,9 @@ def check_all_jobs(self,phase=None,regex=None): The requests response for the GET request for finding all existing jobs. """ - checkalljobs = self.session.get(CosmoSim.QUERY_URL,auth=(self.username,self.password),params={'print':'b'}) + checkalljobs = self._request('GET', CosmoSim.QUERY_URL, + auth=(self.username, self.password), + params={'print':'b'}) self.job_dict={} soup = BeautifulSoup(checkalljobs.content) @@ -311,10 +337,16 @@ def completed_job_info(self,jobid=None,output=False): if jobid is None: completed_jobids = [key for key in self.job_dict.keys() if self.job_dict[key] == 'COMPLETED'] - response_list = [self.session.get(CosmoSim.QUERY_URL+"/{}".format(completed_jobids[i]),auth=(self.username,self.password)) for i in range(len(completed_jobids))] + response_list = [self._request('GET', + CosmoSim.QUERY_URL+"/{}".format(completed_jobids[i]), + auth=(self.username, self.password)) + for i in range(len(completed_jobids))] else: if self.job_dict[jobid] == 'COMPLETED': - response_list = [self.session.get(CosmoSim.QUERY_URL+"/{}".format(jobid),auth=(self.username,self.password))] + response_list = [self._request('GET', + CosmoSim.QUERY_URL+"/{}".format(jobid), + auth=(self.username, + self.password))] else: logging.warning("JobID must refer to a query with a phase of 'COMPLETED'.") return @@ -349,11 +381,17 @@ def general_job_info(self,jobid=None,output=False): self.check_all_jobs() general_jobids = [key for key in self.job_dict.keys() if self.job_dict[key] in ['COMPLETED','ABORTED','ERROR']] if jobid in general_jobids: - response_list = [self.session.get(CosmoSim.QUERY_URL+"/{}".format(jobid),auth=(self.username,self.password))] + response_list = [self._request('GET', + CosmoSim.QUERY_URL+"/{}".format(jobid), + auth=(self.username, + self.password))] else: try: hasattr(self,current_job) - response_list = [self.session.get(CosmoSim.QUERY_URL+"/{}".format(self.current_job),auth=(self.username,self.password))] + response_list = [self._request('GET', + CosmoSim.QUERY_URL+"/{}".format(self.current_job), + auth=(self.username, + self.password))] except (AttributeError, NameError): logging.warning("No current job has been defined, and no jobid has been provided.") @@ -400,7 +438,9 @@ def delete_job(self,jobid=None,squash=None): del self.current_job if self.job_dict[jobid] in ['COMPLETED','ERROR','ABORTED','PENDING']: - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(jobid),auth=(self.username,self.password),data={'follow':''}) + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(jobid), + auth=(self.username, self.password), + data={'follow':''}) else: print("Can only delete a job with phase: 'COMPLETED', 'ERROR', 'ABORTED', or 'PENDING'.") return @@ -446,14 +486,20 @@ def delete_all_jobs(self,phase=None,regex=None): if self.job_dict[key] in phase: if key in self.table_dict.keys(): if self.table_dict[key] in matching_tables: - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key),auth=(self.username,self.password),data={'follow':''}) + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), + auth=(self.username, + self.password), + data={'follow':''}) if not result.ok: result.raise_for_status() print("Deleted job: {} (Table: {})".format(key,self.table_dict[key])) if not regex: for key in self.job_dict.keys(): if self.job_dict[key] in phase: - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key),auth=(self.username,self.password),data={'follow':''}) + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), + auth=(self.username, + self.password), + data={'follow':''}) if not result.ok: result.raise_for_status() print("Deleted job: {}".format(key)) @@ -463,13 +509,19 @@ def delete_all_jobs(self,phase=None,regex=None): for key in self.job_dict.keys(): if key in self.table_dict.keys(): if self.table_dict[key] in matching_tables: - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key),auth=(self.username,self.password),data={'follow':''}) + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), + auth=(self.username, + self.password), + data={'follow':''}) if not result.ok: result.raise_for_status() print("Deleted job: {} (Table: {})".format(key,self.table_dict[key])) if not regex: for key in self.job_dict.keys(): - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key),auth=(self.username,self.password),data={'follow':''}) + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), + auth=(self.username, + self.password), + data={'follow':''}) if not result.ok: result.raise_for_status() print("Deleted job: {}".format(key)) @@ -483,9 +535,10 @@ def _generate_schema(self): the database (in the form of a dictionary). """ - response = requests.get(CosmoSim.SCHEMA_URL, - auth=(self.username,self.password), - headers = {'Accept': 'application/json'}) + response = self._request('GET', CosmoSim.SCHEMA_URL, + auth=(self.username,self.password), + headers={'Accept': 'application/json'}, + cache=True) data = response.json() self.db_dict = {} @@ -609,7 +662,8 @@ def download(self,jobid=None,filename=None,format=None): tableurl = soup.find("uws:result").get("xlink:href") # This is where the request.content parsing happens - raw_table_data = self.session.get(tableurl,auth=(self.username,self.password)) + raw_table_data = self._request('GET', tableurl, auth=(self.username, + self.password)) raw_headers = raw_table_data.content.split('\n')[0] num_cols = len(raw_headers.split(',')) num_rows = len(raw_table_data.content.split('\n'))-2 @@ -636,7 +690,10 @@ def download(self,jobid=None,filename=None,format=None): return headers, data else: with open(filename, 'wb') as fh: - raw_table_data = self.session.get(tableurl,auth=(self.username,self.password),stream=True) + raw_table_data = self._request('GET', tableurl, + auth=(self.username, + self.password), + stream=True) for block in raw_table_data.iter_content(1024): if not block: break @@ -797,3 +854,5 @@ def _alert(self,jobid,queue='short'): self._text(self._smsaddress,self.alert_text,"Job {} Completed with phase {}.".format(jobid,phase)) time.sleep(deltat) + +CosmoSim = CosmoSimClass() From 874a583c4230e85aaf2f81507dbb603df5b5efbc Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Thu, 11 Sep 2014 12:35:42 -0400 Subject: [PATCH 25/42] Added a bit to test_cosmosim_remote. --- .../cosmosim/tests/test_cosmosim_remote.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/astroquery/cosmosim/tests/test_cosmosim_remote.py b/astroquery/cosmosim/tests/test_cosmosim_remote.py index c485a2f991..205c6060bc 100644 --- a/astroquery/cosmosim/tests/test_cosmosim_remote.py +++ b/astroquery/cosmosim/tests/test_cosmosim_remote.py @@ -1,5 +1,24 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import os +import tempfile +import shutil from astropy.tests.helper import pytest, remote_data -from ...cosmosim import CosmoSim +try: + import keyring + HAS_KEYRING = True +except ImportError: + HAS_KEYRING = False +try: + from ...cosmosim import CosmoSim + COSMOSIM_IMPORTED = True +except ImportError: + COSMOSIM_IMPORTED = False from ...exceptions import LoginError + +SKIP_TESTS = not(HAS_KEYRING and ESO_IMPORTED) + +@pytest.mark.skipif('SKIP_TESTS') +@remote_data +class TestEso: + def __init__(): + From df28e34384c5d59287ee2a49f12cf550741d7ec2 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Thu, 11 Sep 2014 16:52:24 +0200 Subject: [PATCH 26/42] add support for 'auth' keyword --- astroquery/query.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/astroquery/query.py b/astroquery/query.py index de9ef2d7f4..1318d2a6e7 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -25,7 +25,7 @@ def to_cache(response, cache_file): class AstroQuery(object): def __init__(self, method, url, params=None, data=None, headers=None, - files=None, timeout=None): + auth=None, files=None, timeout=None): self.method = method self.url = url self.params = params @@ -34,6 +34,7 @@ def __init__(self, method, url, params=None, data=None, headers=None, self.files = files self._hash = None self.timeout = timeout + self.auth = auth @property def timeout(self): @@ -108,7 +109,11 @@ def __call__(self, *args, **kwargs): def _request(self, method, url, params=None, data=None, headers=None, files=None, save=False, savedir='', timeout=None, cache=True, +<<<<<<< HEAD stream=False, auth=None): +======= + auth=None, stream=False): +>>>>>>> 982dded... add support for 'auth' keyword """ A generic HTTP request method, similar to `requests.Session.request` but with added caching-related tools @@ -153,7 +158,8 @@ def _request(self, method, url, params=None, data=None, headers=None, return local_filepath else: query = AstroQuery(method, url, params=params, data=data, - headers=headers, files=files, timeout=timeout) + auth=auth, headers=headers, files=files, + timeout=timeout) if ((self.cache_location is None) or (not self._cache_active) or (not cache)): with suspend_cache(self): From d816ad2b1de45a473e97432ba59987f11a7ea6fe Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Thu, 11 Sep 2014 14:30:23 -0400 Subject: [PATCH 27/42] Attempted to add ok and raise_for_status to AstroResponse. --- astroquery/cosmosim/core.py | 20 ++++++++------------ astroquery/query.py | 4 ---- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 0e6a77b777..b78a2bcf4f 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -11,7 +11,7 @@ from six.moves.email_mime_base import MIMEBase from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import message - +import ipdb # Astropy imports from astropy.table import Table import astropy.units as u @@ -75,11 +75,6 @@ def _login(self, username, password=None, store_password=False): if authenticated.status_code == 200 and password_from_keyring is None and store_password: keyring.set_password("astroquery:www.cosmosim.org", self.username, self.password) - - # Delete job - soup = BeautifulSoup(authenticated.content) - if authenticated.status_code == 200: - self.delete_job(jobid="{}".format(soup.find("uws:jobid").string),squash=True) return authenticated @@ -438,13 +433,13 @@ def delete_job(self,jobid=None,squash=None): del self.current_job if self.job_dict[jobid] in ['COMPLETED','ERROR','ABORTED','PENDING']: - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(jobid), + result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(jobid), auth=(self.username, self.password), data={'follow':''}) else: print("Can only delete a job with phase: 'COMPLETED', 'ERROR', 'ABORTED', or 'PENDING'.") return - + ipdb.set_trace() if not result.ok: result.raise_for_status() if squash is None: @@ -486,7 +481,7 @@ def delete_all_jobs(self,phase=None,regex=None): if self.job_dict[key] in phase: if key in self.table_dict.keys(): if self.table_dict[key] in matching_tables: - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), + result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(key), auth=(self.username, self.password), data={'follow':''}) @@ -496,7 +491,7 @@ def delete_all_jobs(self,phase=None,regex=None): if not regex: for key in self.job_dict.keys(): if self.job_dict[key] in phase: - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), + result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(key), auth=(self.username, self.password), data={'follow':''}) @@ -509,7 +504,7 @@ def delete_all_jobs(self,phase=None,regex=None): for key in self.job_dict.keys(): if key in self.table_dict.keys(): if self.table_dict[key] in matching_tables: - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), + result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(key), auth=(self.username, self.password), data={'follow':''}) @@ -518,10 +513,11 @@ def delete_all_jobs(self,phase=None,regex=None): print("Deleted job: {} (Table: {})".format(key,self.table_dict[key])) if not regex: for key in self.job_dict.keys(): - result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), + result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(key), auth=(self.username, self.password), data={'follow':''}) + ipdb.set_trace() if not result.ok: result.raise_for_status() print("Deleted job: {}".format(key)) diff --git a/astroquery/query.py b/astroquery/query.py index 1318d2a6e7..9c99466d32 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -109,11 +109,7 @@ def __call__(self, *args, **kwargs): def _request(self, method, url, params=None, data=None, headers=None, files=None, save=False, savedir='', timeout=None, cache=True, -<<<<<<< HEAD stream=False, auth=None): -======= - auth=None, stream=False): ->>>>>>> 982dded... add support for 'auth' keyword """ A generic HTTP request method, similar to `requests.Session.request` but with added caching-related tools From fa29a45821565b86e3cd15c9096ec3b13d3bd7d1 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Fri, 12 Sep 2014 11:33:20 -0400 Subject: [PATCH 28/42] Made some fixes to things now that _request() is being used. --- astroquery/cosmosim/core.py | 63 ++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index b78a2bcf4f..13ad9894ef 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -11,7 +11,7 @@ from six.moves.email_mime_base import MIMEBase from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import message -import ipdb + # Astropy imports from astropy.table import Table import astropy.units as u @@ -40,10 +40,18 @@ def __init__(self): self.session = self._BaseQuery__session def _login(self, username, password=None, store_password=False): - + + # login after logging out (interactive) if not hasattr(self,'session'): self.session = requests.session() self._BaseQuery__session = self.session # NOTE FROM AG: I hope this works... + + # login after login (interactive) + if hasattr(self,'username'): + logging.warning("Attempting to login while another use is already logged in.") + self.check_login_status() + return + self.username = username # Get password from keyring or prompt @@ -75,6 +83,11 @@ def _login(self, username, password=None, store_password=False): if authenticated.status_code == 200 and password_from_keyring is None and store_password: keyring.set_password("astroquery:www.cosmosim.org", self.username, self.password) + + # Delete job; prevent them from piling up with phase PENDING + if authenticated.status_code == 200: + soup = BeautifulSoup(authenticated.content) + self.delete_job(jobid=str(soup.find("uws:jobid").string),squash=True) return authenticated @@ -116,16 +129,14 @@ def check_login_status(self): cache=False) if authenticated.status_code == 200: print("Status: You are logged in as {}.".format(self.username)) + soup = BeautifulSoup(authenticated.content) + self.delete_job(jobid=str(soup.find("uws:jobid").string),squash=True) else: logging.warning("Status: The username/password combination for {} appears to be incorrect.".format(self.username)) print("Please re-attempt to login with your cosmosim credentials.") else: print("Status: You are not logged in.") - # Clean up job - soup = BeautifulSoup(authenticated.content) - self.delete_job(jobid="{}".format(soup.find("uws:jobid").string),squash=True) - def run_sql_query(self, query_string, tablename=None, queue=None, mail=None, text=None, cache=True): @@ -172,13 +183,15 @@ def run_sql_query(self, query_string, tablename=None, queue=None, result = self._request('POST', CosmoSim.QUERY_URL, auth=(self.username, self.password), data={'query':query_string, 'phase':'run', - 'queue':queue}) + 'queue':queue}, + cache=cache) else: result = self._request('POST', CosmoSim.QUERY_URL, auth=(self.username, self.password), data={'query':query_string, 'table':'{}'.format(tablename), - 'phase':'run', 'queue':queue}) + 'phase':'run', 'queue':queue}, + cache=cache) self._existing_tables() soup = BeautifulSoup(result.content) @@ -238,7 +251,7 @@ def check_job_status(self,jobid=None): response = self._request('GET', CosmoSim.QUERY_URL+'/{}'.format(jobid)+'/phase', auth=(self.username, self.password), - data={'print':'b'}) + data={'print':'b'},cache=False) logging.info("Job {}: {}".format(jobid,response.content)) return response.content @@ -263,7 +276,8 @@ def check_all_jobs(self, phase=None, regex=None): checkalljobs = self._request('GET', CosmoSim.QUERY_URL, auth=(self.username, self.password), - params={'print':'b'}) + params={'print':'b'},cache=False) + self.job_dict={} soup = BeautifulSoup(checkalljobs.content) @@ -273,7 +287,7 @@ def check_all_jobs(self, phase=None, regex=None): self.job_dict['{}'.format(i.get('xlink:href').split('/')[-1])] = i_phase else: self.job_dict['{}'.format(i.get('id'))] = i_phase - + if regex: pattern = re.compile("{}".format(regex)) groups = [pattern.match(self.table_dict.values()[i]).group() for i in range(len(self.table_dict.values()))] @@ -334,14 +348,14 @@ def completed_job_info(self,jobid=None,output=False): completed_jobids = [key for key in self.job_dict.keys() if self.job_dict[key] == 'COMPLETED'] response_list = [self._request('GET', CosmoSim.QUERY_URL+"/{}".format(completed_jobids[i]), - auth=(self.username, self.password)) + auth=(self.username, self.password),cache=False) for i in range(len(completed_jobids))] else: if self.job_dict[jobid] == 'COMPLETED': response_list = [self._request('GET', CosmoSim.QUERY_URL+"/{}".format(jobid), auth=(self.username, - self.password))] + self.password),cache=False)] else: logging.warning("JobID must refer to a query with a phase of 'COMPLETED'.") return @@ -379,14 +393,14 @@ def general_job_info(self,jobid=None,output=False): response_list = [self._request('GET', CosmoSim.QUERY_URL+"/{}".format(jobid), auth=(self.username, - self.password))] + self.password),cache=False)] else: try: hasattr(self,current_job) response_list = [self._request('GET', CosmoSim.QUERY_URL+"/{}".format(self.current_job), auth=(self.username, - self.password))] + self.password),cache=False)] except (AttributeError, NameError): logging.warning("No current job has been defined, and no jobid has been provided.") @@ -433,13 +447,13 @@ def delete_job(self,jobid=None,squash=None): del self.current_job if self.job_dict[jobid] in ['COMPLETED','ERROR','ABORTED','PENDING']: - result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(jobid), + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(jobid), auth=(self.username, self.password), data={'follow':''}) else: print("Can only delete a job with phase: 'COMPLETED', 'ERROR', 'ABORTED', or 'PENDING'.") return - ipdb.set_trace() + if not result.ok: result.raise_for_status() if squash is None: @@ -481,7 +495,7 @@ def delete_all_jobs(self,phase=None,regex=None): if self.job_dict[key] in phase: if key in self.table_dict.keys(): if self.table_dict[key] in matching_tables: - result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(key), + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), auth=(self.username, self.password), data={'follow':''}) @@ -491,7 +505,7 @@ def delete_all_jobs(self,phase=None,regex=None): if not regex: for key in self.job_dict.keys(): if self.job_dict[key] in phase: - result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(key), + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), auth=(self.username, self.password), data={'follow':''}) @@ -504,7 +518,7 @@ def delete_all_jobs(self,phase=None,regex=None): for key in self.job_dict.keys(): if key in self.table_dict.keys(): if self.table_dict[key] in matching_tables: - result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(key), + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), auth=(self.username, self.password), data={'follow':''}) @@ -513,11 +527,10 @@ def delete_all_jobs(self,phase=None,regex=None): print("Deleted job: {} (Table: {})".format(key,self.table_dict[key])) if not regex: for key in self.job_dict.keys(): - result = self._request('DELETE',CosmoSim.QUERY_URL+"/{}".format(key), + result = self.session.delete(CosmoSim.QUERY_URL+"/{}".format(key), auth=(self.username, self.password), data={'follow':''}) - ipdb.set_trace() if not result.ok: result.raise_for_status() print("Deleted job: {}".format(key)) @@ -630,7 +643,7 @@ def explore_db(self,db=None,table=None,col=None): return - def download(self,jobid=None,filename=None,format=None): + def download(self,jobid=None,filename=None,format=None,cache=True): """ A public function to download data from a job with COMPLETED phase. @@ -659,7 +672,7 @@ def download(self,jobid=None,filename=None,format=None): # This is where the request.content parsing happens raw_table_data = self._request('GET', tableurl, auth=(self.username, - self.password)) + self.password),cache=cache) raw_headers = raw_table_data.content.split('\n')[0] num_cols = len(raw_headers.split(',')) num_rows = len(raw_table_data.content.split('\n'))-2 @@ -689,7 +702,7 @@ def download(self,jobid=None,filename=None,format=None): raw_table_data = self._request('GET', tableurl, auth=(self.username, self.password), - stream=True) + stream=True,cache=cache) for block in raw_table_data.iter_content(1024): if not block: break From e6b40beec8e1402825751f02048b49e63175f134 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Fri, 12 Sep 2014 12:04:04 -0400 Subject: [PATCH 29/42] Fixing explore_db() --- astroquery/cosmosim/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 13ad9894ef..7a96494320 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -48,7 +48,7 @@ def _login(self, username, password=None, store_password=False): # login after login (interactive) if hasattr(self,'username'): - logging.warning("Attempting to login while another use is already logged in.") + logging.warning("Attempting to login while another user ({}) is already logged in.".format(self.username)) self.check_login_status() return @@ -547,7 +547,7 @@ def _generate_schema(self): response = self._request('GET', CosmoSim.SCHEMA_URL, auth=(self.username,self.password), headers={'Accept': 'application/json'}, - cache=True) + cache=False) data = response.json() self.db_dict = {} From 2c2039fb98f5be1ac614a13cedcb18e7af8050fc Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Fri, 12 Sep 2014 20:43:24 -0400 Subject: [PATCH 30/42] Added auth in the same manner. --- astroquery/cosmosim/core.py | 2 +- astroquery/query.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 7a96494320..335a23f19d 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -73,7 +73,7 @@ def _login(self, username, password=None, store_password=False): cache=False) if authenticated.status_code == 200: print("Authentication successful!") - elif authenticated.status_code == 401: + elif authenticated.status_code == 401 or authenticated.status_code == 403: print("Authentication failed!") elif authenticated.status_code == 503: print("Service Temporarily Unavailable...") diff --git a/astroquery/query.py b/astroquery/query.py index 9c99466d32..bd0d8fe7b1 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -22,10 +22,11 @@ def to_cache(response, cache_file): with open(cache_file, "wb") as f: pickle.dump(response, f) + class AstroQuery(object): def __init__(self, method, url, params=None, data=None, headers=None, - auth=None, files=None, timeout=None): + files=None, timeout=None): self.method = method self.url = url self.params = params @@ -34,7 +35,6 @@ def __init__(self, method, url, params=None, data=None, headers=None, self.files = files self._hash = None self.timeout = timeout - self.auth = auth @property def timeout(self): @@ -154,8 +154,7 @@ def _request(self, method, url, params=None, data=None, headers=None, return local_filepath else: query = AstroQuery(method, url, params=params, data=data, - auth=auth, headers=headers, files=files, - timeout=timeout) + headers=headers, files=files, timeout=timeout) if ((self.cache_location is None) or (not self._cache_active) or (not cache)): with suspend_cache(self): From 3bd3367da7c781cde12e0aa8a67018c83d4f2ec2 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Thu, 18 Sep 2014 01:17:43 -0400 Subject: [PATCH 31/42] Switched out the old explore_db function for a new one, using astropy.table to format. Still needs a lot of work. --- astroquery/cosmosim/core.py | 192 ++++++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 39 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 335a23f19d..9f32a8db22 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -1,5 +1,6 @@ from __future__ import print_function import requests +import numpy as np import sys from bs4 import BeautifulSoup import keyring @@ -549,7 +550,6 @@ def _generate_schema(self): headers={'Accept': 'application/json'}, cache=False) data = response.json() - self.db_dict = {} for i in range(len(data['databases'])): self.db_dict['{}'.format(data['databases'][i]['name'])] = {} @@ -599,49 +599,163 @@ def explore_db(self,db=None,table=None,col=None): self.db_dict except AttributeError: self._generate_schema() - + + projects = np.sort(self.db_dict.keys()) + largest = max([len(projects[i]) for i in range(len(projects))]) + t = Table() + # db not specified + if not db: + print("Must first specify a database.") + proj_list = [] + attr_list = [] + info_list = [] + tmp2_largest = 0 + for proj in projects: + size = len(self.db_dict['{}'.format(proj)].keys()) + proj_list += ['@ {}'.format(proj)] + ['' for i in range(size-1)] + ['-'*(largest+2)] + tmp_largest = max([len('{}'.format(key)) + for key in self.db_dict[proj].keys()]) + attr_list += ['@ {}'.format(key) + if isinstance(self.db_dict[proj][key],dict) + else '{}:'.format(key) + for key in self.db_dict[proj].keys()] + ['-'*(tmp_largest+2)] + tmpinfosize = max([len(self.db_dict[proj][key]) + if isinstance(self.db_dict[proj][key],str) + else 0 + for key in self.db_dict[proj].keys()]) + if tmpinfosize > tmp2_largest: + tmp2_largest = tmpinfosize + for proj in projects: + info_list += [self.db_dict[proj][key] + if isinstance(self.db_dict[proj][key],str) + else "" + for key in self.db_dict[proj].keys()] + ['-'*tmp2_largest] + t['Projects'] = proj_list + t['Project Items'] = attr_list + t['Information'] = info_list + t.pprint() + # db specified if db: + try: + size1 = len(self.db_dict['{}'.format(db)].keys()) + slist = [self.db_dict[db][key].keys() + if isinstance(self.db_dict[db][key],dict) + else key + for key in self.db_dict[db].keys()] + size2 = len(max(slist,key=np.size)) + except (KeyError, NameError): + logging.error("Must first specify a valid database.") + return + # if col is specified, table must be specified, and I need to + # check the max size of any given column in the structure if table: - if col: - print("#"*(len(db)+4) + "\n# {} #\n".format(db) + "#"*(len(db)+4)) - print("@ {}".format("tables")) - print(" @ {}".format(table)) - print(" "*6 + "@ {}".format("columns")) - print(" "*9 + "@ {}".format('{}'.format(col))) - for i in self.db_dict['{}'.format(db)]['tables']['{}'.format(table)]['columns']['{}'.format(col)].keys(): - print(" "*12 + "--> {}:{}".format(i,self.db_dict['{}'.format(db)]['tables']['{}'.format(table)]['columns']['{}'.format(col)][i])) + try: + if len(self.db_dict[db]['tables'][table]['columns'].keys()) > size2: + size2 = len(self.db_dict[db]['tables'][table]['columns'].keys()) + + if col: + try: + if len(self.db_dict[db]['tables'][table]['columns'][col].keys()) > size2: + size2 = len(self.db_dict[db]['tables'][table]['columns'][col].keys()) + except(KeyError, NameError): + logging.error("Must first specify a valid column of the `{}` table within the `{}` db.".format(table,db)) + return + except (KeyError, NameError): + logging.error("Must first specify a valid table within the `{}` db.".format(db)) + return + t['Projects'] = ['--> @ {}:'.format(db)] + ['' for i in range(size2-1)] + t['Project Items'] = ['--> @ {}:'.format(key) + if isinstance(self.db_dict[db][key],dict) + and len(self.db_dict[db][key].keys()) == len(self.db_dict[db]['tables'].keys()) + else '@ {}'.format(key) + if isinstance(self.db_dict[db][key],dict) + and len(self.db_dict[db][key].keys()) != len(self.db_dict[db]['tables'].keys()) + else '{}'.format(key) + for key in self.db_dict[db].keys()] + ['' for i in range(size2-size1)] + # if only db is specified + if not table: + if not col: + reordered = sorted(max(slist,key=np.size),key=len) + t['Tables'] = ['@ {}'.format(i) + if isinstance(self.db_dict[db]['tables'][i],dict) + else '{}'.format(i) + for i in reordered] + # if table has been specified + else: + reordered = ['{}'.format(table)] + sorted([key + for key in self.db_dict[db]['tables'].keys() + if key != table],key=len) + t['Tables'] = ['--> @ {}:'.format(i) + if i == table + and isinstance(self.db_dict[db]['tables'][i],dict) + else '@ {}'.format(i) + if i != table + and isinstance(self.db_dict[db]['tables'][i],dict) + else '{}'.format(i) + for i in reordered] + ['' for j in range(size2-len(reordered))] + # if column has been specified + if col: + tblcols_dict = self.db_dict[db]['tables'][table].keys() + t['Table Items'] = ['--> @ columns:'] + [i for i in tblcols_dict if i != 'columns'] + ['' for j in range(size2-len(tblcols_dict))] + col_dict = self.db_dict[db]['tables'][table]['columns'].keys() + reordered = ['{}'.format(col)] + [i for i in col_dict if i != col] + if len(col_dict) < size2: + t['Columns'] = ['--> @ {}:'.format(i) + if isinstance(self.db_dict[db]['tables'][table]['columns'][i],dict) + and i == col + else '--> {}:'.format(i) + if not isinstance(self.db_dict[db]['tables'][table]['columns'][i],dict) + and i == col + else '{}'.format(i) + if not isinstance(self.db_dict[db]['tables'][table]['columns'][i],dict) + and i != col + else '@ {}'.format(i) + if isinstance(self.db_dict[db]['tables'][table]['columns'][i],dict) + and i != col + else '{}'.format(i) + for i in reordered] + ['' for j in range(size2-len(col_dict))] + colinfo_dict = col_dict = self.db_dict[db]['tables'][table]['columns'][col] + t['Col. Info'] = ['{} : {}'.format(i,colinfo_dict[i]) for i in colinfo_dict.keys()] + ['' for j in range(size2-len(colinfo_dict))] + else: + t['Columns'] = ['--> @ {}:'.format(i) + if isinstance(self.db_dict[db]['tables'][table]['columns'][i],dict) + and i == col + else '--> {}:'.format(i) + if not isinstance(self.db_dict[db]['tables'][table]['columns'][i],dict) + and i == col + else '{}'.format(i) + if not isinstance(self.db_dict[db]['tables'][table]['columns'][i],dict) + and i != col + else '@ {}'.format(i) + if isinstance(self.db_dict[db]['tables'][table]['columns'][i],dict) + and i != col + else '{}'.format(i) + for i in reordered] + # if column has not been specified else: - print("#"*(len(db)+4) + "\n# {} #\n".format(db) + "#"*(len(db)+4)) - print("@ {}".format("tables")) - print(" @ {}".format(table)) - for i in self.db_dict['{}'.format(db)]['tables']['{}'.format(table)].keys(): - if type(self.db_dict['{}'.format(db)]['tables']['{}'.format(table)][i]) == dict: - print(" "*6 + "@ {}".format(i)) - for j in self.db_dict['{}'.format(db)]['tables']['{}'.format(table)][i].keys(): - print(" "*9 + "--> {}".format(j)) + tblcols_dict = self.db_dict[db]['tables'][table].keys() + col_dict = self.db_dict[db]['tables'][table]['columns'].keys() + reordered = sorted(col_dict,key=len) + if len(tblcols_dict) < size2: + t['Table Items'] = ['@ {}'.format(i) + if isinstance(self.db_dict[db]['tables'][table][i],dict) + else '{}:'.format(i) + for i in tblcols_dict] + ['' for i in range(size2-len(tblcols_dict))] + t['Table Info'] = ['{}'.format(self.db_dict[db]['tables'][table][i]) + if not isinstance(self.db_dict[db]['tables'][table][i],dict) + else "" + for i in tblcols_dict] + ['' for i in range(size2-len(tblcols_dict))] + if len(col_dict) < size2: + t['Columns'] = ['@ {}'.format(i) + if isinstance(self.db_dict[db]['tables'][table]['columns'][i],dict) + else '{}'.format(i) + for i in reordered] + ['' for i in range(size2-len(col_dict))] else: - print(" "*6 + "$ {}".format(i)) - print(" "*9 + "--> {}".format(self.db_dict['{}'.format(db)]['tables']['{}'.format(table)][i])) - - - else: - print("#"*(len(db)+4) + "\n# {} #\n".format(db) + "#"*(len(db)+4)) - for i in self.db_dict['{}'.format(db)].keys(): - if type(self.db_dict['{}'.format(db)][i]) == dict: - print("@ {}".format(i)) - for j in self.db_dict['{}'.format(db)][i].keys(): - print(" --> {}".format(j)) - else: - print("$ {}".format(i)) - print(" --> {}".format(self.db_dict['{}'.format(db)][i])) - - else: - print("Must choose a database to explore:") - for i in self.db_dict.keys(): - print(" ## " + "{}".format(i)) - - return + t['Columns'] = reordered + else: + t['Table Items'] = tblcols_dict + t.pprint() def download(self,jobid=None,filename=None,format=None,cache=True): """ From 6eb9923ae2702056201334908c1128c2e21f9f4b Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Thu, 18 Sep 2014 13:12:35 -0400 Subject: [PATCH 32/42] Fixed up explore_db in a couple of places. No longer prints only partial tables in the terminal. --- astroquery/cosmosim/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 9f32a8db22..729ecbc6a5 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -22,6 +22,9 @@ from astropy.io import fits from astropy.io import votable import astropy.utils.data as aud +from astropy.table.pprint import conf +conf.max_lines = -1 +conf.max_width = -1 # Astroquery imports from ..utils import commons From aadfd1ca3d87f3effd30cffc56eacf72f5a2359c Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Thu, 18 Sep 2014 15:20:43 -0400 Subject: [PATCH 33/42] Function completed_job_info now creates a global dictionary: response_dict_curent, which contains all of the information for the job(s) fed into completed_job_info. This will break the download function, but that needs an overhaul anyway. --- astroquery/cosmosim/core.py | 56 ++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 729ecbc6a5..475bda92d6 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -12,7 +12,7 @@ from six.moves.email_mime_base import MIMEBase from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import message - +import ipdb # Astropy imports from astropy.table import Table import astropy.units as u @@ -342,8 +342,8 @@ def completed_job_info(self,jobid=None,output=False): Returns ------- - result : list - A list of response object(s) + response_dict_current : dict + A dictionary of completed jobs """ self.check_all_jobs() @@ -354,23 +354,65 @@ def completed_job_info(self,jobid=None,output=False): CosmoSim.QUERY_URL+"/{}".format(completed_jobids[i]), auth=(self.username, self.password),cache=False) for i in range(len(completed_jobids))] + self.response_dict_current = {} + for i,vals in enumerate(completed_jobids): + self.response_dict_current[vals] = self._generate_response_dict(response_list[i]) else: if self.job_dict[jobid] == 'COMPLETED': response_list = [self._request('GET', CosmoSim.QUERY_URL+"/{}".format(jobid), auth=(self.username, self.password),cache=False)] + self.response_dict_current = {} + self.response_dict_current[jobid] = self._generate_response_dict(response_list[0]) else: logging.warning("JobID must refer to a query with a phase of 'COMPLETED'.") return + if output is True: - for i in response_list: - print(i.content) + dictkeys = self.response_dict_current.keys() + if len(dictkeys) > 1: + for i in self.response_dict_current.keys(): + print("{} : COMPLETED".format(i)) + print("Use specific jobid to get more information, or explore `self.response_dict_current`.") + elif len(dictkeys) == 1: + print(self.response_dict_current[dictkeys[0]]['content']) + else: + logging.error('No completed jobs found.') + return else: - print(response_list) + return - return response_list + + + def _generate_response_dict(self,response): + """ + A private function which takes in a response object and creates a response + dictionary . + + Parameters + ---------- + response : requests.models.Response + requests response object + + Returns + ------- + response_dict : dict + A dictionary of some of the repspnse object's methods + """ + + R = response + response_dict = {'{}'.format('content'):R.content, + '{}'.format('cookies'):R.cookies, + '{}'.format('elapsed'):R.elapsed, + '{}'.format('encoding'):R.encoding, + '{}'.format('headers'):R.headers, + '{}'.format('ok'):R.ok, + '{}'.format('request'):R.request, + '{}'.format('url'):R.url} + + return response_dict def general_job_info(self,jobid=None,output=False): """ From 9aa69126ac2648ee0fe0a54b8b4f83896f21d25b Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Thu, 18 Sep 2014 15:21:52 -0400 Subject: [PATCH 34/42] Removed ipdb. --- astroquery/cosmosim/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 475bda92d6..d3ac7a785f 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -12,7 +12,7 @@ from six.moves.email_mime_base import MIMEBase from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import message -import ipdb + # Astropy imports from astropy.table import Table import astropy.units as u From 09167a85f1bda3fdfa9b4bc91772b209d17c24ac Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Sat, 20 Sep 2014 04:26:39 -0400 Subject: [PATCH 35/42] Updated a lot. In the process of switching check_all_jobs over to astropy.table, as well as adding sorting by tablename and starttime. --- astroquery/cosmosim/core.py | 204 +++++++++++++++++++++++++----------- 1 file changed, 140 insertions(+), 64 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index d3ac7a785f..32e7f4ca6c 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -12,7 +12,7 @@ from six.moves.email_mime_base import MIMEBase from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import message - +import ipdb # Astropy imports from astropy.table import Table import astropy.units as u @@ -258,8 +258,8 @@ def check_job_status(self,jobid=None): data={'print':'b'},cache=False) logging.info("Job {}: {}".format(jobid,response.content)) return response.content - - def check_all_jobs(self, phase=None, regex=None): + + def check_all_jobs(self, phase=None, regex=None, sortby=None): """ Public function which builds a dictionary whose keys are each jobid for a given set of user credentials and whose values are the phase status (e.g. - @@ -270,7 +270,11 @@ def check_all_jobs(self, phase=None, regex=None): phase : list A list of phase(s) of jobs to be checked on. If nothing provided, all are checked. regex : string - A regular expression to match all tablenames to. Matching table names will be deleted. + A regular expression to match all tablenames to. Matching table names will be included. + Note - Only tables/starttimes are associated with jobs which have phase COMPLETED. + sortby : string + An option to sort jobs (after phase and regex criteria have been taken into account) + by either the execution start time (`starttime`), or by the table name ('tablename'). Returns ------- @@ -297,6 +301,40 @@ def check_all_jobs(self, phase=None, regex=None): groups = [pattern.match(self.table_dict.values()[i]).group() for i in range(len(self.table_dict.values()))] matching_tables = [groups[i] for i in range(len(groups)) if groups[i] in self.table_dict.values()] self._existing_tables() # creates a fresh up-to-date table_dict + + if sortby: + if sortby.upper() == "TABLENAME": + self._starttime_dict() + if not 'matching_tables' in locals(): + matching_tables = sorted(self.table_dict.values()) + else: + matching_tables = sorted(matching_tables) + matching = zip(*[[(i,self.job_dict[i],self.starttime_dict[i]) + for i in self.table_dict.keys() + if self.table_dict[i] == miter][0] + for miter in matching_tables]) + matching_jobids,matching_phases,matching_starttimes = (matching[0],matching[1],matching[2]) + + elif sortby.upper() == 'STARTTIME': + self._starttime_dict() + if not 'matching_tables' in locals(): + matching_starttimes = sorted(self.starttime_dict.values()) + matching = zip(*[[(i,self.job_dict[i],self.table_dict[i]) + for i in self.starttime_dict.keys() + if self.starttime_dict[i] == miter][0] + for miter in matching_starttimes]) + matching_jobids,matching_phases,matching_tables = (matching[0],matching[1],matching[2]) + else: + matching_tables = matching_tables + matching_starttimes = [[self.starttime_dict[i] + for i in self.table_dict.keys() + if self.table_dict[i] == miter][0] + for miter in matching_tables] + matching = zip(*[[(i,self.job_dict[i],self.table_dict[i]) + for i in self.starttime_dict.keys() + if self.starttime_dict[i] == miter][0] + for miter in matching_starttimes]) + matching_jobids,matching_phases,matching_tables = (matching[0],matching[1],matching[2]) frame = sys._getframe(1) do_not_print_job_dict = ['completed_job_info','general_job_info','delete_all_jobs', @@ -304,34 +342,57 @@ def check_all_jobs(self, phase=None, regex=None): if frame.f_code.co_name in do_not_print_job_dict: return checkalljobs else: - if not phase: - if not regex: - for i in self.job_dict.keys(): - print("{} : {}".format(i,self.job_dict[i])) - if regex: - for i in self.job_dict.keys(): - if i in self.table_dict.keys(): - if self.table_dict[i] in matching_tables: - print("{} : {} (Table: {})".format(i,self.job_dict[i],self.table_dict[i])) - elif phase: + if not phase and not regex: + if not sortby: + t = Table() + t['JobID'] = self.job_dict.keys() + t['Phase'] = self.job_dict.values() + t.pprint() + else: + if sortby.upper() == 'TABLENAME': + t = Table() + t['Tablename'] = matching_tables + t['Starttime'] = matching_starttimes + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t.pprint() + if sortby.upper() == 'STARTTIME': + t = Table() + t['Starttime'] = matching_starttimes + t['Tablename'] = matching_tables + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t.pprint() + + elif not phase and regex: + tmp = [print("{} : {} (Table: {})".format(i,self.job_dict[i],self.table_dict[i])) + if i in self.table_dict.keys() + and self.table_dict[i] in matching_tables + else "" + for i in self.job_dict.keys()] + if phase and not regex: phase = [phase[i].upper() for i in range(len(phase))] - if not regex: - for i in self.job_dict.keys(): - if self.job_dict[i] in phase: - print("{} : {}".format(i,self.job_dict[i])) - if regex: - for i in self.job_dict.keys(): - if self.job_dict[i] in phase: - if i in self.table_dict.keys(): - if self.table_dict[i] in matching_tables: - print("{} : {} (Table: {})".format(i,self.job_dict[i],self.table_dict[i])) - return checkalljobs + tmp = [print("{} : {}".format(i,self.job_dict[i])) + for i in self.job_dict.keys() + if self.job_dict[i] in phase] + if phase and regex: + phase = [phase[i].upper() for i in range(len(phase))] + tmp = [print("{} : {} (Table: {})".format(i,self.job_dict[i],self.table_dict[i])) + if self.job_dict[i] in phase + and i in self.table_dict.keys() + and self.table_dict[i] in matching_tables + else "" + for i in self.job_dict.keys()] + + return checkalljobs def completed_job_info(self,jobid=None,output=False): """ - A public function which sends an http GET request for a given jobid with phase - COMPLETED, and returns a list containing the response object. If no jobid is provided, - a list of all responses with phase COMPLETED is generated. + A public function which sends an http GET request for a given + jobid with phase COMPLETED. If output is True, the function prints + a dictionary to the screen, while always generating a global + dictionary `response_dict_current`. If no jobid is provided, + a visual of all responses with phase COMPLETED is generated. Parameters ---------- @@ -339,11 +400,6 @@ def completed_job_info(self,jobid=None,output=False): The jobid of the sql query. output : bool Print output of response(s) to the terminal - - Returns - ------- - response_dict_current : dict - A dictionary of completed jobs """ self.check_all_jobs() @@ -413,12 +469,31 @@ def _generate_response_dict(self,response): '{}'.format('url'):R.url} return response_dict + + def _starttime_dict(self): + """ + A private function which generates a dictionary of jobids (must have phase + COMPLETED) linked to starttimes. + """ + + completed_ids = [key + for key in self.job_dict.keys() + if self.job_dict[key] == 'COMPLETED'] + response_list = [self._request('GET', + CosmoSim.QUERY_URL+"/{}".format(i), + auth=(self.username,self.password),cache=False) + for i in completed_ids] + soups = [BeautifulSoup(response_list[i].content) for i in range(len(response_list))] + self.starttime_dict = {} + for i in range(len(soups)): + self.starttime_dict['{}'.format(completed_ids[i])] = '{}'.format(soups[i].find('uws:starttime').string) + def general_job_info(self,jobid=None,output=False): """ - A public function which sends an http GET request for a given jobid with phase COMPLETED, - ERROR, or ABORTED, and returns a list containing the response object. If no jobid is provided, - the current job is used (if it exists). + A public function which sends an http GET request for a given + jobid in any phase. If no jobid is provided, a summary of all + jobs is generated. Parameters ---------- @@ -426,39 +501,40 @@ def general_job_info(self,jobid=None,output=False): The jobid of the sql query. output : bool Print output of response(s) to the terminal - - Returns - ------- - result : list - A list of response object(s) """ self.check_all_jobs() - general_jobids = [key for key in self.job_dict.keys() if self.job_dict[key] in ['COMPLETED','ABORTED','ERROR']] - if jobid in general_jobids: - response_list = [self._request('GET', - CosmoSim.QUERY_URL+"/{}".format(jobid), - auth=(self.username, - self.password),cache=False)] + + if jobid is None: + print("Job Summary:") + print("There are {} jobs with phase: COMPLETED.".format(self.job_dict.values().count('COMPLETED'))) + print("There are {} jobs with phase: ERROR.".format(self.job_dict.values().count('ERROR'))) + print("There are {} jobs with phase: ABORTED.".format(self.job_dict.values().count('ABORTED'))) + print("There are {} jobs with phase: PENDING.".format(self.job_dict.values().count('PENDING'))) + print("There are {} jobs with phase: EXECUTING.".format(self.job_dict.values().count('EXECUTING'))) + print("There are {} jobs with phase: QUEUED.".format(self.job_dict.values().count('QUEUED'))) + print("Try providing a jobid for the job you'd like to know more about.") + print("To see a list of all jobs, use `check_all_jobs()`.") + return else: - try: - hasattr(self,current_job) - response_list = [self._request('GET', - CosmoSim.QUERY_URL+"/{}".format(self.current_job), - auth=(self.username, + response_list = [self._request('GET', + CosmoSim.QUERY_URL+"/{}".format(jobid), + auth=(self.username, self.password),cache=False)] - except (AttributeError, NameError): - logging.warning("No current job has been defined, and no jobid has been provided.") - - - if output: - for i in response_list: - print(i.content) + if response_list[0].ok is False: + logging.error('Must provide a valid jobid.') + return + else: + self.response_dict_current = {} + self.response_dict_current[jobid] = self._generate_response_dict(response_list[0]) + + if output is True: + ipdb.set_trace() + dictkeys = self.response_dict_current.keys() + print(self.response_dict_current[dictkeys[0]]['content']) + return else: - print(response_list) - - return response_list - + return def delete_job(self,jobid=None,squash=None): """ @@ -825,7 +901,7 @@ def download(self,jobid=None,filename=None,format=None,cache=True): raise self.check_all_jobs() - completed_job_responses = self.completed_job_info(jobid) + completed_job_responses = self.completed_job_info(jobid) # Re-do this; completed_job_info does not return anything, but rather creates/re-creates a dictionary soup = BeautifulSoup(completed_job_responses[0].content) tableurl = soup.find("uws:result").get("xlink:href") From c56fbcd1960ad4dc0e37adf2733ce93da896bf45 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Sat, 20 Sep 2014 14:26:33 -0400 Subject: [PATCH 36/42] Finished adding sorting as well as astropy.table style output for check_all_jobs. --- astroquery/cosmosim/core.py | 141 ++++++++++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 22 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 32e7f4ca6c..d42ba45a0f 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -12,7 +12,7 @@ from six.moves.email_mime_base import MIMEBase from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import message -import ipdb + # Astropy imports from astropy.table import Table import astropy.units as u @@ -295,16 +295,47 @@ def check_all_jobs(self, phase=None, regex=None, sortby=None): self.job_dict['{}'.format(i.get('xlink:href').split('/')[-1])] = i_phase else: self.job_dict['{}'.format(i.get('id'))] = i_phase + + if phase: + phase = [phase[i].upper() for i in range(len(phase))] if regex: pattern = re.compile("{}".format(regex)) - groups = [pattern.match(self.table_dict.values()[i]).group() for i in range(len(self.table_dict.values()))] - matching_tables = [groups[i] for i in range(len(groups)) if groups[i] in self.table_dict.values()] + try: + groups = [pattern.match(self.table_dict.values()[i]).group() + for i in range(len(self.table_dict.values())) + if pattern.match(self.table_dict.values()[i]) is not None] + matching_tables = [groups[i] + for i in range(len(groups)) + if groups[i] in self.table_dict.values()] + except AttributeError: + print('No tables matching the regular expression `{}` were found.'.format(regex)) + matching_tables = self.table_dict.values() + + if phase: + if "COMPLETED" not in phase: + print("No jobs found with phase `{}` matching the regular expression `{}` were found.".format(phase,regex)) + print("Matching regular expression `{}` to all jobs with phase `COMPLETED` instead (unsorted):".format(regex)) + else: + matching_tables = [[self.table_dict[i] + for i in self.table_dict.keys() + if self.table_dict[i] == miter + and self.job_dict[i] in phase + ][0] + for miter in matching_tables] self._existing_tables() # creates a fresh up-to-date table_dict + self._starttime_dict() + + if not sortby: + if regex: + matching = zip(*[[(i,self.job_dict[i],self.starttime_dict[i]) + for i in self.table_dict.keys() + if self.table_dict[i] == miter][0] + for miter in matching_tables]) + matching_jobids,matching_phases,matching_starttimes = (matching[0],matching[1],matching[2]) if sortby: if sortby.upper() == "TABLENAME": - self._starttime_dict() if not 'matching_tables' in locals(): matching_tables = sorted(self.table_dict.values()) else: @@ -316,7 +347,6 @@ def check_all_jobs(self, phase=None, regex=None, sortby=None): matching_jobids,matching_phases,matching_starttimes = (matching[0],matching[1],matching[2]) elif sortby.upper() == 'STARTTIME': - self._starttime_dict() if not 'matching_tables' in locals(): matching_starttimes = sorted(self.starttime_dict.values()) matching = zip(*[[(i,self.job_dict[i],self.table_dict[i]) @@ -365,24 +395,92 @@ def check_all_jobs(self, phase=None, regex=None, sortby=None): t.pprint() elif not phase and regex: - tmp = [print("{} : {} (Table: {})".format(i,self.job_dict[i],self.table_dict[i])) - if i in self.table_dict.keys() - and self.table_dict[i] in matching_tables - else "" - for i in self.job_dict.keys()] + t = Table() + if sortby: + if sortby.upper() == 'STARTTIME': + t['Starttime'] = matching_starttimes + t['Tablename'] = matching_tables + if sortby.upper() == 'TABLENAME': + t['Tablename'] = matching_tables + t['Starttime'] = matching_starttimes + if not sortby: + t['Tablename'] = matching_tables + t['Starttime'] = matching_starttimes + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t.pprint() + + if phase and not regex: - phase = [phase[i].upper() for i in range(len(phase))] - tmp = [print("{} : {}".format(i,self.job_dict[i])) - for i in self.job_dict.keys() - if self.job_dict[i] in phase] + if len(phase) == 1 and "COMPLETED" in phase: + if not sortby: + matching_jobids = [key + for key in self.job_dict.keys() + if self.job_dict[key] in phase] + matching = zip(*[[(self.table_dict[i],self.job_dict[i],self.starttime_dict[i]) + for i in self.table_dict.keys() + if i == miter][0] + for miter in matching_jobids]) + matching_tables,matching_phases,matching_starttimes = (matching[0],matching[1],matching[2]) + t = Table() + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t['Tablename'] = matching_tables + t['Starttime'] = matching_starttimes + t.pprint() + if sortby: + if sortby.upper() == 'TABLENAME': + t = Table() + t['Tablename'] = matching_tables + t['Starttime'] = matching_starttimes + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t.pprint() + if sortby.upper() == 'STARTTIME': + t = Table() + t['Starttime'] = matching_starttimes + t['Tablename'] = matching_tables + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t.pprint() + else: + if sortby: + print('Sorting can only be applied to jobs with phase `COMPLETED`.') + if not sortby: + matching_jobids = [key + for key in self.job_dict.keys() + if self.job_dict[key] in phase] + matching_phases = [self.job_dict[key] + for key in self.job_dict.keys() + if self.job_dict[key] in phase] + t = Table() + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t.pprint() + if phase and regex: - phase = [phase[i].upper() for i in range(len(phase))] - tmp = [print("{} : {} (Table: {})".format(i,self.job_dict[i],self.table_dict[i])) - if self.job_dict[i] in phase - and i in self.table_dict.keys() - and self.table_dict[i] in matching_tables - else "" - for i in self.job_dict.keys()] + if not sortby: + t = Table() + t['Tablename'] = matching_tables + t['Starttime'] = matching_starttimes + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t.pprint() + else: + if sortby.upper() == 'TABLENAME': + t = Table() + t['Tablename'] = matching_tables + t['Starttime'] = matching_starttimes + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t.pprint() + if sortby.upper() == 'STARTTIME': + t = Table() + t['Starttime'] = matching_starttimes + t['Tablename'] = matching_tables + t['JobID'] = matching_jobids + t['Phase'] = matching_phases + t.pprint() return checkalljobs @@ -529,7 +627,6 @@ def general_job_info(self,jobid=None,output=False): self.response_dict_current[jobid] = self._generate_response_dict(response_list[0]) if output is True: - ipdb.set_trace() dictkeys = self.response_dict_current.keys() print(self.response_dict_current[dictkeys[0]]['content']) return From e223bb5a0189dc41270d97dc953f2bc88c496693 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Sat, 20 Sep 2014 18:33:28 -0400 Subject: [PATCH 37/42] Finished switching out printing for astropy.table output. --- astroquery/cosmosim/core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index d42ba45a0f..ee5b96e824 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -527,8 +527,12 @@ def completed_job_info(self,jobid=None,output=False): if output is True: dictkeys = self.response_dict_current.keys() if len(dictkeys) > 1: - for i in self.response_dict_current.keys(): - print("{} : COMPLETED".format(i)) + keys = [i for i in self.response_dict_current.keys()] + phases = [self.job_dict[key] for key in keys] + t = Table() + t['JobID'] = keys + t['Phase'] = phases + t.pprint() print("Use specific jobid to get more information, or explore `self.response_dict_current`.") elif len(dictkeys) == 1: print(self.response_dict_current[dictkeys[0]]['content']) From dd6218db96b9f52e99f66401811ca0cd1a072ab9 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Sun, 21 Sep 2014 22:05:06 -0400 Subject: [PATCH 38/42] Overhauled the download function to take one of four data formats. --- astroquery/cosmosim/core.py | 132 +++++++++++++++++++++++------------- 1 file changed, 83 insertions(+), 49 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index ee5b96e824..6919673411 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -20,7 +20,6 @@ import astropy.io.votable as votable from astropy import log as logging from astropy.io import fits -from astropy.io import votable import astropy.utils.data as aud from astropy.table.pprint import conf conf.max_lines = -1 @@ -987,64 +986,99 @@ def download(self,jobid=None,filename=None,format=None,cache=True): ---------- jobid : Completed jobid to be downloaded - filename : string - If left blank, downloaded to the terminal. If specified, data is written out to file (directory can be included here). + filename : str + If left blank, downloaded to the terminal. If specified, data is + written out to file (directory can be included here). + format : str + The format of the data to be downloaded. Options are `csv`, `votable`, + `votableB1`, and `votableB2`. + cache : bool + Whether to cache the data. By default, this is set to True. Returns ------- headers, data : list, list """ + self.check_all_jobs() + if not jobid: try: jobid = self.current_job - except: - raise - - self.check_all_jobs() - completed_job_responses = self.completed_job_info(jobid) # Re-do this; completed_job_info does not return anything, but rather creates/re-creates a dictionary - soup = BeautifulSoup(completed_job_responses[0].content) - tableurl = soup.find("uws:result").get("xlink:href") - - # This is where the request.content parsing happens - raw_table_data = self._request('GET', tableurl, auth=(self.username, - self.password),cache=cache) - raw_headers = raw_table_data.content.split('\n')[0] - num_cols = len(raw_headers.split(',')) - num_rows = len(raw_table_data.content.split('\n'))-2 - headers = [raw_headers.split(',')[i].strip('"') for i in range(num_cols)] - raw_data = [raw_table_data.content.split('\n')[i+1].split(",") for i in range(num_rows)] - data = [map(eval,raw_data[i]) for i in range(num_rows)] - - if format: - tbl = Table(data=map(list, zip(*data)),names=headers) - if format in ['VOTable','votable']: - votbl = votable.from_table(tbl) - if not filename: - return votbl - else: - if '.xml' in filename: - filename = filename.split('.')[0] - votable.writeto(votbl, "{}.xml".format(filename)) - print("Data written to file: {}.xml".format(filename)) - elif format in ['FITS','fits']: - print("Need to implement...") + except AttributeError: + print("No current job has been defined for this session.") + return - else: - if not filename: - return headers, data - else: - with open(filename, 'wb') as fh: - raw_table_data = self._request('GET', tableurl, - auth=(self.username, - self.password), - stream=True,cache=cache) - for block in raw_table_data.iter_content(1024): - if not block: - break - fh.write(block) - print("Data written to file: {}".format(filename)) - return headers, data + if self.job_dict['{}'.format(jobid)] == 'COMPLETED': + if not format: + print("Must specify a format:") + t = Table() + t['Format'] = ['csv','votable','votableB1','votableB2'] + t['Description'] = ['Comma-separated values file', + 'Put In Description', + 'Put In Description', + 'Put In Description'] + t.pprint() + if format: + results = self._request('GET', + self.QUERY_URL+"/{}/results".format(jobid), + auth=(self.username,self.password)) + soup = BeautifulSoup(results.content) + urls = [i.get('xlink:href') for i in soup.findAll('uws:result')] + formatlist = [urls[i].split('/')[-1].upper() for i in range(len(urls))] + + if format.upper() in formatlist: + index = formatlist.index(format.upper()) + downloadurl = urls[index] + if filename: + self._download_file(downloadurl, + local_filepath=filename, + auth=(self.username,self.password)) + elif not filename: + if format.upper() == 'CSV': + raw_table_data = self._request('GET', + downloadurl, + auth=(self.username,self.password), + cache=cache).content + raw_headers = raw_table_data.split('\n')[0] + num_cols = len(raw_headers.split(',')) + num_rows = len(raw_table_data.split('\n'))-2 + headers = [raw_headers.split(',')[i].strip('"') for i in range(num_cols)] + raw_data = [raw_table_data.split('\n')[i+1].split(",") for i in range(num_rows)] + data = [map(eval,raw_data[i]) for i in range(num_rows)] + return headers,data + elif format.upper() in ['VOTABLEB1','VOTABLEB2']: + print("Cannot view binary versions of votable within the terminal.") + print("Try saving them to your disk with the `filename` option.") + return + elif format.upper() == 'VOTABLE': + # for terminal output, get data in csv format + tmp_downloadurl = urls[formatlist.index('CSV')] + raw_table_data = self._request('GET', + tmp_downloadurl, + auth=(self.username,self.password), + cache=cache).content + raw_headers = raw_table_data.split('\n')[0] + num_cols = len(raw_headers.split(',')) + num_rows = len(raw_table_data.split('\n'))-2 + headers = [raw_headers.split(',')[i].strip('"') for i in range(num_cols)] + raw_data = [raw_table_data.split('\n')[i+1].split(",") for i in range(num_rows)] + data = [map(eval,raw_data[i]) for i in range(num_rows)] + # store in astropy.Table object + tbl = Table(data=map(list, zip(*data)),names=headers) + # convert to votable format + votbl = votable.from_table(tbl) + return votbl + + elif format.upper() not in formatlist: + print('Format not recognized. Please see formatting options:') + t = Table() + t['Format'] = ['csv','votable','votableB1','votableB2'] + t['Description'] = ['Comma-separated values file', + 'Put In Description', + 'Put In Description', + 'Put In Description'] + t.pprint() def _check_phase(self,jobid): """ From 0d7693be5fa91f366aa46817cc8a1d0762c1b687 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Sun, 21 Sep 2014 23:14:08 -0400 Subject: [PATCH 39/42] Removed _BaseQuery__session from cosomosim/core.py. --- astroquery/cosmosim/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index 6919673411..f64f9e9d60 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -40,14 +40,14 @@ class CosmoSimClass(QueryWithLogin): def __init__(self): super(CosmoSimClass, self).__init__() - self.session = self._BaseQuery__session + #self.session = self._BaseQuery__session def _login(self, username, password=None, store_password=False): # login after logging out (interactive) if not hasattr(self,'session'): self.session = requests.session() - self._BaseQuery__session = self.session # NOTE FROM AG: I hope this works... + #self._BaseQuery__session = self.session # NOTE FROM AG: I hope this works... # login after login (interactive) if hasattr(self,'username'): From 0fcaa4b1868e9c4fca8d59de0656c1b18eb39d48 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Sun, 21 Sep 2014 23:25:07 -0400 Subject: [PATCH 40/42] Fixed a few typos which were causing errors in Travis. --- astroquery/alma/core.py | 2 +- astroquery/cosmosim/tests/test_cosmosim_remote.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index 8c1ccd2c9b..8195b933df 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -1,4 +1,4 @@ -tmp_downloadurl = urls[formatlist.index('CSV')]# Licensed under a 3-clause BSD style license - see LICENSE.rst +# Licensed under a 3-clause BSD style license - see LICENSE.rst from __future__ import print_function import time import sys diff --git a/astroquery/cosmosim/tests/test_cosmosim_remote.py b/astroquery/cosmosim/tests/test_cosmosim_remote.py index 205c6060bc..3ecfef57da 100644 --- a/astroquery/cosmosim/tests/test_cosmosim_remote.py +++ b/astroquery/cosmosim/tests/test_cosmosim_remote.py @@ -17,8 +17,8 @@ SKIP_TESTS = not(HAS_KEYRING and ESO_IMPORTED) -@pytest.mark.skipif('SKIP_TESTS') -@remote_data -class TestEso: - def __init__(): +#@pytest.mark.skipif('SKIP_TESTS') +#@remote_data +#class TestEso: +# def __init__(): From f609b9f4de207968004e15559791d3fa565eff98 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Sun, 21 Sep 2014 23:31:47 -0400 Subject: [PATCH 41/42] One more typo.. --- astroquery/cosmosim/tests/test_cosmosim_remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/cosmosim/tests/test_cosmosim_remote.py b/astroquery/cosmosim/tests/test_cosmosim_remote.py index 3ecfef57da..07fe711419 100644 --- a/astroquery/cosmosim/tests/test_cosmosim_remote.py +++ b/astroquery/cosmosim/tests/test_cosmosim_remote.py @@ -15,7 +15,7 @@ COSMOSIM_IMPORTED = False from ...exceptions import LoginError -SKIP_TESTS = not(HAS_KEYRING and ESO_IMPORTED) +SKIP_TESTS = not(HAS_KEYRING and COSMOSIM_IMPORTED) #@pytest.mark.skipif('SKIP_TESTS') #@remote_data From 8d710d20addaec320831f6b7ab0163f9f205b5e1 Mon Sep 17 00:00:00 2001 From: Austen Groener Date: Mon, 22 Sep 2014 08:11:52 -0400 Subject: [PATCH 42/42] Fixed a remaining typo in logout. --- astroquery/cosmosim/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astroquery/cosmosim/core.py b/astroquery/cosmosim/core.py index f64f9e9d60..e65e677d1b 100644 --- a/astroquery/cosmosim/core.py +++ b/astroquery/cosmosim/core.py @@ -115,7 +115,6 @@ def logout(self,deletepw=False): print("Password for {} was never stored in the keychain.".format(self.username)) del self.session - del self._BaseQuery__session del self.username del self.password else: