From bf79d8e41ef1bdecd0160b4e3b67b0579488f683 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Mon, 4 Oct 2021 15:23:47 -0400 Subject: [PATCH 1/8] get_cloud_uri/s allow for anonymous access --- astroquery/mast/observations.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 433f4cba94..96ae863018 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -752,12 +752,11 @@ def get_cloud_uris(self, data_products, include_bucket=True, full_url=False): if data_products includes products not found in the cloud. """ - if self._cloud_connection is None: - raise AttributeError("Must enable s3 dataset before attempting to query the s3 information") + self.enable_cloud_dataset() return self._cloud_connection.get_cloud_uri_list(data_products, include_bucket, full_url) - def get_cloud_uri(self, data_product, include_bucket=True, full_url=False): + def get_cloud_uri(self, data_product, include_bucket=True, full_url=False, verbose=True): """ For a given data product, returns the associated cloud URI. If the product is from a mission that does not support cloud access an @@ -783,9 +782,10 @@ def get_cloud_uri(self, data_product, include_bucket=True, full_url=False): found in the cloud, None is returned. """ - if self._cloud_connection is None: - raise AttributeError("Must enable s3 dataset before attempting to query the s3 information") + # Instatiate anonymous cloud access + self.enable_cloud_dataset(verbose) + # Query for product URIs return self._cloud_connection.get_cloud_uri(data_product, include_bucket, full_url) From fe7e232b0671b13be39dacb0d9ecc4667a74968c Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Mon, 4 Oct 2021 15:25:33 -0400 Subject: [PATCH 2/8] comments for clarity --- astroquery/mast/observations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 96ae863018..4385adb973 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -752,8 +752,10 @@ def get_cloud_uris(self, data_products, include_bucket=True, full_url=False): if data_products includes products not found in the cloud. """ + # Instatiate anonymous cloud access self.enable_cloud_dataset() + # Query for product URIs return self._cloud_connection.get_cloud_uri_list(data_products, include_bucket, full_url) def get_cloud_uri(self, data_product, include_bucket=True, full_url=False, verbose=True): From 00c238544a2f251ed9ac78482e65ea5216641ebc Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Mon, 4 Oct 2021 18:00:20 -0400 Subject: [PATCH 3/8] cloud will not be enabled automatically. raises an error if user does not have it enabled before running the cloud uri methods --- astroquery/mast/observations.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 4385adb973..55f08150c9 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -752,13 +752,12 @@ def get_cloud_uris(self, data_products, include_bucket=True, full_url=False): if data_products includes products not found in the cloud. """ - # Instatiate anonymous cloud access - self.enable_cloud_dataset() + if self._cloud_connection == None: + raise RemoteServiceError('Please enable anonymous cloud access by calling `enable_cloud_dataset` method. See MAST Labs documentation for an example: https://mast-labs.stsci.io/#example-data-access-with-astroquery-observations') - # Query for product URIs return self._cloud_connection.get_cloud_uri_list(data_products, include_bucket, full_url) - def get_cloud_uri(self, data_product, include_bucket=True, full_url=False, verbose=True): + def get_cloud_uri(self, data_product, include_bucket=True, full_url=False): """ For a given data product, returns the associated cloud URI. If the product is from a mission that does not support cloud access an @@ -784,8 +783,8 @@ def get_cloud_uri(self, data_product, include_bucket=True, full_url=False, verbo found in the cloud, None is returned. """ - # Instatiate anonymous cloud access - self.enable_cloud_dataset(verbose) + if self._cloud_connection == None: + raise RemoteServiceError('Please enable anonymous cloud access by calling `enable_cloud_dataset` method. See MAST Labs documentation for an example: https://mast-labs.stsci.io/#example-data-access-with-astroquery-observations') # Query for product URIs return self._cloud_connection.get_cloud_uri(data_product, include_bucket, full_url) From 52f1b89dba11887628680009635b44f260e65e5f Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Mon, 4 Oct 2021 18:02:39 -0400 Subject: [PATCH 4/8] pep8 --- astroquery/mast/observations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 55f08150c9..a7996b51a7 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -752,7 +752,7 @@ def get_cloud_uris(self, data_products, include_bucket=True, full_url=False): if data_products includes products not found in the cloud. """ - if self._cloud_connection == None: + if self._cloud_connection is None: raise RemoteServiceError('Please enable anonymous cloud access by calling `enable_cloud_dataset` method. See MAST Labs documentation for an example: https://mast-labs.stsci.io/#example-data-access-with-astroquery-observations') return self._cloud_connection.get_cloud_uri_list(data_products, include_bucket, full_url) @@ -783,7 +783,7 @@ def get_cloud_uri(self, data_product, include_bucket=True, full_url=False): found in the cloud, None is returned. """ - if self._cloud_connection == None: + if self._cloud_connection is None: raise RemoteServiceError('Please enable anonymous cloud access by calling `enable_cloud_dataset` method. See MAST Labs documentation for an example: https://mast-labs.stsci.io/#example-data-access-with-astroquery-observations') # Query for product URIs From a550e79f58d319b49c42e7a008fa24c8dc2e3acc Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Mon, 4 Oct 2021 18:44:22 -0400 Subject: [PATCH 5/8] starting a unit test for cloud_uri --- astroquery/mast/tests/test_mast_remote.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index ca98a31cdc..10ee5f0f3b 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -294,6 +294,20 @@ def test_observations_download_file(self, tmpdir): result = mast.Observations.download_file(uri) assert result == ('COMPLETE', None, None) + def test_get_cloud_uri(self, ) + test_obs_id = '2003600312' + + # get a product list + product = mast.Observations.get_product_list(test_obs_id) + + # enable access to public AWS S3 bucket + mast.Observations.enable_cloud_dataset() + + # get uri + mast.Observations.get_cloud_uri(product) + + def test_get_cloud_uris(self, ) + ###################### # CatalogClass tests # ###################### From 21eb0760b75f150af81c10d975565f8ffb0afb21 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Tue, 5 Oct 2021 11:15:32 -0400 Subject: [PATCH 6/8] assert that there are products to feed through get_cloud_uri --- astroquery/mast/tests/test_mast_remote.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 10ee5f0f3b..3700fd47c8 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -294,19 +294,21 @@ def test_observations_download_file(self, tmpdir): result = mast.Observations.download_file(uri) assert result == ('COMPLETE', None, None) - def test_get_cloud_uri(self, ) - test_obs_id = '2003600312' + def test_get_cloud_uri(self) + test_obs_id = '44308' # get a product list product = mast.Observations.get_product_list(test_obs_id) + assert len(product) > 0, 'No products found for OBSID {}. Unable to move forward with getting URIs from the cloud.'.format(test_obs_id) + # enable access to public AWS S3 bucket mast.Observations.enable_cloud_dataset() # get uri mast.Observations.get_cloud_uri(product) - def test_get_cloud_uris(self, ) + #def test_get_cloud_uris(self, ) ###################### # CatalogClass tests # From ff99717e509ab3fcdbe7f9fac5399a1a545b752f Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Wed, 6 Oct 2021 08:23:15 -0400 Subject: [PATCH 7/8] cloud uri tests now passing with products found --- astroquery/mast/tests/test_mast_remote.py | 28 ++++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 3700fd47c8..f23de11c98 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -294,21 +294,37 @@ def test_observations_download_file(self, tmpdir): result = mast.Observations.download_file(uri) assert result == ('COMPLETE', None, None) - def test_get_cloud_uri(self) - test_obs_id = '44308' + def test_get_cloud_uri(self): + test_obs_id = '25568122' # get a product list - product = mast.Observations.get_product_list(test_obs_id) + product = mast.Observations.get_product_list(test_obs_id)[24] - assert len(product) > 0, 'No products found for OBSID {}. Unable to move forward with getting URIs from the cloud.'.format(test_obs_id) + assert len(product) > 0, f'No product found for OBSID {test_obs_id}. Unable to move forward with getting URIs from the cloud.' # enable access to public AWS S3 bucket mast.Observations.enable_cloud_dataset() # get uri - mast.Observations.get_cloud_uri(product) + uri = mast.Observations.get_cloud_uri(product) - #def test_get_cloud_uris(self, ) + assert len(uri) > 0, f'Product for OBSID {test_obs_id} was not found in the cloud.' + + def test_get_cloud_uris(self): + test_obs_id = '25568122' + + # get a product list + products = mast.Observations.get_product_list(test_obs_id)[24:] + + assert len(products) > 0, f'No products found for OBSID {test_obs_id}. Unable to move forward with getting URIs from the cloud.' + + # enable access to public AWS S3 bucket + mast.Observations.enable_cloud_dataset() + + # get uris + uris = mast.Observations.get_cloud_uris(products) + + assert len(uris) > 0, f'Products for OBSID {test_obs_id} were not found in the cloud.' ###################### # CatalogClass tests # From 5b6d5b197e0bb7afb9c7b037b79f2b77e2fab0bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 12 Nov 2021 14:26:08 -0800 Subject: [PATCH 8/8] Adding line breaks --- astroquery/mast/observations.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index a7996b51a7..8ff1b0ba9e 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -753,7 +753,8 @@ def get_cloud_uris(self, data_products, include_bucket=True, full_url=False): """ if self._cloud_connection is None: - raise RemoteServiceError('Please enable anonymous cloud access by calling `enable_cloud_dataset` method. See MAST Labs documentation for an example: https://mast-labs.stsci.io/#example-data-access-with-astroquery-observations') + raise RemoteServiceError('Please enable anonymous cloud access by calling `enable_cloud_dataset` method. ' + 'See MAST Labs documentation for an example: https://mast-labs.stsci.io/#example-data-access-with-astroquery-observations') return self._cloud_connection.get_cloud_uri_list(data_products, include_bucket, full_url) @@ -784,7 +785,8 @@ def get_cloud_uri(self, data_product, include_bucket=True, full_url=False): """ if self._cloud_connection is None: - raise RemoteServiceError('Please enable anonymous cloud access by calling `enable_cloud_dataset` method. See MAST Labs documentation for an example: https://mast-labs.stsci.io/#example-data-access-with-astroquery-observations') + raise RemoteServiceError('Please enable anonymous cloud access by calling `enable_cloud_dataset` method. ' + 'See MAST Labs documentation for an example: https://mast-labs.stsci.io/#example-data-access-with-astroquery-observations') # Query for product URIs return self._cloud_connection.get_cloud_uri(data_product, include_bucket, full_url)