From f9350b4cf9822e6c1a83a7e3f20aaa39e76b3012 Mon Sep 17 00:00:00 2001 From: James Dempsey Date: Mon, 27 Sep 2021 16:18:03 +1000 Subject: [PATCH 1/3] Allow non image data to be retrieved Non image data (e.g. visibilities) will not have a 'cutout_service' in the datalink response but will have an 'async_service'. As this is also present for all image data we should use this unless we are requesting cutouts (which we will add to our astroquery module in the future). In addition I have fixed a bug where if there were no files that the user had access to it would give an obscure error when they were asked to be staged. --- README.rst | 1 + astroquery/casda/core.py | 18 ++++++++--- .../casda/tests/data/datalink_noaccess.xml | 32 +++++++++++++++++++ astroquery/casda/tests/test_casda.py | 19 +++++++++-- docs/casda/casda.rst | 8 ++--- 5 files changed, 67 insertions(+), 11 deletions(-) create mode 100644 astroquery/casda/tests/data/datalink_noaccess.xml diff --git a/README.rst b/README.rst index 356a3af354..077be3c250 100644 --- a/README.rst +++ b/README.rst @@ -109,6 +109,7 @@ The following modules have been completed using a common API: * `Besancon `_: Model of stellar population synthesis in the Galaxy. * `CDS MOC Service `_: A collection of all-sky survey coverage maps. * `CADC `_: Canadian Astronomy Data Centre. + * `CASDA `_: CSIRO ASKAP Science Data Archive. * `ESASky `_: ESASky is a science driven discovery portal providing easy visualizations and full access to the entire sky as observed with ESA Space astronomy missions. * `ESO Archive `_ * `FIRST `_: Faint Images of the Radio Sky at Twenty-cm. 20-cm radio images of the extragalactic sky from the VLA. diff --git a/astroquery/casda/core.py b/astroquery/casda/core.py index cae67836bb..2ae438ca8f 100644 --- a/astroquery/casda/core.py +++ b/astroquery/casda/core.py @@ -184,13 +184,21 @@ def stage_data(self, table, verbose=False): # Use datalink to get authenticated access for each file tokens = [] + soda_url = None for row in table: access_url = row['access_url'] - response = self._request('GET', access_url, auth=self._auth, - timeout=self.TIMEOUT, cache=False) - response.raise_for_status() - soda_url, id_token = self._parse_datalink_for_service_and_id(response, 'cutout_service') - tokens.append(id_token) + if access_url: + response = self._request('GET', access_url, auth=self._auth, + timeout=self.TIMEOUT, cache=False) + response.raise_for_status() + service_url, id_token = self._parse_datalink_for_service_and_id(response, 'async_service') + if id_token: + tokens.append(id_token) + soda_url = service_url + + # Trap a request with no allowed data + if not soda_url: + raise ValueError('You do not have access to any of the requested data files.') # Create job to stage all files job_url = self._create_soda_job(tokens, soda_url=soda_url) diff --git a/astroquery/casda/tests/data/datalink_noaccess.xml b/astroquery/casda/tests/data/datalink_noaccess.xml new file mode 100644 index 0000000000..34d452a32c --- /dev/null +++ b/astroquery/casda/tests/data/datalink_noaccess.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + +
3242 + + DefaultFault: Sorry, but you do not have permission to access this data product + #error + + +
+ + \ No newline at end of file diff --git a/astroquery/casda/tests/test_casda.py b/astroquery/casda/tests/test_casda.py index 244e57613f..2053ebadaf 100644 --- a/astroquery/casda/tests/test_casda.py +++ b/astroquery/casda/tests/test_casda.py @@ -21,7 +21,7 @@ pytest.skip("Install mock for the casda tests.", allow_module_level=True) DATA_FILES = {'CIRCLE': 'cone.xml', 'RANGE': 'box.xml', 'DATALINK': 'datalink.xml', 'RUN_JOB': 'run_job.xml', - 'COMPLETED_JOB': 'completed_job.xml'} + 'COMPLETED_JOB': 'completed_job.xml', 'DATALINK_NOACCESS' : 'datalink_noaccess.xml'} class MockResponse: @@ -59,7 +59,10 @@ def get_mockreturn(self, method, url, data=None, timeout=10, else: raise ValueError("Unexpected SODA async {} call to url {}".format(method, url)) elif 'datalink' in str(url): - key = 'DATALINK' + if 'cube-244' in str(url): + key = 'DATALINK' + else: + key = 'DATALINK_NOACCESS' else: key = params['POS'].split()[0] if params['POS'] else None filename = data_path(DATA_FILES[key]) @@ -257,6 +260,18 @@ def test_stage_data_invalid_credentials(patch_get): casda.stage_data(table) +def test_stage_data_no_link(patch_get): + prefix = 'https://somewhere/casda/datalink/links?' + access_urls = [prefix + 'cube-240'] + table = Table([Column(data=access_urls, name='access_url')]) + casda = Casda('user', 'password') + casda.POLL_INTERVAL = 1 + + with pytest.raises(ValueError) as excinfo: + casda.stage_data(table) + + assert "You do not have access to any of the requested data files." in str(excinfo.value) + def test_stage_data(patch_get): prefix = 'https://somewhere/casda/datalink/links?' access_urls = [prefix + 'cube-244'] diff --git a/docs/casda/casda.rst b/docs/casda/casda.rst index 0354492584..d087b62963 100644 --- a/docs/casda/casda.rst +++ b/docs/casda/casda.rst @@ -27,8 +27,8 @@ For example: >>> from astropy import units as u >>> centre = SkyCoord.from_name('NGC 7232') >>> result_table = Casda.query_region(centre, radius=30*u.arcmin) - >>> print(result_table['obs_publisher_did','s_ra', 's_dec', 'obs_released_date']) - obs_publisher_did s_ra s_dec obs_released_date + >>> print(result_table['obs_publisher_did','s_ra', 's_dec', 'obs_release_date']) + obs_publisher_did s_ra s_dec obs_release_date deg deg ----------------- --------------- ---------------- ------------------------ cube-502 333.16767306594 -45.302084636451 2017-08-02T03:51:19.728Z @@ -64,8 +64,8 @@ For example to filter out the 30 non-public results from the above data set: .. code-block:: python >>> public_results = Casda.filter_out_unreleased(result_table) - >>> print(public_results['obs_publisher_did','s_ra', 's_dec', 'obs_released_date']) - obs_publisher_did s_ra s_dec obs_released_date + >>> print(public_results['obs_publisher_did','s_ra', 's_dec', 'obs_release_date']) + obs_publisher_did s_ra s_dec obs_release_date deg deg ----------------- --------------- ---------------- ------------------------ cube-502 333.16767306594 -45.302084636451 2017-08-02T03:51:19.728Z From 20f725104de63d5e593ae3504dbad6ee476966d9 Mon Sep 17 00:00:00 2001 From: James Dempsey Date: Mon, 27 Sep 2021 16:31:02 +1000 Subject: [PATCH 2/3] Style fixes --- astroquery/casda/tests/test_casda.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astroquery/casda/tests/test_casda.py b/astroquery/casda/tests/test_casda.py index 2053ebadaf..c8b81fae83 100644 --- a/astroquery/casda/tests/test_casda.py +++ b/astroquery/casda/tests/test_casda.py @@ -21,7 +21,7 @@ pytest.skip("Install mock for the casda tests.", allow_module_level=True) DATA_FILES = {'CIRCLE': 'cone.xml', 'RANGE': 'box.xml', 'DATALINK': 'datalink.xml', 'RUN_JOB': 'run_job.xml', - 'COMPLETED_JOB': 'completed_job.xml', 'DATALINK_NOACCESS' : 'datalink_noaccess.xml'} + 'COMPLETED_JOB': 'completed_job.xml', 'DATALINK_NOACCESS': 'datalink_noaccess.xml'} class MockResponse: @@ -272,6 +272,7 @@ def test_stage_data_no_link(patch_get): assert "You do not have access to any of the requested data files." in str(excinfo.value) + def test_stage_data(patch_get): prefix = 'https://somewhere/casda/datalink/links?' access_urls = [prefix + 'cube-244'] From 2dd51b464ec1fbc129ad4412534f6961999727ce Mon Sep 17 00:00:00 2001 From: James Dempsey Date: Mon, 27 Sep 2021 16:32:30 +1000 Subject: [PATCH 3/3] Update CHANGES.rst --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index a8bf4c85f6..43788437df 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,12 @@ New Tools and Services Service fixes and enhancements ------------------------------ +casda +^^^^^^ + +- Add ability to stage and download non image data which have been found + through the CASDA obscore table. [#2158] + vizier ^^^^^^