From 415a436151f6f83a26972e888ae51569b414e11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Wed, 21 Aug 2019 10:36:09 +0200 Subject: [PATCH 01/11] Add tests for schema and country list. --- backend/tests/test_application.py | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 backend/tests/test_application.py diff --git a/backend/tests/test_application.py b/backend/tests/test_application.py new file mode 100644 index 000000000..ba5a9fd67 --- /dev/null +++ b/backend/tests/test_application.py @@ -0,0 +1,62 @@ +""" +Test the browser handlers +""" +import json + +import requests + +BASE_URL = "http://localhost:4000" + +def test_get_schema(): + """ + Test GetSchema.get() + """ + response = requests.get(f'{BASE_URL}/api/schema') + data = json.loads(response.text) + expected = {'@context': 'http://schema.org/', + '@type': 'DataCatalog', + 'name': 'SweFreq', + 'alternateName': ['The Swedish Frequency resource for genomics']} + assert len(data) == 10 + for value in expected: + assert data[value] == expected[value] + + ds_name = 'SweGen' + response = requests.get(f'{BASE_URL}/api/schema?url={BASE_URL}/dataset/{ds_name}/browser') + data = json.loads(response.text) + expected = {"@type": "Dataset", + "url": f"{BASE_URL}/dataset/{ds_name}", + "@id": f"{BASE_URL}/dataset/{ds_name}", + "name": f"{ds_name}", + "description": "desc", + "identifier": f"{ds_name}", + "citation": "doi"} + assert data['dataset'] == expected + + response = requests.get(f'{BASE_URL}/api/schema?url={BASE_URL}/dataset/{ds_name}/beacon') + data = json.loads(response.text) + expected = {'@id': 'https://swefreq.nbis.se/api/beacon-elixir/', + '@type': 'Beacon', + 'dct:conformsTo': 'https://bioschemas.org/specifications/drafts/Beacon/', + 'name': 'Swefreq Beacon', + 'provider': {'@type': 'Organization', + 'name': 'National Bioinformatics Infrastructure Sweden', + 'alternateName': ['NBIS', 'ELIXIR Sweden'], + 'logo': 'http://nbis.se/assets/img/logos/nbislogo-green.svg', + 'url': 'https://nbis.se/'}, + 'supportedRefs': ['GRCh37'], + 'description': 'Beacon API Web Server based on the GA4GH Beacon API', + 'aggregator': False, + 'url': 'https://swefreq.nbis.se/api/beacon-elixir/'} + for value in expected: + assert data[value] == expected[value] + + +def test_get_countrylist(): + """ + Test CountryList.get() + """ + response = requests.get(f'{BASE_URL}/api/countries') + data = json.loads(response.text) + + assert len(data['countries']) == 240 From 4c16acb367c0e1120dc7fdc293efa2995aeebcc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Wed, 21 Aug 2019 13:02:31 +0200 Subject: [PATCH 02/11] No longer necessecary to handle exceptions as they are managed within get_dataset_version(). --- backend/application.py | 44 ++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/backend/application.py b/backend/application.py index 99bd49672..d5ffde0da 100644 --- a/backend/application.py +++ b/backend/application.py @@ -102,33 +102,27 @@ def get(self): if dataset: dataset_schema = {'@type': "Dataset"} - try: - dataset_version = db.get_dataset_version(dataset, version) - if dataset_version is None: - self.send_error(status_code=404) + dataset_version = db.get_dataset_version(dataset, version) + if dataset_version is None: + self.send_error(status_code=404) + return + + if dataset_version.available_from > datetime.now(): + # If it's not available yet, only return if user is admin. + if not (self.current_user and + self.current_user.is_admin(dataset_version.dataset)): + self.send_error(status_code=403) return - if dataset_version.available_from > datetime.now(): - # If it's not available yet, only return if user is admin. - if not (self.current_user and - self.current_user.is_admin(dataset_version.dataset)): - self.send_error(status_code=403) - return - - base_url = "%s://%s" % (self.request.protocol, self.request.host) - dataset_schema['url'] = base_url + "/dataset/" + dataset_version.dataset.short_name - dataset_schema['@id'] = dataset_schema['url'] - dataset_schema['name'] = dataset_version.dataset.short_name - dataset_schema['description'] = dataset_version.description - dataset_schema['identifier'] = dataset_schema['name'] - dataset_schema['citation'] = dataset_version.ref_doi - - base["dataset"] = dataset_schema - - except db.DatasetVersion.DoesNotExist as err: - logging.error(f"Dataset version does not exist: {err}") - except db.DatasetVersionCurrent.DoesNotExist as err: - logging.error(f"Dataset does not exist: {err}") + base_url = "%s://%s" % (self.request.protocol, self.request.host) + dataset_schema['url'] = base_url + "/dataset/" + dataset_version.dataset.short_name + dataset_schema['@id'] = dataset_schema['url'] + dataset_schema['name'] = dataset_version.dataset.short_name + dataset_schema['description'] = dataset_version.description + dataset_schema['identifier'] = dataset_schema['name'] + dataset_schema['citation'] = dataset_version.ref_doi + + base["dataset"] = dataset_schema if beacon: base = {"@context": "http://schema.org", From 08fc825ddd3f3b9b322e443fd448824889979a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Wed, 21 Aug 2019 14:10:44 +0200 Subject: [PATCH 03/11] Don't crash GetVariant.get() if there is a dataset with only non-released versions. --- backend/modules/browser/browser_handlers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/modules/browser/browser_handlers.py b/backend/modules/browser/browser_handlers.py index 568064cd9..2586998c1 100644 --- a/backend/modules/browser/browser_handlers.py +++ b/backend/modules/browser/browser_handlers.py @@ -355,6 +355,9 @@ def get(self, dataset: str, variant: str, ds_version: str = None): curr_dsv = db.get_dataset_version(dataset, ds_version) dsvs = [db.get_dataset_version(dset.short_name) for dset in db.Dataset.select() if dset.short_name != dataset] + # if the only available version is not released yet + dsvs = [dsv for dsv in dsvs if dsv] + logging.error(dsvs) dsvs = [dsv for dsv in dsvs if dsv.reference_set == curr_dsv.reference_set] dsv_groups = [(curr_dsv, variant)] for dsv in dsvs: From c7bfa94f7fe83970d8e81687c24f83f0f95b9355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Wed, 21 Aug 2019 14:16:51 +0200 Subject: [PATCH 04/11] Test GetDataset. --- backend/tests/test_application.py | 76 +++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/backend/tests/test_application.py b/backend/tests/test_application.py index ba5a9fd67..b00df0a4a 100644 --- a/backend/tests/test_application.py +++ b/backend/tests/test_application.py @@ -33,6 +33,19 @@ def test_get_schema(): "citation": "doi"} assert data['dataset'] == expected + response = requests.get(f'{BASE_URL}/api/schema?url={BASE_URL}/dataset/{ds_name}/version/123456/browser') + assert not response.text + assert response.status_code == 404 + + response = requests.get(f'{BASE_URL}/api/schema?url={BASE_URL}/dataset/bad_ds_name/browser') + assert not response.text + assert response.status_code == 404 + + ds_name = 'SweGen2' + response = requests.get(f'{BASE_URL}/api/schema?url={BASE_URL}/dataset/{ds_name}/version/UNRELEASED/browser') + assert not response.text + assert response.status_code == 403 + response = requests.get(f'{BASE_URL}/api/schema?url={BASE_URL}/dataset/{ds_name}/beacon') data = json.loads(response.text) expected = {'@id': 'https://swefreq.nbis.se/api/beacon-elixir/', @@ -60,3 +73,66 @@ def test_get_countrylist(): data = json.loads(response.text) assert len(data['countries']) == 240 + + +def test_get_dataset(): + """ + Test GetDataset.get() + """ + ds_name = 'SweGen' + response = requests.get(f'{BASE_URL}/api/dataset/{ds_name}') + data = json.loads(response.text) + expected = {"study": 1, + "shortName": "SweGen", + "fullName": "SweGen", + "version": {"version": "20180409", + "description": "desc", + "terms": "terms", + "availableFrom": "2001-01-04", + "refDoi": "doi", + "dataContactName": "place", + "dataContactLink": "email", + "numVariants": None, + "coverageLevels": [1, 5, 10, 15, 20, 25, 30, 50, 100], + "portalAvail": True, + "fileAccess": "REGISTERED", + "beaconAccess": "PUBLIC", + "dataset": 1, + "referenceSet": 1, + "varCallRef": None}, + "future": False} + for value in expected: + assert data[value] == expected[value] + assert len(data) == 14 + + ds_name = 'SweGen2' + response = requests.get(f'{BASE_URL}/api/dataset/{ds_name}') + data = json.loads(response.text) + expected = {"study": 1, + "shortName": "SweGen2", + "fullName": "SweGen2", + "version": {"version": "20190409", + "description": "desc", + "terms": "terms", + "availableFrom": "2001-01-05", + "refDoi": "doi", + "dataContactName": "place", + "dataContactLink": "email", + "numVariants": None, + "coverageLevels": [1, 5, 10, 15, 20, 25, 30, 50, 100], + "portalAvail": True, + "fileAccess": "REGISTERED", + "beaconAccess": "PUBLIC", + "dataset": 2, + "referenceSet": 1, + "varCallRef":None}, + "future": False} + for value in expected: + assert data[value] == expected[value] + assert len(data) == 14 + + ds_name = 'Unrel' + response = requests.get(f'{BASE_URL}/api/dataset/{ds_name}') + assert not response.text + assert response.status_code == 404 + From 96042ad2dd67d5b70bd415d5d10da0871f3ed9d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Wed, 21 Aug 2019 14:18:32 +0200 Subject: [PATCH 05/11] Add test dataset and versions to test unreleased versions. --- test/data/browser_test_data.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/data/browser_test_data.sql b/test/data/browser_test_data.sql index c5966b1f6..aac3f713b 100644 --- a/test/data/browser_test_data.sql +++ b/test/data/browser_test_data.sql @@ -20,6 +20,7 @@ COPY data.studies (id, pi_name, pi_email, contact_name, contact_email, title, st COPY data.datasets (id, study, short_name, full_name, browser_uri, beacon_uri, beacon_description, avg_seq_depth, seq_type, seq_tech, seq_center, dataset_size) FROM stdin; 1 1 SweGen SweGen url \N \N 0 type method place 0 2 1 SweGen2 SweGen2 url \N \N 0 type method place 0 +3 1 Unrel Unreleased dataset url \N \N 0 type method place 0 \. COPY data.reference_sets (id, reference_build, reference_name, ensembl_version, gencode_version, dbnsfp_version) FROM stdin; @@ -32,6 +33,8 @@ COPY data.dataset_versions (id, dataset, reference_set, dataset_version, dataset 3 1 1 20171025 desc terms 2001-01-03 00:00:00 doi place email \N {1,5,10,15,20,25,30,50,100} TRUE REGISTERED PUBLIC 4 1 1 20180409 desc terms 2001-01-04 00:00:00 doi place email \N {1,5,10,15,20,25,30,50,100} TRUE REGISTERED PUBLIC 5 2 1 20190409 desc terms 2001-01-05 00:00:00 doi place email \N {1,5,10,15,20,25,30,50,100} TRUE REGISTERED PUBLIC +6 2 1 UNRELEASED desc terms 9999-12-31 00:00:00 doi place email \N {1,5,10,15,20,25,30,50,100} TRUE REGISTERED PUBLIC +7 3 1 UNRELEASED desc terms 9999-12-31 00:00:00 doi place email \N {1,5,10,15,20,25,30,50,100} TRUE REGISTERED PUBLIC \. COPY data.coverage (id, dataset_version, chrom, pos, mean, median, coverage) FROM stdin; From ca424d2d74c14ade939dc5726129f52952b3feec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 22 Aug 2019 08:15:55 +0200 Subject: [PATCH 06/11] Use filter instead of a list comprehension. Co-Authored-By: MalinAhlberg --- backend/modules/browser/browser_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/modules/browser/browser_handlers.py b/backend/modules/browser/browser_handlers.py index 2586998c1..7bcc52fa8 100644 --- a/backend/modules/browser/browser_handlers.py +++ b/backend/modules/browser/browser_handlers.py @@ -356,7 +356,7 @@ def get(self, dataset: str, variant: str, ds_version: str = None): dsvs = [db.get_dataset_version(dset.short_name) for dset in db.Dataset.select() if dset.short_name != dataset] # if the only available version is not released yet - dsvs = [dsv for dsv in dsvs if dsv] + dsvs = list(filter(lambda dsv: dsv, dsvs)) logging.error(dsvs) dsvs = [dsv for dsv in dsvs if dsv.reference_set == curr_dsv.reference_set] dsv_groups = [(curr_dsv, variant)] From 1058bd3f7b408f8430447b4689f245f11ce7585f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 22 Aug 2019 08:42:22 +0200 Subject: [PATCH 07/11] Fix docstring style. --- backend/modules/browser/browser_handlers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/modules/browser/browser_handlers.py b/backend/modules/browser/browser_handlers.py index 7bcc52fa8..eea827275 100644 --- a/backend/modules/browser/browser_handlers.py +++ b/backend/modules/browser/browser_handlers.py @@ -21,6 +21,7 @@ def get(self, dataset: str, query: str, ds_version: str = None): dataset (str): dataset short name query (str): query ds_version (str): dataset version + """ dataset, ds_version = utils.parse_dataset(dataset, ds_version) ret = {} @@ -47,6 +48,7 @@ def get(self, dataset: str, datatype: str, item: str, # pylint: disable=too-man item (str): query item ds_version (str): dataset version filter_type (str): type of filter to apply + """ # ctrl.filterVariantsBy~ctrl.filterIncludeNonPass dataset, ds_version = utils.parse_dataset(dataset, ds_version) @@ -89,6 +91,7 @@ def get(self, dataset: str, datatype: str, item: str, ds_version: str = None): datatype (str): type of data item (str): query item ds_version (str): dataset version + """ dataset, ds_version = utils.parse_dataset(dataset, ds_version) try: @@ -114,6 +117,7 @@ def get(self, dataset: str, datatype: str, item: str, ds_version: str = None): datatype (str): type of data item (str): query item ds_version (str): dataset version + """ dataset, ds_version = utils.parse_dataset(dataset, ds_version) try: @@ -139,6 +143,7 @@ def get(self, dataset: str, gene: str, ds_version: str = None): dataset (str): short name of the dataset gene (str): the gene id ds_version (str): dataset version + """ dataset, ds_version = utils.parse_dataset(dataset, ds_version) gene_id = gene @@ -193,6 +198,7 @@ def get(self, dataset: str, region: str, ds_version: str = None): dataset (str): short name of the dataset region (str): the region in the format chr-startpos-endpos ds_version (str): dataset version + """ dataset, ds_version = utils.parse_dataset(dataset, ds_version) @@ -232,9 +238,6 @@ def get(self, dataset: str, transcript: str, ds_version: str = None): dataset (str): short name of the dataset transcript (str): the transcript id - Returns: - dict: transcript (transcript and exons), gene (gene information) - """ dataset, ds_version = utils.parse_dataset(dataset, ds_version) transcript_id = transcript @@ -282,6 +285,7 @@ def get(self, dataset: str, variant: str, ds_version: str = None): Args: dataset (str): short name of the dataset variant (str): variant in the format chrom-pos-ref-alt + """ # pylint: disable=too-many-locals,too-many-branches,too-many-statements dataset, ds_version = utils.parse_dataset(dataset, ds_version) @@ -413,6 +417,7 @@ def get(self, dataset: str, datatype: str, item: str, ds_version: str = None): dataset (str): short name of the dataset datatype (str): gene, region, or transcript item (str): item to query + """ dataset, ds_version = utils.parse_dataset(dataset, ds_version) try: @@ -443,6 +448,7 @@ def get(self, dataset: str, query: str, ds_version: str = None): Args: dataset (str): short name of the dataset query (str): search query + """ dataset, ds_version = utils.parse_dataset(dataset, ds_version) ret = {"dataset": dataset, "value": None, "type": None} From 0d1483996c9c537be80329069c4e674955b933f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 22 Aug 2019 09:11:46 +0200 Subject: [PATCH 08/11] Check no longer needed as its handled by the earlier exception. --- backend/modules/browser/browser_handlers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/modules/browser/browser_handlers.py b/backend/modules/browser/browser_handlers.py index eea827275..e1021560b 100644 --- a/backend/modules/browser/browser_handlers.py +++ b/backend/modules/browser/browser_handlers.py @@ -160,9 +160,6 @@ def get(self, dataset: str, gene: str, ds_version: str = None): self.send_error(status_code=400, reason=str(err)) return - if not gene: - self.send_error(status_code=404, reason='Gene not found') - return ret['gene'] = gene # Add exons from transcript From b07993fd6bfeae8c5d6c0936b10b978f2be21fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 22 Aug 2019 09:15:35 +0200 Subject: [PATCH 09/11] Don't handle exceptions that are not raised by the function. --- backend/modules/browser/browser_handlers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/modules/browser/browser_handlers.py b/backend/modules/browser/browser_handlers.py index e1021560b..aa760c766 100644 --- a/backend/modules/browser/browser_handlers.py +++ b/backend/modules/browser/browser_handlers.py @@ -156,9 +156,6 @@ def get(self, dataset: str, gene: str, ds_version: str = None): except error.NotFoundError as err: self.send_error(status_code=404, reason=str(err)) return - except (error.ParsingError, error.MalformedRequest) as err: - self.send_error(status_code=400, reason=str(err)) - return ret['gene'] = gene From 2c8e251f434816fcb69baa2af2b4d20104369fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 22 Aug 2019 09:23:52 +0200 Subject: [PATCH 10/11] Add negative tests for coverage pos and variant. --- backend/modules/browser/tests/test_browser_handlers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/modules/browser/tests/test_browser_handlers.py b/backend/modules/browser/tests/test_browser_handlers.py index e0c2c025e..95f3fd489 100644 --- a/backend/modules/browser/tests/test_browser_handlers.py +++ b/backend/modules/browser/tests/test_browser_handlers.py @@ -81,6 +81,12 @@ def test_get_coverage_pos(): assert cov_pos['stop'] == 100101 assert cov_pos['chrom'] == '22' + dataset = 'SweGen' + data_type = 'transcript' + data_item = 'BAD_TRANSCRIPT' + response = requests.get('{}/api/dataset/{}/browser/coverage_pos/{}/{}'.format(BASE_URL, dataset, data_type, data_item)) + assert response.status_code == 404 + def test_get_gene(): """ @@ -197,6 +203,10 @@ def test_get_variant(): response = requests.get('{}/api/dataset/{}/browser/variant/{}'.format(BASE_URL, dataset, variant_id)) assert response.status_code == 400 + variant_id = '1-2-3-4-5-6' + response = requests.get('{}/api/dataset/{}/browser/variant/{}'.format(BASE_URL, dataset, variant_id)) + assert response.status_code == 400 + def test_get_variants(): """ From 811c0707f1f891a36d6cf70f8dceaed2f36c0ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 22 Aug 2019 09:43:09 +0200 Subject: [PATCH 11/11] More bad request tests for coverage pos. --- backend/modules/browser/tests/test_browser_handlers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/modules/browser/tests/test_browser_handlers.py b/backend/modules/browser/tests/test_browser_handlers.py index 95f3fd489..debcc9bbd 100644 --- a/backend/modules/browser/tests/test_browser_handlers.py +++ b/backend/modules/browser/tests/test_browser_handlers.py @@ -81,6 +81,16 @@ def test_get_coverage_pos(): assert cov_pos['stop'] == 100101 assert cov_pos['chrom'] == '22' + data_type = 'region' + data_item = '22-100001-200101' + response = requests.get('{}/api/dataset/{}/browser/coverage_pos/{}/{}'.format(BASE_URL, dataset, data_type, data_item)) + assert response.status_code == 400 + + data_type = 'region' + data_item = '22-1-11-101' + response = requests.get('{}/api/dataset/{}/browser/coverage_pos/{}/{}'.format(BASE_URL, dataset, data_type, data_item)) + assert response.status_code == 400 + dataset = 'SweGen' data_type = 'transcript' data_item = 'BAD_TRANSCRIPT'