From 81e064ccf68f1edb7a1bd417af7e786fe68b4bb1 Mon Sep 17 00:00:00 2001 From: allthatilk Date: Tue, 29 Aug 2017 12:51:55 +0100 Subject: [PATCH 01/12] Add new api call --- iati_datastore/iatilib/frontend/api1.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/iati_datastore/iatilib/frontend/api1.py b/iati_datastore/iatilib/frontend/api1.py index ba04bcc..ffd8dfb 100644 --- a/iati_datastore/iatilib/frontend/api1.py +++ b/iati_datastore/iatilib/frontend/api1.py @@ -67,6 +67,28 @@ def about_dataset(dataset): resources=resources, ) + +@api.route('/about/dataset/nest') +def nest_about_dataset(): + datasets_query = db.session.query(Dataset.name) + datasets = dict() + for ds in datasets_query.all(): + resources = [] + for r in ds.resources: + resources.append({ + 'url': r.url, + 'last_fetch': r.last_fetch.isoformat() if r.last_fetch else None, + 'last_status_code': r.last_status_code, + 'last_successful_fetch': r.last_succ.isoformat() if r.last_succ else None, + 'last_parsed': r.last_parsed.isoformat() if r.last_parsed else None, + 'num_of_activities': r.activities.count(), + }) + + datasets.add({ds.name: {ds.resources}}) + + return jsonify(datasets=datasets) + + @api.route('/about/deleted') def deleted_activities(): deleted_activities = db.session.query(DeletedActivity)\ From 9b5c0f6b9c73a27b3c0c6b03942f1c876b6f7ae3 Mon Sep 17 00:00:00 2001 From: allthatilk Date: Thu, 7 Sep 2017 12:24:18 +0100 Subject: [PATCH 02/12] Add new api call --- iati_datastore/iatilib/frontend/api1.py | 32 ++++++++++++------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/iati_datastore/iatilib/frontend/api1.py b/iati_datastore/iatilib/frontend/api1.py index ffd8dfb..609aaca 100644 --- a/iati_datastore/iatilib/frontend/api1.py +++ b/iati_datastore/iatilib/frontend/api1.py @@ -68,25 +68,23 @@ def about_dataset(dataset): ) -@api.route('/about/dataset/nest') +@api.route('/about/datasets/nest') def nest_about_dataset(): - datasets_query = db.session.query(Dataset.name) + dataset_resources = db.session.query(Dataset).join(Dataset.resources) datasets = dict() - for ds in datasets_query.all(): - resources = [] - for r in ds.resources: - resources.append({ - 'url': r.url, - 'last_fetch': r.last_fetch.isoformat() if r.last_fetch else None, - 'last_status_code': r.last_status_code, - 'last_successful_fetch': r.last_succ.isoformat() if r.last_succ else None, - 'last_parsed': r.last_parsed.isoformat() if r.last_parsed else None, - 'num_of_activities': r.activities.count(), - }) - - datasets.add({ds.name: {ds.resources}}) - - return jsonify(datasets=datasets) + + for dataset in dataset_resources: + resources = { + 'url': dataset.resources[0].url, + 'last_fetch': dataset.resources[0].last_fetch.isoformat() if dataset.resources[0].last_fetch else None, + 'last_status_code': dataset.resources[0].last_status_code, + 'last_successful_fetch': dataset.resources[0].last_succ.isoformat() if dataset.resources[0].last_succ else None, + 'last_parsed': dataset.resources[0].last_parsed.isoformat() if dataset.resources[0].last_parsed else None, + 'num_of_activities': dataset.resources[0].activities.count(), + } + datasets[dataset.name] = resources + + return jsonify(datasets=[{dataset: datasets[dataset]} for dataset in datasets]) @api.route('/about/deleted') From 30fcf013f454c4e38a6616bb2a34022d44939fbb Mon Sep 17 00:00:00 2001 From: allthatilk Date: Thu, 7 Sep 2017 12:27:25 +0100 Subject: [PATCH 03/12] Update documentation --- iati_datastore/iatilib/frontend/api1.py | 1 + 1 file changed, 1 insertion(+) diff --git a/iati_datastore/iatilib/frontend/api1.py b/iati_datastore/iatilib/frontend/api1.py index 609aaca..8e1dc79 100644 --- a/iati_datastore/iatilib/frontend/api1.py +++ b/iati_datastore/iatilib/frontend/api1.py @@ -70,6 +70,7 @@ def about_dataset(dataset): @api.route('/about/datasets/nest') def nest_about_dataset(): + """Output a JSON formatted list of dataset dictionaries containing their resource details.""" dataset_resources = db.session.query(Dataset).join(Dataset.resources) datasets = dict() From 92a507b307c9c04dc1c88e6c11accde83e07ebf7 Mon Sep 17 00:00:00 2001 From: allthatilk Date: Thu, 7 Sep 2017 13:22:11 +0100 Subject: [PATCH 04/12] Add check for new API call --- iati_datastore/iatilib/test/test_api.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/iati_datastore/iatilib/test/test_api.py b/iati_datastore/iatilib/test/test_api.py index d692f86..f927865 100644 --- a/iati_datastore/iatilib/test/test_api.py +++ b/iati_datastore/iatilib/test/test_api.py @@ -23,6 +23,14 @@ def test_about_http(self): resp = self.client.get('/api/1/about') self.assertEquals(200, resp.status_code) +class TestAboutDatasets(ClientTestCase): + def test_about_datasets_nest(self): + """Check that the `about/datasets/nest` page has a 200 response and contains expected data.""" + resp = self.client.get('/api/1/about/datasets/nest') + data = json.loads(resp.data) + self.assertEquals(200, resp.status_code) + self.assertIn("datasets", data) + class TestDeletedActivitiesView(ClientTestCase): def test_deleted_activities(self): db.session.add(model.DeletedActivity( @@ -35,7 +43,7 @@ def test_deleted_activities(self): deleted_activities = data['deleted_activities'] self.assertEquals("test", deleted_activities[0]['iati_identifier']) self.assertEquals("2000-01-01", deleted_activities[0]['deletion_date']) - + class TestEmptyDb_JSON(ClientTestCase): url = '/api/1/access/activity' @@ -341,7 +349,7 @@ def test_receiver_org_activity_id_output(self): csv_headers = output[0] i = csv_headers.index('transaction_receiver-org_receiver-activity-id') self.assertEquals(u'GB-CHC-1068839-dfid_ag_11-13', output[1][i]) - + def test_description(self): load_fix("transaction_provider.xml") output = list(csv.reader(StringIO(self.client.get(self.base_url).data))) @@ -358,14 +366,14 @@ def test_flow_type(self): csv_headers = output[0] i = csv_headers.index('transaction_flow-type_code') self.assertEquals(u'30', output[1][i]) - + def test_finance_type(self): load_fix("transaction_fields_code_lists.xml") output = list(csv.reader(StringIO(self.client.get(self.base_url).data))) csv_headers = output[0] i = csv_headers.index('transaction_finance-type_code') self.assertEquals(u'110', output[1][i]) - + def test_aid_type(self): load_fix("transaction_fields_code_lists.xml") output = list(csv.reader(StringIO(self.client.get(self.base_url).data))) @@ -379,7 +387,7 @@ def test_tied_status(self): csv_headers = output[0] i = csv_headers.index('transaction_tied-status_code') self.assertEquals(u'5', output[1][i]) - + def test_disbursement_channel_status(self): load_fix("transaction_fields_code_lists.xml") output = list(csv.reader(StringIO(self.client.get(self.base_url).data))) @@ -421,4 +429,3 @@ class TestBudgetBySectorView(ClientTestCase, ApiViewMixin): base_url = '/api/1/access/budget/by_sector.csv' filter = 'iatilib.frontend.api1.BudgetsBySectorView.filter' serializer = 'iatilib.frontend.api1.BudgetsBySectorView.serializer' - From d411f368793a046d8e490a5b4a1db32e76837d27 Mon Sep 17 00:00:00 2001 From: allthatilk Date: Thu, 7 Sep 2017 13:36:26 +0100 Subject: [PATCH 05/12] Update documentation to include new API call and remove reference to non-existent API call --- docs/api/error.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/api/error.rst b/docs/api/error.rst index ea0ba04..88ca6a5 100644 --- a/docs/api/error.rst +++ b/docs/api/error.rst @@ -10,9 +10,6 @@ To retrieve metadata on the datasets known to the datastore. the datastore - ``/api/1/about/dataset/`` retreives specific details on the dataset -- ``/api/1/about/resource?url=`` retreives specific details on - the dataset - - ``last_modified`` - the date stamp that the dataset was updated in the iati registry. - ``resources`` @@ -25,6 +22,8 @@ To retrieve metadata on the datasets known to the datastore. - ``last_parsed`` - last date resource was successfully parsed - ``num_of_activities`` number of activities datastore has successfully parsed and stored +- ``/api/1/about/datasets/nest`` retrieves the list of datasets currently + in the datastore with specific details (as shown above) for each dataset. Errors ------ From 52c8b2a51a7542006c650557ae7771fd2c80c278 Mon Sep 17 00:00:00 2001 From: allthatilk Date: Thu, 7 Sep 2017 13:39:28 +0100 Subject: [PATCH 06/12] Fix spelling and capitalisation errors in errors.rst --- docs/api/error.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/error.rst b/docs/api/error.rst index 88ca6a5..18c66b9 100644 --- a/docs/api/error.rst +++ b/docs/api/error.rst @@ -8,10 +8,10 @@ To retrieve metadata on the datasets known to the datastore. - ``/api/1/about/dataset`` retrieves the list of datasets currently in the datastore -- ``/api/1/about/dataset/`` retreives specific details on the +- ``/api/1/about/dataset/`` retrieves specific details on the dataset - ``last_modified`` - the date stamp that the dataset was updated in - the iati registry. + the IATI registry. - ``resources`` - ``url`` - url datastore used to fetch the resource @@ -44,7 +44,7 @@ In JSON - ``/api/1/error/dataset/`` retrieves the list of datasets that have errored (in no order) - - ``datestamp`` - date/time that the error occured + - ``datestamp`` - date/time that the error occurred - ``resource_url`` - url of the errored resource/dataset - ``msg`` - error message - ``traceback`` - detailed traceback of error, useful if you are a From c9e44e282977a1ec26ee80d5300979e8407f0f8f Mon Sep 17 00:00:00 2001 From: allthatilk Date: Fri, 8 Sep 2017 11:09:31 +0100 Subject: [PATCH 07/12] Add suggested changes to improve loading time --- iati_datastore/iatilib/frontend/api1.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/iati_datastore/iatilib/frontend/api1.py b/iati_datastore/iatilib/frontend/api1.py index 8e1dc79..7e78397 100644 --- a/iati_datastore/iatilib/frontend/api1.py +++ b/iati_datastore/iatilib/frontend/api1.py @@ -71,17 +71,20 @@ def about_dataset(dataset): @api.route('/about/datasets/nest') def nest_about_dataset(): """Output a JSON formatted list of dataset dictionaries containing their resource details.""" - dataset_resources = db.session.query(Dataset).join(Dataset.resources) + dataset_resources = db.session.query(Dataset).options(db.subqueryload(Dataset.resources)) datasets = dict() for dataset in dataset_resources: + if len(dataset.resources) == 0: + continue + ds_r = dataset.resources[0] resources = { - 'url': dataset.resources[0].url, - 'last_fetch': dataset.resources[0].last_fetch.isoformat() if dataset.resources[0].last_fetch else None, - 'last_status_code': dataset.resources[0].last_status_code, - 'last_successful_fetch': dataset.resources[0].last_succ.isoformat() if dataset.resources[0].last_succ else None, - 'last_parsed': dataset.resources[0].last_parsed.isoformat() if dataset.resources[0].last_parsed else None, - 'num_of_activities': dataset.resources[0].activities.count(), + 'url': ds_r.url, + 'last_fetch': ds_r.last_fetch.isoformat() if ds_r.last_fetch else None, + 'last_status_code': ds_r.last_status_code, + 'last_successful_fetch': ds_r.last_succ.isoformat() if ds_r.last_succ else None, + 'last_parsed': ds_r.last_parsed.isoformat() if ds_r.last_parsed else None, + 'num_of_activities': ds_r.activities.count(), } datasets[dataset.name] = resources From 220c9a595cce462694e4a0edff8ddeb4db4a8a54 Mon Sep 17 00:00:00 2001 From: allthatilk Date: Wed, 27 Sep 2017 12:34:34 +0100 Subject: [PATCH 08/12] Comment out dataset count to speed up call --- iati_datastore/iatilib/frontend/api1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iati_datastore/iatilib/frontend/api1.py b/iati_datastore/iatilib/frontend/api1.py index 7e78397..8490eec 100644 --- a/iati_datastore/iatilib/frontend/api1.py +++ b/iati_datastore/iatilib/frontend/api1.py @@ -84,7 +84,7 @@ def nest_about_dataset(): 'last_status_code': ds_r.last_status_code, 'last_successful_fetch': ds_r.last_succ.isoformat() if ds_r.last_succ else None, 'last_parsed': ds_r.last_parsed.isoformat() if ds_r.last_parsed else None, - 'num_of_activities': ds_r.activities.count(), + # 'num_of_activities': ds_r.activities.count(), } datasets[dataset.name] = resources From b7d4f7f0748e192bed4e39a8565f8733e333204d Mon Sep 17 00:00:00 2001 From: allthatilk Date: Wed, 27 Sep 2017 15:35:17 +0100 Subject: [PATCH 09/12] Remove 'number of activites' from new API call --- iati_datastore/iatilib/frontend/api1.py | 1 - 1 file changed, 1 deletion(-) diff --git a/iati_datastore/iatilib/frontend/api1.py b/iati_datastore/iatilib/frontend/api1.py index 8490eec..d93c2c7 100644 --- a/iati_datastore/iatilib/frontend/api1.py +++ b/iati_datastore/iatilib/frontend/api1.py @@ -84,7 +84,6 @@ def nest_about_dataset(): 'last_status_code': ds_r.last_status_code, 'last_successful_fetch': ds_r.last_succ.isoformat() if ds_r.last_succ else None, 'last_parsed': ds_r.last_parsed.isoformat() if ds_r.last_parsed else None, - # 'num_of_activities': ds_r.activities.count(), } datasets[dataset.name] = resources From c55118254f395b252e6c6806e2866f5b3befb3bd Mon Sep 17 00:00:00 2001 From: allthatilk Date: Mon, 9 Oct 2017 14:58:51 +0100 Subject: [PATCH 10/12] Update documentation --- iati_datastore/iatilib/frontend/api1.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iati_datastore/iatilib/frontend/api1.py b/iati_datastore/iatilib/frontend/api1.py index d93c2c7..f8fcccf 100644 --- a/iati_datastore/iatilib/frontend/api1.py +++ b/iati_datastore/iatilib/frontend/api1.py @@ -70,7 +70,10 @@ def about_dataset(dataset): @api.route('/about/datasets/nest') def nest_about_dataset(): - """Output a JSON formatted list of dataset dictionaries containing their resource details.""" + """Output a JSON formatted list of dataset dictionaries containing their resource details. + This is an experimental API call and not intended for general use. + + """ dataset_resources = db.session.query(Dataset).options(db.subqueryload(Dataset.resources)) datasets = dict() From f600d13c75353db0786785aeff7f505bd31df64e Mon Sep 17 00:00:00 2001 From: allthatilk Date: Mon, 9 Oct 2017 15:23:44 +0100 Subject: [PATCH 11/12] Change API call name --- iati_datastore/iatilib/frontend/api1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iati_datastore/iatilib/frontend/api1.py b/iati_datastore/iatilib/frontend/api1.py index f8fcccf..2a1d1ab 100644 --- a/iati_datastore/iatilib/frontend/api1.py +++ b/iati_datastore/iatilib/frontend/api1.py @@ -68,7 +68,7 @@ def about_dataset(dataset): ) -@api.route('/about/datasets/nest') +@api.route('/about/datasets/fetch_status') def nest_about_dataset(): """Output a JSON formatted list of dataset dictionaries containing their resource details. This is an experimental API call and not intended for general use. From d8e6e2a2a4526995d9d0b8c5c9e97671008790b3 Mon Sep 17 00:00:00 2001 From: allthatilk Date: Mon, 9 Oct 2017 15:35:44 +0100 Subject: [PATCH 12/12] Update API call name, remove info from docs, update tests. --- docs/api/error.rst | 2 -- iati_datastore/iatilib/frontend/api1.py | 6 ++++-- iati_datastore/iatilib/test/test_api.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/api/error.rst b/docs/api/error.rst index 18c66b9..e020e13 100644 --- a/docs/api/error.rst +++ b/docs/api/error.rst @@ -22,8 +22,6 @@ To retrieve metadata on the datasets known to the datastore. - ``last_parsed`` - last date resource was successfully parsed - ``num_of_activities`` number of activities datastore has successfully parsed and stored -- ``/api/1/about/datasets/nest`` retrieves the list of datasets currently - in the datastore with specific details (as shown above) for each dataset. Errors ------ diff --git a/iati_datastore/iatilib/frontend/api1.py b/iati_datastore/iatilib/frontend/api1.py index 2a1d1ab..8183223 100644 --- a/iati_datastore/iatilib/frontend/api1.py +++ b/iati_datastore/iatilib/frontend/api1.py @@ -69,9 +69,11 @@ def about_dataset(dataset): @api.route('/about/datasets/fetch_status') -def nest_about_dataset(): +def fetch_status_about_dataset(): """Output a JSON formatted list of dataset dictionaries containing their resource details. - This is an experimental API call and not intended for general use. + + Warning: + This is an experimental API call and not intended for general use. """ dataset_resources = db.session.query(Dataset).options(db.subqueryload(Dataset.resources)) diff --git a/iati_datastore/iatilib/test/test_api.py b/iati_datastore/iatilib/test/test_api.py index f927865..ee56711 100644 --- a/iati_datastore/iatilib/test/test_api.py +++ b/iati_datastore/iatilib/test/test_api.py @@ -24,9 +24,9 @@ def test_about_http(self): self.assertEquals(200, resp.status_code) class TestAboutDatasets(ClientTestCase): - def test_about_datasets_nest(self): - """Check that the `about/datasets/nest` page has a 200 response and contains expected data.""" - resp = self.client.get('/api/1/about/datasets/nest') + def test_about_datasets_fetch_status(self): + """Check that the `about/datasets/fetch_status` page has a 200 response and contains expected data.""" + resp = self.client.get('/api/1/about/datasets/fetch_status') data = json.loads(resp.data) self.assertEquals(200, resp.status_code) self.assertIn("datasets", data)