Merge pull request #281 from IATI/master

New API call

docs/api/error.rst

@@ -8,13 +8,10 @@ To retrieve metadata on the datasets known to the datastore.
 
 -  ``/api/1/about/dataset`` retrieves the list of datasets currently in
    the datastore
--  ``/api/1/about/dataset/<dataset>`` retreives specific details on the
+-  ``/api/1/about/dataset/<dataset>`` retrieves specific details on the
    dataset
--  ``/api/1/about/resource?url=<dataset>`` retreives specific details on
-   the dataset
-
    -  ``last_modified`` - the date stamp that the dataset was updated in
-      the iati registry.
+      the IATI registry.
    -  ``resources``
 
       -  ``url`` - url datastore used to fetch the resource
@@ -45,7 +42,7 @@ In JSON
 -  ``/api/1/error/dataset/<dataset_id>`` retrieves the list of datasets
    that have errored (in no order)
 
-   -  ``datestamp`` - date/time that the error occured
+   -  ``datestamp`` - date/time that the error occurred
    -  ``resource_url`` - url of the errored resource/dataset
    -  ``msg`` - error message
    -  ``traceback`` - detailed traceback of error, useful if you are a

iati_datastore/iatilib/frontend/api1.py

@@ -67,6 +67,34 @@ def about_dataset(dataset):
             resources=resources,
     )
 
+
+@api.route('/about/datasets/fetch_status')
+def fetch_status_about_dataset():
+    """Output a JSON formatted list of dataset dictionaries containing their resource details.
+
+    Warning:
+        This is an experimental API call and not intended for general use.
+
+    """
+    dataset_resources = db.session.query(Dataset).options(db.subqueryload(Dataset.resources))
+    datasets = dict()
+
+    for dataset in dataset_resources:
+        if len(dataset.resources) == 0:
+            continue
+        ds_r = dataset.resources[0]
+        resources = {
+            'url': ds_r.url,
+            'last_fetch': ds_r.last_fetch.isoformat() if ds_r.last_fetch else None,
+            'last_status_code': ds_r.last_status_code,
+            'last_successful_fetch': ds_r.last_succ.isoformat() if ds_r.last_succ else None,
+            'last_parsed': ds_r.last_parsed.isoformat() if ds_r.last_parsed else None,
+        }
+        datasets[dataset.name] = resources
+
+    return jsonify(datasets=[{dataset: datasets[dataset]} for dataset in datasets])
+
+
 @api.route('/about/deleted')
 def deleted_activities():
     deleted_activities = db.session.query(DeletedActivity)\

iati_datastore/iatilib/test/test_api.py

@@ -23,6 +23,14 @@ def test_about_http(self):
         resp = self.client.get('/api/1/about')
         self.assertEquals(200, resp.status_code)
 
+class TestAboutDatasets(ClientTestCase):
+    def test_about_datasets_fetch_status(self):
+        """Check that the `about/datasets/fetch_status` page has a 200 response and contains expected data."""
+        resp = self.client.get('/api/1/about/datasets/fetch_status')
+        data = json.loads(resp.data)
+        self.assertEquals(200, resp.status_code)
+        self.assertIn("datasets", data)
+
 class TestDeletedActivitiesView(ClientTestCase):
     def test_deleted_activities(self):
         db.session.add(model.DeletedActivity(
@@ -35,7 +43,7 @@ def test_deleted_activities(self):
         deleted_activities = data['deleted_activities']
         self.assertEquals("test", deleted_activities[0]['iati_identifier'])
         self.assertEquals("2000-01-01", deleted_activities[0]['deletion_date'])
-        
+
 
 class TestEmptyDb_JSON(ClientTestCase):
     url = '/api/1/access/activity'
@@ -341,7 +349,7 @@ def test_receiver_org_activity_id_output(self):
         csv_headers = output[0]
         i = csv_headers.index('transaction_receiver-org_receiver-activity-id')
         self.assertEquals(u'GB-CHC-1068839-dfid_ag_11-13', output[1][i])
-        
+
     def test_description(self):
         load_fix("transaction_provider.xml")
         output = list(csv.reader(StringIO(self.client.get(self.base_url).data)))
@@ -358,14 +366,14 @@ def test_flow_type(self):
         csv_headers = output[0]
         i = csv_headers.index('transaction_flow-type_code')
         self.assertEquals(u'30', output[1][i])
-        
+
     def test_finance_type(self):
         load_fix("transaction_fields_code_lists.xml")
         output = list(csv.reader(StringIO(self.client.get(self.base_url).data)))
         csv_headers = output[0]
         i = csv_headers.index('transaction_finance-type_code')
         self.assertEquals(u'110', output[1][i])
-        
+
     def test_aid_type(self):
         load_fix("transaction_fields_code_lists.xml")
         output = list(csv.reader(StringIO(self.client.get(self.base_url).data)))
@@ -379,7 +387,7 @@ def test_tied_status(self):
         csv_headers = output[0]
         i = csv_headers.index('transaction_tied-status_code')
         self.assertEquals(u'5', output[1][i])
-        
+
     def test_disbursement_channel_status(self):
         load_fix("transaction_fields_code_lists.xml")
         output = list(csv.reader(StringIO(self.client.get(self.base_url).data)))
@@ -421,4 +429,3 @@ class TestBudgetBySectorView(ClientTestCase, ApiViewMixin):
     base_url = '/api/1/access/budget/by_sector.csv'
     filter = 'iatilib.frontend.api1.BudgetsBySectorView.filter'
     serializer = 'iatilib.frontend.api1.BudgetsBySectorView.serializer'
-