Merge pull request #252 from IATI/master

Update live branch from master

.gitignore

@@ -5,5 +5,7 @@ _testdata
 .coverage
 .noseids
 1
+pyenv/
 *.egg-info
 docs/_build
+pyenv/

.travis.yml

@@ -3,6 +3,6 @@ python:
   - "2.7"
 before_install:
   sudo apt-get install -y libevent-dev
-install: "pip install -r requirements_dev.txt --use-mirrors"
+install: "pip install -r requirements_dev.txt"
 script: nosetests iati_datastore --with-coverage
 after_success: coveralls

README.markdown → README.md

@@ -1,11 +1,12 @@
 IATI Datastore
 ==============
 
-[![Build Status](https://travis-ci.org/IATI/iati-datastore.svg?branch=master)](https://travis-ci.org/IATI/iati-datastore)
+[![Build Status](https://travis-ci.org/IATI/IATI-Datastore.svg?branch=master)](https://travis-ci.org/IATI/IATI-Datastore)
 [![Coverage Status](https://img.shields.io/coveralls/IATI/iati-datastore.svg)](https://coveralls.io/r/IATI/iati-datastore?branch=master)
 [![Code Health](https://landscape.io/github/IATI/iati-datastore/master/landscape.png)](https://landscape.io/github/IATI/iati-datastore/master)
 [![License: MIT](https://img.shields.io/badge/license-AGPLv3-blue.svg)](https://github.com/IATI/iati-datastore/blob/master/LICENSE.txt)
 
+
 Introduction
 ------------
 
@@ -19,6 +20,7 @@ extracts they are interested in. A public instance is available here:
 
 http://datastore.iatistandard.org
 
+
 Requirements
 ------------
 
@@ -30,21 +32,47 @@ For example, on Ubuntu:
 Installing for development
 --------------------------
 
-[![Build Status](https://api.travis-ci.org/IATI/iati-datastore.png)](https://travis-ci.org/IATI/iati-datastore)
+[![Build Status](https://api.travis-ci.org/IATI/IATI-Datastore.png)](https://travis-ci.org/IATI/iati-datastore)
+
+```
+# Clone the source
+git clone https://github.com/IATI/IATI-Datastore.git
+
+# Install development dependencies 
+pip install -r requirements_dev.txt
+
+# Run the tests  (these tests use an in-memory sqlite db)
+nosetests iati_datastore
+
+# Create a new PostgreSQL database
+sudo -u postgres psql -c "CREATE DATABASE iati_datastore"
+
+# Set an environment variable for `DATABASE_URL` linking to the database created
+export DATABASE_URL=postgres:///iati_datastore
+
+# Create the db tables
+iati create_database
+
+# Note: To create the tables the new database may need access privileges granted to your system user
+# See http://dba.stackexchange.com/questions/117109/how-to-manage-default-privileges-for-users-on-a-database-vs-schema/117661#117661
+sudo -u postgres psql -c "CREATE USER [SYSTEM USER]"
+sudo -u postgres psql -c "GRANT ALL ON DATABASE iati_datastore TO [SYSTEM USER]"
+
+# Start the process of grabbing the source data
+iati crawl update
+
+# Start a development server – this should be run in a seperate terminal window
+iati runserver
+
+# Run a worker. This will download and index the datafiles
+iati queue background
+
+# The progess of the worker can be checked using:
+iati crawl status
+
+# A local API is available at: http://127.0.0.1:5000
+```
 
-* Clone the source
-* Install `pip install -r requirements_dev.txt`
-* Run the tests `nosetests iati_datastore`
-  (the tests use an in-memory sqlite db)
-* Create a database (in postgres), and set an environment variable
-  `DATABASE_URL` to something like `postgres:///iati-ds`.
-* Run `iati create_database` to create the db tables
-* Run `iati crawl update` to start the process of grabbing the source data
-* Run `iati runserver` to start a development server
-* Run a worker with `iati queue background`
-  - this will download and index the datafiles,
-    check progess with `iati crawl status`.
-* Go to http://127.0.0.1:5000
 
 Deploying with apache
 ---------------------
@@ -84,10 +112,19 @@ Deploying with apache
         WSGIProcessGroup datastore
         WSGIScriptAlias / /home/datastore/datastore.wsgi
 
+
 Updating activities after changing import code
 ----------------------------------------------
 
 * Run this SQL query on the database - `UPDATE resource SET last_succ=NULL;`
 * Restart background process
 * Run `iati crawl update` (or wait for cron to run it for you)
 
+
+Generation of Documentation
+---------------------------
+
+API documentation in the docs folder is generated using [Spinx](http://www.sphinx-doc.org).
+
+    cd iati-datastore/docs
+    make dirhtml

docs/api.rst

@@ -115,8 +115,6 @@ Example API call:
 policy-marker
 `````````````
 
-*The IATI Tech team have uncovered issues with this filter (‘500 internal server error’). `This issue has been logged <https://github.com/IATI/iati-datastore/issues/237>`__ and we will endeavour to resolve this shortly.*
-
 Returns activities containing a `policy-marker <http://iatistandard.org/201/activity-standard/iati-activities/iati-activity/policy-marker/>`__ element which matches a your specified policy-marker value.
 
 Parameters:

iati_datastore/iatilib/codelists/enum.py

@@ -86,6 +86,8 @@ def copy(self):
     def process_bind_param(self, value, dialect):
         if value is None:
             return None
+        elif isinstance(value, basestring):
+            return value
         return value.value
 
     def process_result_value(self, value, dialect):

iati_datastore/iatilib/crawler.py

@@ -16,8 +16,8 @@
 
 log = logging.getLogger("crawler")
 
-CKAN_WEB_BASE = 'http://www.iatiregistry.org/dataset/%s'
-CKAN_API = 'http://www.iatiregistry.org/'
+CKAN_WEB_BASE = 'https://www.iatiregistry.org/dataset/%s'
+CKAN_API = 'https://www.iatiregistry.org/'
 
 registry = ckanapi.RemoteCKAN(CKAN_API)
 

iati_datastore/iatilib/frontend/api1.py

@@ -58,8 +58,8 @@ def about_dataset(dataset):
             'last_successful_fetch': r.last_succ.isoformat() if r.last_succ else None,
             'last_parsed': r.last_parsed.isoformat() if r.last_parsed else None,
             'num_of_activities': r.activities.count(),
-        }) 
-        
+        })
+
     return jsonify(
             dataset=dataset.name,
             last_modified=None if dataset.last_modified is None else dataset.last_modified.isoformat(),
@@ -72,11 +72,11 @@ def deleted_activities():
     deleted_activities = db.session.query(DeletedActivity)\
                                    .order_by(DeletedActivity.deletion_date)
     return jsonify(
-        deleted_activities=[ 
+        deleted_activities=[
           {
             'iati_identifier' : da.iati_identifier,
             'deletion_date' : da.deletion_date.isoformat(),
-          } 
+          }
           for da in deleted_activities
         ],
     )
@@ -132,7 +132,7 @@ def dataset_error(dataset_id):
 def dataset_log():
     logs = db.session.query(Log.dataset).distinct()
     return render_template('datasets.log', logs=logs)
-    
+
 @api.route('/error/dataset.log/<dataset_id>')
 def dataset_log_error(dataset_id):
     error_logs = db.session.query(Log).order_by(sa.desc(Log.created_at)).\

iati_datastore/iatilib/frontend/serialize/csv.py

@@ -96,7 +96,7 @@ def transaction_tied_status(transaction):
 
 def transaction_disbursement_channel(transaction):
     if transaction.disbursement_channel:
-        return transaction.disbursement_channel.value 
+        return transaction.disbursement_channel.value
     return ""
 
 def transaction_org(field, transaction):
@@ -147,10 +147,16 @@ def recipient_region_percentage(activity):
         for rrp in activity.recipient_region_percentages)
 
 def reporting_org_ref(activity):
-    return activity.reporting_org.ref
+    try:
+        return activity.reporting_org.ref
+    except AttributeError:
+        return ""
 
 def reporting_org_name(activity):
-    return activity.reporting_org.name
+    try:
+        return activity.reporting_org.name
+    except AttributeError:
+        return ""
 
 def reporting_org_type(activity):
     try:
@@ -209,7 +215,7 @@ def value_currency(transaction):
     if transaction.value_currency:
         return transaction.value_currency.value
     else:
-        return u"" 
+        return u""
 
 def activity_default_currency(activity):
     if activity.default_currency:
@@ -439,7 +445,7 @@ def wrapper(args):
 
 
 csv_activity_by_country = CSVSerializer((
-    (u"recipient-country-code", lambda (a, c): c.country.value if c.country else ""), 
+    (u"recipient-country-code", lambda (a, c): c.country.value if c.country else ""),
     (u"recipient-country", lambda (a, c): c.country.description.title() if c.country and c.country.description else ""),
     (u"recipient-country-percentage", lambda (a, c): c.percentage if c.percentage else ""),
     "iati-identifier",