Merge pull request #6 from LibCrowds/fix-csv-export

Fix csv export

.gitignore

@@ -0,0 +1,63 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Local
+settings.py
+pybossa/

README.md

@@ -15,6 +15,21 @@ Copy the [libcrowds_data](libcrowds_data) folder into your PyBossa
 [plugins](https://github.com/PyBossa/pybossa/tree/master/pybossa/plugins) directory. The
 plugin will be available after you next restart the server.
 
+## Configuration
+
+The default configuration settings for libcrowds_data are:
+
+``` Python
+DATA_DISPLAY_TASKS = True
+DATA_DISPLAY_TASK_RUNS = True
+DATA_DISPLAY_RESULTS = True
+DATA_DISPLAY_FLICKR = True
+```
+
+Each settings defines whether or not a particular item will be made available for
+download via the data page. You can modify these settings by adding them to your
+main PyBossa configuration file.
+
 
 ## Contributing
 

libcrowds_data/__init__.py

@@ -7,6 +7,7 @@
 """
 
 import os
+import default_settings
 from flask import current_app as app
 from flask.ext.plugins import Plugin
 
@@ -19,8 +20,16 @@ class LibCrowdsData(Plugin):
 
     def setup(self):
         """Setup the plugin."""
+        self.load_config()
         self.setup_blueprint()
 
+    def load_config(self):
+        """Configure the plugin."""
+        settings = [key for key in dir(default_settings) if key.isupper()]
+        for s in settings:
+            if not app.config.get(s):
+                app.config[s] = getattr(default_settings, s)
+
     def setup_blueprint(self):
         """Setup blueprint."""
         from .blueprint import DataBlueprint

libcrowds_data/default_settings.py

@@ -0,0 +1,4 @@
+DATA_DISPLAY_TASKS = True
+DATA_DISPLAY_TASK_RUNS = True
+DATA_DISPLAY_RESULTS = True
+DATA_DISPLAY_FLICKR = True

libcrowds_data/templates/index.html

@@ -55,6 +55,8 @@ <h1 class="inset-text-grey">Data</h1>
 
 <section class="padding-top-sm">
     <div class="container container-padded">
+
+        {% if display['tasks'] %}
         <div class="row padding-bottom-md">
             <div class="col-xs-12">
                 <div class="white-panel inset-shadow-white">
@@ -90,7 +92,9 @@ <h3 class="text-center">Raw task data</h3>
                 </div>
             </div>
         </div>
+        {% endif %}
 
+        {% if display['task_runs'] %}
         <div class="row padding-bottom-md">
             <div class="col-xs-12">
                 <div class="white-panel inset-shadow-white">
@@ -126,7 +130,9 @@ <h3 class="text-center">Raw contribution data</h3>
                 </div>
             </div>
         </div>
+        {% endif %}
 
+        {% if display['results'] %}
         <div class="row padding-bottom-md">
             <div class="col-xs-12">
                 <div class="white-panel inset-shadow-white">
@@ -156,7 +162,9 @@ <h3 class="text-center">Result data</h3>
                 </div>
             </div>
         </div>
+        {% endif %}
 
+        {% if display['flickr'] %}
         <div class="row">
             <div class="col-xs-12">
                 <div class="white-panel inset-shadow-white text-center">
@@ -166,6 +174,7 @@ <h4>Visit our <a href="http://www.flickr.com/photos/132066275@N04/albums">Flickr
                 </div>
             </div>
         </div>
+        {% endif %}
     </div>
 </section>
 {% endblock %}

libcrowds_data/view.py

@@ -2,7 +2,8 @@
 """Views modules for libcrowds-data."""
 
 import StringIO
-from flask import render_template, make_response
+import itertools
+from flask import render_template, make_response, current_app
 from pybossa.core import project_repo, result_repo
 from pybossa.util import UnicodeWriter
 from pybossa.exporter import Exporter
@@ -16,7 +17,11 @@ def index():
     title = "Data"
     description = """Download open datasets of all crowdsourced data produced
                   via LibCrowds."""
-    return render_template('/index.html', projects=projects)
+    display = {'tasks': current_app.config['DATA_DISPLAY_TASKS'],
+               'task_runs': current_app.config['DATA_DISPLAY_TASK_RUNS'],
+               'results': current_app.config['DATA_DISPLAY_RESULTS'],
+               'flickr': current_app.config['DATA_DISPLAY_FLICKR']}
+    return render_template('/index.html', projects=projects, display=display)
 
 
 def csv_export(short_name):
@@ -27,29 +32,30 @@ def csv_export(short_name):
     project = project_repo.get_by_shortname(short_name)
     if project is None:  # pragma: no cover
         abort(404)
-
     si = StringIO.StringIO()
     writer = UnicodeWriter(si)
     exporter = Exporter()
     name = exporter._project_name_latin_encoded(project)
     secure_name = secure_filename('{0}_{1}.csv'.format(name, 'results'))
     results = result_repo.filter_by(project_id=project.id)
-    if len(results) > 0:
-        keys = [r.info.keys() for r in results if r.info]
-        headers = results[0].dictize().keys()
-        headers += list(set(list(itertools.chain(*keys))))
-        writer.writerow([h for h in headers])
-
-        for row in results:
-            values = results[0].dictize().items()
-            if row.info is not None:
-                values.extend([row.info.get(h, '') for h in headers])
-            writer.writerow(values)
+    data = []
+
+    for r in results:
+        row = {k: v for k, v in r.dictize().items()}
+        if isinstance(row['info'], dict):  # Explode info
+            keys = row['info'].keys()
+            for k in keys:
+                row['info_{0}'.format(k)] = row['info'][k]
+        data.append(row)
+    headers = set(itertools.chain(*[row.keys() for row in data]))
+    writer.writerow([h for h in headers])
+    for row in data:
+        writer.writerow([row.get(h, '') for h in headers])
 
     fn = "filename={0}".format(secure_name)
     resp = make_response(si.getvalue())
     resp.headers["Content-Disposition"] = "attachment; {0}".format(fn)
     resp.headers["Content-type"] = "text/csv"
     resp.headers['Cache-Control'] = "no-store, no-cache, must-revalidate, \
                                     post-check=0, pre-check=0, max-age=0"
-    return resp
+    return resp

tests/__init__.py

@@ -3,13 +3,16 @@
 import sys
 import os
 import libcrowds_data as plugin
+from mock import patch
 
 
 # Use the PyBossa test suite
 sys.path.append(os.path.abspath("./pybossa/test"))
 
 from default import with_context
 from helper import web
+from factories import ProjectFactory, TaskFactory, TaskRunFactory
+from pybossa.core import result_repo
 
 
 def setUpPackage():
@@ -22,30 +25,43 @@ def setUpPackage():
 
 class TestPlugin(web.Helper):
 
-    def test_blueprint_registered(self):
-        assert 'data' in self.flask_app.blueprints
-
-    def test_static_folder_exists(self):
-        bp = self.flask_app.blueprints['data']
-        static_folder = os.path.abspath(bp.static_folder)
-        assert os.path.isdir(static_folder), static_folder
-
-    def test_templates_folder_exists(self):
-        bp = self.flask_app.blueprints['data']
-        template_folder = os.path.abspath(bp.template_folder)
-        assert os.path.isdir(template_folder), template_folder
+    def setUp(self):
+        super(TestPlugin, self).setUp()
+        self.project = ProjectFactory.create(short_name='project')
+        self.task = TaskFactory.create(n_answers=1, state='completed')
 
     @with_context
-    def test_view_renders_at_expected_route(self):
+    def test_get_main_view(self):
         res = self.app.get('/data', follow_redirects=True)
-        assert res.status_code == 200
+        assert res.status_code == 200, res.status_code
+
+    @with_context
+    def test_get_csv_export_view(self):
+        res = self.app.get('/data/project/csv_export', follow_redirects=True)
+        assert res.status_code == 200, res.status_code
 
     @with_context
     def test_csv_file_exported(self):
         self.signin(email='owner@a.com', password='1234')
-        url = u'{0}/csv_eport'.format(self.base_url)
-        res = self.app.get(url, follow_redirects=True)
+        res = self.app.get('/data/project/csv_export', follow_redirects=True)
         content = res.headers['Content-Disposition']
         content_type = res.headers['Content-Type']
         fn = "{0}_results.csv".format(self.project.short_name)
-        assert fn in content and "text/csv" in content_type
+        assert fn in content, content
+        assert "text/csv" in content_type, content_type
+
+    @with_context
+    @patch('libcrowds_data.view.UnicodeWriter.writerow')
+    def test_correct_data_written_to_csv(self, mock_writer):
+        TaskRunFactory.create(project=self.project, task=self.task)
+        result = result_repo.filter_by(project_id=self.project.id)[0]
+        result.info = {'n': 42}
+        result_repo.update(result)
+        res = self.app.get('/data/project/csv_export', follow_redirects=True)
+        expected_headers = ['info', 'task_id', 'created', 'last_version',
+                            'task_run_ids', 'project_id', 'id', 'info_n']
+        expected_row = result.dictize().values() + [42]
+        headers = mock_writer.call_args_list[0][0][0]
+        row = mock_writer.call_args_list[1][0][0]
+        assert sorted(headers) == sorted(expected_headers)
+        assert sorted(row) == sorted(expected_row)