Merge pull request #35 from davidbgk/23-csv-issues-discussions-export

Add CSV export for issues and discussions, refs #23

udata/core/dataset/csv.py

@@ -61,3 +61,17 @@ class ResourcesCsvAdapter(csv.NestedAdapter):
         ('downloads', lambda o: int(o.metrics.get('views', 0))),
     )
     attribute = 'resources'
+
+
+class IssuesOrDiscussionCsvAdapter(csv.Adapter):
+    fields = (
+        'id',
+        'user',
+        'subject',
+        'title',
+        ('size', lambda o: len(o.discussion)),
+        ('messages', lambda o: '\n'.join(msg.content for msg in o.discussion)),
+        'created',
+        'closed',
+        'closed_by',
+    )

udata/core/organization/views.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import itertools
 from datetime import datetime
 
 from flask import request, g, jsonify, redirect, url_for, abort
@@ -9,12 +10,19 @@
 from udata import search
 from udata.app import nav
 from udata.frontend import csv
-from udata.frontend.views import DetailView, CreateView, EditView, SearchView, BaseView, SingleObject
+from udata.frontend.views import (
+    DetailView, CreateView, EditView, SearchView, BaseView, SingleObject
+)
 from udata.i18n import I18nBlueprint, lazy_gettext as _
-from udata.models import db, Organization, Member, Reuse, Dataset, ORG_ROLES, User, Issue, FollowOrg
+from udata.models import (
+    db, Organization, Member, Reuse, Dataset, ORG_ROLES, User, Issue, FollowOrg,
+    DatasetIssue, DatasetDiscussion
+)
 from udata.utils import get_by
 
-from udata.core.dataset.csv import DatasetCsvAdapter, ResourcesCsvAdapter
+from udata.core.dataset.csv import (
+    DatasetCsvAdapter, IssuesOrDiscussionCsvAdapter, ResourcesCsvAdapter
+)
 from udata.core.activity.views import ActivityView
 
 from .forms import OrganizationForm, OrganizationMemberForm, OrganizationExtraForm
@@ -33,7 +41,6 @@ def set_g_user_orgs():
 
 navbar = nav.Bar('edit_org', [
     nav.Item(_('Descrition'), 'organizations.edit'),
-    # nav.Item(_('Additional informations'), 'organizations.edit_extras'),
     nav.Item(_('Members'), 'organizations.edit_members'),
     nav.Item(_('Membership request'), 'organizations.edit_membership_requests'),
     nav.Item(_('Teams'), 'organizations.edit_teams'),
@@ -163,8 +170,6 @@ def get_context(self):
                     'title': _('Permitted reuses'),
                     'metric': 'permitted_reuses',
                     'type': 'line',
-                    # 'endpoint': 'reuses.list',
-                    # 'args': {'org': self.organization}
                 },
                 {
                     'title': _('Followers'),
@@ -291,6 +296,26 @@ def datasets_csv(org):
     return csv.stream(adapter, '{0}-datasets'.format(org.slug))
 
 
+@blueprint.route('/<org:org>/issues.csv')
+def issues_csv(org):
+    datasets = Dataset.objects.filter(organization=str(org.id))
+    issues = [DatasetIssue.objects.filter(subject=dataset)
+              for dataset in datasets]
+    # Turns a list of lists into a flat list.
+    adapter = IssuesOrDiscussionCsvAdapter(itertools.chain(*issues))
+    return csv.stream(adapter, '{0}-issues'.format(org.slug))
+
+
+@blueprint.route('/<org:org>/discussions.csv')
+def discussions_csv(org):
+    datasets = Dataset.objects.filter(organization=str(org.id))
+    discussions = [DatasetDiscussion.objects.filter(subject=dataset)
+                   for dataset in datasets]
+    # Turns a list of lists into a flat list.
+    adapter = IssuesOrDiscussionCsvAdapter(itertools.chain(*discussions))
+    return csv.stream(adapter, '{0}-discussions'.format(org.slug))
+
+
 @blueprint.route('/<org:org>/datasets-resources.csv')
 def datasets_resources_csv(org):
     datasets = search.iter(Dataset, organization=str(org.id))

udata/templates/organization/display.html

@@ -112,6 +112,16 @@ <h1>
                             {{ _('Resources list') }}
                             </a>
                         </li>
+                        <li role="presentation">
+                            <a role="menuitem" tabindex="-1" href="{{ url_for('organizations.issues_csv', org=org) }}">
+                            {{ _('Issues list') }}
+                            </a>
+                        </li>
+                        <li role="presentation">
+                            <a role="menuitem" tabindex="-1" href="{{ url_for('organizations.discussions_csv', org=org) }}">
+                            {{ _('Discussions list') }}
+                            </a>
+                        </li>
                     </ul>
                 </div>
             </div>

udata/tests/__init__.py

@@ -67,6 +67,11 @@ def assert201(self, response):
     def assert204(self, response):
         self.assertStatus(response, 204)
 
+    def assertStartswith(self, haystack, needle):
+        self.assertEqual(
+            haystack.startswith(needle), True,
+            '{haystack} does not start with {needle}'.format(haystack=haystack, needle=needle))
+
     @contextmanager
     def assert_emit(self, *signals):
         specs = []

udata/tests/factories.py

@@ -156,6 +156,20 @@ def resources(self):
         return [ResourceFactory()]
 
 
+class DatasetIssueFactory(MongoEngineFactory):
+    class Meta:
+        model = models.DatasetIssue
+
+    title = factory.LazyAttribute(lambda o: faker.sentence())
+
+
+class DatasetDiscussionFactory(MongoEngineFactory):
+    class Meta:
+        model = models.DatasetDiscussion
+
+    title = factory.LazyAttribute(lambda o: faker.sentence())
+
+
 class OrganizationFactory(MongoEngineFactory):
     class Meta:
         model = models.Organization

udata/tests/test_discussions.py

@@ -10,8 +10,12 @@
 from udata.core.discussions.models import Message, Discussion
 from udata.core.discussions.signals import on_new_discussion
 
+from frontend import FrontTestCase
+
 from .api import APITestCase
-from .factories import faker, UserFactory
+from .factories import (
+    faker, UserFactory, OrganizationFactory, DatasetFactory, DatasetDiscussionFactory
+)
 
 
 class DiscussionsTest(APITestCase):
@@ -258,3 +262,27 @@ def test_close_discussion_permissions(self):
         dataset.reload()
         # Metrics unchanged after attempt to close the discussion.
         self.assertEqual(dataset.metrics['discussions'], 1)
+
+
+class DiscussionCsvTest(FrontTestCase):
+
+    def test_discussions_csv_content_empty(self):
+        organization = OrganizationFactory()
+        response = self.get(url_for('organizations.discussions_csv', org=organization))
+        self.assert200(response)
+
+        self.assertEqual(
+            response.data,
+            '"id";"user";"subject";"title";"size";"messages";"created";"closed";"closed_by"\r\n'
+        )
+
+    def test_discussions_csv_content_filled(self):
+        organization = OrganizationFactory()
+        dataset = DatasetFactory(organization=organization)
+        user = UserFactory(first_name='John', last_name='Snow')
+        discussion = DatasetDiscussionFactory(subject=dataset, user=user)
+        response = self.get(url_for('organizations.discussions_csv', org=organization))
+        self.assert200(response)
+
+        headers, data = response.data.strip().split('\r\n')
+        self.assertStartswith(data, '"{discussion.id}";"{discussion.user}"'.format(discussion=discussion))

udata/tests/test_issues.py

@@ -10,8 +10,12 @@
 from udata.core.issues.models import Issue, Message
 from udata.core.issues.signals import on_new_issue
 
+from frontend import FrontTestCase
+
 from .api import APITestCase
-from .factories import faker, UserFactory
+from .factories import (
+    faker, UserFactory, OrganizationFactory, DatasetFactory, DatasetIssueFactory
+)
 
 
 class IssuesTest(APITestCase):
@@ -258,3 +262,27 @@ def test_close_issue_permissions(self):
         dataset.reload()
         # Metrics unchanged after attempt to close the discussion.
         self.assertEqual(dataset.metrics['issues'], 1)
+
+
+class IssueCsvTest(FrontTestCase):
+
+    def test_issues_csv_content_empty(self):
+        organization = OrganizationFactory()
+        response = self.get(url_for('organizations.issues_csv', org=organization))
+        self.assert200(response)
+
+        self.assertEqual(
+            response.data,
+            '"id";"user";"subject";"title";"size";"messages";"created";"closed";"closed_by"\r\n'
+        )
+
+    def test_issues_csv_content_filled(self):
+        organization = OrganizationFactory()
+        dataset = DatasetFactory(organization=organization)
+        user = UserFactory(first_name='John', last_name='Snow')
+        issue = DatasetIssueFactory(subject=dataset, user=user)
+        response = self.get(url_for('organizations.issues_csv', org=organization))
+        self.assert200(response)
+
+        headers, data = response.data.strip().split('\r\n')
+        self.assertStartswith(data, '"{issue.id}";"{issue.user}"'.format(issue=issue))