Merge pull request #200 from rapidpro/develop

Updates for deploy for 1.5.0

docs/conf.py

@@ -12,6 +12,7 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+import datetime
 import sys
 import os
 
@@ -45,18 +46,18 @@
 
 # General information about the project.
 project = u'TracPro'
-copyright = u'2015, UNICEF'
+copyright = u'%s, UNICEF' % datetime.datetime.now().year
 author = u'UNICEF'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-release = "1.4.3"
+release = "1.5.0"
 
 # The short X.Y version.
-version = "1.4"
+version = "1.5"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/releases/changelog.rst

@@ -7,6 +7,15 @@ Tracpro's version is incremented upon each merge to master according to our
 We recommend reviewing the release notes and code diffs before upgrading
 between versions.
 
+v1.5.0 (released 2017-02-27)
+----------------------------
+
+Code diff: https://github.com/rapidpro/tracpro/compare/v1.5.0...develop
+
+* Many-to-many contact to group relation update in the back-end
+* Allow admins to set super user status on the front end
+* Web interface to manage fetch runs
+
 v1.4.3 (released 2016-04-06)
 ----------------------------
 

docs/users/getting-started.rst

@@ -37,20 +37,13 @@ There won't be much to see until you tell TracPro about which flows and groups t
 Fetching old runs
 ------------------
 
-If a new poll is added, TracPro will only track runs made after the poll has been added. If you need to fetch older runs, then there is a management task which allows you to do this. The command takes two parameters:
-
-#. The database id of the organization
-#. A number of minutes, hours or days::
-
-    # fetches all runs for org #1 made in the last 45 minutes
-    $ ./manage.py fetchruns 1 --minutes=45
-
-    # fetches all runs for org #2 made in the last 6 hours
-    $ ./manage.py fetchruns 2 --hours=6
-
-    # fetches all runs for org #3 made in the last 2 days (48 hours)
-    $ ./manage.py fetchruns 3 --days=2
+If a new poll is added, TracPro will only track runs made after the poll has been added.
+If you need to fetch older runs, then there is a button which allows you to do this.
 
+ * Navigate to `http://SUBDOMAIN.yourtracprodomain/`.
+ * Navigate to **Administration** > **Organization** and click **Fetch runs** (on the right side near the top).
+ * Enter how many days in the past the fetch should go. For example, to fetch runs from the last two weeks, enter 14.
+ * Click **Submit**.
 
 **One should use this command with caution as it could potentially try to download a very high number of runs**
 

tracpro/__init__.py

@@ -6,7 +6,7 @@
 
 
 # NOTE: Version must be updated in docs/conf.py as well.
-VERSION = (1, 4, 3, "final")
+VERSION = (1, 5, 0, "final")
 
 
 def get_version(version):

tracpro/contacts/migrations/0012_contact_groups.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('groups', '0008_uuid_is_unique_to_org'),
+        ('contacts', '0011_uuid_is_unique_to_org'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='contact',
+            name='groups',
+            field=models.ManyToManyField(help_text='All groups to which this contact belongs.', related_name='all_contacts', null=True, verbose_name='Groups', to='groups.Group'),
+        ),
+    ]

tracpro/contacts/migrations/0013_auto_20161118_1516.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contacts', '0012_contact_groups'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='contact',
+            name='groups',
+            field=models.ManyToManyField(help_text='All groups to which this contact belongs.', related_name='all_contacts', verbose_name='Groups', to='groups.Group'),
+        ),
+    ]

tracpro/contacts/models.py

@@ -89,6 +89,10 @@ class Contact(models.Model):
         'groups.Group', null=True, verbose_name=_("Reporter group"),
         related_name='contacts',
         help_text=_("Reporter group to which this contact belongs."))
+    groups = models.ManyToManyField(
+        'groups.Group', verbose_name=_("Groups"),
+        related_name='all_contacts',
+        help_text=_("All groups to which this contact belongs."))
     language = models.CharField(
         max_length=3, verbose_name=_("Language"), null=True, blank=True,
         help_text=_("Language for this contact"))
@@ -129,17 +133,13 @@ def __str__(self):
 
     def as_temba(self):
         """Return a Temba object representing this Contact."""
-        groups = [self.region.uuid]
-        if self.group_id:
-            groups.append(self.group.uuid)
-
         fields = {f.field.key: f.get_value() for f in self.contactfield_set.all()}
 
         temba_contact = TembaContact()
         temba_contact.name = self.name
         temba_contact.urns = [self.urn]
         temba_contact.fields = fields
-        temba_contact.groups = groups
+        temba_contact.groups = list(self.groups.all().values_list('uuid', flat=True))
         temba_contact.language = self.language
         temba_contact.uuid = self.uuid
 
@@ -198,8 +198,7 @@ def _get_first(model_class, temba_uuids):
 
         # Use the first Temba group that matches one of the org's Groups.
         group = _get_first(Group, temba_contact.groups)
-
-        return {
+        kwargs = {
             'org': org,
             'name': temba_contact.name or "",
             'urn': temba_contact.urns[0],
@@ -210,6 +209,9 @@ def _get_first(model_class, temba_uuids):
             'temba_modified_on': temba_contact.modified_on,
             '_data_field_values': temba_contact.fields,  # managed by post-save signal
         }
+        if cls.objects.filter(org=org, uuid=temba_contact.uuid).exists():
+            kwargs['groups'] = [Group.objects.get(uuid=group_uuid) for group_uuid in temba_contact.groups]
+        return kwargs
 
     def push(self, change_type):
         push_contact_change.delay(self.pk, change_type)
@@ -241,6 +243,9 @@ def save(self, *args, **kwargs):
 
         return contact
 
+    def fields(self):
+        return {f.field.key: f.value for f in self.contactfield_set.all()}
+
 
 class DataFieldQuerySet(models.QuerySet):
 

tracpro/contacts/signals.py

@@ -2,7 +2,9 @@
 
 from django.db.models.signals import post_save
 from django.dispatch import receiver
+from temba_client.base import TembaNoSuchObjectError
 
+from tracpro.groups.models import Group
 from .models import Contact, ContactField
 
 
@@ -32,3 +34,16 @@ def set_data_field_values(sender, instance, **kwargs):
             contact_field.save()
 
     del instance._data_field_values
+
+
+@receiver(post_save, sender=Contact)
+def set_groups_to_new_contact(sender, instance, created, **kwargs):
+    """Hook to set the groups of a temba contact when a Contact is created after sync."""
+    if created:
+        try:
+            temba_contact = instance.org.get_temba_client().get_contact(uuid=instance.uuid)
+            for group_uuid in temba_contact.groups:
+                instance.groups.add(Group.objects.get(uuid=group_uuid))
+        except TembaNoSuchObjectError:
+            # The contact was created locally
+            pass

tracpro/contacts/tests/factories.py

@@ -27,6 +27,12 @@ def region(self):
         from tracpro.test.factories import Region
         return Region(org=self.org)
 
+    @factory.post_generation
+    def groups(self, create, extracted, **kwargs):
+        if create and extracted:
+            # A list of groups were passed in, use them
+            self.groups.add(*extracted)
+
 
 class TwitterContact(Contact):
     urn = factory.Sequence(lambda n: "twitter:contact" + str(n))

tracpro/contacts/tests/test_models.py

@@ -99,7 +99,7 @@ def test_kwargs_from_temba(self):
             'name': "Jan",
             'urn': 'tel:123',
             'region': self.region1,
-            'group': self.group3,
+            'group': self.group5,
             'language': 'eng',
             'temba_modified_on': modified_date,
             '_data_field_values': {
@@ -115,7 +115,7 @@ def test_as_temba(self):
         self.assertEqual(temba_contact.name, "Ann")
         self.assertEqual(temba_contact.urns, ['tel:1234'])
         self.assertEqual(temba_contact.fields, {})
-        self.assertEqual(temba_contact.groups, ['G-001', 'G-005'])
+        self.assertEqual(temba_contact.groups, ['G-005', 'G-001'])
         self.assertEqual(temba_contact.uuid, 'C-001')
 
     def test_by_org(self):

tracpro/groups/tests/test_models.py

@@ -209,7 +209,7 @@ def test_create(self):
         self.assertEqual(group.uuid, 'G-101')
 
     def test_get_all(self):
-        self.assertEqual(len(models.Group.get_all(self.unicef)), 3)
+        self.assertEqual(len(models.Group.get_all(self.unicef)), 4)
         self.assertEqual(len(models.Group.get_all(self.nyaruka)), 1)
 
 

tracpro/groups/tests/test_views.py

@@ -532,7 +532,7 @@ def test_admin(self):
         self.login(self.admin)
         response = self.url_get('unicef', reverse(self.url_name))
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.context['object_list']), 3)
+        self.assertEqual(len(response.context['object_list']), 4)
 
 
 class TestGroupMostActive(TracProDataTest):

tracpro/orgs_ext/forms.py

@@ -96,3 +96,7 @@ def save(self, *args, **kwargs):
             self.instance._visible_data_fields = self.cleaned_data.get('contact_fields')
 
         return super(OrgExtForm, self).save(*args, **kwargs)
+
+
+class FetchRunsForm(forms.Form):
+    days = forms.IntegerField(initial=1, min_value=1)

tracpro/orgs_ext/tasks.py

@@ -6,14 +6,16 @@
 from celery import signature
 from celery.exceptions import SoftTimeLimitExceeded
 from celery.utils.log import get_task_logger
+from dash.orgs.models import Org
 
 from django.apps import apps
 from django.conf import settings
 from django.core.cache import cache
 from django.core.mail import send_mail
 from django.utils import timezone
+from django.utils.translation import ugettext_lazy as _
 
-from djcelery_transactions import PostTransactionTask
+from djcelery_transactions import PostTransactionTask, task
 
 from temba_client.base import TembaAPIError
 from temba_client.utils import parse_iso8601, format_iso8601
@@ -229,3 +231,72 @@ def wrap_logger(self, level, org, msg, *args, **kwargs):
         kwargs.setdefault('org', org.name)
         msg = "{task} for {org}: " + msg
         return super(OrgTask, self).wrap_logger(level, msg, *args, **kwargs)
+
+
+@task(ignore_result=True)
+def fetch_runs(org_id, since, email=None):
+    """
+    Fetch responses for the org with id=org_id, going back
+    to `since` (datetime).
+
+    Creates or updates Response objects for each run.
+
+    If `email` is provided, an email is sent to that address at the end to
+    report the results.
+    """
+    from tracpro.polls.models import Poll, Response  # Avoid circular imports
+
+    try:
+        org = Org.objects.get(id=org_id)
+    except Org.DoesNotExist:
+        raise ValueError("No such org with id %d" % org_id)
+
+    # Collect our log messages so we can email them at the end if we want to.
+    messages = []
+
+    def log(s):
+        messages.append(s)
+        logger.info(s)
+
+    # These will show up on stdout when this is called from the management command
+    # (without an `email`).
+    log(_('Fetching responses for org {org_name} since {time}.')
+        .format(org_name=org.name, time=since.strftime('%b %d, %Y %H:%M')))
+
+    client = org.get_temba_client()
+
+    polls_by_flow_uuids = {p.flow_uuid: p for p in Poll.objects.active().by_org(org)}
+
+    runs = client.get_runs(flows=polls_by_flow_uuids.keys(), after=since)
+
+    log(_("Fetched {num} runs for org {org_name}.").format(num=len(runs), org_name=org.name))
+
+    created = 0
+    updated = 0
+    for run in runs:
+        if run.flow not in polls_by_flow_uuids:
+            continue  # Response is for a Poll not tracked for this org.
+
+        poll = polls_by_flow_uuids[run.flow]
+        try:
+            response = Response.from_run(org, run, poll=poll)
+        except ValueError as e:
+            log(_("Unable to save run #{num} due to error: {message}.").format(num=run.id, message=e.message))
+            continue
+
+        if getattr(response, 'is_new', False):
+            created += 1
+        else:
+            updated += 1
+
+    log(_("Created {created} new responses and updated {updated} existing responses.")
+        .format(created=created, updated=updated))
+
+    if email:
+        send_mail(
+            subject=(_("Results from fetching runs for organization {org_name} since {time}")
+                     .format(org_name=org.name, time=since.strftime('%b %d, %Y %H:%M'))),
+            message="\n".join(messages) + "\n",
+            from_email=settings.DEFAULT_FROM_EMAIL,
+            recipient_list=[email],
+            fail_silently=True)