Merge pull request #208 from rapidpro/develop

Updates for deploy for 1.5.2

docs/conf.py

@@ -54,7 +54,7 @@
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-release = "1.5.1"
+release = "1.5.2"
 
 # The short X.Y version.
 version = "1.5"

docs/releases/changelog.rst

@@ -7,6 +7,19 @@ Tracpro's version is incremented upon each merge to master according to our
 We recommend reviewing the release notes and code diffs before upgrading
 between versions.
 
+v1.5.2 (released 2017-03-17)
+----------------------------
+
+Code diff: https://github.com/rapidpro/tracpro/compare/v1.5.2...develop
+
+* Larger, sorted multi-select drop-downs
+* Fixes for error emails from tasks
+* Permissions fix for fetch runs front-end: Admins can now use this feature, in addition to super users.
+* Fix redirect loop on change password
+* Fixes for admins:
+    - When someone is added to an org as an admin, make sure they're flagged as staff.
+    - In the users list, add a column showing whether a user is staff or not.
+
 v1.5.1 (released 2017-02-28)
 ----------------------------
 

tracpro/__init__.py

@@ -6,7 +6,7 @@
 
 
 # NOTE: Version must be updated in docs/conf.py as well.
-VERSION = (1, 5, 1, "final")
+VERSION = (1, 5, 2, "final")
 
 
 def get_version(version):

tracpro/baseline/forms.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from django import forms
+from django.db.models.functions import Lower
 from django.utils.translation import ugettext_lazy as _
 
 from tracpro.charts import filters
@@ -32,7 +33,7 @@ def __init__(self, *args, **kwargs):
         super(BaselineTermForm, self).__init__(*args, **kwargs)
 
         if org:
-            polls = Poll.objects.active().by_org(org).order_by('name')
+            polls = Poll.objects.active().by_org(org).order_by(Lower('name'))
             self.fields['baseline_poll'].queryset = polls
             self.fields['follow_up_poll'].queryset = polls
 
@@ -62,20 +63,22 @@ def label_from_instance(self, obj):
 class SpoofDataForm(forms.Form):
     """ Form to create spoofed poll data """
     contacts = forms.ModelMultipleChoiceField(
-        queryset=Contact.objects.all(),
-        help_text=_("Select contacts for this set of spoofed data."))
+        queryset=Contact.objects.order_by(Lower('name')),
+        help_text=_("Select contacts for this set of spoofed data."),
+        widget=forms.widgets.SelectMultiple(attrs={'size': '20'}),
+    )
     start_date = forms.DateField(
         help_text=_("Baseline poll data will be submitted on this date. "
                     "Follow up data will start on this date."))
     end_date = forms.DateField(
         help_text=_("Follow up data will end on this date. "))
     baseline_question = QuestionModelChoiceField(
-        queryset=Question.objects.all().order_by('poll__name', 'text'),
+        queryset=Question.objects.order_by(Lower('poll__name'), 'text'),
         help_text=_("Select a baseline question which will have numeric "
                     "answers only."))
     follow_up_question = QuestionModelChoiceField(
         label=_("Follow Up Question"),
-        queryset=Question.objects.all().order_by('poll__name', 'text'),
+        queryset=Question.objects.order_by(Lower('poll__name'), 'text'),
         help_text=_("Select a follow up question which will have "
                     "numeric answers only."))
     baseline_minimum = forms.IntegerField(
@@ -99,7 +102,7 @@ def __init__(self, *args, **kwargs):
         super(SpoofDataForm, self).__init__(*args, **kwargs)
 
         if org:
-            contacts = Contact.objects.active().by_org(org).order_by('name')
+            contacts = Contact.objects.active().by_org(org).order_by(Lower('name'))
             self.fields['contacts'].queryset = contacts
             questions = Question.objects.filter(poll__in=Poll.objects.active().by_org(org))
             self.fields['baseline_question'].queryset = questions
@@ -168,8 +171,7 @@ def __init__(self, baseline_term, data_regions, *args, **kwargs):
             queryset = self.org.regions.filter(is_active=True)
         else:
             queryset = data_regions
-        queryset = queryset.filter(pk__in=baseline_term.get_regions())
-        queryset = queryset.order_by('name')
+        queryset = queryset.filter(pk__in=baseline_term.get_regions()).order_by(Lower('name'))
         self.fields['region'].queryset = queryset
 
     def filter_contacts(self, queryset=None):

tracpro/contacts/forms.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 
 from django import forms
+from django.db.models.functions import Lower
 from django.utils.translation import ugettext_lazy as _
 
 from tracpro.groups.models import Group, Region
@@ -35,9 +36,10 @@ def __init__(self, *args, **kwargs):
         self.fields['name'].required = True
         self.fields['group'].required = True
 
-        self.fields['region'].queryset = self.user.get_all_regions(org)
+        regions = self.user.get_all_regions(org).order_by(Lower('name'))
+        self.fields['region'].queryset = regions
         self.fields['group'].empty_label = ""
-        self.fields['group'].queryset = Group.get_all(org).order_by('name')
+        self.fields['group'].queryset = Group.get_all(org).order_by(Lower('name'))
 
         # Add form fields to update contact's DataField values.
         self.data_field_keys = []

tracpro/groups/forms.py

@@ -9,7 +9,9 @@
 class ContactGroupsForm(forms.Form):
     groups = forms.MultipleChoiceField(
         choices=(), label=_("Groups"),
-        help_text=_("Contact groups to use."))
+        help_text=_("Contact groups to use."),
+        widget=forms.widgets.SelectMultiple(attrs={'size': '20'}),
+    )
 
     def __init__(self, model, org, *args, **kwargs):
         self.model = model
@@ -20,6 +22,8 @@ def __init__(self, model, org, *args, **kwargs):
         # Retrieve Contact Group choices from RapidPro.
         choices = [(group.uuid, "%s (%d)" % (group.name, group.count))
                    for group in get_client(org).get_groups()]
+        # Sort choices by the labels, case-insensitively
+        choices.sort(key=lambda item: item[1].lower())
         self.fields['groups'].choices = choices
 
         # Set initial group values from the org.

tracpro/groups/views.py

@@ -10,6 +10,7 @@
 from django.core.urlresolvers import reverse
 from django.db import transaction
 from django.db.models import Prefetch
+from django.db.models.functions import Lower
 from django.http import (
     HttpResponseBadRequest, HttpResponseRedirect, JsonResponse)
 from django.shortcuts import redirect
@@ -130,7 +131,7 @@ def derive_queryset(self, **kwargs):
             return regions
 
         def get_context_data(self, **kwargs):
-            org_boundaries = Boundary.objects.by_org(self.request.org).order_by('name')
+            org_boundaries = Boundary.objects.by_org(self.request.org).order_by(Lower('name'))
             kwargs.setdefault('org_boundaries', org_boundaries)
             return super(RegionCRUDL.List, self).get_context_data(**kwargs)
 

tracpro/home/views.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 
 from dash.orgs.views import OrgPermsMixin
+from django.db.models.functions import Lower
 
 from django.utils.translation import ugettext_lazy as _
 
@@ -22,8 +23,7 @@ def has_permission(self, request, *args, **kwargs):
         return request.user.is_authenticated()
 
     def get_context_data(self, **kwargs):
-        polls = Poll.objects.active().by_org(self.request.org)
-        polls = polls.order_by('name')
+        polls = Poll.objects.active().by_org(self.request.org).order_by(Lower('name'))
 
         indicators = BaselineTerm.objects.by_org(self.request.org)
         indicators = indicators.order_by('-end_date')

tracpro/orgs_ext/forms.py

@@ -1,6 +1,8 @@
 from __future__ import unicode_literals
 
 from dash.orgs.forms import OrgForm
+from django.contrib.auth.models import User
+from django.db.models.functions import Lower
 
 from temba_client.base import TembaAPIError
 
@@ -13,6 +15,25 @@
 from . import utils
 
 
+class UserMultipleChoiceField(forms.ModelMultipleChoiceField):
+    def label_from_instance(self, user):
+        """
+        Display a user as their full name from their profile,
+        if they have one, else "No profile", and then add on
+        their email to distinguish multiple accounts under the
+        same full name (or no profile).
+        """
+        try:
+            full_name = user.profile.full_name
+        except User.profile.RelatedObjectDoesNotExist:
+            full_name = "No profile"
+
+        return "{full_name} ({email})".format(
+            full_name=full_name,
+            email=user.email
+        )
+
+
 class OrgExtForm(OrgForm):
     """Also configure available languages for this organization.
 
@@ -22,15 +43,24 @@ class OrgExtForm(OrgForm):
     """
 
     available_languages = forms.MultipleChoiceField(
-        choices=settings.LANGUAGES,
-        help_text=_("The languages used by administrators in your organization"))
+        choices=settings.LANGUAGES,  # Rely on settings to be in a useful order.
+        help_text=_("The languages used by administrators in your organization"),
+        widget=forms.widgets.SelectMultiple(attrs={'size': str(len(settings.LANGUAGES))}),
+    )
     show_spoof_data = forms.BooleanField(
         required=False,
         help_text=_("Whether to show spoof data for this organization."))
     contact_fields = forms.ModelMultipleChoiceField(
         queryset=None, required=False,
         help_text=_("Custom contact data fields that should be visible "
-                    "and editable in TracPro."))
+                    "and editable in TracPro."),
+        widget=forms.widgets.SelectMultiple(attrs={'size': '20'}),
+    )
+    administrators = UserMultipleChoiceField(
+        queryset=None,  # OrgForm.__init__ will set this
+        widget=forms.widgets.SelectMultiple(attrs={'size': '20'}),
+        help_text=_("The administrators for this organization")
+    )
 
     def __init__(self, *args, **kwargs):
         super(OrgExtForm, self).__init__(*args, **kwargs)
@@ -48,6 +78,10 @@ def __init__(self, *args, **kwargs):
         self.fields['available_languages'].initial = self.instance.available_languages or []
         self.fields['show_spoof_data'].initial = self.instance.show_spoof_data or False
 
+        # Sort the administrators
+        self.fields['administrators'].queryset \
+            = self.fields['administrators'].queryset.order_by(Lower('profile__full_name'))
+
         if not self.instance.pk:
             # We don't have this org's API key yet,
             # so we can't get available fields from the RapidPro API.
@@ -66,7 +100,7 @@ def __init__(self, *args, **kwargs):
                     raise
 
             data_fields = self.instance.datafield_set.all()
-            self.fields['contact_fields'].queryset = data_fields
+            self.fields['contact_fields'].queryset = data_fields.order_by(Lower('label'))
             self.fields['contact_fields'].initial = data_fields.visible()
 
     def clean(self):

tracpro/orgs_ext/tasks.py

@@ -200,7 +200,7 @@ def run(self, org_pk):
                 elif isinstance(e, SoftTimeLimitExceeded):
                     msg = "Time limit exceeded (#{count})."
                     full_msg = self.log_error(org, msg, count=fail_count)
-                    self.send_error_email(org, full_msg)
+                    self.send_org_error_email(org, full_msg)
                     return None
                 else:
                     msg = "Unknown failure (#{count})."
@@ -217,16 +217,16 @@ def run(self, org_pk):
             self.log_info(org, msg)
             return None
 
-    def send_error_email(self, org, msg):
+    def send_org_error_email(self, org, msg):
         # FIXME: Logging is not sending us regular logging error emails.
-        self.log_debug(org, "Starting to send error email.")
+        self.log_debug(org, "Starting to send org error email.")
         send_mail(
             subject="{}{}".format(settings.EMAIL_SUBJECT_PREFIX, msg),
             message=msg,
             from_email=settings.DEFAULT_FROM_EMAIL,
             recipient_list=dict(settings.ADMINS).values(),
-            fail_silently=True)
-        self.log_debug(org, "Finished sending error email.")
+            fail_silently=False)
+        self.log_debug(org, "Finished sending org error email.")
 
     def wrap_logger(self, level, org, msg, *args, **kwargs):
         kwargs.setdefault('org', org.name)
@@ -301,4 +301,4 @@ def log(s):
             message="\n".join(messages) + "\n",
             from_email=settings.DEFAULT_FROM_EMAIL,
             recipient_list=[email],
-            fail_silently=True)
+            fail_silently=False)

tracpro/orgs_ext/tests/test_views.py

@@ -35,10 +35,16 @@ def test_get_not_logged_in(self):
         self.assertEqual(response.status_code, 302)
         self.assertIn(settings.LOGIN_URL + "?next", response['Location'])
 
-    def test_get_not_superuser(self):
+    def test_get_as_admin(self):
         self.client.logout()
         self.login(self.admin)
         response = self.url_get('unicef', reverse(self.url_name))
+        self.assertEqual(response.status_code, 200)
+
+    def test_get_not_superuser_or_admin(self):
+        self.client.logout()
+        self.login(self.user1)
+        response = self.url_get('unicef', reverse(self.url_name))
         self.assertEqual(response.status_code, 302)
         self.assertIn(settings.LOGIN_URL + "?next", response['Location'])
 

tracpro/orgs_ext/views.py

@@ -16,11 +16,18 @@
 from . import tasks
 
 
+class MakeAdminsIntoStaffMixin(object):
+    # Make sure all admins are staff users.
+    def post_save(self, obj):
+        obj.get_org_admins().filter(is_staff=False).update(is_staff=True)
+        return super(MakeAdminsIntoStaffMixin, self).post_save(obj)
+
+
 class OrgExtCRUDL(OrgCRUDL):
     actions = ('create', 'update', 'list', 'home', 'edit', 'chooser',
                'choose', 'fetchruns')
 
-    class Create(OrgCRUDL.Create):
+    class Create(MakeAdminsIntoStaffMixin, OrgCRUDL.Create):
         form_class = forms.OrgExtForm
         fields = ('name', 'available_languages', 'language',
                   'timezone', 'subdomain', 'api_token', 'show_spoof_data',
@@ -29,7 +36,7 @@ class Create(OrgCRUDL.Create):
     class List(OrgCRUDL.List):
         default_order = ('name',)
 
-    class Update(OrgCRUDL.Update):
+    class Update(MakeAdminsIntoStaffMixin, OrgCRUDL.Update):
         form_class = forms.OrgExtForm
         fields = ('is_active', 'name', 'available_languages', 'language',
                   'contact_fields', 'timezone', 'subdomain', 'api_token',
@@ -76,12 +83,9 @@ class Edit(InferOrgMixin, OrgPermsMixin, SmartUpdateView):
         success_url = '@orgs_ext.org_home'
         title = _("Edit My Organization")
 
-    class Fetchruns(InferOrgMixin, SmartFormView):
+    class Fetchruns(InferOrgMixin, OrgPermsMixin, SmartFormView):
         form_class = forms.FetchRunsForm
-        # Hack: has_perm always returns true for a superuser.  Since this
-        # isn't a valid permission name, it'll always be false for non superuser.
-        # If there's a real way to require superuser in SmartView, I'm all ears.
-        permission = 'must be superuser'
+        permission = 'orgs.org_fetch_runs'
         success_url = '@orgs_ext.org_home'
         title = _("Fetch past runs for my organization")
 

tracpro/polls/charts.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 
 from django.core.urlresolvers import reverse
+from django.db.models.functions import Lower
 from django.utils.http import urlencode
 
 from tracpro.charts.formatters import format_series, format_x_axis
@@ -168,7 +169,8 @@ def multiple_pollruns_numeric_split(pollruns, answers, responses, question, cont
     sum_data = []
     avg_data = []
     rate_data = []
-    for region in Region.objects.filter(pk__in=data.keys()).order_by('name'):
+    regions = (Region.objects.filter(pk__in=data.keys()).order_by(Lower('name')))
+    for region in regions:
         answer_sums, answer_avgs, answer_stdevs, response_rates = data.get(region.pk)
         region_answer_sums = []
         region_answer_avgs = []