Upmerge from Nyaruka

.gitignore

@@ -41,6 +41,7 @@ media/orgs
 media/recordings
 media/test_orgs
 media/tmp
+media/attachments
 
 
 # External projects

CHANGELOG.md

@@ -1,16 +1,17 @@
-v3.0.57
+v3.0.62
 ----------
- * Fix inbound calls on Nexmo to use conversation_uuid
- * Style tweaks for zapier widget
+ * Fix preferred channels for non-msg channels
 
-v3.0.54
+v3.0.61
 ----------
+ * Make migrations to populate new export task fields non-atomic
+ * Add indexes for admin boundaries and aliases
+ * Nexmo: make sure calls are ended on hangup, log hangups and media
+ * Fix inbound calls on Nexmo to use conversation_uuid
+ * Style tweaks for zapier widget
  * Use shorter timeout for IVR
  * Issue hangups on expiration during IVR runs
  * Catch all exceptions and log them when initiating call
-
-v3.0.49
-----------
  * Fix update status for Nexmo calls
 
 v3.0.48

temba/channels/models.py

@@ -831,9 +831,9 @@ def generate_ivr_response(self):
     def get_ivr_client(self):
         if self.channel_type == Channel.TYPE_TWILIO:
             return self.org.get_twilio_client()
-        if self.channel_type == Channel.TYPE_TWIML:
+        elif self.channel_type == Channel.TYPE_TWIML:
             return self.get_twiml_client()
-        if self.channel_type == Channel.TYPE_VERBOICE:  # pragma: no cover
+        elif self.channel_type == Channel.TYPE_VERBOICE:  # pragma: no cover
             return self.org.get_verboice_client()
         elif self.channel_type == Channel.TYPE_NEXMO:
             return self.org.get_nexmo_client()

temba/contacts/migrations/0052_baseexporttask_2.py

@@ -20,6 +20,7 @@ def populate_uuid(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
+    atomic = False
 
     dependencies = [
         ('contacts', '0051_baseexporttask_1'),

temba/contacts/migrations/0051_auto_20170208_1450.py → temba/contacts/migrations/0053_auto_20170208_1450.py

@@ -8,7 +8,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('contacts', '0050_contactgroupcount_is_squashed'),
+        ('contacts', '0052_baseexporttask_2'),
     ]
 
     operations = [

temba/contacts/migrations/0052_contacturn_auth.py → temba/contacts/migrations/0054_contacturn_auth.py

@@ -8,7 +8,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('contacts', '0051_auto_20170208_1450'),
+        ('contacts', '0053_auto_20170208_1450'),
     ]
 
     operations = [

temba/contacts/models.py

@@ -1011,7 +1011,7 @@ def create_instance(cls, field_dict):
             if not value:
                 continue
 
-            value = str(value)
+            value = six.text_type(value)
 
             urn_scheme = ContactURN.IMPORT_HEADER_TO_SCHEME[urn_header]
 

temba/contacts/tests.py

@@ -2521,6 +2521,21 @@ def test_contact_import(self):
         Contact.objects.all().delete()
         ContactGroup.user_groups.all().delete()
 
+        self.assertContactImport('%s/test_imports/sample_contacts_bad_unicode.xls' % settings.MEDIA_ROOT,
+                                 dict(records=2, errors=0, creates=2, updates=0, error_messages=[]))
+
+        self.assertEquals(1, Contact.objects.filter(name='John Doe').count())
+        self.assertEquals(1, Contact.objects.filter(name='Mary Smith').count())
+
+        contact = Contact.objects.filter(name='John Doe').first()
+        contact2 = Contact.objects.filter(name='Mary Smith').first()
+
+        self.assertEqual(list(contact.get_urns().values_list('path', flat=True)), ['+250788123123'])
+        self.assertEqual(list(contact2.get_urns().values_list('path', flat=True)), ['+250788345345'])
+
+        Contact.objects.all().delete()
+        ContactGroup.user_groups.all().delete()
+
         # import a spreadsheet with phone, name and twitter columns
         self.assertContactImport('%s/test_imports/sample_contacts_twitter_and_phone.xls' % settings.MEDIA_ROOT,
                                  dict(records=3, errors=0, error_messages=[], creates=3, updates=0))

temba/flows/migrations/0090_baseexporttask_2.py

@@ -20,6 +20,7 @@ def populate_uuid(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
+    atomic = False
 
     dependencies = [
         ('flows', '0089_baseexporttask_1'),

temba/ivr/clients.py

@@ -21,6 +21,7 @@
 from twilio import TwilioRestException
 from twilio.rest import TwilioRestClient
 from twilio.util import RequestValidator
+from nexmo import AuthenticationError, ClientError, ServerError
 
 
 class IVRException(Exception):
@@ -105,8 +106,37 @@ def download_media(self, media_url):
 
         return None
 
-    def hangup(self, call_id):
-        self.update_call(call_id, action='hangup')
+    def hangup(self, call):
+        return self.update_call(call.external_id, action='hangup', call_id=call.external_id)
+
+    def parse(self, host, response):
+        try:
+            request = response.request
+            request_body = json.loads(response.request.body)
+            call_id = request_body.get('call_id', None)
+            if call_id:
+                call = IVRCall.objects.filter(external_id=call_id).first()
+                if call:
+                    ChannelLog.log_ivr_interaction(call, "Nexmo call update", request.body, response.content, request.url,
+                                                   request.method, status_code=response.status_code)
+        except:
+            pass
+
+        # Nexmo client doesn't extend object, so can't call super
+        if response.status_code == 401:
+            raise AuthenticationError
+        elif response.status_code == 204:  # pragma: no cover
+            return None
+        elif 200 <= response.status_code < 300:
+            return response.json()
+        elif 400 <= response.status_code < 500:  # pragma: no cover
+            message = "{code} response from {host}".format(code=response.status_code, host=host)
+
+            raise ClientError(message)
+        elif 500 <= response.status_code < 600:  # pragma: no cover
+            message = "{code} response from {host}".format(code=response.status_code, host=host)
+
+            raise ServerError(message)
 
 
 class TwilioClient(TwilioRestClient):
@@ -181,8 +211,8 @@ def download_media(self, media_url):
 
         return None  # pragma: needs cover
 
-    def hangup(self, sid):
-        self.calls.hangup(sid)
+    def hangup(self, call):
+        return self.calls.hangup(call.external_id)
 
 
 class VerboiceClient:  # pragma: needs cover

temba/ivr/models.py

@@ -52,11 +52,12 @@ def close(self):
 
             # mark us as interrupted
             self.status = ChannelSession.INTERRUPTED
+            self.ended_on = timezone.now()
             self.save()
 
             client = self.channel.get_ivr_client()
             if client and self.external_id:
-                client.hangup(self.external_id)
+                client.hangup(self)
 
     def do_start_call(self, qs=None):
         client = self.channel.get_ivr_client()

temba/ivr/tests.py

@@ -42,6 +42,61 @@ def tearDown(self):
         super(IVRTests, self).tearDown()
         settings.SEND_CALLS = False
 
+    @patch('nexmo.Client.create_application')
+    @patch('nexmo.Client.create_call')
+    @patch('nexmo.Client.update_call')
+    @patch('temba.orgs.models.TwilioRestClient', MockTwilioClient)
+    @patch('temba.ivr.clients.TwilioClient', MockTwilioClient)
+    @patch('twilio.util.RequestValidator', MockRequestValidator)
+    def test_preferred_channel(self, mock_update_call, mock_create_call, mock_create_application):
+        mock_create_application.return_value = dict(id='app-id', keys=dict(private_key='private-key'))
+        mock_create_call.return_value = dict(uuid='12345')
+        mock_update_call.return_value = dict(uuid='12345')
+
+        flow = self.get_flow('call_me_maybe')
+
+        # start our flow
+        contact = self.create_contact('Chuck D', number='+13603621737')
+        flow.start([], [contact])
+
+        call = IVRCall.objects.get()
+        self.assertEquals(IVRCall.PENDING, call.status)
+
+        # call should be on a Twilio channel since that's all we have
+        self.assertEquals(Channel.TYPE_TWILIO, call.channel.channel_type)
+
+        # connect Nexmo instead
+        self.org.connect_nexmo('123', '456', self.admin)
+        self.org.save()
+
+        # manually create a Nexmo channel
+        nexmo = Channel.create(self.org, self.user, 'RW', Channel.TYPE_NEXMO, role=Channel.ROLE_CALL + Channel.ROLE_ANSWER + Channel.ROLE_SEND,
+                               name="Nexmo Channel", address="+250785551215")
+
+        # set the preferred channel on this contact to Twilio
+        contact.set_preferred_channel(self.channel)
+
+        # restart the flow
+        flow.start([], [contact], restart_participants=True)
+
+        call = IVRCall.objects.all().last()
+        self.assertEquals(IVRCall.PENDING, call.status)
+        self.assertEquals(Channel.TYPE_TWILIO, call.channel.channel_type)
+
+        # switch back to Nexmo being the preferred channel
+        contact.set_preferred_channel(nexmo)
+
+        # clear open calls and runs
+        IVRCall.objects.all().delete()
+        FlowRun.objects.all().delete()
+
+        # restart the flow
+        flow.start([], [contact], restart_participants=True)
+
+        call = IVRCall.objects.all().last()
+        self.assertEquals(IVRCall.PENDING, call.status)
+        self.assertEquals(Channel.TYPE_NEXMO, call.channel.channel_type)
+
     @patch('temba.orgs.models.TwilioRestClient', MockTwilioClient)
     @patch('temba.ivr.clients.TwilioClient', MockTwilioClient)
     @patch('twilio.util.RequestValidator', MockRequestValidator)
@@ -616,13 +671,21 @@ def test_ivr_digital_gather_with_nexmo(self, mock_create_call, mock_create_appli
 
         self.assertContains(response, '"eventUrl": ["https://%s%s"]}]' % (settings.TEMBA_HOST, callback_url))
 
-    @patch('nexmo.Client.update_call')
+    @patch('jwt.encode')
+    @patch('requests.put')
     @patch('nexmo.Client.create_application')
     @patch('nexmo.Client.create_call')
-    def test_hangup(self, mock_create_call, mock_create_application, mock_update_call):
+    def test_expiration_hangup(self, mock_create_call, mock_create_application, mock_put, mock_jwt):
         mock_create_application.return_value = dict(id='app-id', keys=dict(private_key='private-key'))
         mock_create_call.return_value = dict(uuid='12345')
-        mock_update_call.return_value = dict(uuid='12345')
+        mock_jwt.return_value = "Encoded data"
+
+        import mock
+        request = mock.MagicMock()
+        request.body = json.dumps(dict(call_id='12345'))
+        request.url = "http://api.nexmo.com/../"
+        request.method = "PUT"
+        mock_put.return_value = mock.MagicMock(call_id='12345', request=request, status_code=200, content='response')
 
         self.org.connect_nexmo('123', '456', self.admin)
         self.org.save()
@@ -644,10 +707,14 @@ def test_hangup(self, mock_create_call, mock_create_application, mock_update_cal
         run = FlowRun.objects.get()
         run.expire()
 
-        mock_update_call.assert_called()
+        mock_put.assert_called()
         call = IVRCall.objects.filter(direction=IVRCall.OUTGOING).first()
         self.assertEqual(ChannelSession.INTERRUPTED, call.status)
 
+        # call initiation and timeout should both be logged
+        self.assertEqual(2, ChannelLog.objects.filter(session=call).count())
+        self.assertIsNotNone(call.ended_on)
+
     @patch('nexmo.Client.create_application')
     @patch('nexmo.Client.create_call')
     def test_ivr_subflow_with_nexmo(self, mock_create_call, mock_create_application):

temba/ivr/views.py

@@ -40,7 +40,7 @@ def post(self, request, *args, **kwargs):
             if not request.user.is_anonymous():
                 user_org = request.user.get_org()
                 if user_org and user_org.pk == call.org.pk:
-                    client.hangup(call.external_id)
+                    client.hangup(call)
                     return HttpResponse(json.dumps(dict(status='Canceled')), content_type="application/json")
                 else:  # pragma: no cover
                     return HttpResponse("Not found", status=404)
@@ -120,9 +120,10 @@ def post(self, request, *args, **kwargs):
                         cache.delete('last_call:media_url:%d' % call.pk)
                     else:
                         response_msg = 'Saved media for call %s' % call.external_id
-                        ChannelLog.log_ivr_interaction(call, response_msg, request_body, six.text_type(response_msg),
+                        response = dict(message=response_msg)
+                        ChannelLog.log_ivr_interaction(call, response_msg, request_body, json.dumps(response),
                                                        request_path, request_method)
-                        return HttpResponse(six.text_type(response_msg))
+                        return JsonResponse(response)
 
             if call.status not in IVRCall.DONE or hangup:
                 if call.is_ivr():

temba/locations/migrations/0008_auto_20170221_1424.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-02-21 14:24
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ('locations', '0007_reset_2'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'CREATE INDEX CONCURRENTLY locations_adminboundary_name on locations_adminboundary(upper("name"))'),
+        migrations.RunSQL(
+            'CREATE INDEX CONCURRENTLY locations_boundaryalias_name on locations_boundaryalias(upper("name"))')
+    ]

temba/msgs/migrations/0082_baseexporttask_2.py

@@ -20,6 +20,7 @@ def populate_uuid(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
+    atomic = False
 
     dependencies = [
         ('msgs', '0081_baseexporttask_1'),

temba/orgs/models.py

@@ -482,7 +482,7 @@ def get_channel_for_role(self, role, scheme=None, contact_urn=None, country_code
                 scheme = contact_urn.scheme
 
                 # if URN has a previously used channel that is still active, use that
-                if contact_urn.channel and contact_urn.channel.is_active and role == Channel.ROLE_SEND:
+                if contact_urn.channel and contact_urn.channel.is_active:
                     previous_sender = self.get_channel_delegate(contact_urn.channel, role)
                     if previous_sender:
                         return previous_sender

temba/sql/current_functions.sql

@@ -1,4 +1,4 @@
--- Generated by collect_sql on 2017-02-03 08:33 UTC
+-- Generated by collect_sql on 2017-02-21 14:28 UTC
 
 ----------------------------------------------------------------------
 -- Trigger procedure to prevent illegal state changes to contacts

temba/sql/current_indexes.sql

@@ -1,4 +1,8 @@
--- Generated by collect_sql on 2017-02-03 08:33 UTC
+-- Generated by collect_sql on 2017-02-21 14:28 UTC
+
+CREATE INDEX CONCURRENTLY locations_adminboundary_name on locations_adminboundary(upper("name"))
+
+CREATE INDEX CONCURRENTLY locations_boundaryalias_name on locations_boundaryalias(upper("name"))
 
 -- index for fast fetching of unsquashed rows
 CREATE INDEX channels_channelcount_unsquashed

temba/sql/current_triggers.sql

@@ -1,4 +1,4 @@
--- Generated by collect_sql on 2017-02-03 08:33 UTC
+-- Generated by collect_sql on 2017-02-21 14:28 UTC
 
 CREATE TRIGGER contact_check_update_trg
    BEFORE UPDATE OF is_test, is_blocked, is_stopped ON contacts_contact

temba/utils/management/commands/make_test_db.py

@@ -142,7 +142,8 @@ def load_locations(self, path):
         # load dump into current db with pg_restore
         db_config = settings.DATABASES['default']
         try:
-            check_call('pg_restore -U%s -w -d %s %s' % (db_config['USER'], db_config['NAME'], path), shell=True)
+            check_call('export PGPASSWORD=%s && pg_restore -U%s -w -d %s %s' %
+                       (db_config['PASSWORD'], db_config['USER'], db_config['NAME'], path), shell=True)
         except CalledProcessError:  # pragma: no cover
             raise CommandError("Error occurred whilst calling pg_restore to load locations dump")