Merge branch 'twilio_log_checks'

.gitignore

@@ -47,6 +47,7 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+coverage_html_report/
 
 # Translations
 *.mo

apostello/logs.py

@@ -7,66 +7,129 @@
 from apostello.models import Keyword, Recipient, SmsInbound, SmsOutbound
 
 
-def import_incoming_sms():
+def handle_incoming_sms(msg):
+    sms, created = SmsInbound.objects.get_or_create(sid=msg.sid)
+    if created:
+        check_next_page = True
+        sender, s_created = Recipient.objects.get_or_create(number=msg.from_)
+        if s_created:
+            sender.first_name = 'Unknown'
+            sender.last_name = 'Person'
+            sender.save()
+
+        sms.content = msg.body
+        sms.time_received = timezone.make_aware(
+            msg.date_created,
+            timezone.get_current_timezone()
+        )
+        sms.sender_name = str(sender)
+        sms.sender_num = msg.from_
+        matched_keyword = Keyword.match(msg.body)
+        sms.matched_keyword = str(matched_keyword)
+        sms.matched_colour = Keyword.lookup_colour(msg.body)
+        sms.matched_link = Keyword.get_log_link(matched_keyword)
+        sms.save()
+        return check_next_page
+
+
+def handle_outgoing_sms(msg):
+    try:
+        sms, created = SmsOutbound.objects.get_or_create(sid=msg.sid)
+        if created:
+            check_next_page = True
+            recip, r_created = Recipient.objects.get_or_create(number=msg.to)
+            if r_created:
+                recip.first_name = 'Unknown'
+                recip.last_name = 'Person'
+                recip.save()
+
+            sms.content = msg.body
+            sms.time_sent = timezone.make_aware(
+                msg.date_sent,
+                timezone.get_current_timezone()
+            )
+            sms.sent_by = "[Imported]"
+            sms.recipient = recip
+            sms.save()
+            return check_next_page
+    except Exception as e:
+        print(e)
+
+
+def fetch_generator(direction):
+    if direction == 'in':
+        return twilio_client.messages.iter(
+            to_=settings.TWILIO_FROM_NUM
+        )
+    if direction == 'out':
+        return twilio_client.messages.iter(
+            from_=settings.TWILIO_FROM_NUM
+        )
+    return []
+
+
+def fetch_list(direction, page_id):
+    if direction == 'in':
+        return twilio_client.messages.list(
+            page=page_id,
+            page_size=50,
+            to=settings.TWILIO_FROM_NUM
+        )
+    if direction == 'out':
+        return twilio_client.messages.list(
+            page=page_id,
+            page_size=50,
+            from_=settings.TWILIO_FROM_NUM
+        )
+    return []
+
+
+def check_log(direction, sms_handler, page_id, fetch_all):
+    check_next_page = False
+    if fetch_all:
+        # we want to iterate over all the incoming messages
+        sms_page = fetch_generator(direction)
+    else:
+        # we only want to iterate over the most recent messages to begin with
+        try:
+            sms_page = fetch_list(direction, page_id)
+        except TwilioRestException as e:
+            if e.msg == "Page number out of range":
+                # last page
+                return []
+            else:
+                raise e
+
+    for msg in sms_page:
+        check_next_page = sms_handler(msg) or check_next_page
+
+    if fetch_all:
+        # have looped over all messages and we are done
+        return
+
+    if check_next_page:
+        check_log(direction, sms_handler, page_id + 1, fetch_all)
+
+
+def check_incoming_log(page_id=0, fetch_all=False):
     """
-    Loops over all incoming messages in Twilio's logs and adds them to our db.
+    Checks Twilio's logs for messages that have been sent to our number.
+
+    page_id: Twilio log page to start with.
+    fetch_all: If set to True, all messages on Twilio will be checked. If False,
+    only the first 50 messages will be checked. If a missing message is found in
+    these 50, the next 50 will also be checked.
     """
-    try:
-        sms_page = twilio_client.messages.iter(to_=settings.TWILIO_FROM_NUM)
-        for x in sms_page:
-            try:
-                sms, created = SmsInbound.objects.get_or_create(
-                    sid=x.sid,
-                    time_received=timezone.now()
-                )
-                if created:
-                    sender, s_created = Recipient.objects.get_or_create(number=x.from_)
-                    if s_created:
-                        sender.first_name = 'Unknown'
-                        sender.last_name = 'Person'
-                        sender.save()
-
-                    sms.content = x.body
-                    sms.time_received = timezone.make_aware(x.date_sent, timezone.get_current_timezone())
-                    sms.sender_name = str(sender)
-                    sms.sender_num = x.from_
-                    matched_keyword = Keyword.match(x.body)
-                    sms.matched_keyword = str(matched_keyword)
-                    sms.matched_colour = Keyword.lookup_colour(x.body)
-                    sms.matched_link = Keyword.get_log_link(matched_keyword)
-                    sms.save()
-            except Exception as e:
-                print(e)
-
-    except TwilioRestException as e:
-        if e.code == 20008:
-            return 'test credentials used'
-
-
-def import_outgoing_sms():
+    check_log('in', handle_incoming_sms, page_id, fetch_all)
+
+
+def check_outgoing_log(page_id=0, fetch_all=False):
     """
-    Loops over all outgoing messages in Twilio's logs and adds them to our db.
+    Checks Twilio's logs for messages that we have sent.
+
+    page_id: Twilio log page to start with.
+    fetch_all: If set to True, all messages on Twilio will be checked. If False,
+    only the first 50 messages will be checked. If a missing message is found in
+    these 50, the next 50 will also be checked.
     """
-    try:
-        sms_page = twilio_client.messages.iter(from_=settings.TWILIO_FROM_NUM)
-        for x in sms_page:
-            try:
-                sms, created = SmsOutbound.objects.get_or_create(sid=x.sid)
-                if created:
-                    recip, r_created = Recipient.objects.get_or_create(number=x.to)
-                    if r_created:
-                        recip.first_name = 'Unknown'
-                        recip.last_name = 'Person'
-                        recip.save()
-
-                    sms.content = x.body
-                    sms.time_sent = timezone.make_aware(x.date_sent, timezone.get_current_timezone())
-                    sms.sent_by = "Unknown - imported"
-                    sms.recipient = recip
-                    sms.save()
-            except Exception as e:
-                print(e)
-
-    except TwilioRestException as e:
-        if e.code == 20008:
-            return 'test credentials used'
+    check_log('out', handle_outgoing_sms, page_id, fetch_all)

apostello/management/commands/import_incoming_sms.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from django.core.management.base import BaseCommand
 
-from apostello.logs import import_incoming_sms
+from apostello.logs import check_incoming_log
 
 
 class Command(BaseCommand):
@@ -13,4 +13,4 @@ class Command(BaseCommand):
     help = 'Import incoming messages from twilio'
 
     def handle(self, *args, **options):
-        import_incoming_sms()
+        check_incoming_log(fetch_all=True)

apostello/management/commands/import_outgoing_sms.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from django.core.management.base import BaseCommand
 
-from apostello.logs import import_outgoing_sms
+from apostello.logs import check_outgoing_log
 
 
 class Command(BaseCommand):
@@ -13,4 +13,4 @@ class Command(BaseCommand):
     help = 'Import outgoing messages from twilio'
 
     def handle(self, *args, **options):
-        import_outgoing_sms()
+        check_outgoing_log(fetch_all=True)

apostello/reply.py

@@ -3,7 +3,7 @@
 from django.core.exceptions import ValidationError
 
 from apostello.models import Recipient
-from apostello.tasks import (check_recent_outgoing_log, notify_office_mail,
+from apostello.tasks import (check_outgoing_log, notify_office_mail,
                              update_msgs_name, warn_on_blacklist,
                              warn_on_blacklist_receipt)
 from apostello.utils import fetch_default_reply
@@ -51,7 +51,7 @@ def get_person_or_ask_for_name(from_, sms_body, keyword_obj):
 def reply_to_incoming(person_from, from_, sms_body, keyword):
     # update outgoing log 1 minute from now:
     if not settings.TESTING:
-        check_recent_outgoing_log.apply_async(args=[0], countdown=60)
+        check_outgoing_log.apply_async(countdown=60)
 
     if keyword == "start":
         person_from.is_blocking = False

apostello/tasks.py

@@ -66,56 +66,15 @@ def recipient_send_message_task(recipient_pk, body, group, sent_by):
 # SMS logging and consistency checks
 
 @task()
-def check_log_consistent(page_id):
-    from apostello.models import Keyword, Recipient, SmsInbound
-    check_next_page = False
-    for x in twilio_client.messages.list(page=page_id, page_size=50, to=settings.TWILIO_FROM_NUM):
-        sms, created = SmsInbound.objects.get_or_create(sid=x.sid)
-        if created:
-            sender, s_created = Recipient.objects.get_or_create(number=x.from_)
-            if s_created:
-                sender.first_name = 'Unknown'
-                sender.last_name = 'Person'
-                sender.save()
-
-            sms.content = x.body
-            sms.time_received = timezone.make_aware(x.date_created,
-                                                    timezone.get_current_timezone())
-            sms.sender_name = str(sender)
-            sms.sender_num = x.from_
-            sms.matched_keyword = str(Keyword.match(x.body.strip()))
-            sms.matched_colour = Keyword.lookup_colour(x.body.strip())
-            sms.is_archived = True
-            sms.save()
-            check_next_page = True
-
-    if check_next_page:
-        check_log_consistent.delay(page_id + 1)
+def check_incoming_log(page_id=0, fetch_all=False):
+    from apostello.logs import check_incoming_log
+    check_incoming_log(page_id=page_id, fetch_all=fetch_all)
 
 
 @task()
-def check_recent_outgoing_log(page_id):
-    from apostello.models import Recipient, SmsOutbound
-    check_next_page = False
-    for x in twilio_client.messages.list(page=page_id, page_size=50, from_=settings.TWILIO_FROM_NUM):
-        recip, r_created = Recipient.objects.get_or_create(number=x.to)
-        if r_created:
-            recip.first_name = 'Unknown'
-            recip.last_name = 'Person'
-            recip.save()
-
-        sms, created = SmsOutbound.objects.get_or_create(sid=x.sid)
-        if created:
-            sms.content = x.body
-            sms.time_sent = timezone.make_aware(x.date_sent,
-                                                timezone.get_current_timezone())
-            sms.sent_by = "Unknown - imported"
-            sms.recipient = recip
-            sms.save()
-            check_next_page = True
-
-    if check_next_page:
-        check_recent_outgoing_log.delay(page_id + 1)
+def check_outgoing_log(page_id=0, fetch_all=False):
+    from apostello.logs import check_outgoing_log
+    check_outgoing_log(page_id=page_id, fetch_all=fetch_all)
 
 
 @task()
@@ -132,7 +91,7 @@ def log_msg_in(p, t, from_pk):
                               matched_link=Keyword.get_log_link(matched_keyword),
                               matched_colour=Keyword.lookup_colour(p['Body'].strip()))
     # check log is consistent:
-    check_log_consistent.delay(0)
+    check_incoming_log.delay()
 
 
 @task()
@@ -252,16 +211,3 @@ def pull_elvanto_groups(force=False):
     if force or config.sync_elvanto:
         from apostello.models import ElvantoGroup
         ElvantoGroup.pull_all_groups()
-
-
-# import twilio log
-@task()
-def import_incoming_sms_task():
-    from apostello.logs import import_incoming_sms
-    import_incoming_sms()
-
-
-@task
-def import_outgoing_sms_task():
-    from apostello.logs import import_outgoing_sms
-    import_outgoing_sms()

apostello/tests/integration_tests/test_login.py

@@ -4,6 +4,7 @@
 
 
 @pytest.mark.django_db
+@pytest.mark.slow
 class TestLogin:
     @pytest.mark.parametrize("uri", [
         '/',

apostello/tests/integration_tests/test_signup.py

@@ -3,6 +3,7 @@
 
 
 @pytest.mark.django_db
+@pytest.mark.slow
 class TestSignup:
     def test_sign_up(self, live_server, browser, users):
         """

apostello/tests/test_elvanto.py

@@ -40,6 +40,7 @@ def test_neither_good(self):
             try_both_num_fields('+448902537905', '+457902537905')
 
 
+@pytest.mark.slow
 @pytest.mark.django_db
 class TestApi:
     """