add basic support for importing from onebody (no ui yet)

 - import from onebody using the onebody csv export
 - you need to setup an api key in onebody: https://github.com/churchio/onebody/wiki/API
 - then set the following 3 env vars in your apostello deployment:
    * `ONEBODY_API_KEY` - the API key from the previous step
    * `ONEBODY_USER_EMAIL` - the email associated with the user from the previous step
    * `ONEBODY_BASE_URL` - the url for your onebody site. must be of the form: `http://onebody.example.com` - note the protocol and no trailing space
 - apostello will then fetch your directory from onebody once a day - around 02.30 each morning
 - all the imported users will be added to a group called `[onebody]`

 - caveats
    * if someone changes their number in onebody, you will have to remove the old one from apostello
    * archived contacts will be updated, but remain archived
    * errors can be seen in the site admin

apostello/management/commands/setup_periodic_tasks.py

@@ -49,6 +49,14 @@ def handle(self, *args, **options):
                 next_run=next_230am,
             )
 
+        if Schedule.objects.filter(func='apostello.tasks.pull_onebody_csv').count() < 1:
+            Schedule.objects.create(
+                func='apostello.tasks.pull_onebody_csv',
+                schedule_type=Schedule.DAILY,
+                repeats=-1,
+                next_run=next_230am,
+            )
+
         if Schedule.objects.filter(func='apostello.tasks.send_keyword_digest').count() < 1:
             Schedule.objects.create(
                 func='apostello.tasks.send_keyword_digest',

apostello/tasks.py

@@ -254,6 +254,10 @@ def pull_elvanto_groups(force=False):
         from elvanto.models import ElvantoGroup
         ElvantoGroup.pull_all_groups()
 
+# Onebody import
+def pull_onebody_csv():
+    from onebody.importer import import_onebody_csv
+    import_onebody_csv()
 
 #
 def add_new_contact_to_groups(contact_pk):

onebody/importer.py

@@ -0,0 +1,77 @@
+import csv
+import io
+import logging
+from time import sleep
+
+import requests
+from django.core.exceptions import ValidationError
+from django.conf import settings
+from django.utils import timezone
+from phonenumber_field.validators import validate_international_phonenumber
+
+from apostello.models import Recipient, RecipientGroup
+from elvanto.exceptions import NotValidPhoneNumber
+
+logger = logging.getLogger('apostello')
+
+WAIT_TIME = 10
+
+
+class OnebodyException(Exception):
+    pass
+
+
+def import_onebody_csv():
+    base_url = settings.ONEBODY_BASE_URL
+    user_email = settings.ONEBODY_USER_EMAIL
+    key = settings.ONEBODY_API_KEY
+    if not any([base_url, user_email, key]):
+        logger.info('Onebody Sync Disabled')
+        return
+
+    resp = requests.get(
+        base_url + '/people.csv',
+        auth=(user_email, key),
+        allow_redirects=False,
+    )
+    resp.raise_for_status()
+
+    csv_url = resp.headers['Location']
+
+    sleep(WAIT_TIME) # wait for csv to be generated
+    tries = 0
+    max_tries = 10
+    while tries <= max_tries:
+        try:
+            csv_resp = requests.get(
+                csv_url,
+                auth=(user_email, key),
+            )
+            csv_resp.raise_for_status()
+            data = csv.DictReader(io.StringIO(csv_resp.text))
+            break
+        except Exception:
+            sleep(WAIT_TIME)
+            if tries >= max_tries:
+                logger.warning('Failed to get CSV from onebody')
+                raise OnebodyException('Failed to get CSV from onebody')
+            tries += 1
+
+    # we now have the good data, let's import it:
+    grp, _ = RecipientGroup.objects.get_or_create(name='[onebody]', description='imported from onebody')
+    for row in data:
+        try:
+            number = row['mobile_phone']
+            if not number.startswith('+'):
+                number = '+' + number
+            validate_international_phonenumber(number)
+            prsn_obj = Recipient.objects.get_or_create(number=number)[0]
+            prsn_obj.first_name = row['first_name'].strip()
+            prsn_obj.last_name = row['last_name'].strip()
+            prsn_obj.save()
+            # add person to group
+            grp.recipient_set.add(prsn_obj)
+        except ValidationError:
+            logger.warning('Failed to import - bad number: %s %s (%s)', row['first_name'], row['last_name'], number)
+        except Exception:
+            logging.exception('Failed to import %s %s (%s)', row['first_name'], row['last_name'], number)

settings/common.py

@@ -160,6 +160,13 @@
 
 # Elvanto credentials
 ELVANTO_KEY = os.environ.get('ELVANTO_KEY', '')
+# Onebody credentials - if left blank, no syncing will be done, otherwise, data
+# is pulled automatically once a day
+# details on how to obtain the api key:
+# https://github.com/churchio/onebody/wiki/API
+ONEBODY_BASE_URL = os.environ.get('ONEBODY_BASE_URL')
+ONEBODY_USER_EMAIL = os.environ.get('ONEBODY_USER_EMAIL')
+ONEBODY_API_KEY = os.environ.get('ONEBODY_API_KEY')
 
 # Twilio credentials
 TWILIO_ACCOUNT_SID = os.environ.get('TWILIO_ACCOUNT_SID', 'no SID found')

settings/test.py

@@ -72,3 +72,33 @@
     RuntimeWarning,
     r'django\.db\.models\.fields',
 )
+
+# logging
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': True,
+    'formatters': {
+        'verbose': {
+            'format': '[%(asctime)s][%(levelname)s][%(module)s.py][%(process)d][%(thread)d] %(message)s'
+        },
+    },
+    'handlers': {
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose'
+        }
+    },
+    'loggers': {
+        'django': {
+            'level': 'ERROR',
+            'handlers': ['console'],
+            'propagate': False,
+        },
+        'apostello': {
+            'level': 'ERROR',
+            'handlers': ['console'],
+            'propagate': False,
+        },
+    },
+}

tests/conftest.py

@@ -419,6 +419,16 @@ def pytest_runtest_makereport(item, call):
     'tests/fixtures/vcr_cass/elv.yaml',
     filter_headers=['authorization'],
 )
+onebody_vcr = base_vcr.use_cassette(
+    'tests/fixtures/vcr_cass/onebody.yaml',
+    filter_headers=['authorization'],
+    ignore_localhost=False,
+)
+onebody_no_csv_vcr = base_vcr.use_cassette(
+    'tests/fixtures/vcr_cass/onebody_no_csv.yaml',
+    filter_headers=['authorization'],
+    ignore_localhost=False,
+)
 
 
 def post_json(client, url, data):

tests/fixtures/vcr_cass/onebody.yaml

@@ -0,0 +1,83 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [python-requests/2.13.0]
+    method: GET
+    uri: http://127.0.0.1:4000/people.csv
+  response:
+    body: {string: '<html><body>You are being <a href="http://127.0.0.1:4000/generated_files/80c2f420-d74a-4a83-bec1-ee4f4dc1255f">redirected</a>.</body></html>'}
+    headers:
+      Cache-Control: [no-cache]
+      Connection: [close]
+      Content-Type: [text/csv; charset=utf-8]
+      Location: ['http://127.0.0.1:4000/generated_files/80c2f420-d74a-4a83-bec1-ee4f4dc1255f']
+      Server: [thin]
+      X-Content-Type-Options: [nosniff]
+      X-Frame-Options: [SAMEORIGIN]
+      X-Request-Id: [3e7b3895-bc1d-4de5-92f1-90ee60ffcdf7]
+      X-Runtime: ['0.103560']
+      X-XSS-Protection: [1; mode=block]
+    status: {code: 302, message: Moved Temporarily}
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [python-requests/2.13.0]
+    method: GET
+    uri: http://127.0.0.1:4000/generated_files/80c2f420-d74a-4a83-bec1-ee4f4dc1255f
+  response:
+    body: {string: 'family_id,position,gender,first_name,last_name,mobile_phone,work_phone,fax,birthday,email,website,classes,shepherd,mail_group,about,testimony,share_mobile_phone,share_work_phone,share_fax,share_email,share_birthday,share_address,share_anniversary,share_home_phone,business_category,business_name,business_description,business_address,business_phone,business_email,business_website,legacy_id,suffix,anniversary,updated_at,alternate_email,account_frozen,messages_enabled,visible,parental_consent,friends_enabled,member,staff,elder,deacon,status,legacy_family_id,share_activity,child,custom_type,description,family_name,family_last_name,family_address1,family_address2,family_city,family_state,family_zip,family_home_phone,family_legacy_id,family_updated_at,family_visible
+
+        1,1,,test,test,,,,,test@example.com,,,,,,,false,false,false,false,true,true,true,true,,,,,,,,,,,2017-06-29
+        11:42:08 UTC,,false,true,true,,true,false,false,false,false,active,,true,false,,,test
+        family,test,,,,,,,,2017-06-29 11:42:07 UTC,true
+
+        2,1,,John,Calvin,447927401749,,,,john.calvin@example.com,,,,,,,false,false,false,false,true,true,true,true,,,,,,,,,,,2017-06-29
+        11:42:08 UTC,,false,true,true,,true,false,false,false,false,active,,true,false,,,Calvin,Calvin,,,,,,,,2017-06-29
+        11:42:08 UTC,true
+
+        3,1,,Johannes,Oecolampadius,447927401740,,,,johannes.oecolampadius@example.com,,,,,,,false,false,false,false,true,true,true,true,,,,,,,,,,,2017-06-29
+        11:42:08 UTC,,false,true,true,,true,false,false,false,false,active,,true,false,,,Oecolampadius,Oecolampadius,,,,,,,,2017-06-29
+        11:42:08 UTC,true
+
+        4,1,,John,Knox,447928401745,,,,john.knox@example.com,,,,,,,false,false,false,false,true,true,true,true,,,,,,,,,,,2017-06-29
+        11:42:08 UTC,,false,true,true,,true,false,false,false,false,active,,true,false,,,Knox,Knox,,,,,,,,2017-06-29
+        11:42:08 UTC,true
+
+        5,1,,John,Wesley,447927401745,,,,john.wesley@example.com,,,,,,,false,false,false,false,true,true,true,true,,,,,,,,,,,2017-06-29
+        11:42:08 UTC,,false,true,true,,true,false,false,false,false,active,,true,false,,,Wesley,Wesley,,,,,,,,2017-06-29
+        11:42:08 UTC,true
+
+        6,1,,John,Owen,15005550004,,,,john.owen@example.com,,,,,,,false,false,false,false,true,true,true,true,,,,,,,,,,,2017-06-29
+        11:42:08 UTC,,false,true,true,,true,false,false,false,false,active,,true,false,,,Owen,Owen,,,,,,,,2017-06-29
+        11:42:08 UTC,true
+
+        7,1,,Thomas,Chalmers,15005550009,,,,thomas.chalmers@example.com,,,,,,,false,false,false,false,true,true,true,true,,,,,,,,,,,2017-06-29
+        11:42:08 UTC,,false,true,true,,true,false,false,false,false,active,,true,false,,,Chalmers,Chalmers,,,,,,,,2017-06-29
+        11:42:08 UTC,true
+
+        8,1,,Theodore,Beza,447927411115,,,,theodore.beza@example.com,,,,,,,false,false,false,false,true,true,true,true,,,,,,,,,,,2017-06-29
+        11:42:08 UTC,,false,true,true,,true,false,false,false,false,active,,true,false,,,Beza,Beza,,,,,,,,2017-06-29
+        11:42:08 UTC,true
+
+'}
+    headers:
+      Cache-Control: [private]
+      Connection: [close]
+      Content-Disposition: [attachment; filename="people.csv"]
+      Content-Transfer-Encoding: [binary]
+      Content-Type: [text/plain]
+      Server: [thin]
+      X-Content-Type-Options: [nosniff]
+      X-Frame-Options: [SAMEORIGIN]
+      X-Request-Id: [617c3a99-7cb5-4389-9cc3-5075de9905eb]
+      X-Runtime: ['0.089237']
+      X-XSS-Protection: [1; mode=block]
+    status: {code: 200, message: OK}
+version: 1

tests/fixtures/vcr_cass/onebody_no_csv.yaml

@@ -0,0 +1,25 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [python-requests/2.13.0]
+    method: GET
+    uri: http://127.0.0.1:4000/people.csv
+  response:
+    body: {string: '<html><body>You are being <a href="http://127.0.0.1:4000/generated_files/80c2f420-d74a-4a83-bec1-ee4f4dc1255f">redirected</a>.</body></html>'}
+    headers:
+      Cache-Control: [no-cache]
+      Connection: [close]
+      Content-Type: [text/csv; charset=utf-8]
+      Location: ['http://127.0.0.1:4000/generated_files/80c2f420-d74a-4a83-bec1-ee4f4dc1255f']
+      Server: [thin]
+      X-Content-Type-Options: [nosniff]
+      X-Frame-Options: [SAMEORIGIN]
+      X-Request-Id: [3e7b3895-bc1d-4de5-92f1-90ee60ffcdf7]
+      X-Runtime: ['0.103560']
+      X-XSS-Protection: [1; mode=block]
+    status: {code: 302, message: Moved Temporarily}
+version: 1

tests/test_management.py

@@ -23,6 +23,6 @@ def test_import_out(self):
     def test_setup_scheduled_tasks(self):
         """Test setup of perdiodic tasks and ensure function is idempotent."""
         call_command('setup_periodic_tasks')
-        assert Schedule.objects.all().count() == 5
+        assert Schedule.objects.all().count() == 6
         call_command('setup_periodic_tasks')
-        assert Schedule.objects.all().count() == 5
+        assert Schedule.objects.all().count() == 6

tests/test_onebody.py

@@ -0,0 +1,25 @@
+import pytest
+from tests.conftest import onebody_no_csv_vcr, onebody_vcr
+
+from apostello import models
+from onebody.importer import OnebodyException, import_onebody_csv
+
+
+@pytest.mark.onebody_api
+@pytest.mark.django_db
+class TestImporting:
+    @onebody_vcr
+    def test_ok(self):
+        """Test fetching people from onebody."""
+        import_onebody_csv()
+        assert models.RecipientGroup.objects.count() == 1
+        assert models.Recipient.objects.count() == 7
+        assert models.RecipientGroup.objects.get(name='[onebody]').recipient_set.count() == 7
+
+    @onebody_no_csv_vcr
+    def test_csv_fails(self):
+        """Test fetching people from onebody."""
+        with pytest.raises(OnebodyException):
+            import_onebody_csv()
+        assert models.RecipientGroup.objects.count() == 0
+        assert models.Recipient.objects.count() == 0

tests/test_tasks.py

@@ -1,7 +1,7 @@
 from datetime import datetime, timedelta
 
 import pytest
-from tests.conftest import twilio_vcr
+from tests.conftest import onebody_vcr, twilio_vcr
 from twilio.base.exceptions import TwilioRestException
 
 from apostello.models import *
@@ -104,3 +104,11 @@ def test_add_new_ppl_to_groups(self, groups):
         )
         assert grp.recipient_set.count() == 1
         assert new_c in grp.recipient_set.all()
+
+    @onebody_vcr
+    def test_onebody_csv(self):
+        """Test fetching people from onebody."""
+        pull_onebody_csv()
+        assert RecipientGroup.objects.count() == 1
+        assert Recipient.objects.count() == 7
+        assert RecipientGroup.objects.get(name='[onebody]').recipient_set.count() == 7

tox.ini

@@ -29,6 +29,9 @@ setenv =
   TWILIO_SENDING_COST=0.04
   DJANGO_FROM_EMAIL=test@apostello.ninja
   DJANGO_EMAIL_HOST=smtp.test.apostello
+  ONEBODY_API_KEY=d20d0004ab0ae52af6254d39846d9637d2bb963aa380d035c0
+  ONEBODY_BASE_URL=http://127.0.0.1:4000
+  ONEBODY_USER_EMAIL=test@example.com
 passenv =
   DATABASE_POSTGRESQL_USERNAME
   DATABASE_POSTGRESQL_PASSWORD