Split up DNS integration module (#89)

* Simplify email sender interface to callable

* Move mailbox setup to sendgrid module

* Move dns setup to cloudflare module

* Remove thin dns integration module

* Remove unused CLI

* Make import style consistent

* Remove test action calling code duplication

opwen_email_server/actions.py

@@ -6,7 +6,7 @@
 
 from opwen_email_server.constants import events
 from opwen_email_server.services.auth import AzureAuth
-from opwen_email_server.services.sendgrid import SendgridEmailSender
+from opwen_email_server.services.sendgrid import SendSendgridEmail
 from opwen_email_server.services.storage import AzureObjectsStorage
 from opwen_email_server.services.storage import AzureObjectStorage
 from opwen_email_server.services.storage import AzureTextStorage
@@ -28,15 +28,15 @@ def __call__(self) -> Response:
 class SendOutboundEmails(LogMixin):
     def __init__(self,
                  email_storage: AzureObjectStorage,
-                 email_sender: SendgridEmailSender):
+                 send_email: SendSendgridEmail):
 
         self._email_storage = email_storage
-        self._email_sender = email_sender
+        self._send_email = send_email
 
     def __call__(self, resource_id: str) -> Response:
         email = self._email_storage.fetch_object(resource_id)
 
-        success = self._email_sender.send_email(email)
+        success = self._send_email(email)
         if not success:
             return 'error', 500
 
@@ -221,12 +221,14 @@ class RegisterClient(LogMixin):
     def __init__(self,
                  auth: AzureAuth,
                  client_storage: AzureObjectsStorage,
-                 setup_email_dns: Callable[[str, str], None],
+                 setup_mailbox: Callable[[str, str], None],
+                 setup_mx_records: Callable[[str], None],
                  client_id_source: Callable[[], str] = None):
 
         self._auth = auth
         self._client_storage = client_storage
-        self._setup_email_dns = setup_email_dns
+        self._setup_mailbox = setup_mailbox
+        self._setup_mx_records = setup_mx_records
         self._client_id_source = client_id_source or self._new_client_id
 
     def __call__(self, client: dict) -> Response:
@@ -237,7 +239,8 @@ def __call__(self, client: dict) -> Response:
         client_id = self._client_id_source()
         access_info = self._client_storage.access_info()
 
-        self._setup_email_dns(client_id, domain)
+        self._setup_mailbox(client_id, domain)
+        self._setup_mx_records(domain)
         self._client_storage.store_objects([{'client_id': client_id}], domain)
         self._auth.insert(client_id, domain)
 

opwen_email_server/constants/cloudflare.py

@@ -0,0 +1,3 @@
+from typing_extensions import Final  # noqa: F401
+
+DNS_URL = 'https://api.cloudflare.com/client/v4/zones/{}/dns_records'  # type: Final  # noqa: E501

opwen_email_server/constants/sendgrid.py

@@ -0,0 +1,7 @@
+from typing_extensions import Final  # noqa: F401
+
+MAILBOX_URL = 'https://api.sendgrid.com/v3/user/webhooks/parse/settings'  # type: Final  # noqa: E501
+
+INBOX_URL = 'http://mailserver.lokole.ca/api/email/sendgrid/{}'  # type: Final
+
+MX_RECORD = 'mx.sendgrid.net'  # type: Final

opwen_email_server/integration/azure.py

@@ -4,7 +4,9 @@
 from opwen_email_server.constants import azure as constants
 from opwen_email_server.constants.cache import PENDING_STORAGE_CACHE_SIZE
 from opwen_email_server.services.auth import AzureAuth
-from opwen_email_server.services.sendgrid import SendgridEmailSender
+from opwen_email_server.services.cloudflare import SetupCloudflareMxRecords
+from opwen_email_server.services.sendgrid import SendSendgridEmail
+from opwen_email_server.services.sendgrid import SetupSendgridMailbox
 from opwen_email_server.services.storage import AzureFileStorage
 from opwen_email_server.services.storage import AzureObjectStorage
 from opwen_email_server.services.storage import AzureObjectsStorage
@@ -42,11 +44,25 @@ def get_raw_email_storage() -> AzureTextStorage:
 
 
 @singleton
-def get_email_sender() -> SendgridEmailSender:
-    return SendgridEmailSender(
+def get_email_sender() -> SendSendgridEmail:
+    return SendSendgridEmail(
         key=config.SENDGRID_KEY)
 
 
+@singleton
+def get_mailbox_setup() -> SetupSendgridMailbox:
+    return SetupSendgridMailbox(
+        key=config.SENDGRID_KEY)
+
+
+@singleton
+def get_mx_setup() -> SetupCloudflareMxRecords:
+    return SetupCloudflareMxRecords(
+        user=config.CLOUDFLARE_USER,
+        key=config.CLOUDFLARE_KEY,
+        zone=config.CLOUDFLARE_ZONE)
+
+
 @singleton
 def get_email_storage() -> AzureObjectStorage:
     return AzureObjectStorage(

opwen_email_server/integration/celery.py

@@ -40,7 +40,7 @@ def written_store(resource_id: str) -> None:
 def send(resource_id: str) -> None:
     action = SendOutboundEmails(
         email_storage=get_email_storage(),
-        email_sender=get_email_sender())
+        send_email=get_email_sender())
 
     action(resource_id)
 

opwen_email_server/integration/connexion.py

@@ -6,11 +6,12 @@
 from opwen_email_server.integration.azure import get_auth
 from opwen_email_server.integration.azure import get_client_storage
 from opwen_email_server.integration.azure import get_email_storage
+from opwen_email_server.integration.azure import get_mailbox_setup
+from opwen_email_server.integration.azure import get_mx_setup
 from opwen_email_server.integration.azure import get_pending_storage
 from opwen_email_server.integration.azure import get_raw_email_storage
 from opwen_email_server.integration.celery import inbound_store
 from opwen_email_server.integration.celery import written_store
-from opwen_email_server.integration.dns import SetupEmailDns
 
 email_receive = ReceiveInboundEmail(
     auth=get_auth(),
@@ -30,6 +31,7 @@
 client_register = RegisterClient(
     auth=get_auth(),
     client_storage=get_client_storage(),
-    setup_email_dns=SetupEmailDns())
+    setup_mailbox=get_mailbox_setup(),
+    setup_mx_records=get_mx_setup())
 
 healthcheck = Ping()

opwen_email_server/services/cloudflare.py

@@ -0,0 +1,36 @@
+from requests import post as http_post
+
+from opwen_email_server.constants.cloudflare import DNS_URL
+from opwen_email_server.constants.sendgrid import MX_RECORD
+from opwen_email_server.utils.log import LogMixin
+
+
+class SetupCloudflareMxRecords(LogMixin):
+    def __init__(self, user: str, key: str, zone: str) -> None:
+        self._user = user
+        self._key = key
+        self._zone = zone
+
+    def __call__(self, domain: str) -> None:
+        if not self._key:
+            self.log_warning('No key, skipping MX setup for %s', domain)
+            return
+
+        client_name = domain.split('.')[0]
+
+        http_post(
+            url=DNS_URL.format(self._zone),
+            json={
+                'type': 'MX',
+                'content': MX_RECORD,
+                'proxied': False,
+                'priority': 1,
+                'name': client_name,
+            },
+            headers={
+                'X-Auth-Key': self._key,
+                'X-Auth-Email': self._user,
+            }
+        ).raise_for_status()
+
+        self.log_debug('Set up mx records for %s', domain)

opwen_email_server/services/sendgrid.py

@@ -4,17 +4,20 @@
 from urllib.error import URLError
 
 from cached_property import cached_property
+from requests import post as http_post
 from sendgrid import SendGridAPIClient
 from sendgrid.helpers.mail import Attachment
 from sendgrid.helpers.mail import Content
 from sendgrid.helpers.mail import Email
 from sendgrid.helpers.mail import Mail
 from sendgrid.helpers.mail import Personalization
 
+from opwen_email_server.constants.sendgrid import INBOX_URL
+from opwen_email_server.constants.sendgrid import MAILBOX_URL
 from opwen_email_server.utils.log import LogMixin
 
 
-class SendgridEmailSender(LogMixin):
+class SendSendgridEmail(LogMixin):
     def __init__(self, key: str) -> None:
         self._key = key
 
@@ -34,7 +37,7 @@ def send_email(email: dict) -> int:
 
         return send_email
 
-    def send_email(self, email: dict) -> bool:
+    def __call__(self, email: dict) -> bool:
         email_id = email.get('_uid', '')
         email = self._create_email(email, email_id)
         return self._send_email(email, email_id)
@@ -108,35 +111,26 @@ def _create_attachment(cls, attachment: dict) -> Attachment:
         return mail_attachment
 
 
-def _cli():  # pragma: no cover
-    from argparse import ArgumentParser
-    from argparse import FileType
-    from base64 import b64encode
-    from json import loads
-    from os.path import basename
-    from uuid import uuid4
-
-    from opwen_email_server.config import SENDGRID_KEY
-
-    parser = ArgumentParser()
-    parser.add_argument('email')
-    parser.add_argument('--attachment', type=FileType('rb'))
-    parser.add_argument('--key', default=SENDGRID_KEY)
-    args = parser.parse_args()
-
-    email = loads(args.email)
-    email.setdefault('_uid', str(uuid4()))
-
-    if args.attachment:
-        email.setdefault('attachments', []).append({
-            'filename': basename(args.attachment.name),
-            'content': b64encode(args.attachment.read()).decode('ascii')
-        })
-        args.attachment.close()
-
-    sender = SendgridEmailSender(args.key)
-    sender.send_email(email)
-
+class SetupSendgridMailbox(LogMixin):
+    def __init__(self, key: str) -> None:
+        self._key = key
 
-if __name__ == '__main__':
-    _cli()
+    def __call__(self, client_id: str, domain: str) -> None:
+        if not self._key:
+            self.log_warning('No key, skipping mailbox setup for %s', domain)
+            return
+
+        http_post(
+            url=MAILBOX_URL,
+            json={
+                'hostname': domain,
+                'url': INBOX_URL.format(client_id),
+                'spam_check': True,
+                'send_raw': True,
+            },
+            headers={
+                'Authorization': 'Bearer {}'.format(self._key),
+            }
+        ).raise_for_status()
+
+        self.log_debug('Set up mailbox for %s', domain)

tests/opwen_email_server/services/test_sendgrid.py

@@ -1,6 +1,6 @@
 from unittest import TestCase
 
-from opwen_email_server.services.sendgrid import SendgridEmailSender
+from opwen_email_server.services.sendgrid import SendSendgridEmail
 
 
 class SendgridEmailSenderTests(TestCase):
@@ -9,9 +9,9 @@ class SendgridEmailSenderTests(TestCase):
     sender = 'sendgridtests@lokole.ca'
 
     def test_sends_email(self):
-        sender = SendgridEmailSender(key='')
+        send_email = SendSendgridEmail(key='')
 
-        success = sender.send_email({
+        success = send_email({
             'to': [self.recipient1],
             'from': self.sender,
             'subject': self.test_sends_email.__name__,
@@ -20,9 +20,9 @@ def test_sends_email(self):
         self.assertTrue(success)
 
     def test_sends_email_with_attachments(self):
-        sender = SendgridEmailSender(key='')
+        send_email = SendSendgridEmail(key='')
 
-        success = sender.send_email({
+        success = send_email({
             'to': [self.recipient1],
             'from': self.sender,
             'subject': self.test_sends_email_with_attachments.__name__,
@@ -36,9 +36,9 @@ def test_sends_email_with_attachments(self):
         self.assertTrue(success)
 
     def test_sends_email_to_multiple_recipients(self):
-        sender = SendgridEmailSender(key='')
+        send_email = SendSendgridEmail(key='')
 
-        success = sender.send_email({
+        success = send_email({
             'to': [self.recipient1, self.recipient2],
             'from': self.sender,
             'subject': self.test_sends_email_to_multiple_recipients.__name__,