Store attachments as BLOB column in database (#114)

See https://github.com/ascoderu/opwen-webapp/issues/112

.env

@@ -1,11 +1,12 @@
-OPWEN_COMPRESSION=gz
 OPWEN_EMAIL_SERVER_HOSTNAME='localhost:8080'
 OPWEN_SIM_TYPE=Ethernet
 OPWEN_SESSION_KEY='session-secret'
 OPWEN_CLIENT_ID='dev-client-id'
 OPWEN_CLIENT_NAME='dev-client-name'
 OPWEN_ADMIN_SECRET='admin-secret'
 OPWEN_PASSWORD_SALT='password-salt'
-OPWEN_REMOTE_RESOURCE_CONTAINER='azure-storage-container'
-OPWEN_REMOTE_ACCOUNT_NAME='azure-storage-account-name'
-OPWEN_REMOTE_ACCOUNT_KEY='azure-storage-account-key'
+OPWEN_ENABLE_DEBUG=True
+LOKOLE_STORAGE_PROVIDER=LOCAL
+OPWEN_COMPRESSION=gz
+OPWEN_REMOTE_ACCOUNT_NAME=./tests/files/opwen_email_client
+OPWEN_REMOTE_RESOURCE_CONTAINER=compressedpackages

README.rst

@@ -111,23 +111,13 @@ any of the Flask code or Jinja templates are changed.
 The routes of the app are defined in `views.py <https://github.com/ascoderu/opwen-webapp/blob/master/opwen_email_client/webapp/views.py>`_
 so take a look there for an overview of the entrypoints into the code.
 
-For local development, you can set the following additional environment
-variables:
-
-.. sourcecode :: sh
-
-  OPWEN_ENABLE_DEBUG="True"
-  LOKOLE_STORAGE_PROVIDER="LOCAL"
-  OPWEN_REMOTE_ACCOUNT_NAME="./tests/files/opwen_email_client"
-  OPWEN_REMOTE_RESOURCE_CONTAINER="downloads"
-
-With these environment variables set, when the Lokole exchanges data with the
-server, it will not make any calls to Azure and instead depend on the files
-in the `./tests/files/opwen_email_client` directory. Any files uploaded to the
-server will be written to the `compressedpackages` subdirectory so that they
-can be inspected. To test sending emails from the server to the Lokole, a
-sample email batch file is included in the `downloads` directory. This file
-will be ingested by the client when the `/sync` endpoint is called.
+When the Lokole exchanges data with the server, it will not make any calls to Azure
+and instead depend on the files in the `./tests/files/opwen_email_client` directory.
+Any files uploaded to the server will be written to the `compressedpackages`
+subdirectory so that they can be inspected. To test sending emails from the server
+to the Lokole, a sample email batch file is included in the `compressedpackages`
+directory. This file will be ingested by the client when the `/admin/sync` endpoint
+is called.
 
 Production setup
 ----------------

opwen_email_client/domain/email/sql_store.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+from sqlalchemy import BLOB
 from sqlalchemy import Boolean
 from sqlalchemy import Column
 from sqlalchemy import DateTime
@@ -70,7 +71,7 @@ class _Attachment(_Base):
     id = Column(Integer, primary_key=True)
 
     filename = Column(Text)
-    content = Column(Text)
+    content = Column(BLOB)
     cid = Column(Text)
 
 

opwen_email_client/util/serialization.py

@@ -1,5 +1,8 @@
 from abc import ABCMeta
 from abc import abstractmethod
+from base64 import b64decode
+from base64 import b64encode
+from copy import deepcopy
 from json import dumps
 from json import loads
 from typing import TypeVar
@@ -21,10 +24,39 @@ class JsonSerializer(Serializer):
     _encoding = 'utf-8'
     _separators = (',', ':')
 
-    def serialize(self, obj):
-        serialized = dumps(obj, separators=self._separators)
+    def serialize(self, email: dict) -> bytes:
+        email = self._encode_attachments(email)
+        serialized = dumps(email, separators=self._separators)
         return serialized.encode(self._encoding)
 
-    def deserialize(self, serialized):
+    def deserialize(self, serialized: bytes) -> dict:
         decoded = serialized.decode(self._encoding)
-        return loads(decoded)
+        email = loads(decoded)
+        email = self._decode_attachments(email)
+        return email
+
+    @classmethod
+    def _encode_attachments(cls, email: dict) -> dict:
+        attachments = email.get('attachments', [])
+        if not attachments:
+            return email
+
+        email = deepcopy(email)
+        for attachment in email['attachments']:
+            content = attachment.get('content', b'')
+            if content:
+                attachment['content'] = b64encode(content).decode('ascii')
+        return email
+
+    @classmethod
+    def _decode_attachments(cls, email: dict) -> dict:
+        attachments = email.get('attachments', [])
+        if not attachments:
+            return email
+
+        email = deepcopy(email)
+        for attachment in email['attachments']:
+            content = attachment.get('content', '')
+            if content:
+                attachment['content'] = b64decode(content)
+        return email

opwen_email_client/webapp/forms/email.py

@@ -15,7 +15,6 @@
 from wtforms.validators import DataRequired
 from wtforms.validators import Optional as DataOptional
 
-from opwen_email_client.domain.email.attachment import AttachmentEncoder
 from opwen_email_client.domain.email.store import EmailStore
 from opwen_email_client.util.wtforms import Emails
 from opwen_email_client.util.wtforms import HtmlTextAreaField
@@ -51,7 +50,7 @@ class NewEmailForm(FlaskForm):
 
     submit = SubmitField()
 
-    def as_dict(self, attachment_encoder: AttachmentEncoder) -> dict:
+    def as_dict(self) -> dict:
         form = {key: value for (key, value) in self.data.items() if value}
         form.pop('submit', None)
 
@@ -72,8 +71,7 @@ def as_dict(self, attachment_encoder: AttachmentEncoder) -> dict:
         form['bcc'] = bcc
         form['body'] = form.get('body')
         form['subject'] = form.get('subject', i8n.EMAIL_NO_SUBJECT)
-        form['attachments'] = list(_attachments_as_dict(attachments,
-                                                        attachment_encoder))
+        form['attachments'] = list(_attachments_as_dict(attachments))
         return form
 
     def _populate(self, email: Optional[dict], to: Optional[str]):
@@ -157,13 +155,12 @@ def _populate(self, email: Optional[dict], to: Optional[str]):
         self.body.data = render_template('emails/forward.html', email=email)
 
 
-def _attachments_as_dict(
-        filestorages: Iterable[FileStorage],
-        attachment_encoder: AttachmentEncoder) -> Iterable[dict]:
+def _attachments_as_dict(filestorages: Iterable[FileStorage]) \
+        -> Iterable[dict]:
 
     for filestorage in filestorages:
         filename = filestorage.filename
-        content = attachment_encoder.encode(filestorage.stream.read())
+        content = filestorage.stream.read()
         if filename and content:
             yield {'filename': filename, 'content': content}
 

opwen_email_client/webapp/ioc.py

@@ -1,7 +1,6 @@
 from flask import Flask
 from flask_babelex import Babel
 
-from opwen_email_client.domain.email.attachment import Base64AttachmentEncoder as AttachmentEncoder  # noqa
 from opwen_email_client.domain.email.client import HttpEmailServerClient as EmailServerClient  # noqa
 from opwen_email_client.domain.email.sql_store import SqliteEmailStore as EmailStore  # noqa
 from opwen_email_client.domain.email.sync import AzureSync as Sync  # noqa
@@ -35,11 +34,8 @@ class Ioc(object):
         provider=AppConfig.STORAGE_PROVIDER,
         serializer=serializer)
 
-    attachment_encoder = AttachmentEncoder()
-
     attachments_session = AttachmentsStore(
-        email_store=email_store,
-        attachment_encoder=attachment_encoder)
+        email_store=email_store)
 
 
 def create_app() -> Flask:

opwen_email_client/webapp/session.py

@@ -7,7 +7,6 @@
 from flask import request
 from flask import session
 
-from opwen_email_client.domain.email.attachment import AttachmentEncoder
 from opwen_email_client.domain.email.store import EmailStore
 
 
@@ -17,9 +16,7 @@ class FileInfo(namedtuple('FileInfo', 'name content')):
 
 
 class AttachmentsStore(object):
-    def __init__(self, attachment_encoder: AttachmentEncoder,
-                 email_store: EmailStore):
-        self._attachment_encoder = attachment_encoder
+    def __init__(self, email_store: EmailStore):
         self._email_store = email_store
 
     @property
@@ -54,13 +51,13 @@ def lookup(self, attachment_id: str) -> Optional[FileInfo]:
         if attachment_idx >= len(attachments):
             return None
 
-        attachment = attachments[attachment_idx]  # type: Dict
-        filename = attachment.get('filename')
-        content = attachment.get('content')
+        attachment = attachments[attachment_idx]  # type: dict
+        filename = attachment.get('filename')  # type: str
+        content = attachment.get('content')  # type: bytes
         if not filename or not content:
             return None
 
-        return FileInfo(filename, self._attachment_encoder.decode(content))
+        return FileInfo(filename, content)
 
 
 class Session(object):

opwen_email_client/webapp/views.py

@@ -137,14 +137,13 @@ def email_delete(email_uid: str) -> Response:
 @track_history
 def email_new() -> Response:
     email_store = app.ioc.email_store
-    attachment_encoder = app.ioc.attachment_encoder
 
     form = NewEmailForm.from_request(email_store)
     if form is None:
         return abort(404)
 
     if form.validate_on_submit():
-        email_store.create([form.as_dict(attachment_encoder)])
+        email_store.create([form.as_dict()])
         flash(i8n.EMAIL_SENT, category='success')
         return redirect(url_for('email_inbox'))
 

tests/files/opwen_email_client/compressedpackages/.gitignore

@@ -1,2 +1,3 @@
 *
 !.gitignore
+!sync.tar.gz

tests/opwen_email_client/domain/email/test_store.py

@@ -240,7 +240,7 @@ def test_get(self):
             given = self.given_emails(
                 {'to': ['foo@bar.com'], 'subject': 'foo',
                  'attachments': [{'filename': 'foo.txt',
-                                  'content': 'Zm9vLnR4dA==',
+                                  'content': b'foo.txt',
                                   'cid': None}]},
                 {'to': ['baz@bar.com'], 'subject': 'bar'})
 

tests/opwen_email_client/util/test_serialization.py

@@ -16,10 +16,8 @@ def create_serializer(self) -> Serializer:
 
         @property
         def serializable_objects(self) -> Iterable:
-            yield 'some string'
-            yield 123.4
-            yield [1, "two"]
             yield {'key1': 1, 'key2': ["value2"]}
+            yield {'attachments': [{'content': b'content'}]}
 
         def setUp(self):
             self.serializer = self.create_serializer()