Replace azure-storage-blob with libcloud

.travis.yml

@@ -10,13 +10,6 @@ services:
 
 cache: pip
 
-addons:
-  apt:
-    sources:
-      - debian-sid
-    packages:
-      - docker-ce
-
 script: travis/ci.sh
 
 install:

opwen_email_server/config.py

@@ -1,5 +1,6 @@
 from os import environ
 
+STORAGE_PROVIDER = environ.get('LOKOLE_STORAGE_PROVIDER', 'AZURE_BLOBS')
 BLOBS_ACCOUNT = environ.get('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_NAME', '')
 BLOBS_KEY = environ.get('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_KEY', '')
 TABLES_ACCOUNT = environ.get('LOKOLE_EMAIL_SERVER_AZURE_TABLES_NAME', '')

opwen_email_server/services/storage.py

@@ -1,12 +1,18 @@
 from gzip import open as gzip_open
+from io import BytesIO
 from json import loads
-from typing import Callable
 from typing import Iterable
 from typing import Optional
 from uuid import uuid4
 
-from azure.storage.blob import BlockBlobService
+from cached_property import cached_property
+from libcloud.storage.base import Container
+from libcloud.storage.base import StorageDriver  # noqa
+from libcloud.storage.providers import get_driver
+from libcloud.storage.types import ContainerDoesNotExistError
+from libcloud.storage.types import Provider
 
+from opwen_email_server.config import STORAGE_PROVIDER
 from opwen_email_server.utils.log import LogMixin
 from opwen_email_server.utils.serialization import gunzip_string
 from opwen_email_server.utils.serialization import gzip_string
@@ -17,24 +23,21 @@
 
 class _BaseAzureStorage(LogMixin):
     def __init__(self, account: str, key: str, container: str,
-                 client: BlockBlobService=None,
-                 factory: Callable[..., BlockBlobService]=BlockBlobService
-                 ) -> None:
-
+                 provider: Optional[str]=None) -> None:
         self._account = account
         self._key = key
         self._container = container
-        self.__client = client
-        self._client_factory = factory
+        self._provider = getattr(Provider, provider or STORAGE_PROVIDER)
 
-    @property
-    def _client(self) -> BlockBlobService:
-        if self.__client is not None:
-            return self.__client
-        client = self._client_factory(self._account, self._key)
-        client.create_container(self._container)
-        self.__client = client
-        return client
+    @cached_property
+    def _client(self) -> Container:
+        driver = get_driver(self._provider)
+        client = driver(self._account, self._key)  # type: StorageDriver
+        try:
+            container = client.get_container(self._container)
+        except ContainerDoesNotExistError:
+            container = client.create_container(self._container)
+        return container
 
     @property
     def container(self) -> str:
@@ -44,30 +47,37 @@ def extra_log_args(self):
         yield 'container %s', self._container
 
     def delete(self, resource_id: str):
-        self._client.delete_blob(self._container, resource_id)
+        self._client.delete_object(resource_id)
 
 
 class _AzureFileStorage(_BaseAzureStorage):
     def store_file(self, resource_id: str, path: str):
         self.log_debug('storing file %s at %s', path, resource_id)
-        self._client.create_blob_from_path(self._container, resource_id, path)
+        self._client.upload_object(path, resource_id)
 
     def fetch_file(self, resource_id: str) -> str:
         path = create_tempfilename()
-        self._client.get_blob_to_path(self._container, resource_id, path)
+        resource = self._client.get_object(resource_id)
+        resource.download(path)
         self.log_debug('fetched file %s from %s', path, resource_id)
         return path
 
 
 class AzureTextStorage(_BaseAzureStorage):
     def store_text(self, resource_id: str, text: str):
         self.log_debug('storing %d characters at %s', len(text), resource_id)
-        self._client.create_blob_from_bytes(self._container, resource_id,
-                                            gzip_string(text))
+        upload = BytesIO()
+        upload.write(gzip_string(text))
+        upload.seek(0)
+        self._client.upload_object_via_stream(upload, resource_id)
 
     def fetch_text(self, resource_id: str) -> str:
-        blob = self._client.get_blob_to_bytes(self._container, resource_id)
-        text = gunzip_string(blob.content)
+        download = BytesIO()
+        resource = self._client.get_object(resource_id)
+        for chunk in resource.as_stream():
+            download.write(chunk)
+        download.seek(0)
+        text = gunzip_string(download.read())
         self.log_debug('fetched %d characters from %s', len(text), resource_id)
         return text
 
@@ -76,12 +86,10 @@ class AzureObjectStorage(LogMixin):
     _encoding = 'utf-8'
 
     def __init__(self, account: str, key: str, container: str,
-                 client: BlockBlobService=None,
-                 factory: Callable[..., BlockBlobService]=BlockBlobService,
-                 file_storage: _AzureFileStorage=None) -> None:
-        self._file_storage = file_storage or _AzureFileStorage(
+                 provider: Optional[str]=None) -> None:
+        self._file_storage = _AzureFileStorage(
             account=account, key=key, container=container,
-            client=client, factory=factory)
+            provider=provider)
 
     @property
     def container(self) -> str:

opwen_email_server/utils/temporary.py

@@ -1,15 +1,14 @@
 from contextlib import contextmanager
 from contextlib import suppress
-from os import close
 from os import remove
-from tempfile import mkstemp
+from os.path import join
+from tempfile import gettempdir
 from typing import Generator
+from uuid import uuid4
 
 
 def create_tempfilename() -> str:
-    file_descriptor, filename = mkstemp()
-    close(file_descriptor)
-    return filename
+    return join(gettempdir(), str(uuid4()))
 
 
 @contextmanager

requirements-dev.txt

@@ -1,5 +1,6 @@
 coverage==4.5.1
 flake8==3.5.0
+lockfile==0.12.2
 mypy==0.620
 nose==1.3.7
 Pillow==5.2.0

requirements.txt

@@ -1,10 +1,11 @@
 applicationinsights==0.11.6
 azure-cosmosdb-table==1.0.4
 azure-servicebus==0.21.1
-azure-storage-blob==1.3.1
 beautifulsoup4==4.6.3
+cached-property==1.5.1
 connexion==1.5.2
 pyzmail36==1.0.3
 requests==2.19.1
 sendgrid==5.6.0
 typing==3.6.6
+https://github.com/apache/libcloud/archive/7d5eb6fffc4521824c87b2cf71d47bdcd2161cae.zip