From 5a8a11b0586b125140a97cee9bd0c168c82e4ed5 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Sun, 16 Sep 2018 02:41:57 -0400 Subject: [PATCH 1/2] Avoid newing up a storage instance in the auth --- opwen_email_server/api/client_read.py | 11 +++++++---- opwen_email_server/api/client_write.py | 11 +++++++---- opwen_email_server/api/email_receive.py | 10 ++++++---- opwen_email_server/services/auth.py | 10 ++-------- registerclient.py | 18 ++++++++++++++---- tests/opwen_email_server/services/test_auth.py | 9 +++++++-- 6 files changed, 43 insertions(+), 26 deletions(-) diff --git a/opwen_email_server/api/client_read.py b/opwen_email_server/api/client_read.py index a4f52569..6f50362f 100644 --- a/opwen_email_server/api/client_read.py +++ b/opwen_email_server/api/client_read.py @@ -7,6 +7,7 @@ from opwen_email_server.backend import server_datastore from opwen_email_server.services.auth import AzureAuth from opwen_email_server.services.storage import AzureObjectStorage +from opwen_email_server.services.storage import AzureTextStorage from opwen_email_server.utils.log import LogMixin STORAGE = AzureObjectStorage(account=config.CLIENT_STORAGE_ACCOUNT, @@ -14,10 +15,12 @@ container=constants.CONTAINER_CLIENT_PACKAGES, provider=config.STORAGE_PROVIDER) -CLIENTS = AzureAuth(account=config.TABLES_ACCOUNT, - key=config.TABLES_KEY, - table=constants.TABLE_AUTH, - provider=config.STORAGE_PROVIDER) +CLIENTS = AzureAuth( + storage=AzureTextStorage( + account=config.TABLES_ACCOUNT, + key=config.TABLES_KEY, + container=constants.TABLE_AUTH, + provider=config.STORAGE_PROVIDER)) class _Downloader(LogMixin): diff --git a/opwen_email_server/api/client_write.py b/opwen_email_server/api/client_write.py index 26c74bce..a7471f82 100644 --- a/opwen_email_server/api/client_write.py +++ b/opwen_email_server/api/client_write.py @@ -5,6 +5,7 @@ from opwen_email_server import events from opwen_email_server.services.auth import AzureAuth from opwen_email_server.services.queue import AzureQueue +from opwen_email_server.services.storage import AzureTextStorage from opwen_email_server.utils.log import LogMixin QUEUE = AzureQueue(namespace=config.QUEUES_NAMESPACE, @@ -12,10 +13,12 @@ sas_name=config.QUEUES_SAS_NAME, name=constants.QUEUE_CLIENT_PACKAGE) -CLIENTS = AzureAuth(account=config.TABLES_ACCOUNT, - key=config.TABLES_KEY, - table=constants.TABLE_AUTH, - provider=config.STORAGE_PROVIDER) +CLIENTS = AzureAuth( + storage=AzureTextStorage( + account=config.TABLES_ACCOUNT, + key=config.TABLES_KEY, + container=constants.TABLE_AUTH, + provider=config.STORAGE_PROVIDER)) class _Uploader(LogMixin): diff --git a/opwen_email_server/api/email_receive.py b/opwen_email_server/api/email_receive.py index 4c8e7677..75c37c31 100644 --- a/opwen_email_server/api/email_receive.py +++ b/opwen_email_server/api/email_receive.py @@ -19,10 +19,12 @@ sas_name=config.QUEUES_SAS_NAME, name=constants.QUEUE_SENDGRID_MIME) -CLIENTS = AzureAuth(account=config.TABLES_ACCOUNT, - key=config.TABLES_KEY, - table=constants.TABLE_AUTH, - provider=config.STORAGE_PROVIDER) +CLIENTS = AzureAuth( + storage=AzureTextStorage( + account=config.TABLES_ACCOUNT, + key=config.TABLES_KEY, + container=constants.TABLE_AUTH, + provider=config.STORAGE_PROVIDER)) class _Receiver(LogMixin): diff --git a/opwen_email_server/services/auth.py b/opwen_email_server/services/auth.py index c3aa87db..8574f260 100644 --- a/opwen_email_server/services/auth.py +++ b/opwen_email_server/services/auth.py @@ -1,6 +1,4 @@ from functools import lru_cache -from json import loads -from os import getenv from typing import Optional from libcloud.storage.types import ObjectDoesNotExistError @@ -10,12 +8,8 @@ class AzureAuth(LogMixin): - def __init__(self, account: str, key: str, table: str, - provider: str) -> None: - self._storage = AzureTextStorage(account, key, table, provider) - - for client in loads(getenv('LOKOLE_DEFAULT_CLIENTS', '[]')): - self.insert(client['id'], client['domain']) + def __init__(self, storage: AzureTextStorage) -> None: + self._storage = storage def insert(self, client_id: str, domain: str): self._storage.store_text(client_id, domain) diff --git a/registerclient.py b/registerclient.py index 5490887a..e4ce43d8 100644 --- a/registerclient.py +++ b/registerclient.py @@ -6,6 +6,7 @@ from opwen_email_server import config from opwen_email_server.services.auth import AzureAuth from opwen_email_server.services.storage import AzureObjectStorage +from opwen_email_server.services.storage import AzureTextStorage parser = ArgumentParser() parser.add_argument('--tables_account', default=config.TABLES_ACCOUNT) @@ -14,15 +15,24 @@ parser.add_argument('--client_key', default=config.CLIENT_STORAGE_KEY) parser.add_argument('--table', default=constants.TABLE_AUTH) parser.add_argument('--container', default=constants.CONTAINER_CLIENT_PACKAGES) +parser.add_argument('--provider', default=config.STORAGE_PROVIDER) parser.add_argument('--client', required=True) parser.add_argument('--domain', required=True) args = parser.parse_args() -auth = AzureAuth(account=args.tables_account, key=args.tables_key, - table=args.table) +auth = AzureAuth( + storage=AzureTextStorage( + account=args.tables_account, + key=args.tables_key, + container=args.table, + provider=args.provider)) + auth.insert(client_id=args.client, domain=args.domain) # noinspection PyStatementEffect,PyProtectedMember # hack to ensure that the container gets created before the client accesses it -storage = AzureObjectStorage(account=args.client_account, key=args.client_key, - container=args.container)._file_storage._client +storage = AzureObjectStorage( + account=args.client_account, + key=args.client_key, + container=args.container, + provider=args.provider)._file_storage._client diff --git a/tests/opwen_email_server/services/test_auth.py b/tests/opwen_email_server/services/test_auth.py index 87093e94..7d27ae2a 100644 --- a/tests/opwen_email_server/services/test_auth.py +++ b/tests/opwen_email_server/services/test_auth.py @@ -3,13 +3,18 @@ from unittest import TestCase from opwen_email_server.services.auth import AzureAuth +from opwen_email_server.services.storage import AzureTextStorage class AzureAuthTests(TestCase): def setUp(self): self._folder = mkdtemp() - self._auth = AzureAuth(account=self._folder, key='key', - table='auth', provider='LOCAL') + self._auth = AzureAuth( + storage=AzureTextStorage( + account=self._folder, + key='key', + container='auth', + provider='LOCAL')) def tearDown(self): rmtree(self._folder) From 1d343f0303e3a958801cca7bc3f148b45571693b Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Sun, 16 Sep 2018 02:41:57 -0400 Subject: [PATCH 2/2] Avoid newing up storage instance in object store --- opwen_email_server/api/client_read.py | 11 +++++++---- .../api/store_written_client_emails.py | 11 +++++++---- opwen_email_server/services/storage.py | 9 +++------ registerclient.py | 10 ++++++---- tests/opwen_email_server/services/test_storage.py | 12 +++++++----- 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/opwen_email_server/api/client_read.py b/opwen_email_server/api/client_read.py index 6f50362f..bb2ed722 100644 --- a/opwen_email_server/api/client_read.py +++ b/opwen_email_server/api/client_read.py @@ -6,14 +6,17 @@ from opwen_email_server import events from opwen_email_server.backend import server_datastore from opwen_email_server.services.auth import AzureAuth +from opwen_email_server.services.storage import AzureFileStorage from opwen_email_server.services.storage import AzureObjectStorage from opwen_email_server.services.storage import AzureTextStorage from opwen_email_server.utils.log import LogMixin -STORAGE = AzureObjectStorage(account=config.CLIENT_STORAGE_ACCOUNT, - key=config.CLIENT_STORAGE_KEY, - container=constants.CONTAINER_CLIENT_PACKAGES, - provider=config.STORAGE_PROVIDER) +STORAGE = AzureObjectStorage( + file_storage=AzureFileStorage( + account=config.CLIENT_STORAGE_ACCOUNT, + key=config.CLIENT_STORAGE_KEY, + container=constants.CONTAINER_CLIENT_PACKAGES, + provider=config.STORAGE_PROVIDER)) CLIENTS = AzureAuth( storage=AzureTextStorage( diff --git a/opwen_email_server/api/store_written_client_emails.py b/opwen_email_server/api/store_written_client_emails.py index 5dadae32..eecb3573 100644 --- a/opwen_email_server/api/store_written_client_emails.py +++ b/opwen_email_server/api/store_written_client_emails.py @@ -3,6 +3,7 @@ from opwen_email_server import events from opwen_email_server.backend import server_datastore from opwen_email_server.services.queue import AzureQueue +from opwen_email_server.services.storage import AzureFileStorage from opwen_email_server.services.storage import AzureObjectStorage from opwen_email_server.utils.email_parser import get_domain from opwen_email_server.utils.log import LogMixin @@ -12,10 +13,12 @@ sas_name=config.QUEUES_SAS_NAME, name=constants.QUEUE_EMAIL_SEND) -STORAGE = AzureObjectStorage(account=config.CLIENT_STORAGE_ACCOUNT, - key=config.CLIENT_STORAGE_KEY, - container=constants.CONTAINER_CLIENT_PACKAGES, - provider=config.STORAGE_PROVIDER) +STORAGE = AzureObjectStorage( + file_storage=AzureFileStorage( + account=config.CLIENT_STORAGE_ACCOUNT, + key=config.CLIENT_STORAGE_KEY, + container=constants.CONTAINER_CLIENT_PACKAGES, + provider=config.STORAGE_PROVIDER)) class _WrittenStorer(LogMixin): diff --git a/opwen_email_server/services/storage.py b/opwen_email_server/services/storage.py index 73f2d9a3..0d547dfc 100644 --- a/opwen_email_server/services/storage.py +++ b/opwen_email_server/services/storage.py @@ -55,7 +55,7 @@ def __iter__(self) -> Iterator[str]: yield resource.name -class _AzureFileStorage(_BaseAzureStorage): +class AzureFileStorage(_BaseAzureStorage): def store_file(self, resource_id: str, path: str): self.log_debug('storing file %s at %s', path, resource_id) self._client.upload_object(path, resource_id) @@ -90,11 +90,8 @@ def fetch_text(self, resource_id: str) -> str: class AzureObjectStorage(LogMixin): _encoding = 'utf-8' - def __init__(self, account: str, key: str, container: str, - provider: str) -> None: - self._file_storage = _AzureFileStorage( - account=account, key=key, container=container, - provider=provider) + def __init__(self, file_storage: AzureFileStorage) -> None: + self._file_storage = file_storage @property def container(self) -> str: diff --git a/registerclient.py b/registerclient.py index e4ce43d8..baee259d 100644 --- a/registerclient.py +++ b/registerclient.py @@ -5,6 +5,7 @@ from opwen_email_server import azure_constants as constants from opwen_email_server import config from opwen_email_server.services.auth import AzureAuth +from opwen_email_server.services.storage import AzureFileStorage from opwen_email_server.services.storage import AzureObjectStorage from opwen_email_server.services.storage import AzureTextStorage @@ -32,7 +33,8 @@ # noinspection PyStatementEffect,PyProtectedMember # hack to ensure that the container gets created before the client accesses it storage = AzureObjectStorage( - account=args.client_account, - key=args.client_key, - container=args.container, - provider=args.provider)._file_storage._client + file_storage=AzureFileStorage( + account=args.client_account, + key=args.client_key, + container=args.container, + provider=args.provider))._file_storage._client diff --git a/tests/opwen_email_server/services/test_storage.py b/tests/opwen_email_server/services/test_storage.py index 1b550162..5030aebc 100644 --- a/tests/opwen_email_server/services/test_storage.py +++ b/tests/opwen_email_server/services/test_storage.py @@ -11,10 +11,9 @@ from libcloud.storage.types import ObjectDoesNotExistError +from opwen_email_server.services.storage import AzureFileStorage from opwen_email_server.services.storage import AzureObjectStorage from opwen_email_server.services.storage import AzureTextStorage -# noinspection PyProtectedMember -from opwen_email_server.services.storage import _AzureFileStorage class AzureTextStorageTests(TestCase): @@ -76,7 +75,7 @@ def _given_file(self, resource_id: str, expected_content: str): def setUp(self): self._folder = mkdtemp() self._container = 'container' - self._storage = _AzureFileStorage( + self._storage = AzureFileStorage( account=self._folder, key='key', container=self._container, provider='LOCAL') self._extra_files = set() @@ -148,8 +147,11 @@ def setUp(self): self._container = 'container' mkdir(join(self._folder, self._container)) self._storage = AzureObjectStorage( - account=self._folder, key='unused', - container=self._container, provider='LOCAL') + file_storage=AzureFileStorage( + account=self._folder, + key='unused', + container=self._container, + provider='LOCAL')) def tearDown(self): rmtree(self._folder)