From c6e881f8dde8449c951c2102f791240298e89c8d Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Mon, 3 Feb 2020 20:48:14 -0500 Subject: [PATCH 1/6] Move queue names to config --- opwen_email_server/config.py | 8 ++++++++ opwen_email_server/constants/queues.py | 8 -------- opwen_email_server/integration/celery.py | 21 +++++++-------------- 3 files changed, 15 insertions(+), 22 deletions(-) delete mode 100644 opwen_email_server/constants/queues.py diff --git a/opwen_email_server/config.py b/opwen_email_server/config.py index 3a97955a..8c7023ee 100644 --- a/opwen_email_server/config.py +++ b/opwen_email_server/config.py @@ -22,6 +22,7 @@ CLIENT_STORAGE_SECURE = env.bool('LOKOLE_CLIENT_AZURE_STORAGE_SECURE', True) resource_suffix = env('LOKOLE_RESOURCE_SUFFIX', '') + CONTAINER_CLIENT_PACKAGES = f"compressedpackages{resource_suffix}" CONTAINER_EMAILS = f"emails{resource_suffix}" CONTAINER_MAILBOX = f"mailbox{resource_suffix}" @@ -30,6 +31,13 @@ CONTAINER_PENDING = f"pendingemails{resource_suffix}" CONTAINER_AUTH = f"clientsauth{resource_suffix}" +REGISTER_CLIENT_QUEUE = f'register{resource_suffix}' +INBOUND_STORE_QUEUE = f'inbound{resource_suffix}' +WRITTEN_STORE_QUEUE = f'written{resource_suffix}' +SEND_QUEUE = f'send{resource_suffix}' +MAILBOX_RECEIVED_QUEUE = f'mailboxreceived{resource_suffix}' +MAILBOX_SENT_QUEUE = f'mailboxsent{resource_suffix}' + SENDGRID_MAX_RETRIES = env.int('LOKOLE_SENDGRID_MAX_RETRIES', 20) SENDGRID_RETRY_INTERVAL_SECONDS = env.float('LOKOLE_SENDGRID_RETRY_INTERVAL_SECONDS', 5) SENDGRID_KEY = env('LOKOLE_SENDGRID_KEY', '') diff --git a/opwen_email_server/constants/queues.py b/opwen_email_server/constants/queues.py deleted file mode 100644 index ddc55dee..00000000 --- a/opwen_email_server/constants/queues.py +++ /dev/null @@ -1,8 +0,0 @@ -from typing_extensions import Final # noqa: F401 - -REGISTER_CLIENT_QUEUE = 'register' # type: Final -INBOUND_STORE_QUEUE = 'inbound' # type: Final -WRITTEN_STORE_QUEUE = 'written' # type: Final -SEND_QUEUE = 'send' # type: Final -MAILBOX_RECEIVED_QUEUE = 'mailboxreceived' # type: Final -MAILBOX_SENT_QUEUE = 'mailboxsent' # type: Final diff --git a/opwen_email_server/integration/celery.py b/opwen_email_server/integration/celery.py index 998d37f7..9d81ff97 100644 --- a/opwen_email_server/integration/celery.py +++ b/opwen_email_server/integration/celery.py @@ -7,13 +7,6 @@ from opwen_email_server.actions import SendOutboundEmails from opwen_email_server.actions import StoreInboundEmails from opwen_email_server.actions import StoreWrittenClientEmails -from opwen_email_server.config import QUEUE_BROKER -from opwen_email_server.constants.queues import INBOUND_STORE_QUEUE -from opwen_email_server.constants.queues import MAILBOX_RECEIVED_QUEUE -from opwen_email_server.constants.queues import MAILBOX_SENT_QUEUE -from opwen_email_server.constants.queues import REGISTER_CLIENT_QUEUE -from opwen_email_server.constants.queues import SEND_QUEUE -from opwen_email_server.constants.queues import WRITTEN_STORE_QUEUE from opwen_email_server.integration.azure import get_auth from opwen_email_server.integration.azure import get_client_storage from opwen_email_server.integration.azure import get_email_storage @@ -25,7 +18,7 @@ from opwen_email_server.services.sendgrid import SendSendgridEmail from opwen_email_server.services.sendgrid import SetupSendgridMailbox -celery = Celery(broker=QUEUE_BROKER) +celery = Celery(broker=config.QUEUE_BROKER) @celery.task(ignore_result=True) @@ -112,12 +105,12 @@ def _fqn(task): task_routes = { - _fqn(register_client): {'queue': REGISTER_CLIENT_QUEUE}, - _fqn(index_received_email_for_mailbox): {'queue': MAILBOX_RECEIVED_QUEUE}, - _fqn(index_sent_email_for_mailbox): {'queue': MAILBOX_SENT_QUEUE}, - _fqn(inbound_store): {'queue': INBOUND_STORE_QUEUE}, - _fqn(written_store): {'queue': WRITTEN_STORE_QUEUE}, - _fqn(send): {'queue': SEND_QUEUE} + _fqn(register_client): {'queue': config.REGISTER_CLIENT_QUEUE}, + _fqn(index_received_email_for_mailbox): {'queue': config.MAILBOX_RECEIVED_QUEUE}, + _fqn(index_sent_email_for_mailbox): {'queue': config.MAILBOX_SENT_QUEUE}, + _fqn(inbound_store): {'queue': config.INBOUND_STORE_QUEUE}, + _fqn(written_store): {'queue': config.WRITTEN_STORE_QUEUE}, + _fqn(send): {'queue': config.SEND_QUEUE} } celery.conf.update(task_routes=task_routes) From 9aa9c715b4e91cb0fcb0ea3c571becca82f7218f Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Mon, 3 Feb 2020 21:00:06 -0500 Subject: [PATCH 2/6] Avoid hard-coding queue names in celery script --- docker/app/run-celery.sh | 2 +- opwen_email_server/integration/cli.py | 26 ++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 opwen_email_server/integration/cli.py diff --git a/docker/app/run-celery.sh b/docker/app/run-celery.sh index 1339c721..6aac0a46 100755 --- a/docker/app/run-celery.sh +++ b/docker/app/run-celery.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash if [[ "${CELERY_QUEUE_NAMES}" = "all" ]]; then - CELERY_QUEUE_NAMES="register,inbound,written,send,mailboxreceived,mailboxsent" + CELERY_QUEUE_NAMES="$("${PY_ENV}/bin/python" -m opwen_email_server.integration.cli print_queues --separator=,)" fi "${PY_ENV}/bin/celery" \ diff --git a/opwen_email_server/integration/cli.py b/opwen_email_server/integration/cli.py new file mode 100644 index 00000000..06ed8fe6 --- /dev/null +++ b/opwen_email_server/integration/cli.py @@ -0,0 +1,26 @@ +import click + +from opwen_email_server import config + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.option('--separator', '-s', default='\n') +def print_queues(separator): + click.echo( + separator.join(( + config.REGISTER_CLIENT_QUEUE, + config.INBOUND_STORE_QUEUE, + config.WRITTEN_STORE_QUEUE, + config.SEND_QUEUE, + config.MAILBOX_RECEIVED_QUEUE, + config.MAILBOX_SENT_QUEUE, + ))) + + +if __name__ == '__main__': + cli() diff --git a/requirements.txt b/requirements.txt index a9b15039..2dfd804a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ apache-libcloud==3.0.0rc1 applicationinsights==0.11.9 beautifulsoup4==4.8.2 cached-property==1.5.1 +click==6.7 connexion[swagger-ui]==2.6.0 environs==7.1.0 msgpack==0.6.2 From 19f4589c93fbabb313b9869a9e98de9400a4c9ec Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Mon, 3 Feb 2020 22:25:42 -0500 Subject: [PATCH 3/6] Clean up queues after CI --- makefile | 2 ++ opwen_email_server/integration/cli.py | 42 +++++++++++++++++++++------ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/makefile b/makefile index 89110e49..9f3af36b 100644 --- a/makefile +++ b/makefile @@ -61,6 +61,8 @@ clean: clean-storage: docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml build integtest && \ docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml run --rm integtest ./clean.sh "$(SUFFIX)" + docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml exec api sh -c \ + '"$${PY_ENV}/bin/python" -m opwen_email_server.integration.cli delete_queues' build: BUILD_TARGET=builder docker-compose build api && \ diff --git a/opwen_email_server/integration/cli.py b/opwen_email_server/integration/cli.py index 06ed8fe6..304bb054 100644 --- a/opwen_email_server/integration/cli.py +++ b/opwen_email_server/integration/cli.py @@ -1,7 +1,20 @@ +from urllib.parse import unquote +from urllib.parse import urlparse + import click +from azure.servicebus import ServiceBusClient from opwen_email_server import config +_QUEUES = ( + config.REGISTER_CLIENT_QUEUE, + config.INBOUND_STORE_QUEUE, + config.WRITTEN_STORE_QUEUE, + config.SEND_QUEUE, + config.MAILBOX_RECEIVED_QUEUE, + config.MAILBOX_SENT_QUEUE, +) + @click.group() def cli(): @@ -11,15 +24,26 @@ def cli(): @cli.command() @click.option('--separator', '-s', default='\n') def print_queues(separator): - click.echo( - separator.join(( - config.REGISTER_CLIENT_QUEUE, - config.INBOUND_STORE_QUEUE, - config.WRITTEN_STORE_QUEUE, - config.SEND_QUEUE, - config.MAILBOX_RECEIVED_QUEUE, - config.MAILBOX_SENT_QUEUE, - ))) + click.echo(separator.join(_QUEUES)) + + +@cli.command() +@click.option('--queues', '-q', default=_QUEUES, multiple=True) +def delete_queues(queues): + queue_broker = urlparse(config.QUEUE_BROKER) + if queue_broker.scheme != 'azureservicebus': + click.echo(f'Skipping queue cleanup for {queue_broker.scheme}') + return + + client = ServiceBusClient( + service_namespace=queue_broker.hostname, + shared_access_key_name=unquote(queue_broker.username), + shared_access_key_value=unquote(queue_broker.password), + ) + + for queue in queues: + click.echo(f'Deleting queue {queue}') + client.delete_queue(queue) if __name__ == '__main__': From 1ad3ca6615f59e100197340c66617b6426d65040 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Mon, 3 Feb 2020 22:33:57 -0500 Subject: [PATCH 4/6] Fix quoting style --- opwen_email_server/config.py | 14 +++++++------- opwen_email_server/integration/cli.py | 5 ++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/opwen_email_server/config.py b/opwen_email_server/config.py index 8c7023ee..92f8bc73 100644 --- a/opwen_email_server/config.py +++ b/opwen_email_server/config.py @@ -23,13 +23,13 @@ resource_suffix = env('LOKOLE_RESOURCE_SUFFIX', '') -CONTAINER_CLIENT_PACKAGES = f"compressedpackages{resource_suffix}" -CONTAINER_EMAILS = f"emails{resource_suffix}" -CONTAINER_MAILBOX = f"mailbox{resource_suffix}" -CONTAINER_USERS = f"users{resource_suffix}" -CONTAINER_SENDGRID_MIME = f"sendgridinboundemails{resource_suffix}" -CONTAINER_PENDING = f"pendingemails{resource_suffix}" -CONTAINER_AUTH = f"clientsauth{resource_suffix}" +CONTAINER_CLIENT_PACKAGES = f'compressedpackages{resource_suffix}' +CONTAINER_EMAILS = f'emails{resource_suffix}' +CONTAINER_MAILBOX = f'mailbox{resource_suffix}' +CONTAINER_USERS = f'users{resource_suffix}' +CONTAINER_SENDGRID_MIME = f'sendgridinboundemails{resource_suffix}' +CONTAINER_PENDING = f'pendingemails{resource_suffix}' +CONTAINER_AUTH = f'clientsauth{resource_suffix}' REGISTER_CLIENT_QUEUE = f'register{resource_suffix}' INBOUND_STORE_QUEUE = f'inbound{resource_suffix}' diff --git a/opwen_email_server/integration/cli.py b/opwen_email_server/integration/cli.py index 304bb054..a5f0256d 100644 --- a/opwen_email_server/integration/cli.py +++ b/opwen_email_server/integration/cli.py @@ -28,8 +28,7 @@ def print_queues(separator): @cli.command() -@click.option('--queues', '-q', default=_QUEUES, multiple=True) -def delete_queues(queues): +def delete_queues(): queue_broker = urlparse(config.QUEUE_BROKER) if queue_broker.scheme != 'azureservicebus': click.echo(f'Skipping queue cleanup for {queue_broker.scheme}') @@ -41,7 +40,7 @@ def delete_queues(queues): shared_access_key_value=unquote(queue_broker.password), ) - for queue in queues: + for queue in _QUEUES: click.echo(f'Deleting queue {queue}') client.delete_queue(queue) From 872e068e0f693f594679770c505c8621f4fce1fa Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Mon, 3 Feb 2020 22:54:20 -0500 Subject: [PATCH 5/6] Move container deletion to CLI utility --- docker/integtest/clean.sh | 20 ----------- makefile | 6 ++-- opwen_email_server/config.py | 6 ++-- opwen_email_server/integration/cli.py | 50 ++++++++++++++++++++++++--- 4 files changed, 52 insertions(+), 30 deletions(-) delete mode 100755 docker/integtest/clean.sh diff --git a/docker/integtest/clean.sh b/docker/integtest/clean.sh deleted file mode 100755 index 6559c130..00000000 --- a/docker/integtest/clean.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -scriptdir="$(dirname "$0")" -# shellcheck disable=SC1090 -. "${scriptdir}/utils.sh" - -mapfile -t containers < <(\ - az storage container list \ - --connection-string "$(az_connection_string)" \ - --query "[].name" \ - --output tsv \ - | grep "$1$") - -for container in "${containers[@]}"; do - log "Deleting container ${container}" - az storage container delete \ - --connection-string "$(az_connection_string)" \ - --name "${container}" -done diff --git a/makefile b/makefile index 9f3af36b..35fac4d3 100644 --- a/makefile +++ b/makefile @@ -59,9 +59,9 @@ clean: find . -name '__pycache__' -type d -print0 | xargs -0 rm -rf clean-storage: - docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml build integtest && \ - docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml run --rm integtest ./clean.sh "$(SUFFIX)" - docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml exec api sh -c \ + docker-compose exec api sh -c \ + '"$${PY_ENV}/bin/python" -m opwen_email_server.integration.cli delete_containers --suffix "$(SUFFIX)"' + docker-compose exec api sh -c \ '"$${PY_ENV}/bin/python" -m opwen_email_server.integration.cli delete_queues' build: diff --git a/opwen_email_server/config.py b/opwen_email_server/config.py index 92f8bc73..58f3a4c2 100644 --- a/opwen_email_server/config.py +++ b/opwen_email_server/config.py @@ -8,17 +8,17 @@ BLOBS_ACCOUNT = env('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_NAME', '') BLOBS_KEY = env('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_KEY', '') -BLOBS_HOST = env('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_HOST', '') +BLOBS_HOST = env('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_HOST', '') or None BLOBS_SECURE = env.bool('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_SECURE', True) TABLES_ACCOUNT = env('LOKOLE_EMAIL_SERVER_AZURE_TABLES_NAME', '') TABLES_KEY = env('LOKOLE_EMAIL_SERVER_AZURE_TABLES_KEY', '') -TABLES_HOST = env('LOKOLE_EMAIL_SERVER_AZURE_TABLES_HOST', '') +TABLES_HOST = env('LOKOLE_EMAIL_SERVER_AZURE_TABLES_HOST', '') or None TABLES_SECURE = env.bool('LOKOLE_EMAIL_SERVER_AZURE_TABLES_SECURE', True) CLIENT_STORAGE_ACCOUNT = env('LOKOLE_CLIENT_AZURE_STORAGE_NAME', '') CLIENT_STORAGE_KEY = env('LOKOLE_CLIENT_AZURE_STORAGE_KEY', '') -CLIENT_STORAGE_HOST = env('LOKOLE_CLIENT_AZURE_STORAGE_HOST', '') +CLIENT_STORAGE_HOST = env('LOKOLE_CLIENT_AZURE_STORAGE_HOST', '') or None CLIENT_STORAGE_SECURE = env.bool('LOKOLE_CLIENT_AZURE_STORAGE_SECURE', True) resource_suffix = env('LOKOLE_RESOURCE_SUFFIX', '') diff --git a/opwen_email_server/integration/cli.py b/opwen_email_server/integration/cli.py index a5f0256d..37d1a4b8 100644 --- a/opwen_email_server/integration/cli.py +++ b/opwen_email_server/integration/cli.py @@ -3,6 +3,7 @@ import click from azure.servicebus import ServiceBusClient +from libcloud.storage.providers import get_driver from opwen_email_server import config @@ -15,6 +16,27 @@ config.MAILBOX_SENT_QUEUE, ) +_STORAGES = ( + ( + config.BLOBS_ACCOUNT, + config.BLOBS_KEY, + config.BLOBS_HOST, + config.BLOBS_SECURE, + ), + ( + config.TABLES_ACCOUNT, + config.TABLES_KEY, + config.TABLES_HOST, + config.TABLES_SECURE, + ), + ( + config.CLIENT_STORAGE_ACCOUNT, + config.CLIENT_STORAGE_KEY, + config.CLIENT_STORAGE_HOST, + config.CLIENT_STORAGE_SECURE, + ), +) + @click.group() def cli(): @@ -22,7 +44,7 @@ def cli(): @cli.command() -@click.option('--separator', '-s', default='\n') +@click.option("--separator", "-s", default="\n") def print_queues(separator): click.echo(separator.join(_QUEUES)) @@ -30,8 +52,8 @@ def print_queues(separator): @cli.command() def delete_queues(): queue_broker = urlparse(config.QUEUE_BROKER) - if queue_broker.scheme != 'azureservicebus': - click.echo(f'Skipping queue cleanup for {queue_broker.scheme}') + if queue_broker.scheme != "azureservicebus": + click.echo(f"Skipping queue cleanup for {queue_broker.scheme}") return client = ServiceBusClient( @@ -41,9 +63,29 @@ def delete_queues(): ) for queue in _QUEUES: - click.echo(f'Deleting queue {queue}') + click.echo(f"Deleting queue {queue}") client.delete_queue(queue) +@cli.command() +@click.option('-s', '--suffix') +def delete_containers(suffix): + for account, key, host, secure in _STORAGES: + client = get_driver(config.STORAGE_PROVIDER)(account, key, host=host, secure=secure) + + for container in client.iterate_containers(): + if container.name.endswith(suffix): + click.echo(f'Deleting container {container.name}') + + response = client.connection.request( + f'/{container.name}', + params={'restype': 'container'}, + method='DELETE', + ) + + if response.status != 202: + raise ValueError(f'Unable to delete container {container.name}') + + if __name__ == '__main__': cli() From 437fd147448f9e9950f71f72c711bbb52ddab1c7 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Mon, 3 Feb 2020 23:08:49 -0500 Subject: [PATCH 6/6] Make queue deletion consistent with container --- makefile | 2 +- opwen_email_server/integration/cli.py | 37 +++++++++++++++------------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/makefile b/makefile index 35fac4d3..827f3447 100644 --- a/makefile +++ b/makefile @@ -62,7 +62,7 @@ clean-storage: docker-compose exec api sh -c \ '"$${PY_ENV}/bin/python" -m opwen_email_server.integration.cli delete_containers --suffix "$(SUFFIX)"' docker-compose exec api sh -c \ - '"$${PY_ENV}/bin/python" -m opwen_email_server.integration.cli delete_queues' + '"$${PY_ENV}/bin/python" -m opwen_email_server.integration.cli delete_queues --suffix "$(SUFFIX)"' build: BUILD_TARGET=builder docker-compose build api && \ diff --git a/opwen_email_server/integration/cli.py b/opwen_email_server/integration/cli.py index 37d1a4b8..01c2360e 100644 --- a/opwen_email_server/integration/cli.py +++ b/opwen_email_server/integration/cli.py @@ -7,15 +7,6 @@ from opwen_email_server import config -_QUEUES = ( - config.REGISTER_CLIENT_QUEUE, - config.INBOUND_STORE_QUEUE, - config.WRITTEN_STORE_QUEUE, - config.SEND_QUEUE, - config.MAILBOX_RECEIVED_QUEUE, - config.MAILBOX_SENT_QUEUE, -) - _STORAGES = ( ( config.BLOBS_ACCOUNT, @@ -44,16 +35,27 @@ def cli(): @cli.command() -@click.option("--separator", "-s", default="\n") +@click.option('--separator', '-s', default='\n') def print_queues(separator): - click.echo(separator.join(_QUEUES)) + click.echo( + separator.join(( + config.REGISTER_CLIENT_QUEUE, + config.INBOUND_STORE_QUEUE, + config.WRITTEN_STORE_QUEUE, + config.SEND_QUEUE, + config.MAILBOX_RECEIVED_QUEUE, + config.MAILBOX_SENT_QUEUE, + ))) @cli.command() -def delete_queues(): +@click.option('-s', '--suffix', default='') +def delete_queues(suffix): + suffix = suffix.replace('-', '_') + queue_broker = urlparse(config.QUEUE_BROKER) - if queue_broker.scheme != "azureservicebus": - click.echo(f"Skipping queue cleanup for {queue_broker.scheme}") + if queue_broker.scheme != 'azureservicebus': + click.echo(f'Skipping queue cleanup for {queue_broker.scheme}') return client = ServiceBusClient( @@ -62,9 +64,10 @@ def delete_queues(): shared_access_key_value=unquote(queue_broker.password), ) - for queue in _QUEUES: - click.echo(f"Deleting queue {queue}") - client.delete_queue(queue) + for queue in client.list_queues(): + if queue.name.endswith(suffix): + click.echo(f'Deleting queue {queue.name}') + client.delete_queue(queue.name) @cli.command()