Add suffix to ServiceBus queues used in CI

docker/app/run-celery.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 if [[ "${CELERY_QUEUE_NAMES}" = "all" ]]; then
-  CELERY_QUEUE_NAMES="register,inbound,written,send,mailboxreceived,mailboxsent"
+  CELERY_QUEUE_NAMES="$("${PY_ENV}/bin/python" -m opwen_email_server.integration.cli print_queues --separator=,)"
 fi
 
 "${PY_ENV}/bin/celery" \

makefile

@@ -59,8 +59,10 @@ clean:
 	find . -name '__pycache__' -type d -print0 | xargs -0 rm -rf
 
 clean-storage:
-	docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml build integtest && \
-  docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml run --rm integtest ./clean.sh "$(SUFFIX)"
+	docker-compose exec api sh -c \
+    '"$${PY_ENV}/bin/python" -m opwen_email_server.integration.cli delete_containers --suffix "$(SUFFIX)"'
+	docker-compose exec api sh -c \
+    '"$${PY_ENV}/bin/python" -m opwen_email_server.integration.cli delete_queues --suffix "$(SUFFIX)"'
 
 build:
 	BUILD_TARGET=builder docker-compose build api && \

opwen_email_server/config.py

@@ -8,27 +8,35 @@
 
 BLOBS_ACCOUNT = env('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_NAME', '')
 BLOBS_KEY = env('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_KEY', '')
-BLOBS_HOST = env('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_HOST', '')
+BLOBS_HOST = env('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_HOST', '') or None
 BLOBS_SECURE = env.bool('LOKOLE_EMAIL_SERVER_AZURE_BLOBS_SECURE', True)
 
 TABLES_ACCOUNT = env('LOKOLE_EMAIL_SERVER_AZURE_TABLES_NAME', '')
 TABLES_KEY = env('LOKOLE_EMAIL_SERVER_AZURE_TABLES_KEY', '')
-TABLES_HOST = env('LOKOLE_EMAIL_SERVER_AZURE_TABLES_HOST', '')
+TABLES_HOST = env('LOKOLE_EMAIL_SERVER_AZURE_TABLES_HOST', '') or None
 TABLES_SECURE = env.bool('LOKOLE_EMAIL_SERVER_AZURE_TABLES_SECURE', True)
 
 CLIENT_STORAGE_ACCOUNT = env('LOKOLE_CLIENT_AZURE_STORAGE_NAME', '')
 CLIENT_STORAGE_KEY = env('LOKOLE_CLIENT_AZURE_STORAGE_KEY', '')
-CLIENT_STORAGE_HOST = env('LOKOLE_CLIENT_AZURE_STORAGE_HOST', '')
+CLIENT_STORAGE_HOST = env('LOKOLE_CLIENT_AZURE_STORAGE_HOST', '') or None
 CLIENT_STORAGE_SECURE = env.bool('LOKOLE_CLIENT_AZURE_STORAGE_SECURE', True)
 
 resource_suffix = env('LOKOLE_RESOURCE_SUFFIX', '')
-CONTAINER_CLIENT_PACKAGES = f"compressedpackages{resource_suffix}"
-CONTAINER_EMAILS = f"emails{resource_suffix}"
-CONTAINER_MAILBOX = f"mailbox{resource_suffix}"
-CONTAINER_USERS = f"users{resource_suffix}"
-CONTAINER_SENDGRID_MIME = f"sendgridinboundemails{resource_suffix}"
-CONTAINER_PENDING = f"pendingemails{resource_suffix}"
-CONTAINER_AUTH = f"clientsauth{resource_suffix}"
+
+CONTAINER_CLIENT_PACKAGES = f'compressedpackages{resource_suffix}'
+CONTAINER_EMAILS = f'emails{resource_suffix}'
+CONTAINER_MAILBOX = f'mailbox{resource_suffix}'
+CONTAINER_USERS = f'users{resource_suffix}'
+CONTAINER_SENDGRID_MIME = f'sendgridinboundemails{resource_suffix}'
+CONTAINER_PENDING = f'pendingemails{resource_suffix}'
+CONTAINER_AUTH = f'clientsauth{resource_suffix}'
+
+REGISTER_CLIENT_QUEUE = f'register{resource_suffix}'
+INBOUND_STORE_QUEUE = f'inbound{resource_suffix}'
+WRITTEN_STORE_QUEUE = f'written{resource_suffix}'
+SEND_QUEUE = f'send{resource_suffix}'
+MAILBOX_RECEIVED_QUEUE = f'mailboxreceived{resource_suffix}'
+MAILBOX_SENT_QUEUE = f'mailboxsent{resource_suffix}'
 
 SENDGRID_MAX_RETRIES = env.int('LOKOLE_SENDGRID_MAX_RETRIES', 20)
 SENDGRID_RETRY_INTERVAL_SECONDS = env.float('LOKOLE_SENDGRID_RETRY_INTERVAL_SECONDS', 5)

opwen_email_server/integration/celery.py

@@ -7,13 +7,6 @@
 from opwen_email_server.actions import SendOutboundEmails
 from opwen_email_server.actions import StoreInboundEmails
 from opwen_email_server.actions import StoreWrittenClientEmails
-from opwen_email_server.config import QUEUE_BROKER
-from opwen_email_server.constants.queues import INBOUND_STORE_QUEUE
-from opwen_email_server.constants.queues import MAILBOX_RECEIVED_QUEUE
-from opwen_email_server.constants.queues import MAILBOX_SENT_QUEUE
-from opwen_email_server.constants.queues import REGISTER_CLIENT_QUEUE
-from opwen_email_server.constants.queues import SEND_QUEUE
-from opwen_email_server.constants.queues import WRITTEN_STORE_QUEUE
 from opwen_email_server.integration.azure import get_auth
 from opwen_email_server.integration.azure import get_client_storage
 from opwen_email_server.integration.azure import get_email_storage
@@ -25,7 +18,7 @@
 from opwen_email_server.services.sendgrid import SendSendgridEmail
 from opwen_email_server.services.sendgrid import SetupSendgridMailbox
 
-celery = Celery(broker=QUEUE_BROKER)
+celery = Celery(broker=config.QUEUE_BROKER)
 
 
 @celery.task(ignore_result=True)
@@ -112,12 +105,12 @@ def _fqn(task):
 
 
 task_routes = {
-    _fqn(register_client): {'queue': REGISTER_CLIENT_QUEUE},
-    _fqn(index_received_email_for_mailbox): {'queue': MAILBOX_RECEIVED_QUEUE},
-    _fqn(index_sent_email_for_mailbox): {'queue': MAILBOX_SENT_QUEUE},
-    _fqn(inbound_store): {'queue': INBOUND_STORE_QUEUE},
-    _fqn(written_store): {'queue': WRITTEN_STORE_QUEUE},
-    _fqn(send): {'queue': SEND_QUEUE}
+    _fqn(register_client): {'queue': config.REGISTER_CLIENT_QUEUE},
+    _fqn(index_received_email_for_mailbox): {'queue': config.MAILBOX_RECEIVED_QUEUE},
+    _fqn(index_sent_email_for_mailbox): {'queue': config.MAILBOX_SENT_QUEUE},
+    _fqn(inbound_store): {'queue': config.INBOUND_STORE_QUEUE},
+    _fqn(written_store): {'queue': config.WRITTEN_STORE_QUEUE},
+    _fqn(send): {'queue': config.SEND_QUEUE}
 }
 
 celery.conf.update(task_routes=task_routes)

opwen_email_server/integration/cli.py

@@ -0,0 +1,94 @@
+from urllib.parse import unquote
+from urllib.parse import urlparse
+
+import click
+from azure.servicebus import ServiceBusClient
+from libcloud.storage.providers import get_driver
+
+from opwen_email_server import config
+
+_STORAGES = (
+    (
+        config.BLOBS_ACCOUNT,
+        config.BLOBS_KEY,
+        config.BLOBS_HOST,
+        config.BLOBS_SECURE,
+    ),
+    (
+        config.TABLES_ACCOUNT,
+        config.TABLES_KEY,
+        config.TABLES_HOST,
+        config.TABLES_SECURE,
+    ),
+    (
+        config.CLIENT_STORAGE_ACCOUNT,
+        config.CLIENT_STORAGE_KEY,
+        config.CLIENT_STORAGE_HOST,
+        config.CLIENT_STORAGE_SECURE,
+    ),
+)
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@click.option('--separator', '-s', default='\n')
+def print_queues(separator):
+    click.echo(
+        separator.join((
+            config.REGISTER_CLIENT_QUEUE,
+            config.INBOUND_STORE_QUEUE,
+            config.WRITTEN_STORE_QUEUE,
+            config.SEND_QUEUE,
+            config.MAILBOX_RECEIVED_QUEUE,
+            config.MAILBOX_SENT_QUEUE,
+        )))
+
+
+@cli.command()
+@click.option('-s', '--suffix', default='')
+def delete_queues(suffix):
+    suffix = suffix.replace('-', '_')
+
+    queue_broker = urlparse(config.QUEUE_BROKER)
+    if queue_broker.scheme != 'azureservicebus':
+        click.echo(f'Skipping queue cleanup for {queue_broker.scheme}')
+        return
+
+    client = ServiceBusClient(
+        service_namespace=queue_broker.hostname,
+        shared_access_key_name=unquote(queue_broker.username),
+        shared_access_key_value=unquote(queue_broker.password),
+    )
+
+    for queue in client.list_queues():
+        if queue.name.endswith(suffix):
+            click.echo(f'Deleting queue {queue.name}')
+            client.delete_queue(queue.name)
+
+
+@cli.command()
+@click.option('-s', '--suffix')
+def delete_containers(suffix):
+    for account, key, host, secure in _STORAGES:
+        client = get_driver(config.STORAGE_PROVIDER)(account, key, host=host, secure=secure)
+
+        for container in client.iterate_containers():
+            if container.name.endswith(suffix):
+                click.echo(f'Deleting container {container.name}')
+
+                response = client.connection.request(
+                    f'/{container.name}',
+                    params={'restype': 'container'},
+                    method='DELETE',
+                )
+
+                if response.status != 202:
+                    raise ValueError(f'Unable to delete container {container.name}')
+
+
+if __name__ == '__main__':
+    cli()

requirements.txt

@@ -5,6 +5,7 @@ apache-libcloud==3.0.0rc1
 applicationinsights==0.11.9
 beautifulsoup4==4.8.2
 cached-property==1.5.1
+click==6.7
 connexion[swagger-ui]==2.6.0
 environs==7.1.0
 msgpack==0.6.2