diff --git a/.env b/.env index 77a53f88..61db94ca 100644 --- a/.env +++ b/.env @@ -10,6 +10,7 @@ LOKOLE_EMAIL_SERVER_QUEUES_SAS_NAME= LOKOLE_EMAIL_SERVER_QUEUES_SAS_KEY= LOKOLE_EMAIL_SERVER_QUEUES_NAMESPACE= LOKOLE_SENDGRID_KEY= +LOKOLE_RESOURCE_SUFFIX= REGISTRATION_CREDENTIALS=admin:password WEBAPP_VERSION=0.5.10 diff --git a/.travis.sh b/.travis.sh index c8dc42ad..cfeae84e 100755 --- a/.travis.sh +++ b/.travis.sh @@ -19,6 +19,7 @@ case "$1" in ;; after_script) + SUFFIX="$TRAVIS_SCRIPT_UUID" make clean-storage make stop ;; @@ -45,8 +46,7 @@ case "$1" in ;; install) - export BUILD_TARGET=runtime - make build verify-build + BUILD_TARGET=runtime make build verify-build ;; script) @@ -55,15 +55,6 @@ case "$1" in exit 0 fi - if [[ "$TEST_MODE" = "live" ]]; then - export REGISTRATION_CREDENTIALS="$GITHUB_AUTH_USERNAME:$GITHUB_AUTH_TOKEN" - export LOKOLE_QUEUE_BROKER_SCHEME="azureservicebus" - else - export REGISTRATION_CREDENTIALS="admin:password" - export LOKOLE_QUEUE_BROKER_SCHEME="amqp" - fi - - export CI=true make start make integration-tests ;; diff --git a/.travis.yml b/.travis.yml index 5cecfff9..f739ba58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,24 @@ services: before_install: ./.travis.sh before_install install: ./.travis.sh install +before_script: | + export TRAVIS_SCRIPT_UUID="$(cat /proc/sys/kernel/random/uuid)" + export CI=true + + if [[ "$TEST_MODE" = "live" ]]; then + export REGISTRATION_CREDENTIALS="$GITHUB_AUTH_USERNAME:$GITHUB_AUTH_TOKEN" + export LOKOLE_QUEUE_BROKER_SCHEME="azureservicebus" + export LOKOLE_RESOURCE_SUFFIX="$TRAVIS_SCRIPT_UUID" + export APPINSIGHTS_INSTRUMENTATIONKEY="$TRAVIS_SCRIPT_UUID" + export AZURITE_ACCOUNT="$TEST_AZURE_STORAGE_ACCOUNT" + export AZURITE_KEY="$TEST_AZURE_STORAGE_KEY" + export AZURITE_HOST="" + export AZURITE_SECURE="True" + else + export REGISTRATION_CREDENTIALS="admin:password" + export LOKOLE_QUEUE_BROKER_SCHEME="amqp" + fi + script: ./.travis.sh script after_success: ./.travis.sh after_success after_failure: ./.travis.sh after_failure diff --git a/docker-compose.yml b/docker-compose.yml index 7516091f..c97cbf34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,7 @@ x-shared-app-environment: LOKOLE_LOG_LEVEL: ${LOKOLE_LOG_LEVEL} LOKOLE_EMAIL_SERVER_APPINSIGHTS_KEY: ${APPINSIGHTS_INSTRUMENTATIONKEY} LOKOLE_EMAIL_SERVER_APPINSIGHTS_HOST: http://appinsights:8000 + LOKOLE_RESOURCE_SUFFIX: ${LOKOLE_RESOURCE_SUFFIX} LOKOLE_STORAGE_PROVIDER: AZURE_BLOBS @@ -108,12 +109,12 @@ services: - ${RABBITMQ_PORT}:15672 appinsights: - image: cwolff/appinsights-on-premises:0.1.3-libcloud + image: cwolff/appinsights-on-premises:0.2.0-libcloud depends_on: - azurite environment: PORT: "8000" - DATABASE_URL: "libcloud://${AZURITE_ACCOUNT}:${AZURITE_KEY}@azure_blobs?endpoint=azurite:10000&ssl=False" + DATABASE_URL: "libcloud://${AZURITE_ACCOUNT}:${AZURITE_KEY}@azure_blobs?endpoint=${AZURITE_HOST}&ssl=${AZURITE_SECURE}" APPINSIGHTS_INSTRUMENTATIONKEY: "${APPINSIGHTS_INSTRUMENTATIONKEY}" azurite: diff --git a/docker/docker-compose.test.yml b/docker/docker-compose.test.yml index e9c9165a..4ccd81c1 100644 --- a/docker/docker-compose.test.yml +++ b/docker/docker-compose.test.yml @@ -9,5 +9,9 @@ services: dockerfile: docker/integtest/Dockerfile environment: REGISTRATION_CREDENTIALS: ${REGISTRATION_CREDENTIALS} + APPINSIGHTS_INSTRUMENTATIONKEY: ${APPINSIGHTS_INSTRUMENTATIONKEY} + AZURITE_ACCOUNT: ${AZURITE_ACCOUNT} + AZURITE_KEY: ${AZURITE_KEY} + AZURITE_HOST: ${AZURITE_HOST} volumes: - /var/run/docker.sock:/var/run/docker.sock diff --git a/docker/integtest/0-wait-for-services.sh b/docker/integtest/0-wait-for-services.sh index c271ac0e..b1c34b48 100755 --- a/docker/integtest/0-wait-for-services.sh +++ b/docker/integtest/0-wait-for-services.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail scriptdir="$(dirname "$0")" # shellcheck disable=SC1090 @@ -31,7 +31,7 @@ wait_for_appinsights() { for i in $(seq 1 "${max_retries}"); do if [[ \ "$(az storage container exists \ - --name "$(appinsights_container)" \ + --name "${APPINSIGHTS_INSTRUMENTATIONKEY}" \ --connection-string "$(az_connection_string)" \ --output tsv)" = "True" \ ]]; then diff --git a/docker/integtest/1-register-client.sh b/docker/integtest/1-register-client.sh index a7e44b91..168866c0 100755 --- a/docker/integtest/1-register-client.sh +++ b/docker/integtest/1-register-client.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail scriptdir="$(dirname "$0")" out_dir="${scriptdir}/files/test.out" diff --git a/docker/integtest/2-client-uploads-emails.sh b/docker/integtest/2-client-uploads-emails.sh index 3de0a3b8..e4796abd 100755 --- a/docker/integtest/2-client-uploads-emails.sh +++ b/docker/integtest/2-client-uploads-emails.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail scriptdir="$(dirname "$0")" in_dir="${scriptdir}/files" diff --git a/docker/integtest/3-receive-email-for-client.sh b/docker/integtest/3-receive-email-for-client.sh index cd67008a..b3d995ca 100755 --- a/docker/integtest/3-receive-email-for-client.sh +++ b/docker/integtest/3-receive-email-for-client.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail scriptdir="$(dirname "$0")" in_dir="${scriptdir}/files" diff --git a/docker/integtest/4-client-downloads-emails.sh b/docker/integtest/4-client-downloads-emails.sh index 67994a6e..9cfca1a4 100755 --- a/docker/integtest/4-client-downloads-emails.sh +++ b/docker/integtest/4-client-downloads-emails.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail scriptdir="$(dirname "$0")" out_dir="${scriptdir}/files/test.out" diff --git a/docker/integtest/5-assert-on-results.sh b/docker/integtest/5-assert-on-results.sh index f8ece5c0..c3b8b5a5 100755 --- a/docker/integtest/5-assert-on-results.sh +++ b/docker/integtest/5-assert-on-results.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail scriptdir="$(dirname "$0")" out_dir="${scriptdir}/files/test.out" @@ -9,7 +9,7 @@ mkdir -p "${out_dir}" num_exceptions="$(az storage blob list \ --prefix "Microsoft.ApplicationInsights.Exception/" \ - --container-name "$(appinsights_container)" \ + --container-name "${APPINSIGHTS_INSTRUMENTATIONKEY}" \ --connection-string "$(az_connection_string)" \ --output tsv | wc -l)" num_exceptions_expected=0 diff --git a/docker/integtest/clean.sh b/docker/integtest/clean.sh new file mode 100755 index 00000000..6559c130 --- /dev/null +++ b/docker/integtest/clean.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eo pipefail + +scriptdir="$(dirname "$0")" +# shellcheck disable=SC1090 +. "${scriptdir}/utils.sh" + +mapfile -t containers < <(\ + az storage container list \ + --connection-string "$(az_connection_string)" \ + --query "[].name" \ + --output tsv \ + | grep "$1$") + +for container in "${containers[@]}"; do + log "Deleting container ${container}" + az storage container delete \ + --connection-string "$(az_connection_string)" \ + --name "${container}" +done diff --git a/docker/integtest/tests.sh b/docker/integtest/tests.sh index a305ff4e..648cc131 100755 --- a/docker/integtest/tests.sh +++ b/docker/integtest/tests.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail scriptdir="$(dirname "$0")" diff --git a/docker/integtest/utils.sh b/docker/integtest/utils.sh index 167973bd..a0708c9b 100755 --- a/docker/integtest/utils.sh +++ b/docker/integtest/utils.sh @@ -1,34 +1,20 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail get_container() { docker ps --format '{{.Names}}' | grep "$1" } -get_dotenv() { - local key dotenv_file - - key="$1" - dotenv_file="$(dirname "$0")/.env" - - grep "^${key}=" "${dotenv_file}" | cut -d'=' -f2- -} - log() { echo "$@" >&2 } -appinsights_container() { - get_dotenv "APPINSIGHTS_INSTRUMENTATIONKEY" | tr -d '-' -} - az_connection_string() { - local storage_account storage_key - - storage_account="$(get_dotenv AZURITE_ACCOUNT)" - storage_key="$(get_dotenv AZURITE_KEY)" - - echo "DefaultEndpointsProtocol=http;AccountName=${storage_account};AccountKey=${storage_key};BlobEndpoint=http://azurite:10000/${storage_account};" + if [[ -z "${AZURITE_HOST}" ]]; then + echo "AccountName=${AZURITE_ACCOUNT};AccountKey=${AZURITE_KEY};" + else + echo "AccountName=${AZURITE_ACCOUNT};AccountKey=${AZURITE_KEY};BlobEndpoint=http://${AZURITE_HOST}/${AZURITE_ACCOUNT};" + fi } az_storage() { diff --git a/makefile b/makefile index 5c4ede04..5861f23a 100644 --- a/makefile +++ b/makefile @@ -58,6 +58,10 @@ integration-tests: clean: find . -name '__pycache__' -type d -print0 | xargs -0 rm -rf +clean-storage: + docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml build integtest && \ + docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml run --rm integtest ./clean.sh "$(SUFFIX)" + build: BUILD_TARGET=builder docker-compose build api && \ docker-compose run --no-deps --rm api cat coverage.xml > coverage.xml diff --git a/opwen_email_server/config.py b/opwen_email_server/config.py index 8fdbbf4d..3a97955a 100644 --- a/opwen_email_server/config.py +++ b/opwen_email_server/config.py @@ -21,6 +21,15 @@ CLIENT_STORAGE_HOST = env('LOKOLE_CLIENT_AZURE_STORAGE_HOST', '') CLIENT_STORAGE_SECURE = env.bool('LOKOLE_CLIENT_AZURE_STORAGE_SECURE', True) +resource_suffix = env('LOKOLE_RESOURCE_SUFFIX', '') +CONTAINER_CLIENT_PACKAGES = f"compressedpackages{resource_suffix}" +CONTAINER_EMAILS = f"emails{resource_suffix}" +CONTAINER_MAILBOX = f"mailbox{resource_suffix}" +CONTAINER_USERS = f"users{resource_suffix}" +CONTAINER_SENDGRID_MIME = f"sendgridinboundemails{resource_suffix}" +CONTAINER_PENDING = f"pendingemails{resource_suffix}" +CONTAINER_AUTH = f"clientsauth{resource_suffix}" + SENDGRID_MAX_RETRIES = env.int('LOKOLE_SENDGRID_MAX_RETRIES', 20) SENDGRID_RETRY_INTERVAL_SECONDS = env.float('LOKOLE_SENDGRID_RETRY_INTERVAL_SECONDS', 5) SENDGRID_KEY = env('LOKOLE_SENDGRID_KEY', '') diff --git a/opwen_email_server/constants/azure.py b/opwen_email_server/constants/azure.py deleted file mode 100644 index 293a3928..00000000 --- a/opwen_email_server/constants/azure.py +++ /dev/null @@ -1,9 +0,0 @@ -from typing_extensions import Final # noqa: F401 - -CONTAINER_CLIENT_PACKAGES = 'compressedpackages' # type: Final -CONTAINER_EMAILS = 'emails' # type: Final -CONTAINER_MAILBOX = 'mailbox' # type: Final -CONTAINER_USERS = 'users' # type: Final -CONTAINER_SENDGRID_MIME = 'sendgridinboundemails' # type: Final -CONTAINER_PENDING = 'pendingemails' # type: Final -CONTAINER_AUTH = 'clientsauth' # type: Final diff --git a/opwen_email_server/integration/azure.py b/opwen_email_server/integration/azure.py index 5ccd27c3..f7a06d6f 100644 --- a/opwen_email_server/integration/azure.py +++ b/opwen_email_server/integration/azure.py @@ -1,5 +1,4 @@ from opwen_email_server import config -from opwen_email_server.constants import azure as constants from opwen_email_server.services.auth import AzureAuth from opwen_email_server.services.storage import AzureFileStorage from opwen_email_server.services.storage import AzureObjectsStorage @@ -15,7 +14,7 @@ def get_auth() -> AzureAuth: key=config.TABLES_KEY, host=config.TABLES_HOST, secure=config.TABLES_SECURE, - container=constants.CONTAINER_AUTH, + container=config.CONTAINER_AUTH, provider=config.STORAGE_PROVIDER, )) @@ -27,7 +26,7 @@ def get_client_storage() -> AzureObjectsStorage: key=config.CLIENT_STORAGE_KEY, host=config.CLIENT_STORAGE_HOST, secure=config.CLIENT_STORAGE_SECURE, - container=constants.CONTAINER_CLIENT_PACKAGES, + container=config.CONTAINER_CLIENT_PACKAGES, provider=config.STORAGE_PROVIDER, )) @@ -39,7 +38,7 @@ def get_raw_email_storage() -> AzureTextStorage: key=config.BLOBS_KEY, host=config.BLOBS_HOST, secure=config.BLOBS_SECURE, - container=constants.CONTAINER_SENDGRID_MIME, + container=config.CONTAINER_SENDGRID_MIME, provider=config.STORAGE_PROVIDER, ) @@ -51,7 +50,7 @@ def get_email_storage() -> AzureObjectStorage: key=config.BLOBS_KEY, host=config.BLOBS_HOST, secure=config.BLOBS_SECURE, - container=constants.CONTAINER_EMAILS, + container=config.CONTAINER_EMAILS, provider=config.STORAGE_PROVIDER, ) @@ -63,7 +62,7 @@ def get_user_storage() -> AzureObjectStorage: key=config.TABLES_KEY, host=config.TABLES_HOST, secure=config.TABLES_SECURE, - container=constants.CONTAINER_USERS, + container=config.CONTAINER_USERS, provider=config.STORAGE_PROVIDER, ) @@ -75,7 +74,7 @@ def get_mailbox_storage() -> AzureTextStorage: key=config.BLOBS_KEY, host=config.BLOBS_HOST, secure=config.BLOBS_SECURE, - container=constants.CONTAINER_MAILBOX, + container=config.CONTAINER_MAILBOX, provider=config.STORAGE_PROVIDER, ) @@ -87,6 +86,6 @@ def get_pending_storage() -> AzureTextStorage: key=config.TABLES_KEY, host=config.TABLES_HOST, secure=config.TABLES_SECURE, - container=constants.CONTAINER_PENDING, + container=config.CONTAINER_PENDING, provider=config.STORAGE_PROVIDER, ) diff --git a/tests/opwen_email_server/constants/__init__.py b/tests/opwen_email_server/constants/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/opwen_email_server/constants/test_azure.py b/tests/opwen_email_server/constants/test_azure.py deleted file mode 100644 index 340a8549..00000000 --- a/tests/opwen_email_server/constants/test_azure.py +++ /dev/null @@ -1,24 +0,0 @@ -from re import match -from unittest import TestCase - -from opwen_email_server.constants import azure - - -class AzureTests(TestCase): - def test_azure_names_are_valid(self): - acceptable_config_value = '^[a-z]{3,63}$' - constants = _get_constants(azure) - - for constant, value in constants: - if not match(acceptable_config_value, value): - self.fail(f'config {constant} is invalid: {value}, ' f'should be {acceptable_config_value}') - - -def _get_constants(container): - for variable_name in dir(container): - if variable_name.upper() != variable_name: - continue - value = getattr(container, variable_name) - if not isinstance(value, str): - continue - yield variable_name, value diff --git a/tests/opwen_email_server/test_config.py b/tests/opwen_email_server/test_config.py index cd594567..c9f6ea84 100644 --- a/tests/opwen_email_server/test_config.py +++ b/tests/opwen_email_server/test_config.py @@ -1,6 +1,7 @@ from contextlib import contextmanager from importlib import reload from os import environ +from re import match from unittest import TestCase from opwen_email_server import config @@ -34,6 +35,23 @@ def test_queue_broker_servicebus_urlsafe(self): with setenvs(envs): self.assertEqual(config.QUEUE_BROKER, 'azureservicebus://us%2Fer:pass@host') + def test_container_names_are_valid(self): + acceptable_container_name = '^[a-z0-9][a-z0-9-]{2,62}$' + + for constant, value in get_constants(config): + if constant.startswith('CONTAINER_') and not match(acceptable_container_name, value): + self.fail(f'config {constant} is invalid: {value}, ' f'should be {acceptable_container_name}') + + +def get_constants(container): + for variable_name in dir(container): + if variable_name.upper() != variable_name: + continue + value = getattr(container, variable_name) + if not isinstance(value, str): + continue + yield variable_name, value + @contextmanager def setenvs(envs):