Make live integration tests use real Azure Storage

.env

@@ -10,6 +10,7 @@ LOKOLE_EMAIL_SERVER_QUEUES_SAS_NAME=
 LOKOLE_EMAIL_SERVER_QUEUES_SAS_KEY=
 LOKOLE_EMAIL_SERVER_QUEUES_NAMESPACE=
 LOKOLE_SENDGRID_KEY=
+LOKOLE_RESOURCE_SUFFIX=
 REGISTRATION_CREDENTIALS=admin:password
 WEBAPP_VERSION=0.5.10
 

.travis.sh

@@ -19,6 +19,7 @@ case "$1" in
     ;;
 
   after_script)
+    SUFFIX="$TRAVIS_SCRIPT_UUID" make clean-storage
     make stop
     ;;
 
@@ -45,8 +46,7 @@ case "$1" in
     ;;
 
   install)
-    export BUILD_TARGET=runtime
-    make build verify-build
+    BUILD_TARGET=runtime make build verify-build
     ;;
 
   script)
@@ -55,15 +55,6 @@ case "$1" in
       exit 0
     fi
 
-    if [[ "$TEST_MODE" = "live" ]]; then
-      export REGISTRATION_CREDENTIALS="$GITHUB_AUTH_USERNAME:$GITHUB_AUTH_TOKEN"
-      export LOKOLE_QUEUE_BROKER_SCHEME="azureservicebus"
-    else
-      export REGISTRATION_CREDENTIALS="admin:password"
-      export LOKOLE_QUEUE_BROKER_SCHEME="amqp"
-    fi
-
-    export CI=true
     make start
     make integration-tests
     ;;

.travis.yml

@@ -11,6 +11,24 @@ services:
 before_install: ./.travis.sh before_install
 install: ./.travis.sh install
 
+before_script: |
+  export TRAVIS_SCRIPT_UUID="$(cat /proc/sys/kernel/random/uuid)"
+  export CI=true
+
+  if [[ "$TEST_MODE" = "live" ]]; then
+    export REGISTRATION_CREDENTIALS="$GITHUB_AUTH_USERNAME:$GITHUB_AUTH_TOKEN"
+    export LOKOLE_QUEUE_BROKER_SCHEME="azureservicebus"
+    export LOKOLE_RESOURCE_SUFFIX="$TRAVIS_SCRIPT_UUID"
+    export APPINSIGHTS_INSTRUMENTATIONKEY="$TRAVIS_SCRIPT_UUID"
+    export AZURITE_ACCOUNT="$TEST_AZURE_STORAGE_ACCOUNT"
+    export AZURITE_KEY="$TEST_AZURE_STORAGE_KEY"
+    export AZURITE_HOST=""
+    export AZURITE_SECURE="True"
+  else
+    export REGISTRATION_CREDENTIALS="admin:password"
+    export LOKOLE_QUEUE_BROKER_SCHEME="amqp"
+  fi
+
 script: ./.travis.sh script
 after_success: ./.travis.sh after_success
 after_failure: ./.travis.sh after_failure

docker-compose.yml

@@ -10,6 +10,7 @@ x-shared-app-environment:
   LOKOLE_LOG_LEVEL: ${LOKOLE_LOG_LEVEL}
   LOKOLE_EMAIL_SERVER_APPINSIGHTS_KEY: ${APPINSIGHTS_INSTRUMENTATIONKEY}
   LOKOLE_EMAIL_SERVER_APPINSIGHTS_HOST: http://appinsights:8000
+  LOKOLE_RESOURCE_SUFFIX: ${LOKOLE_RESOURCE_SUFFIX}
 
   LOKOLE_STORAGE_PROVIDER: AZURE_BLOBS
 
@@ -108,12 +109,12 @@ services:
       - ${RABBITMQ_PORT}:15672
 
   appinsights:
-    image: cwolff/appinsights-on-premises:0.1.3-libcloud
+    image: cwolff/appinsights-on-premises:0.2.0-libcloud
     depends_on:
       - azurite
     environment:
       PORT: "8000"
-      DATABASE_URL: "libcloud://${AZURITE_ACCOUNT}:${AZURITE_KEY}@azure_blobs?endpoint=azurite:10000&ssl=False"
+      DATABASE_URL: "libcloud://${AZURITE_ACCOUNT}:${AZURITE_KEY}@azure_blobs?endpoint=${AZURITE_HOST}&ssl=${AZURITE_SECURE}"
       APPINSIGHTS_INSTRUMENTATIONKEY: "${APPINSIGHTS_INSTRUMENTATIONKEY}"
 
   azurite:

docker/docker-compose.test.yml

@@ -9,5 +9,9 @@ services:
       dockerfile: docker/integtest/Dockerfile
     environment:
       REGISTRATION_CREDENTIALS: ${REGISTRATION_CREDENTIALS}
+      APPINSIGHTS_INSTRUMENTATIONKEY: ${APPINSIGHTS_INSTRUMENTATIONKEY}
+      AZURITE_ACCOUNT: ${AZURITE_ACCOUNT}
+      AZURITE_KEY: ${AZURITE_KEY}
+      AZURITE_HOST: ${AZURITE_HOST}
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

docker/integtest/0-wait-for-services.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 
 scriptdir="$(dirname "$0")"
 # shellcheck disable=SC1090
@@ -31,7 +31,7 @@ wait_for_appinsights() {
   for i in $(seq 1 "${max_retries}"); do
     if [[ \
       "$(az storage container exists \
-      --name "$(appinsights_container)" \
+      --name "${APPINSIGHTS_INSTRUMENTATIONKEY}" \
       --connection-string "$(az_connection_string)" \
       --output tsv)" = "True" \
    ]]; then

docker/integtest/1-register-client.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 
 scriptdir="$(dirname "$0")"
 out_dir="${scriptdir}/files/test.out"

docker/integtest/2-client-uploads-emails.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 
 scriptdir="$(dirname "$0")"
 in_dir="${scriptdir}/files"

docker/integtest/3-receive-email-for-client.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 
 scriptdir="$(dirname "$0")"
 in_dir="${scriptdir}/files"

docker/integtest/4-client-downloads-emails.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 
 scriptdir="$(dirname "$0")"
 out_dir="${scriptdir}/files/test.out"

docker/integtest/5-assert-on-results.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 
 scriptdir="$(dirname "$0")"
 out_dir="${scriptdir}/files/test.out"
@@ -9,7 +9,7 @@ mkdir -p "${out_dir}"
 
 num_exceptions="$(az storage blob list \
   --prefix "Microsoft.ApplicationInsights.Exception/" \
-  --container-name "$(appinsights_container)" \
+  --container-name "${APPINSIGHTS_INSTRUMENTATIONKEY}" \
   --connection-string "$(az_connection_string)" \
   --output tsv | wc -l)"
 num_exceptions_expected=0

docker/integtest/clean.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+scriptdir="$(dirname "$0")"
+# shellcheck disable=SC1090
+. "${scriptdir}/utils.sh"
+
+mapfile -t containers < <(\
+  az storage container list \
+    --connection-string "$(az_connection_string)" \
+    --query "[].name" \
+    --output tsv \
+  | grep "$1$")
+
+for container in "${containers[@]}"; do
+  log "Deleting container ${container}"
+  az storage container delete \
+    --connection-string "$(az_connection_string)" \
+    --name "${container}"
+done

docker/integtest/tests.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 
 scriptdir="$(dirname "$0")"
 

docker/integtest/utils.sh

@@ -1,34 +1,20 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 
 get_container() {
   docker ps --format  '{{.Names}}' | grep "$1"
 }
 
-get_dotenv() {
-  local key dotenv_file
-
-  key="$1"
-  dotenv_file="$(dirname "$0")/.env"
-
-  grep "^${key}=" "${dotenv_file}" | cut -d'=' -f2-
-}
-
 log() {
   echo "$@" >&2
 }
 
-appinsights_container() {
-  get_dotenv "APPINSIGHTS_INSTRUMENTATIONKEY" | tr -d '-'
-}
-
 az_connection_string() {
-  local storage_account storage_key
-
-  storage_account="$(get_dotenv AZURITE_ACCOUNT)"
-  storage_key="$(get_dotenv AZURITE_KEY)"
-
-  echo "DefaultEndpointsProtocol=http;AccountName=${storage_account};AccountKey=${storage_key};BlobEndpoint=http://azurite:10000/${storage_account};"
+  if [[ -z "${AZURITE_HOST}" ]]; then
+    echo "AccountName=${AZURITE_ACCOUNT};AccountKey=${AZURITE_KEY};"
+  else
+    echo "AccountName=${AZURITE_ACCOUNT};AccountKey=${AZURITE_KEY};BlobEndpoint=http://${AZURITE_HOST}/${AZURITE_ACCOUNT};"
+  fi
 }
 
 az_storage() {

makefile

@@ -58,6 +58,10 @@ integration-tests:
 clean:
 	find . -name '__pycache__' -type d -print0 | xargs -0 rm -rf
 
+clean-storage:
+	docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml build integtest && \
+  docker-compose -f docker-compose.yml -f docker/docker-compose.test.yml run --rm integtest ./clean.sh "$(SUFFIX)"
+
 build:
 	BUILD_TARGET=builder docker-compose build api && \
   docker-compose run --no-deps --rm api cat coverage.xml > coverage.xml

opwen_email_server/config.py

@@ -21,6 +21,15 @@
 CLIENT_STORAGE_HOST = env('LOKOLE_CLIENT_AZURE_STORAGE_HOST', '')
 CLIENT_STORAGE_SECURE = env.bool('LOKOLE_CLIENT_AZURE_STORAGE_SECURE', True)
 
+resource_suffix = env('LOKOLE_RESOURCE_SUFFIX', '')
+CONTAINER_CLIENT_PACKAGES = f"compressedpackages{resource_suffix}"
+CONTAINER_EMAILS = f"emails{resource_suffix}"
+CONTAINER_MAILBOX = f"mailbox{resource_suffix}"
+CONTAINER_USERS = f"users{resource_suffix}"
+CONTAINER_SENDGRID_MIME = f"sendgridinboundemails{resource_suffix}"
+CONTAINER_PENDING = f"pendingemails{resource_suffix}"
+CONTAINER_AUTH = f"clientsauth{resource_suffix}"
+
 SENDGRID_MAX_RETRIES = env.int('LOKOLE_SENDGRID_MAX_RETRIES', 20)
 SENDGRID_RETRY_INTERVAL_SECONDS = env.float('LOKOLE_SENDGRID_RETRY_INTERVAL_SECONDS', 5)
 SENDGRID_KEY = env('LOKOLE_SENDGRID_KEY', '')

opwen_email_server/integration/azure.py

@@ -1,5 +1,4 @@
 from opwen_email_server import config
-from opwen_email_server.constants import azure as constants
 from opwen_email_server.services.auth import AzureAuth
 from opwen_email_server.services.storage import AzureFileStorage
 from opwen_email_server.services.storage import AzureObjectsStorage
@@ -15,7 +14,7 @@ def get_auth() -> AzureAuth:
         key=config.TABLES_KEY,
         host=config.TABLES_HOST,
         secure=config.TABLES_SECURE,
-        container=constants.CONTAINER_AUTH,
+        container=config.CONTAINER_AUTH,
         provider=config.STORAGE_PROVIDER,
     ))
 
@@ -27,7 +26,7 @@ def get_client_storage() -> AzureObjectsStorage:
         key=config.CLIENT_STORAGE_KEY,
         host=config.CLIENT_STORAGE_HOST,
         secure=config.CLIENT_STORAGE_SECURE,
-        container=constants.CONTAINER_CLIENT_PACKAGES,
+        container=config.CONTAINER_CLIENT_PACKAGES,
         provider=config.STORAGE_PROVIDER,
     ))
 
@@ -39,7 +38,7 @@ def get_raw_email_storage() -> AzureTextStorage:
         key=config.BLOBS_KEY,
         host=config.BLOBS_HOST,
         secure=config.BLOBS_SECURE,
-        container=constants.CONTAINER_SENDGRID_MIME,
+        container=config.CONTAINER_SENDGRID_MIME,
         provider=config.STORAGE_PROVIDER,
     )
 
@@ -51,7 +50,7 @@ def get_email_storage() -> AzureObjectStorage:
         key=config.BLOBS_KEY,
         host=config.BLOBS_HOST,
         secure=config.BLOBS_SECURE,
-        container=constants.CONTAINER_EMAILS,
+        container=config.CONTAINER_EMAILS,
         provider=config.STORAGE_PROVIDER,
     )
 
@@ -63,7 +62,7 @@ def get_user_storage() -> AzureObjectStorage:
         key=config.TABLES_KEY,
         host=config.TABLES_HOST,
         secure=config.TABLES_SECURE,
-        container=constants.CONTAINER_USERS,
+        container=config.CONTAINER_USERS,
         provider=config.STORAGE_PROVIDER,
     )
 
@@ -75,7 +74,7 @@ def get_mailbox_storage() -> AzureTextStorage:
         key=config.BLOBS_KEY,
         host=config.BLOBS_HOST,
         secure=config.BLOBS_SECURE,
-        container=constants.CONTAINER_MAILBOX,
+        container=config.CONTAINER_MAILBOX,
         provider=config.STORAGE_PROVIDER,
     )
 
@@ -87,6 +86,6 @@ def get_pending_storage() -> AzureTextStorage:
         key=config.TABLES_KEY,
         host=config.TABLES_HOST,
         secure=config.TABLES_SECURE,
-        container=constants.CONTAINER_PENDING,
+        container=config.CONTAINER_PENDING,
         provider=config.STORAGE_PROVIDER,
     )

tests/opwen_email_server/test_config.py

@@ -1,6 +1,7 @@
 from contextlib import contextmanager
 from importlib import reload
 from os import environ
+from re import match
 from unittest import TestCase
 
 from opwen_email_server import config
@@ -34,6 +35,23 @@ def test_queue_broker_servicebus_urlsafe(self):
         with setenvs(envs):
             self.assertEqual(config.QUEUE_BROKER, 'azureservicebus://us%2Fer:pass@host')
 
+    def test_container_names_are_valid(self):
+        acceptable_container_name = '^[a-z0-9][a-z0-9-]{2,62}$'
+
+        for constant, value in get_constants(config):
+            if constant.startswith('CONTAINER_') and not match(acceptable_container_name, value):
+                self.fail(f'config {constant} is invalid: {value}, ' f'should be {acceptable_container_name}')
+
+
+def get_constants(container):
+    for variable_name in dir(container):
+        if variable_name.upper() != variable_name:
+            continue
+        value = getattr(container, variable_name)
+        if not isinstance(value, str):
+            continue
+        yield variable_name, value
+
 
 @contextmanager
 def setenvs(envs):