Move away from Storage Queues

.dockerignore

@@ -1,7 +1,15 @@
+.idea/
+.vscode/
+
 *.pyc
 .mypy_cache/
 __pycache__/
 docs/
 tests/
 travis/
-venv/
+venv*/
+
+.vs/
+bin/
+obj/
+out/

.env

@@ -2,5 +2,4 @@ APP_PORT=8080
 BUILD_TAG=development
 DOCKER_REPO=cwolff
 GUNICORN_WORKERS=1
-LOKOLE_QUEUE_POLL_SECONDS=30
 LOKOLE_LOG_LEVEL=DEBUG

.gitignore

@@ -1,11 +1,17 @@
 .idea/
+.vscode/
 
 *.pyc
 __pycache__/
 /build/
 /dist/
 /*.egg-info/
-venv/
+venv*/
 .mypy_cache/
 
 serviceprincipal.json
+
+.vs/
+bin/
+obj/
+out/

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.3'
 
 services:
 
@@ -22,74 +22,128 @@ services:
       context: .
       dockerfile: docker/api/Dockerfile
     environment:
-      API_NAME: opwen_email_server/static/email-receive-spec.yaml
+      CONNEXION_SPEC: opwen_email_server/static/email-receive-spec.yaml
       GUNICORN_WORKERS: ${GUNICORN_WORKERS}
       LOKOLE_LOG_LEVEL: ${LOKOLE_LOG_LEVEL}
-    env_file:
-      - ./secrets/azure.env
-      - ./secrets/sendgrid.env
+      DOTENV_SECRETS: azure;sendgrid
+    secrets:
+      - azure
+      - sendgrid
 
   apiclientwrite:
     image: ${DOCKER_REPO}/opwenserver_api:${BUILD_TAG}
     build:
       context: .
       dockerfile: docker/api/Dockerfile
     environment:
-      API_NAME: opwen_email_server/static/client-write-spec.yaml
+      CONNEXION_SPEC: opwen_email_server/static/client-write-spec.yaml
       GUNICORN_WORKERS: ${GUNICORN_WORKERS}
       LOKOLE_LOG_LEVEL: ${LOKOLE_LOG_LEVEL}
-    env_file:
-      - ./secrets/azure.env
-      - ./secrets/sendgrid.env
+      DOTENV_SECRETS: azure;sendgrid
+    secrets:
+      - azure
+      - sendgrid
 
   apiclientread:
     image: ${DOCKER_REPO}/opwenserver_api:${BUILD_TAG}
     build:
       context: .
       dockerfile: docker/api/Dockerfile
     environment:
-      API_NAME: opwen_email_server/static/client-read-spec.yaml
+      CONNEXION_SPEC: opwen_email_server/static/client-read-spec.yaml
       GUNICORN_WORKERS: ${GUNICORN_WORKERS}
       LOKOLE_LOG_LEVEL: ${LOKOLE_LOG_LEVEL}
-    env_file:
-      - ./secrets/azure.env
-      - ./secrets/sendgrid.env
+      DOTENV_SECRETS: azure;sendgrid
+    secrets:
+      - azure
+      - sendgrid
 
-  jobsendoutboundemails:
-    image: ${DOCKER_REPO}/opwenserver_job:${BUILD_TAG}
+  apisendoutboundemails:
+    image: ${DOCKER_REPO}/opwenserver_api:${BUILD_TAG}
     build:
       context: .
-      dockerfile: docker/job/Dockerfile
+      dockerfile: docker/api/Dockerfile
     environment:
-      JOB_NAME: opwen_email_server.jobs.send_outbound_emails
-      LOKOLE_QUEUE_POLL_SECONDS: ${LOKOLE_QUEUE_POLL_SECONDS}
+      CONNEXION_SPEC: opwen_email_server/static/send-outbound-emails.yaml
+      GUNICORN_WORKERS: ${GUNICORN_WORKERS}
       LOKOLE_LOG_LEVEL: ${LOKOLE_LOG_LEVEL}
-    env_file:
-      - ./secrets/azure.env
-      - ./secrets/sendgrid.env
+      DOTENV_SECRETS: azure;sendgrid
+    secrets:
+      - azure
+      - sendgrid
 
-  jobstoreinboundemails:
-    image: ${DOCKER_REPO}/opwenserver_job:${BUILD_TAG}
+  apistoreinboundemails:
+    image: ${DOCKER_REPO}/opwenserver_api:${BUILD_TAG}
     build:
       context: .
-      dockerfile: docker/job/Dockerfile
+      dockerfile: docker/api/Dockerfile
     environment:
-      JOB_NAME: opwen_email_server.jobs.store_inbound_emails
-      LOKOLE_QUEUE_POLL_SECONDS: ${LOKOLE_QUEUE_POLL_SECONDS}
+      CONNEXION_SPEC: opwen_email_server/static/store-inbound-emails.yaml
+      GUNICORN_WORKERS: ${GUNICORN_WORKERS}
       LOKOLE_LOG_LEVEL: ${LOKOLE_LOG_LEVEL}
-    env_file:
-      - ./secrets/azure.env
-      - ./secrets/sendgrid.env
+      DOTENV_SECRETS: azure;sendgrid
+    secrets:
+      - azure
+      - sendgrid
 
-  jobstorewritteclientemails:
-    image: ${DOCKER_REPO}/opwenserver_job:${BUILD_TAG}
+  apistorewrittenclientemails:
+    image: ${DOCKER_REPO}/opwenserver_api:${BUILD_TAG}
     build:
       context: .
-      dockerfile: docker/job/Dockerfile
+      dockerfile: docker/api/Dockerfile
     environment:
-      JOB_NAME: opwen_email_server.jobs.store_written_client_emails
-      LOKOLE_QUEUE_POLL_SECONDS: ${LOKOLE_QUEUE_POLL_SECONDS}
+      CONNEXION_SPEC: opwen_email_server/static/store-written-emails.yaml
+      GUNICORN_WORKERS: ${GUNICORN_WORKERS}
       LOKOLE_LOG_LEVEL: ${LOKOLE_LOG_LEVEL}
-    env_file:
-      - ./secrets/azure.env
-      - ./secrets/sendgrid.env
+      DOTENV_SECRETS: azure;sendgrid
+    secrets:
+      - azure
+      - sendgrid
+
+  connectorsendoutboundemails:
+    image: ${DOCKER_REPO}/opwenserver_queueconnector:${BUILD_TAG}
+    build:
+      context: .
+      dockerfile: docker/queueconnector/Dockerfile
+    environment:
+      LOKOLE_POST_URL: http://apisendoutboundemails/job/email/outbound/send
+      LOKOLE_SOURCE_QUEUE: sengridoutboundemails
+      DOTENV_SECRETS: azure
+    secrets:
+      - azure
+    depends_on:
+      - apisendoutboundemails
+
+  connectorstoreinboundemails:
+    image: ${DOCKER_REPO}/opwenserver_queueconnector:${BUILD_TAG}
+    build:
+      context: .
+      dockerfile: docker/queueconnector/Dockerfile
+    environment:
+      LOKOLE_POST_URL: http://apistoreinboundemails/job/email/inbound/store
+      LOKOLE_SOURCE_QUEUE: sengridinboundemails
+      DOTENV_SECRETS: azure
+    secrets:
+      - azure
+    depends_on:
+      - apistoreinboundemails
+
+  connectorstorewrittenclientemails:
+    image: ${DOCKER_REPO}/opwenserver_queueconnector:${BUILD_TAG}
+    build:
+      context: .
+      dockerfile: docker/queueconnector/Dockerfile
+    environment:
+      LOKOLE_POST_URL: http://apistorewrittenclientemails/job/email/client/store
+      LOKOLE_SOURCE_QUEUE: lokoleinboundemails
+      DOTENV_SECRETS: azure
+    secrets:
+      - azure
+    depends_on:
+      - apistorewrittenclientemails
+
+secrets:
+  azure:
+    file: ./secrets/azure.env
+  sendgrid:
+    file: ./secrets/sendgrid.env

docker/api/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.6
 
-ENV API_NAME="SET_ME"
+ENV CONNEXION_SPEC="SET_ME"
 ENV GUNICORN_WORKERS="1"
 
 ADD requirements.txt /app/requirements.txt
@@ -15,8 +15,10 @@ RUN apt-get update \
 ADD opwen_email_server /app/opwen_email_server
 ADD runserver.py /app/server.py
 ADD docker/api/healthcheck.sh /app/healthcheck.sh
+ADD docker/docker-entrypoint.sh /docker-entrypoint.sh
 
 EXPOSE 80
 WORKDIR /app
 HEALTHCHECK --interval=59m --timeout=5s CMD /app/healthcheck.sh
-CMD "gunicorn" "-w" "${GUNICORN_WORKERS}" "-b" "0.0.0.0:80" "server:build_app(apis=['/app/${API_NAME}', '/app/opwen_email_server/static/healthcheck-spec.yaml'], server='tornado')"
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD "gunicorn" "-w" "${GUNICORN_WORKERS}" "-b" "0.0.0.0:80" "server:build_app(apis=['/app/${CONNEXION_SPEC}', '/app/opwen_email_server/static/healthcheck-spec.yaml'], server='tornado')"

docker/docker-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+# This docker-entrypoint populates environment variables from docker secrets.
+# The docker secrets are assumed to be files in dotenv syntax. The requested
+# secrets should be declared in the environment variable DOTENV_SECRETS, with
+# multiple secret names separated by a semi-colon.
+
+if [ ! -d /run/secrets ]; then
+  exec "$@"
+fi
+
+if [ -z "${DOTENV_SECRETS}" ]; then
+  exec "$@"
+fi
+
+eval "$(find /run/secrets -maxdepth 1 -type f | grep "$(echo "${DOTENV_SECRETS}" | sed 's/;/\\|/g')" | xargs cat | grep -v '^#' | sed 's/^\([^=]\+\)=\(.*\)$/if [ -z "$\1" ]; then \1="\2"; export \1; fi/g')"
+
+exec "$@"

docker/queueconnector/Dockerfile

@@ -0,0 +1,16 @@
+FROM microsoft/dotnet:2.0-sdk-stretch AS builder
+
+WORKDIR /app
+COPY opwen_queue_connector/*.csproj ./
+RUN dotnet restore
+COPY opwen_queue_connector/*.cs ./
+RUN dotnet publish -c Release -o out
+
+FROM microsoft/dotnet:2.0-runtime-stretch
+
+WORKDIR /app
+ADD docker/docker-entrypoint.sh /docker-entrypoint.sh
+COPY --from=builder /app/out ./
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["dotnet", "queueconnector.dll"]

docker/setup/arm.parameters.json

@@ -4,6 +4,9 @@
   "parameters": {
     "storageAccountSKU": {
       "value": ""
+    },
+    "serviceBusSKU": {
+      "value": ""
     }
   }
 }