diff --git a/.gitignore b/.gitignore index c7a8ec4d..e1e05211 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,3 @@ cover/ requirements.txt.out serviceprincipal.json -volumes/ -downloaded.tar.gz -register.json -download.json diff --git a/.travis.yml b/.travis.yml index dcbab828..b41b4f8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,15 @@ services: cache: pip +before_script: + - docker-compose build + - docker-compose pull || true + - docker-compose up -d + script: travis/ci.sh +after_script: docker-compose down --volumes + install: - pip install codecov diff --git a/README.rst b/README.rst index 6db9e394..1ea0cef9 100644 --- a/README.rst +++ b/README.rst @@ -110,50 +110,18 @@ application and provide references to the entry points into the code (look for the yaml files in the swagger directory). The various APIs can also be easily called via the testing console that is available by adding /ui to the end of the API's URL. Sample workflows are shown -below. +in the integration tests folder and can be run via: .. sourcecode :: sh - # precondition: - # register a new client - curl "http://localhost:8080/api/email/register/" \ - -H "Content-Type: application/json" \ - -u "admin:password" \ - -d '{"domain":"developer.lokole.ca"}' \ - | tee register.json - - # precondition: - # enable azure cli to talk to azurite - export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" - - # workflow 1: - # simulate delivering emails from client to online email provider - emails_to_send="./tests/files/end_to_end/client-emails.tar.gz" - client_id="$(jq -r '.client_id' < register.json)" - resource_container="$(jq -r '.resource_container' < register.json)" - resource_id="$(python3 -c 'import uuid;print(str(uuid.uuid4()))').tar.gz" - az storage blob upload --name "${resource_id}" --container-name "${resource_container}" --file "${emails_to_send}" - curl "http://localhost:8080/api/email/upload/${client_id}" \ - -H "Content-Type: application/json" \ - -d '{"resource_id":"'"${resource_id}"'"}' - - # workflow 2a: - # simulate receiving email sent from online email provider to client - email_to_receive="./tests/files/end_to_end/inbound-email.mime" - client_id="$(jq -r '.client_id' < register.json)" - curl "http://localhost:8080/api/email/sendgrid/${client_id}" \ - -H "Content-Type: multipart/form-data" \ - -F "email=$(cat "${email_to_receive}")" - - # workflow 2b: - # simulate delivering emails sent from online email provider to client - client_id="$(jq -r '.client_id' < register.json)" - resource_container="$(jq -r '.resource_container' < register.json)" - curl "http://localhost:8080/api/email/download/${client_id}" \ - -H "Accept: application/json" \ - | tee download.json - resource_id="$(jq -r '.resource_id' < download.json)" - az storage blob download --name "${resource_id}" --container-name "${resource_container}" --file "downloaded.tar.gz" + # run the services, wait for them to start + docker-compose up --build + + # in another terminal, run the integration tests + make integration-tests + + # finally, tear down the services + docker-compose down --volumes Note that by default the application is run in a fully local mode, without leveraging any cloud services. For most development purposes this is fine diff --git a/makefile b/makefile index dc71b52c..623eabd0 100644 --- a/makefile +++ b/makefile @@ -36,6 +36,12 @@ bandit: venv ci: tests lint typecheck bandit +integration-tests: + ./tests/integration/0-register-client.sh && \ + ./tests/integration/1-client-uploads-emails.sh && sleep 10s && \ + ./tests/integration/2-receive-email-for-client.sh && sleep 10s && \ + ./tests/integration/3-client-downloads-emails.sh + clean: rm -rf $$(find opwen_email_server -name '__pycache__' -type d) rm -rf $$(find tests -name '__pycache__' -type d) diff --git a/tests/files/end_to_end/.gitignore b/tests/files/end_to_end/.gitignore new file mode 100644 index 00000000..2e48a692 --- /dev/null +++ b/tests/files/end_to_end/.gitignore @@ -0,0 +1,4 @@ +register.json +download.json +downloaded.tar.gz +emails.jsonl diff --git a/tests/integration/0-register-client.sh b/tests/integration/0-register-client.sh new file mode 100755 index 00000000..ece6049f --- /dev/null +++ b/tests/integration/0-register-client.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail +data_dir="$(dirname "$0")/../files/end_to_end" + +# workflow 3: register a new client called "developer" +# normally this endpoint would be called during a new lokole device setup +curl -fs \ + -H "Content-Type: application/json" \ + -u "admin:password" \ + -d '{"domain":"developer.lokole.ca"}' \ + "http://localhost:8080/api/email/register/" \ +| tee "${data_dir}/register.json" diff --git a/tests/integration/1-client-uploads-emails.sh b/tests/integration/1-client-uploads-emails.sh new file mode 100755 index 00000000..5a5a778b --- /dev/null +++ b/tests/integration/1-client-uploads-emails.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -euo pipefail +data_dir="$(dirname "$0")/../files/end_to_end" + +emails_to_send="${data_dir}/client-emails.tar.gz" +client_id="$(jq -r '.client_id' < "${data_dir}/register.json")" +resource_container="$(jq -r '.resource_container' < "${data_dir}/register.json")" +resource_id="$(python3 -c 'import uuid;print(str(uuid.uuid4()))').tar.gz" + +# workflow 1: send emails written on the client to the world +# first we simulate the client uploading its emails to the shared blob storage +curl -fs \ + -X PUT -T "${emails_to_send}" \ + -H "x-ms-blob-type: BlockBlob" \ + -H "Content-Length: $(wc -c "${emails_to_send}" | cut -d' ' -f1)" \ + "http://localhost:10000/devstoreaccount1/${resource_container}/${resource_id}" + +# the client then calls the server to trigger the delivery of the emails +curl -fs \ + -H "Content-Type: application/json" \ + -d '{"resource_id":"'"${resource_id}"'"}' \ + "http://localhost:8080/api/email/upload/${client_id}" diff --git a/tests/integration/2-receive-email-for-client.sh b/tests/integration/2-receive-email-for-client.sh new file mode 100755 index 00000000..ea8d1058 --- /dev/null +++ b/tests/integration/2-receive-email-for-client.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail +data_dir="$(dirname "$0")/../files/end_to_end" + +email_to_receive="${data_dir}/inbound-email.mime" + +client_id="$(jq -r '.client_id' < "${data_dir}/register.json")" + +# workflow 2a: receive an email directed at one of the clients +# this simulates sendgrid delivering an email to the service +curl -fs \ + -H "Content-Type: multipart/form-data" \ + -F "email=$(cat "${email_to_receive}")" \ + "http://localhost:8080/api/email/sendgrid/${client_id}" diff --git a/tests/integration/3-client-downloads-emails.sh b/tests/integration/3-client-downloads-emails.sh new file mode 100755 index 00000000..4b0ddb43 --- /dev/null +++ b/tests/integration/3-client-downloads-emails.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo pipefail +data_dir="$(dirname "$0")/../files/end_to_end" + +client_id="$(jq -r '.client_id' < "${data_dir}/register.json")" +resource_container="$(jq -r '.resource_container' < "${data_dir}/register.json")" + +# workflow 2b: deliver emails written by the world to a lokole client +# first the client makes a request to ask the server to package up all the +# emails sent from the world to the client during the last period; the server +# will package up the emails and store them on the shared blob storage +curl -fs \ + -H "Accept: application/json" \ + "http://localhost:8080/api/email/download/${client_id}" \ +| tee "${data_dir}/download.json" + +resource_id="$(jq -r '.resource_id' < "${data_dir}/download.json")" + +# now we simulate the client downloading the package from the shared blob storage +curl -fs \ + "http://localhost:10000/devstoreaccount1/${resource_container}/${resource_id}" \ +> "${data_dir}/downloaded.tar.gz" + +tar xzf "${data_dir}/downloaded.tar.gz" -C "${data_dir}" + +num_emails_actual="$(wc -l "${data_dir}/emails.jsonl" | cut -d' ' -f1)" +num_emails_expected=1 + +if [[ "${num_emails_actual}" -ne "${num_emails_expected}" ]]; then + echo "Got ${num_emails_actual} but expected ${num_emails_expected}" >&2 + exit 1 +fi diff --git a/travis/ci.sh b/travis/ci.sh index d6282ba9..8da766b7 100755 --- a/travis/ci.sh +++ b/travis/ci.sh @@ -6,4 +6,4 @@ if [[ -z "$TRAVIS_PYTHON_VERSION" ]]; then echo "Build is not targetting a Python version, can't run CI" >&2; exit 1 fi -make ci -e py_env="$HOME/virtualenv/python$TRAVIS_PYTHON_VERSION" +make ci integration-tests -e py_env="$HOME/virtualenv/python$TRAVIS_PYTHON_VERSION"