Allow easily running tests against AWS

.gitignore

@@ -40,13 +40,22 @@ tools/api-simulations/reports
 .DS_Store
 brig-schema.yaml
 brig.yaml
+brig.integration-aws.yaml
 cannon.yaml
+cannon.integration-aws.yaml
 cargohold.yaml
+cargohold.integration-aws.yaml
 galley-schema.yaml
 galley.yaml
+galley.integration-aws.yaml
 gundeck-schema.yaml
 gundeck.yaml
+gundeck.integration-aws.yaml
 proxy.yaml
+proxy.integration-aws.yaml
+spar.yaml
+spar.integration-aws.yaml
+integration-aws.yaml
 DOCKER_ID*
 swagger-ui
 services/spar/spar.cabal

Makefile

@@ -1,4 +1,5 @@
-LANG := en_US.UTF-8
+SHELL            := /usr/bin/env bash
+LANG             := en_US.UTF-8
 HASKELL_SERVICES := proxy cannon cargohold brig galley gundeck
 SERVICES         := $(HASKELL_SERVICES) nginz
 DOCKER_USER      ?= wireserver
@@ -38,6 +39,16 @@ integration: fast
 	$(MAKE) -C services/gundeck i-fake-aws
 	$(MAKE) -C services/spar i
 
+.PHONY: integration-aws
+integration-aws: fast
+	# We run "i" instead of "integration" to avoid useless rebuilds
+	# (since after "fast" everything will be built already)
+	$(MAKE) -C services/cargohold i-aws
+	$(MAKE) -C services/galley i-aws
+	$(MAKE) -C services/brig i-aws
+	$(MAKE) -C services/gundeck i-aws
+	$(MAKE) -C services/spar i-aws
+
 .PHONY: haddock
 haddock:
 	WIRE_STACK_OPTIONS="--haddock --haddock-internal" make fast

services/brig/Makefile

@@ -86,6 +86,10 @@ $(DEB_INDEX): install
 i:
 	../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml
 
+.PHONY: i-aws
+i-aws:
+	INTEGRATION_USE_REAL_AWS=1 ../integration.sh $(EXE_IT) -s $(NAME).integration-aws.yaml -i ../integration.yaml
+
 .PHONY: i-list
 i-list:
 	$(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -l

services/cargohold/Makefile

@@ -64,6 +64,10 @@ $(DEB_IT):
 i:
 	../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml
 
+.PHONY: i-aws
+i-aws:
+	INTEGRATION_USE_REAL_AWS=1 ../integration.sh $(EXE_IT) -s $(NAME).integration-aws.yaml -i ../integration.yaml
+
 .PHONY: i-list
 i-list:
 	$(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -l

services/cargohold/cargohold.integration.yaml

@@ -3,9 +3,6 @@ cargohold:
   port: 8084
 
 aws:
-  # TODO: Check AWS loadCredentialsFromEnv
-  keyId: dummykey # <-- insert-key-id-here
-  secretKey: dummysecret # <-- insert-secret-key-here
   s3Bucket: dummy-bucket # <-- insert-bucket-name-here
   s3Endpoint: http://localhost:4570 # https://s3-eu-west-1.amazonaws.com:443
   # s3DownloadEndpoint: http://fake-s3:4570

services/cargohold/src/CargoHold/App.hs

@@ -98,9 +98,6 @@ newEnv o = do
 
 initAws :: Opts -> Logger -> Manager -> IO AwsEnv
 initAws o l m = do
-    -- TODO: The AWS package can also load them from the env, check the latest API
-    -- https://hackage.haskell.org/package/aws-0.17.1/docs/src/Aws-Core.html#loadCredentialsFromFile
-    -- which would avoid the need to specify them in a config file when running tests
     let awsOpts = o^.optAws
     amz  <- Aws.newEnv l m $ liftM2 (,) (awsOpts^.awsKeyId) (awsOpts^.awsSecretKey)
     sig  <- newCloudFrontEnv (o^.optAws.awsCloudFront) (o^.optSettings.setDownloadLinkTTL)

services/galley/Makefile

@@ -83,6 +83,10 @@ $(DEB_JOURNALER):
 i:
 	../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml
 
+.PHONY: i-aws
+i-aws:
+	INTEGRATION_USE_REAL_AWS=1 ../integration.sh $(EXE_IT) -s $(NAME).integration-aws.yaml -i ../integration.yaml
+
 .PHONY: i-list
 i-list:
 	$(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -l

services/galley/galley.integration.yaml

@@ -32,6 +32,5 @@ settings:
 #
 #journal: # if set, journals; if not set, disables journaling
 #  queueName: integration-team-events.fifo
-#   endpoint: https://sqs.eu-west-1.amazonaws.com
-#  endpoint: http://localhost:4568
+#  endpoint: http://localhost:4568 # https://sqs.eu-west-1.amazonaws.com
 #  region: eu-west-1

services/gen-aws-conf.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/" && pwd )"
+
+which yq >/dev/null || ( echo "*** please install yq ( https://github.com/mikefarah/yq ) in your path."; exit 22 )
+
+# Ensure that we have a file named integration-aws.yaml in the current
+# dir. If not, fetch it from a known location on S3
+if [ ! -f "${DIR}/integration-aws.yaml" ]
+then
+    echo "Could not find AWS config file to override settings, specify a location on S3 to download the file or add one at ${DIR}/integration-aws.yaml and retry: "
+    read -r location
+    aws s3 cp s3://"$location" "${DIR}/integration-aws.yaml"
+fi
+
+services=( brig cargohold galley gundeck cannon proxy spar )
+for service in "${services[@]}"; do
+    yq r "${DIR}/integration-aws.yaml" "${service}" > "/tmp/${service}-aws.yaml"
+    yq m -a "/tmp/${service}-aws.yaml" "${DIR}/${service}/${service}.integration.yaml" > "${DIR}/${service}/${service}.integration-aws.yaml"
+done
+
+# Example of what the integration-aws.yaml could look like
+# brig:
+#   aws:
+#     userJournalQueue: <queue_name>.fifo
+#     prekeyTable: <table_name>
+#     sqsEndpoint: https://sqs.eu-west-1.amazonaws.com
+#     dynamoDBEndpoint: https://dynamodb.eu-west-1.amazonaws.com
+#   internalEvents:
+#     queueType: sqs
+#     queueName: <queue_name>
+#   emailSMS:
+#     email:
+#       sesQueue: <queue_name>
+#       sesEndpoint: https://email.eu-west-1.amazonaws.com
+#
+# cargohold:
+#   aws:
+#     s3Bucket: <bucket_name>
+#     s3Endpoint: https://s3.eu-west-1.amazonaws.com
+#
+# galley:
+#   journal:
+#     queueName: <queue_name>.fifo
+#     endpoint: https://sqs.eu-west-1.amazonaws.com
+#     region: eu-west-1
+#
+# gundeck:
+#   aws:
+#     queueName: integration-gundeck-events
+#     region: eu-west-1
+#     account: "<account_nr>"
+#     arnEnv: <some_arn_env>
+#     sqsEndpoint: https://sqs.eu-west-1.amazonaws.com
+#     snsEndpoint: https://sns.eu-west-1.amazonaws.com

services/gundeck/Makefile

@@ -91,6 +91,10 @@ bench: install
 i:
 	../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml
 
+.PHONY: i-aws
+i-aws:
+	INTEGRATION_USE_REAL_AWS=1 ../integration.sh $(EXE_IT) -s $(NAME).integration-aws.yaml -i ../integration.yaml
+
 .PHONY: i-list
 i-list:
 	$(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -l

services/integration.sh

@@ -52,20 +52,32 @@ orange=3
 yellow=11
 purpleish=13
 
+if [[ $INTEGRATION_USE_REAL_AWS -eq 1 ]]; then
+    echo 'Attempting to run integration tests using real AWS services!'
+    [ -z "$AWS_REGION" ] && echo "Need to set AWS_REGION in your environment" && exit 1;
+    [ -z "$AWS_ACCESS_KEY_ID" ] && echo "Need to set AWS_ACCESS_KEY_ID in your environment" && exit 1;
+    [ -z "$AWS_SECRET_ACCESS_KEY" ] && echo "Need to set AWS_SECRET_ACCESS_KEY in your environment" && exit 1;
+    ${TOP_LEVEL}/services/gen-aws-conf.sh
+    integration_file_extension='-aws.yaml'
+else
+    # brig,gundeck,galley use the amazonka library's 'Discover', which expects AWS credentials
+    # even if those are not used/can be dummy values with the fake sqs/ses/etc containers used
+    # (see deploy/docker-ephemeral/docker-compose.yaml )
+    echo 'Running tests using mocked AWS services'
+    export AWS_REGION=eu-west-1
+    export AWS_ACCESS_KEY_ID=dummykey
+    export AWS_SECRET_ACCESS_KEY=dummysecret
+    integration_file_extension='.yaml'
+fi
+
 function run() {
     service=$1
     colour=$2
     export LOG_LEVEL=$3
-    (cd ${DIR}/${service} && ${TOP_LEVEL}/dist/${service} -c ${service}.integration.yaml || kill_all) \
+    (cd ${DIR}/${service} && ${TOP_LEVEL}/dist/${service} -c ${service}.integration${integration_file_extension} || kill_all) \
         | sed -e "s/^/$(tput setaf ${colour})[${service}] /" -e "s/$/$(tput sgr0)/" &
 }
 
-# brig,gundeck,galley use the amazonka library's 'Discover', which expects AWS credentials
-# even if those are not used/can be dummy values with the fake sqs/ses/etc containers used (see deploy/docker-ephemeral/docker-compose.yaml )
-export AWS_REGION=eu-west-1
-export AWS_ACCESS_KEY_ID=dummykey
-export AWS_SECRET_ACCESS_KEY=dummysecret
-
 check_prerequisites
 
 run brig ${green} Warn

services/spar/Makefile

@@ -66,6 +66,10 @@ $(DEB_SCHEMA): install
 i:
 	../integration.sh $(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml
 
+.PHONY: i-aws
+i-aws:
+	INTEGRATION_USE_REAL_AWS=1 ../integration.sh $(EXE_IT) -s $(NAME).integration-aws.yaml -i ../integration.yaml
+
 .PHONY: i-list
 i-list:
 	$(EXE_IT) -s $(NAME).integration.yaml -i ../integration.yaml -l