From 9c03030e9007dac2e8dbb1e6e4fbf63e010fdcd1 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 16 Mar 2021 15:50:00 -0400 Subject: [PATCH 1/9] removing empty deployment from db creation we could move this back to deployable backend if we see the need this causes an error while spinning up user-auth db because project namespace is not created --- .../db-ops/job-create-db-mysql.yml.tpl | 44 ------------------- .../db-ops/job-create-db-postgres.yml.tpl | 44 ------------------- 2 files changed, 88 deletions(-) diff --git a/templates/scripts/db-ops/job-create-db-mysql.yml.tpl b/templates/scripts/db-ops/job-create-db-mysql.yml.tpl index 2357a8d..6a8cfaa 100644 --- a/templates/scripts/db-ops/job-create-db-mysql.yml.tpl +++ b/templates/scripts/db-ops/job-create-db-mysql.yml.tpl @@ -60,47 +60,3 @@ spec: secretName: db-create-users restartPolicy: Never backoffLimit: 1 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: db-pod - namespace: $PROJECT_NAME -spec: -# this is purposely left at 0 so it can be enabled for troubleshooting purposes - replicas: 0 - selector: - matchLabels: - app: db-pod - template: - metadata: - labels: - app: db-pod - spec: - automountServiceAccountToken: false - containers: - - command: - - sh - args: - - "-c" - # long running task so the pod doesn't exit with 0 - - tail -f /dev/null - image: $DOCKER_IMAGE_TAG - imagePullPolicy: Always - name: db-pod - env: - - name: DB_ENDPOINT - value: $DB_ENDPOINT - - name: DB_NAME - value: $DB_NAME - - name: DB_USERNAME - valueFrom: - secretKeyRef: - name: $SECRET_NAME - key: DATABASE_USERNAME - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: $SECRET_NAME - key: DATABASE_PASSWORD diff --git a/templates/scripts/db-ops/job-create-db-postgres.yml.tpl b/templates/scripts/db-ops/job-create-db-postgres.yml.tpl index 15b3802..b4421b5 100644 --- a/templates/scripts/db-ops/job-create-db-postgres.yml.tpl +++ b/templates/scripts/db-ops/job-create-db-postgres.yml.tpl @@ -62,47 +62,3 @@ spec: secretName: db-create-users restartPolicy: Never backoffLimit: 1 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: db-pod - namespace: $PROJECT_NAME -spec: -# this is purposely left at 0 so it can be enabled for troubleshooting purposes - replicas: 0 - selector: - matchLabels: - app: db-pod - template: - metadata: - labels: - app: db-pod - spec: - automountServiceAccountToken: false - containers: - - command: - - sh - args: - - "-c" - # long running task so the pod doesn't exit with 0 - - tail -f /dev/null - image: $DOCKER_IMAGE_TAG - imagePullPolicy: Always - name: db-pod - env: - - name: DB_ENDPOINT - value: $DB_ENDPOINT - - name: DB_NAME - value: $DB_NAME - - name: DB_USERNAME - valueFrom: - secretKeyRef: - name: $SECRET_NAME - key: DATABASE_USERNAME - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: $SECRET_NAME - key: DATABASE_PASSWORD From 4d066c046b24f5a0d166c92ef9ce74a79af452ab Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 16 Mar 2021 16:16:45 -0400 Subject: [PATCH 2/9] fix reference to undeclared variables --- templates/kubernetes/terraform/environments/prod/main.tf | 4 +++- templates/kubernetes/terraform/environments/stage/main.tf | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/templates/kubernetes/terraform/environments/prod/main.tf b/templates/kubernetes/terraform/environments/prod/main.tf index e993973..29fbb19 100644 --- a/templates/kubernetes/terraform/environments/prod/main.tf +++ b/templates/kubernetes/terraform/environments/prod/main.tf @@ -20,7 +20,9 @@ locals { region = "<% index .Params `region` %>" account_id = "<% index .Params `accountId` %>" domain_name = "<% index .Params `productionHostRoot` %>" + environment = "prod" file_uploads = <% if eq (index .Params `fileUploads`) "yes" %>true<% else %>false<% end %> + random_seed = "<% index .Params `randomSeed` %>" } provider "aws" { @@ -37,7 +39,7 @@ module "kubernetes" { project = local.project region = local.region allowed_account_ids = [local.account_id] - random_seed = "<% index .Params `randomSeed` %>" + random_seed = local.random_seed cf_signing_enabled = local.file_uploads # Authenticate with the EKS cluster via the cluster id diff --git a/templates/kubernetes/terraform/environments/stage/main.tf b/templates/kubernetes/terraform/environments/stage/main.tf index 5635fe2..964364f 100644 --- a/templates/kubernetes/terraform/environments/stage/main.tf +++ b/templates/kubernetes/terraform/environments/stage/main.tf @@ -20,7 +20,9 @@ locals { region = "<% index .Params `region` %>" account_id = "<% index .Params `accountId` %>" domain_name = "<% index .Params `stagingHostRoot` %>" + environment = "stage" file_uploads = <% if eq (index .Params `fileUploads`) "yes" %>true<% else %>false<% end %> + random_seed = "<% index .Params `randomSeed` %>" } provider "aws" { @@ -36,7 +38,7 @@ module "kubernetes" { project = local.project region = local.region allowed_account_ids = [local.account_id] - random_seed = "<% index .Params `randomSeed` %>" + random_seed = local.random_seed cf_signing_enabled = local.file_uploads # Authenticate with the EKS cluster via the cluster id From d0390c0303404f22f65c94ba3859fb7c173fa482 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 16 Mar 2021 16:50:56 -0400 Subject: [PATCH 3/9] allow notification secrets to partially exist --- .../terraform/environments/prod/main.tf | 2 ++ .../terraform/environments/stage/main.tf | 2 ++ .../kubernetes/notification_service.tf | 30 ++++++++++++------- .../terraform/modules/kubernetes/variables.tf | 12 ++++++++ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/templates/kubernetes/terraform/environments/prod/main.tf b/templates/kubernetes/terraform/environments/prod/main.tf index 29fbb19..1274697 100644 --- a/templates/kubernetes/terraform/environments/prod/main.tf +++ b/templates/kubernetes/terraform/environments/prod/main.tf @@ -95,6 +95,8 @@ module "kubernetes" { ## If you need to add another user-auth instance you will have to create another set of these resources ]<% end %> notification_service_enabled = <%if eq (index .Params `notificationServiceEnabled`) "yes" %>true<% else %>false<% end %> + notification_service_sendgrid_enabled = <%if ne (index .Params `sendgridApiKey`) "" %>true<% else %>false<% end %> + notification_service_slack_enabled = <%if ne (index .Params `notificationServiceSlackApiKey`) "" %>true<% else %>false<% end %> notification_service_highly_available = true cache_store = "<% index .Params `cacheStore` %>" diff --git a/templates/kubernetes/terraform/environments/stage/main.tf b/templates/kubernetes/terraform/environments/stage/main.tf index 964364f..67397cf 100644 --- a/templates/kubernetes/terraform/environments/stage/main.tf +++ b/templates/kubernetes/terraform/environments/stage/main.tf @@ -94,6 +94,8 @@ module "kubernetes" { ## If you need to add another user-auth instance you will have to create another set of these resources ]<% end %> notification_service_enabled = <%if eq (index .Params `notificationServiceEnabled`) "yes" %>true<% else %>false<% end %> + notification_service_sendgrid_enabled = <%if ne (index .Params `sendgridApiKey`) "" %>true<% else %>false<% end %> + notification_service_slack_enabled = <%if ne (index .Params `notificationServiceSlackApiKey`) "" %>true<% else %>false<% end %> notification_service_highly_available = false cache_store = "<% index .Params `cacheStore` %>" diff --git a/templates/kubernetes/terraform/modules/kubernetes/notification_service.tf b/templates/kubernetes/terraform/modules/kubernetes/notification_service.tf index a74958a..05d063e 100644 --- a/templates/kubernetes/terraform/modules/kubernetes/notification_service.tf +++ b/templates/kubernetes/terraform/modules/kubernetes/notification_service.tf @@ -5,20 +5,20 @@ locals { } data "aws_secretsmanager_secret" "sendgrid_api_key" { - count = var.notification_service_enabled ? 1 : 0 + count = var.notification_service_enabled && var.notification_service_sendgrid_enabled ? 1 : 0 name = local.sendgrid_api_key_secret_name } data "aws_secretsmanager_secret_version" "sendgrid_api_key" { - count = var.notification_service_enabled ? 1 : 0 + count = var.notification_service_enabled && var.notification_service_sendgrid_enabled ? 1 : 0 secret_id = data.aws_secretsmanager_secret.sendgrid_api_key[0].id } data "aws_secretsmanager_secret" "slack_api_key" { - count = var.notification_service_enabled ? 1 : 0 + count = var.notification_service_enabled && var.notification_service_slack_enabled ? 1 : 0 name = local.slack_api_key_secret_name } data "aws_secretsmanager_secret_version" "slack_api_key" { - count = var.notification_service_enabled ? 1 : 0 + count = var.notification_service_enabled && var.notification_service_slack_enabled ? 1 : 0 secret_id = data.aws_secretsmanager_secret.slack_api_key[0].id } @@ -63,13 +63,23 @@ resource "helm_release" "notification_service" { } # These will become secrets provided as env vars - set_sensitive { - name = "application.sendgridApiKey" - value = data.aws_secretsmanager_secret_version.sendgrid_api_key[0].secret_string + dynamic set_sensitive { + for_each = var.notification_service_enabled && var.notification_service_sendgrid_enabled ? [data.aws_secretsmanager_secret_version.sendgrid_api_key[0].secret_string] : [] + iterator = sendgrid_api_key + content { + + name = "application.sendgridApiKey" + value = sendgrid_api_key.value + } } - set_sensitive { - name = "application.slackApiKey" - value = data.aws_secretsmanager_secret_version.slack_api_key[0].secret_string + dynamic set_sensitive { + for_each = var.notification_service_enabled && var.notification_service_slack_enabled ? [data.aws_secretsmanager_secret_version.slack_api_key[0].secret_string] : [] + iterator = slack_api_key + content { + + name = "application.slackApiKey" + value = slack_api_key.value + } } } diff --git a/templates/kubernetes/terraform/modules/kubernetes/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/variables.tf index 6c34674..0fbbc37 100644 --- a/templates/kubernetes/terraform/modules/kubernetes/variables.tf +++ b/templates/kubernetes/terraform/modules/kubernetes/variables.tf @@ -107,6 +107,18 @@ variable "notification_service_enabled" { default = false } +variable "notification_service_slack_enabled" { + description = "If enabled, will inject slack_api_key env-vars from secret manager to notification service" + type = bool + default = false +} + +variable "notification_service_sendgrid_enabled" { + description = "If enabled, will inject sendgrid_api_key env-vars from secret manager to notification service" + type = bool + default = false +} + variable "notification_service_highly_available" { description = "If enabled, will make sure a minimum of 2 pods are running and use a horizontal pod autoscaler to make scale the number of pods based on CPU. Recommended for Production." type = bool From 8e62df75a8645978c20cb797f7a29981c08d6aa6 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 16 Mar 2021 17:00:13 -0400 Subject: [PATCH 4/9] fixup! allow notification secrets to partially exist --- .../kubernetes/notification_service.tf | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/templates/kubernetes/terraform/modules/kubernetes/notification_service.tf b/templates/kubernetes/terraform/modules/kubernetes/notification_service.tf index 05d063e..416f6f6 100644 --- a/templates/kubernetes/terraform/modules/kubernetes/notification_service.tf +++ b/templates/kubernetes/terraform/modules/kubernetes/notification_service.tf @@ -63,23 +63,13 @@ resource "helm_release" "notification_service" { } # These will become secrets provided as env vars - dynamic set_sensitive { - for_each = var.notification_service_enabled && var.notification_service_sendgrid_enabled ? [data.aws_secretsmanager_secret_version.sendgrid_api_key[0].secret_string] : [] - iterator = sendgrid_api_key - content { - - name = "application.sendgridApiKey" - value = sendgrid_api_key.value - } + set_sensitive { + name = "application.sendgridApiKey" + value = var.notification_service_enabled && var.notification_service_sendgrid_enabled ? data.aws_secretsmanager_secret_version.sendgrid_api_key[0].secret_string : "" } - dynamic set_sensitive { - for_each = var.notification_service_enabled && var.notification_service_slack_enabled ? [data.aws_secretsmanager_secret_version.slack_api_key[0].secret_string] : [] - iterator = slack_api_key - content { - - name = "application.slackApiKey" - value = slack_api_key.value - } + set_sensitive { + name = "application.slackApiKey" + value = var.notification_service_enabled && var.notification_service_slack_enabled ? data.aws_secretsmanager_secret_version.slack_api_key[0].secret_string : "" } } From 2682a8798d9decc35aa0c6d171b546d108d10a0f Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 16 Mar 2021 17:18:32 -0400 Subject: [PATCH 5/9] fix user-auth domain name ended up with extra dot --- templates/kubernetes/terraform/environments/prod/main.tf | 6 +++--- templates/kubernetes/terraform/environments/stage/main.tf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/kubernetes/terraform/environments/prod/main.tf b/templates/kubernetes/terraform/environments/prod/main.tf index 1274697..bc8631a 100644 --- a/templates/kubernetes/terraform/environments/prod/main.tf +++ b/templates/kubernetes/terraform/environments/prod/main.tf @@ -81,9 +81,9 @@ module "kubernetes" { { name = local.project auth_namespace = "user-auth" - frontend_service_domain = "<% index .Params `productionFrontendSubdomain` %>.${local.domain_name}" - backend_service_domain = "<% index .Params `productionBackendSubdomain` %>.${local.domain_name}" - whitelisted_return_urls = ["https://<% index .Params `productionFrontendSubdomain` %>.${local.domain_name}"] + frontend_service_domain = "<% index .Params `productionFrontendSubdomain` %>${local.domain_name}" + backend_service_domain = "<% index .Params `productionBackendSubdomain` %>${local.domain_name}" + whitelisted_return_urls = ["https://<% index .Params `productionFrontendSubdomain` %>${local.domain_name}"] jwks_secret_name = "${local.project}-${local.environment}-oathkeeper-jwks-${local.random_seed}" # This domain or address must be verified by the mail provider (Sendgrid, SES, etc.) user_auth_mail_from_address = "noreply@${local.domain_name}" diff --git a/templates/kubernetes/terraform/environments/stage/main.tf b/templates/kubernetes/terraform/environments/stage/main.tf index 67397cf..ebb6baf 100644 --- a/templates/kubernetes/terraform/environments/stage/main.tf +++ b/templates/kubernetes/terraform/environments/stage/main.tf @@ -80,9 +80,9 @@ module "kubernetes" { { name = local.project auth_namespace = "user-auth" - frontend_service_domain = "<% index .Params `stagingFrontendSubdomain` %>.${local.domain_name}" - backend_service_domain = "<% index .Params `stagingBackendSubdomain` %>.${local.domain_name}" - whitelisted_return_urls = ["https://<% index .Params `stagingFrontendSubdomain` %>.${local.domain_name}"] + frontend_service_domain = "<% index .Params `stagingFrontendSubdomain` %>${local.domain_name}" + backend_service_domain = "<% index .Params `stagingBackendSubdomain` %>${local.domain_name}" + whitelisted_return_urls = ["https://<% index .Params `stagingFrontendSubdomain` %>${local.domain_name}"] jwks_secret_name = "${local.project}-${local.environment}-oathkeeper-jwks-${local.random_seed}" # This domain or address must be verified by the mail provider (Sendgrid, SES, etc.) user_auth_mail_from_address = "noreply@${local.domain_name}" From ab96fe1d655baec35d0f647d394ab5586e696b66 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 16 Mar 2021 17:28:39 -0400 Subject: [PATCH 6/9] fix user-auth db-endpoint failure due to ext-svc --- templates/scripts/create-db-user.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/scripts/create-db-user.sh b/templates/scripts/create-db-user.sh index a8c2f53..d9bd293 100755 --- a/templates/scripts/create-db-user.sh +++ b/templates/scripts/create-db-user.sh @@ -34,7 +34,8 @@ echo "Some environment variables (REGION, SEED, PROJECT_NAME, ENVIRONMENT, NAMES DOCKER_IMAGE_TAG=commitdev/zero-k8s-utilities:0.0.3 # database info preparation -DB_ENDPOINT=database.${PROJECT_NAME} +# this script will run both before and after make-apply-k8s, therefore the database service is not always available +DB_ENDPOINT=$(aws rds describe-db-instances --region=$REGION --db-instance-identifier "${PROJECT_NAME}-${ENVIRONMENT}" --query "DBInstances[0].Endpoint.Address") DB_NAME_LIST=$(echo ${DATABASE_NAME} | tr -dc 'A-Za-z0-9 ') # used by job DB_NAME=$(echo ${DB_NAME_LIST} | cut -d" " -f1) # used by db-pod DB_TYPE=${DATABASE_TYPE} From f6e1ac6c117c4c5dec8f62b707e51ad695ff72c9 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 16 Mar 2021 18:57:26 -0400 Subject: [PATCH 7/9] fixup! fix user-auth db-endpoint failure due to ext-svc --- templates/scripts/create-db-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/scripts/create-db-user.sh b/templates/scripts/create-db-user.sh index d9bd293..9f8f550 100755 --- a/templates/scripts/create-db-user.sh +++ b/templates/scripts/create-db-user.sh @@ -35,7 +35,7 @@ DOCKER_IMAGE_TAG=commitdev/zero-k8s-utilities:0.0.3 # database info preparation # this script will run both before and after make-apply-k8s, therefore the database service is not always available -DB_ENDPOINT=$(aws rds describe-db-instances --region=$REGION --db-instance-identifier "${PROJECT_NAME}-${ENVIRONMENT}" --query "DBInstances[0].Endpoint.Address") +DB_ENDPOINT=$(aws rds describe-db-instances --region=$REGION --db-instance-identifier "${PROJECT_NAME}-${ENVIRONMENT}" --query "DBInstances[0].Endpoint.Address" | jq -r) DB_NAME_LIST=$(echo ${DATABASE_NAME} | tr -dc 'A-Za-z0-9 ') # used by job DB_NAME=$(echo ${DB_NAME_LIST} | cut -d" " -f1) # used by db-pod DB_TYPE=${DATABASE_TYPE} From c3c5e0ecd631a5c0c4a4f8251be0a97470e810e4 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 16 Mar 2021 19:38:27 -0400 Subject: [PATCH 8/9] fixup! fixup! fix user-auth db-endpoint failure due to ext-svc --- templates/scripts/create-db-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/scripts/create-db-user.sh b/templates/scripts/create-db-user.sh index 9f8f550..32f1d83 100755 --- a/templates/scripts/create-db-user.sh +++ b/templates/scripts/create-db-user.sh @@ -35,7 +35,7 @@ DOCKER_IMAGE_TAG=commitdev/zero-k8s-utilities:0.0.3 # database info preparation # this script will run both before and after make-apply-k8s, therefore the database service is not always available -DB_ENDPOINT=$(aws rds describe-db-instances --region=$REGION --db-instance-identifier "${PROJECT_NAME}-${ENVIRONMENT}" --query "DBInstances[0].Endpoint.Address" | jq -r) +DB_ENDPOINT=$(aws rds describe-db-instances --region=$REGION --db-instance-identifier "${PROJECT_NAME}-${ENVIRONMENT}" --query "DBInstances[0].Endpoint.Address" | jq -r '.') DB_NAME_LIST=$(echo ${DATABASE_NAME} | tr -dc 'A-Za-z0-9 ') # used by job DB_NAME=$(echo ${DB_NAME_LIST} | cut -d" " -f1) # used by db-pod DB_TYPE=${DATABASE_TYPE} From 1eb068c6b82c74de1e5fe37bbe7c3872c2fea3a7 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 16 Mar 2021 20:02:00 -0400 Subject: [PATCH 9/9] elasticsearch seems to only allow https --- templates/scripts/files/elasticsearch-setup.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/scripts/files/elasticsearch-setup.sh b/templates/scripts/files/elasticsearch-setup.sh index 1b8a6d4..3985b2d 100644 --- a/templates/scripts/files/elasticsearch-setup.sh +++ b/templates/scripts/files/elasticsearch-setup.sh @@ -6,19 +6,19 @@ set -e echo "Executing Elasticsearch queries to configure the ${ENVIRONMENT} environment" # Create the index pattern -curl -X POST "http://${ES_ENDPOINT}/_plugin/kibana/api/saved_objects/index-pattern" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' \ +curl -X POST "https://${ES_ENDPOINT}/_plugin/kibana/api/saved_objects/index-pattern" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' \ -d'{"attributes":{"title":"fluentd-*","timeFieldName":"@timestamp","fields":"[]"}}' if [ "${ENVIRONMENT}" = "stage" ]; then # Create the policy - curl -X PUT "http://${ES_ENDPOINT}/_opendistro/_ism/policies/hot_cold_delete_workflow?pretty" -H 'Content-Type: application/json' -d@/elasticsearch-index-policy-stage.json + curl -X PUT "https://${ES_ENDPOINT}/_opendistro/_ism/policies/hot_cold_delete_workflow?pretty" -H 'Content-Type: application/json' -d@/elasticsearch-index-policy-stage.json # Make indices use the policy - curl -X PUT "http://${ES_ENDPOINT}/_template/fluentd_template?pretty" -H 'Content-Type: application/json' \ + curl -X PUT "https://${ES_ENDPOINT}/_template/fluentd_template?pretty" -H 'Content-Type: application/json' \ -d'{ "index_patterns": ["fluentd-*"], "settings": { "number_of_shards": 2, "number_of_replicas": 1, "opendistro.index_state_management.policy_id": "hot_cold_delete_workflow" }}' else # Create the policy - curl -X PUT "http://${ES_ENDPOINT}/_opendistro/_ism/policies/hot_warm_cold_delete_workflow?pretty" -H 'Content-Type: application/json' -d@/elasticsearch-index-policy-prod.json + curl -X PUT "https://${ES_ENDPOINT}/_opendistro/_ism/policies/hot_warm_cold_delete_workflow?pretty" -H 'Content-Type: application/json' -d@/elasticsearch-index-policy-prod.json # Make indices use the policy - curl -X PUT "http://${ES_ENDPOINT}/_template/fluentd_template?pretty" -H 'Content-Type: application/json' \ + curl -X PUT "https://${ES_ENDPOINT}/_template/fluentd_template?pretty" -H 'Content-Type: application/json' \ -d'{ "index_patterns": ["fluentd-*"], "settings": { "number_of_shards": 2, "number_of_replicas": 2, "opendistro.index_state_management.policy_id": "hot_warm_cold_delete_workflow" }}' fi