From fa850ba33b183f48b38c4718cc14eb0ef07d8892 Mon Sep 17 00:00:00 2001 From: Karl Baker Date: Thu, 27 May 2021 12:46:37 +0100 Subject: [PATCH 1/2] Log Smokey output to AWS This commit updates the logging for Smokey to be sent to AWS, rather than Splunk. This is necessary so that developers can view the entire output of a particular Smokey job when debugging issues, as logs sent to Splunk are broken up by log line and means navigating multiple search results pages to see the output of a single job. --- concourse/tasks/run-task.sh | 8 ++- concourse/tasks/run-task.yml | 1 + .../govuk-publishing-platform/smokey.tf | 7 +-- .../modules/container-definition/main.tf | 63 ++++++++++++------- .../modules/container-definition/variables.tf | 19 ++++++ 5 files changed, 68 insertions(+), 30 deletions(-) diff --git a/concourse/tasks/run-task.sh b/concourse/tasks/run-task.sh index dcd52a092..bdfb5d58f 100755 --- a/concourse/tasks/run-task.sh +++ b/concourse/tasks/run-task.sh @@ -88,8 +88,12 @@ aws ecs wait tasks-stopped --tasks $task_id --cluster $CLUSTER echo "task finished." task_results=$(aws ecs describe-tasks --tasks $task_id --cluster $CLUSTER) -container_id=$(echo $task_results | jq '.tasks[0].containers[] | select(.name=="app") | .runtimeId') -echo "Check Splunk for logs: https://gds.splunkcloud.com/en-GB/app/gds-006-govuk/search?q=search%20index%3D%22govuk_replatforming%22%20container_id%3D$container_id" +if [[ "${LOG_TO_SPLUNK:-true}" == "true" ]]; then + container_id=$(echo $task_results | jq '.tasks[0].containers[] | select(.name=="app") | .runtimeId') + echo "Check Splunk for logs: https://gds.splunkcloud.com/en-GB/app/gds-006-govuk/search?q=search%20index%3D%22govuk_replatforming%22%20container_id%3D$container_id" +else + ecs-cli logs --cluster $CLUSTER --task-id $task_id --since "60" | head -n 5000 +fi exit_code=$(echo $task_results | jq '[.tasks[0].containers[].exitCode]' | jq add) echo "Exiting with code $exit_code" diff --git a/concourse/tasks/run-task.yml b/concourse/tasks/run-task.yml index c894dc0f6..c74a10d91 100644 --- a/concourse/tasks/run-task.yml +++ b/concourse/tasks/run-task.yml @@ -21,5 +21,6 @@ params: COMMAND: # Place command in run-task-command file or use COMMAND param VARIANT: DISABLE: false + LOG_TO_SPLUNK: true run: path: ./src/concourse/tasks/run-task.sh diff --git a/terraform/deployments/govuk-publishing-platform/smokey.tf b/terraform/deployments/govuk-publishing-platform/smokey.tf index e8da94bad..9011beb92 100644 --- a/terraform/deployments/govuk-publishing-platform/smokey.tf +++ b/terraform/deployments/govuk-publishing-platform/smokey.tf @@ -14,10 +14,9 @@ module "smokey_container_definition" { # TODO: This should be autogenerated. Add to signon bootstrap task. SIGNON_EMAIL = "signon@alphagov.co.uk" # For historical reasons } - splunk_url_secret_arn = local.defaults.splunk_url_secret_arn - splunk_token_secret_arn = local.defaults.splunk_token_secret_arn - splunk_index = local.defaults.splunk_index - splunk_sourcetype = local.defaults.splunk_sourcetype + log_group = local.log_group + log_stream_prefix = "smokey" + log_to_splunk = false secrets_from_arns = { # TODO: These can be autogenerated. AUTH_USERNAME = data.aws_secretsmanager_secret.smokey_auth_username.arn diff --git a/terraform/modules/container-definition/main.tf b/terraform/modules/container-definition/main.tf index adb41fa50..7f1580fbe 100644 --- a/terraform/modules/container-definition/main.tf +++ b/terraform/modules/container-definition/main.tf @@ -1,3 +1,37 @@ +locals { + log_configuration_splunk = { + logDriver = "splunk" + options = { + env = "GOVUK_APP_NAME", + tag = "image_name={{.ImageName}} container_name={{.Name}} container_id={{.FullID}}", + splunk-sourcetype = var.splunk_sourcetype, + splunk-index = var.splunk_index, + splunk-format = "raw" + } + secretOptions = [ + { + name = "splunk-token", + valueFrom = var.splunk_token_secret_arn + }, + { + name = "splunk-url", + valueFrom = var.splunk_url_secret_arn + }, + ], + } + + log_configuration_aws = { + logDriver = "awslogs" + options = { + awslogs-create-group = "true", # TODO create the log group in terraform so we can configure the retention policy + awslogs-group = var.log_group, + awslogs-region = var.aws_region, + awslogs-stream-prefix = var.log_stream_prefix, + }, + secretOptions = [], + } +} + output "json_format" { value = { name = var.name, @@ -14,29 +48,10 @@ output "json_format" { linuxParameters = { initProcessEnabled = true } - logConfiguration = { - logDriver = "splunk", - options = { - env = "GOVUK_APP_NAME", - tag = "image_name={{.ImageName}} container_name={{.Name}} container_id={{.FullID}}", - splunk-sourcetype = var.splunk_sourcetype, - splunk-index = var.splunk_index, - splunk-format = "raw" - } - secretOptions = [ - { - name = "splunk-token", - valueFrom = var.splunk_token_secret_arn - }, - { - name = "splunk-url", - valueFrom = var.splunk_url_secret_arn - }, - ], - }, - mountPoints = [], - portMappings = [for port in var.ports : { containerPort = port, hostPort = port, protocol = "tcp" }], - secrets = [for key, value in var.secrets_from_arns : { name = key, valueFrom = value }] - user = var.user + logConfiguration = var.log_to_splunk ? local.log_configuration_splunk : local.log_configuration_aws + mountPoints = [], + portMappings = [for port in var.ports : { containerPort = port, hostPort = port, protocol = "tcp" }], + secrets = [for key, value in var.secrets_from_arns : { name = key, valueFrom = value }] + user = var.user } } diff --git a/terraform/modules/container-definition/variables.tf b/terraform/modules/container-definition/variables.tf index f6dfd69f4..a4233be42 100644 --- a/terraform/modules/container-definition/variables.tf +++ b/terraform/modules/container-definition/variables.tf @@ -38,16 +38,19 @@ variable "image" { variable "splunk_url_secret_arn" { type = string description = "ARN to the secret containing the URL for the Splunk instance (of the form `https://http-inputs-XXXXXXXX.splunkcloud.com:PORT`)." + default = null } variable "splunk_token_secret_arn" { type = string description = "ARN to the secret containing the HTTP Event Collector (HEC) token." + default = null } variable "splunk_index" { type = string description = "Splunk index to log events to (which HEC token must have access to write to)." + default = null } variable "splunk_sourcetype" { @@ -80,3 +83,19 @@ variable "user" { type = string default = null } + +variable "log_group" { + type = string + default = null +} + +variable "log_stream_prefix" { + type = string + description = "Set log_stream_prefix to an ECS Service name, if applicable. A prefix makes it easier to associate a log with a service." + default = null +} + +variable "log_to_splunk" { + type = bool + default = true +} From 549be0ac9c15ec8cfa451803f2233ba2af0e2e09 Mon Sep 17 00:00:00 2001 From: Karl Baker Date: Mon, 14 Jun 2021 12:28:55 +0100 Subject: [PATCH 2/2] Allow for different log providers in container definition This commit changes the `logConfiguration` schema within the `container_definitions` variable of the task definition to allow for different providers. As it stands, the `logConfiguration` is hard-coded to Splunk, however we wish to log to AWS logs so that we can stream output for non-app deployments to Concourse (such as Smokey output, Rake tasks etc). --- .../modules/task-definition/variables.tf | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/terraform/modules/task-definition/variables.tf b/terraform/modules/task-definition/variables.tf index 28ea1b9d5..be347a3b0 100644 --- a/terraform/modules/task-definition/variables.tf +++ b/terraform/modules/task-definition/variables.tf @@ -11,24 +11,10 @@ variable "container_definitions" { startPeriod = number retries = number }) - image = string - linuxParameters = object({ initProcessEnabled = bool }) - logConfiguration = object({ - logDriver = string - options = object({ - env = string - tag = string - splunk-sourcetype = string - splunk-index = string - splunk-format = string - }) - secretOptions = list(object({ - name = string - valueFrom = string - }) - ) - }) - mountPoints = list(any), + image = string + linuxParameters = object({ initProcessEnabled = bool }) + logConfiguration = any + mountPoints = list(any), portMappings = list( object({ containerPort = number, hostPort = number, protocol = string }) )