Merge pull request #328 from alphagov/log-smokey-output-to-aws

Log Smokey output to AWS

concourse/tasks/run-task.sh

@@ -88,8 +88,12 @@ aws ecs wait tasks-stopped --tasks $task_id --cluster $CLUSTER
 echo "task finished."
 task_results=$(aws ecs describe-tasks --tasks $task_id --cluster $CLUSTER)
 
-container_id=$(echo $task_results | jq '.tasks[0].containers[]  | select(.name=="app") | .runtimeId')
-echo "Check Splunk for logs: https://gds.splunkcloud.com/en-GB/app/gds-006-govuk/search?q=search%20index%3D%22govuk_replatforming%22%20container_id%3D$container_id"
+if [[ "${LOG_TO_SPLUNK:-true}" == "true" ]]; then
+  container_id=$(echo $task_results | jq '.tasks[0].containers[]  | select(.name=="app") | .runtimeId')
+  echo "Check Splunk for logs: https://gds.splunkcloud.com/en-GB/app/gds-006-govuk/search?q=search%20index%3D%22govuk_replatforming%22%20container_id%3D$container_id"
+else
+  ecs-cli logs --cluster $CLUSTER --task-id $task_id --since "60" | head -n 5000
+fi
 
 exit_code=$(echo $task_results | jq '[.tasks[0].containers[].exitCode]' | jq add)
 echo "Exiting with code $exit_code"

concourse/tasks/run-task.yml

@@ -21,5 +21,6 @@ params:
   COMMAND: # Place command in run-task-command file or use COMMAND param
   VARIANT:
   DISABLE: false
+  LOG_TO_SPLUNK: true
 run:
   path: ./src/concourse/tasks/run-task.sh

terraform/deployments/govuk-publishing-platform/smokey.tf

@@ -14,10 +14,9 @@ module "smokey_container_definition" {
     # TODO: This should be autogenerated. Add to signon bootstrap task.
     SIGNON_EMAIL = "signon@alphagov.co.uk" # For historical reasons
   }
-  splunk_url_secret_arn   = local.defaults.splunk_url_secret_arn
-  splunk_token_secret_arn = local.defaults.splunk_token_secret_arn
-  splunk_index            = local.defaults.splunk_index
-  splunk_sourcetype       = local.defaults.splunk_sourcetype
+  log_group         = local.log_group
+  log_stream_prefix = "smokey"
+  log_to_splunk     = false
   secrets_from_arns = {
     # TODO: These can be autogenerated.
     AUTH_USERNAME = data.aws_secretsmanager_secret.smokey_auth_username.arn

terraform/modules/container-definition/main.tf

@@ -1,3 +1,37 @@
+locals {
+  log_configuration_splunk = {
+    logDriver = "splunk"
+    options = {
+      env               = "GOVUK_APP_NAME",
+      tag               = "image_name={{.ImageName}} container_name={{.Name}} container_id={{.FullID}}",
+      splunk-sourcetype = var.splunk_sourcetype,
+      splunk-index      = var.splunk_index,
+      splunk-format     = "raw"
+    }
+    secretOptions = [
+      {
+        name      = "splunk-token",
+        valueFrom = var.splunk_token_secret_arn
+      },
+      {
+        name      = "splunk-url",
+        valueFrom = var.splunk_url_secret_arn
+      },
+    ],
+  }
+
+  log_configuration_aws = {
+    logDriver = "awslogs"
+    options = {
+      awslogs-create-group  = "true", # TODO create the log group in terraform so we can configure the retention policy
+      awslogs-group         = var.log_group,
+      awslogs-region        = var.aws_region,
+      awslogs-stream-prefix = var.log_stream_prefix,
+    },
+    secretOptions = [],
+  }
+}
+
 output "json_format" {
   value = {
     name        = var.name,
@@ -14,29 +48,10 @@ output "json_format" {
     linuxParameters = {
       initProcessEnabled = true
     }
-    logConfiguration = {
-      logDriver = "splunk",
-      options = {
-        env               = "GOVUK_APP_NAME",
-        tag               = "image_name={{.ImageName}} container_name={{.Name}} container_id={{.FullID}}",
-        splunk-sourcetype = var.splunk_sourcetype,
-        splunk-index      = var.splunk_index,
-        splunk-format     = "raw"
-      }
-      secretOptions = [
-        {
-          name      = "splunk-token",
-          valueFrom = var.splunk_token_secret_arn
-        },
-        {
-          name      = "splunk-url",
-          valueFrom = var.splunk_url_secret_arn
-        },
-      ],
-    },
-    mountPoints  = [],
-    portMappings = [for port in var.ports : { containerPort = port, hostPort = port, protocol = "tcp" }],
-    secrets      = [for key, value in var.secrets_from_arns : { name = key, valueFrom = value }]
-    user         = var.user
+    logConfiguration = var.log_to_splunk ? local.log_configuration_splunk : local.log_configuration_aws
+    mountPoints      = [],
+    portMappings     = [for port in var.ports : { containerPort = port, hostPort = port, protocol = "tcp" }],
+    secrets          = [for key, value in var.secrets_from_arns : { name = key, valueFrom = value }]
+    user             = var.user
   }
 }

terraform/modules/container-definition/variables.tf

@@ -38,16 +38,19 @@ variable "image" {
 variable "splunk_url_secret_arn" {
   type        = string
   description = "ARN to the secret containing the URL for the Splunk instance (of the form `https://http-inputs-XXXXXXXX.splunkcloud.com:PORT`)."
+  default     = null
 }
 
 variable "splunk_token_secret_arn" {
   type        = string
   description = "ARN to the secret containing the HTTP Event Collector (HEC) token."
+  default     = null
 }
 
 variable "splunk_index" {
   type        = string
   description = "Splunk index to log events to (which HEC token must have access to write to)."
+  default     = null
 }
 
 variable "splunk_sourcetype" {
@@ -80,3 +83,19 @@ variable "user" {
   type    = string
   default = null
 }
+
+variable "log_group" {
+  type    = string
+  default = null
+}
+
+variable "log_stream_prefix" {
+  type        = string
+  description = "Set log_stream_prefix to an ECS Service name, if applicable. A prefix makes it easier to associate a log with a service."
+  default     = null
+}
+
+variable "log_to_splunk" {
+  type    = bool
+  default = true
+}

terraform/modules/task-definition/variables.tf

@@ -11,24 +11,10 @@ variable "container_definitions" {
       startPeriod = number
       retries     = number
     })
-    image           = string
-    linuxParameters = object({ initProcessEnabled = bool })
-    logConfiguration = object({
-      logDriver = string
-      options = object({
-        env               = string
-        tag               = string
-        splunk-sourcetype = string
-        splunk-index      = string
-        splunk-format     = string
-      })
-      secretOptions = list(object({
-        name      = string
-        valueFrom = string
-        })
-      )
-    })
-    mountPoints = list(any),
+    image            = string
+    linuxParameters  = object({ initProcessEnabled = bool })
+    logConfiguration = any
+    mountPoints      = list(any),
     portMappings = list(
       object({ containerPort = number, hostPort = number, protocol = string })
     )