Use multiple CW agent configuration files.

asg.tf

@@ -1,9 +1,6 @@
 data "aws_region" "current" {}
 
 locals {
-  shared_metrics_fragment = templatefile("${path.module}/config/cw_agent_config_shared_metrics_fragment.json", {
-    "cloudwatch_namespace" = var.cloudwatch_namespace_ec2_metrics
-  })
   web_interpolation_vars = {
     "authorized_worker_keys"                = tls_private_key.worker_key.public_key_openssh
     "session_signing_key"                   = tls_private_key.session_signing_key.private_key_pem
@@ -20,18 +17,8 @@ locals {
     "cred_store_config"                     = var.cred_store_config
     "feature_flags"                         = var.web_feature_flags
     "concourse_base_resource_type_defaults" = yamlencode(var.concourse_base_resource_type_defaults)
-    "cloudwatch_config"                     = templatefile("${path.module}/config/cw_agent_config_web.json", {
-      "region"                    = data.aws_region.current.name
-      "metrics_fragment"          = local.shared_metrics_fragment
-      "prometheus_namespace"      = var.cloudwatch_namespace_prometheus_metrics
-      "prometheus_log_group_name" = aws_cloudwatch_log_group.concourse.name
-    })
-    "metrics_enabled"                       = var.metrics_enabled
     "prometheus_enabled"                    = var.prometheus_enabled
     "prometheus_bind_port"                  = var.prometheus_bind_port
-    "prometheus_config"                     = templatefile("${path.module}/config/prometheus_config.yml", {
-      "prometheus_bind_port" = var.prometheus_bind_port
-    })
   }
 
   web_user_data = <<EOF
@@ -49,11 +36,6 @@ EOF
     "storage_driver"    = var.worker_container_storage_driver
     "dns_servers"       = var.worker_dns_servers
     "feature_flags"     = var.worker_feature_flags
-    "cloudwatch_config" = templatefile("${path.module}/config/cw_agent_config_worker.json", {
-      "region"           = data.aws_region.current.name
-      "metrics_fragment" = local.shared_metrics_fragment
-    })
-    "metrics_enabled"   = var.metrics_enabled
   }
 
   worker_user_data = <<EOF
@@ -65,7 +47,12 @@ EOF
 }
 
 resource "aws_launch_template" "web_template" {
-  depends_on = [aws_s3_object.web_user_data]
+  depends_on = [
+    aws_s3_object.web_user_data,
+    aws_s3_object.cw_agent_init,
+    aws_s3_object.cw_agent_metrics_init,
+    aws_s3_object.cw_agent_prometheus_init
+  ]
 
   name          = "conc-web-tmpl"
   instance_type = var.web_instance_type
@@ -86,7 +73,12 @@ resource "aws_launch_template" "web_template" {
 
   lifecycle {
     create_before_destroy = true
-    replace_triggered_by  = [aws_s3_object.web_user_data]
+    replace_triggered_by  = [
+      aws_s3_object.web_user_data,
+      aws_s3_object.cw_agent_init,
+      aws_s3_object.cw_agent_metrics_init,
+      aws_s3_object.cw_agent_prometheus_init
+    ]
   }
 }
 
@@ -131,7 +123,11 @@ resource "aws_autoscaling_attachment" "web_asg_to_lb" {
 }
 
 resource "aws_launch_template" "worker_template" {
-  depends_on = [aws_s3_object.worker_user_data]
+  depends_on = [
+    aws_s3_object.worker_user_data,
+    aws_s3_object.cw_agent_init,
+    aws_s3_object.cw_agent_metrics_init
+  ]
 
   name          = "conc-worker-tmpl"
   instance_type = var.worker_instance_type
@@ -160,7 +156,11 @@ resource "aws_launch_template" "worker_template" {
 
   lifecycle {
     create_before_destroy = true
-    replace_triggered_by  = [aws_s3_object.worker_user_data]
+    replace_triggered_by  = [
+      aws_s3_object.worker_user_data,
+      aws_s3_object.cw_agent_init,
+      aws_s3_object.cw_agent_metrics_init
+    ]
   }
 }
 

config/cw_agent_config_worker.json → config/cw_agent_config.json

@@ -2,6 +2,5 @@
   "agent": {
     "region": "${region}",
     "metrics_collection_interval": 10
-  },
-  "metrics": ${metrics_fragment}
+  }
 }

config/cw_metrics_config.json

@@ -0,0 +1,18 @@
+{
+  "metrics": {
+    "namespace": "${cloudwatch_namespace}",
+    "aggregation_dimensions": [["InstanceId"], ["Hostname"]],
+    "metrics_collected": {
+      "disk": {
+        "resources": ["/"],
+        "measurement": ["disk_used_percent"]
+      },
+      "swap": {
+        "measurement": ["swap_used_percent"]
+      },
+      "mem": {
+        "measurement": ["mem_used_percent"]
+      }
+    }
+  }
+}

config/cw_agent_config_web.json → config/cw_prometheus_config.json

@@ -1,8 +1,4 @@
 {
-  "agent": {
-    "region": "${region}",
-    "metrics_collection_interval": 10
-  },
   "logs": {
     "metrics_collected": {
       "prometheus": {
@@ -392,6 +388,5 @@
         }
       }
     }
-  },
-  "metrics": ${metrics_fragment}
+  }
 }

s3.tf

@@ -17,6 +17,44 @@ resource "aws_s3_bucket_acl" "user_data_acl" {
   acl    = "private"
 }
 
+resource "aws_s3_object" "cw_agent_init" {
+  bucket  = aws_s3_bucket.user_data.id
+  key     = "cw_agent_init.sh"
+  content = templatefile("${path.module}/templates/cw_agent_init.sh", {
+    metrics_enabled    = var.metrics_enabled
+    prometheus_enabled = var.prometheus_enabled
+    cw_agent_config    = templatefile("${path.module}/config/cw_agent_config.json", {
+      region = data.aws_region.current.name
+    })
+    cw_metrics_config  = templatefile("${path.module}/config/cw_metrics_config.json", {
+      cloudwatch_namespace = var.cloudwatch_namespace_ec2_metrics
+    })
+    cw_prometheus_config = templatefile("${path.module}/config/cw_prometheus_config.json", {
+      prometheus_log_group_name = aws_cloudwatch_log_group.concourse.name
+      prometheus_namespace      = var.cloudwatch_namespace_prometheus_metrics
+    })
+    prometheus_config = templatefile("${path.module}/config/prometheus_config.yml", {
+      prometheus_bind_port = var.prometheus_bind_port
+    })
+  })
+}
+
+resource "aws_s3_object" "cw_agent_metrics_init" {
+  bucket  = aws_s3_bucket.user_data.id
+  key     = "cw_agent_metrics_init.sh"
+  content = templatefile("${path.module}/templates/cw_agent_metrics_init.sh", {
+    metrics_enabled    = var.metrics_enabled
+  })
+}
+
+resource "aws_s3_object" "cw_agent_prometheus_init" {
+  bucket  = aws_s3_bucket.user_data.id
+  key     = "cw_agent_prometheus_init.sh"
+  content = templatefile("${path.module}/templates/cw_agent_prometheus_init.sh", {
+    prometheus_enabled = var.prometheus_enabled
+  })
+}
+
 resource "aws_s3_object" "web_user_data" {
   bucket  = aws_s3_bucket.user_data.id
   key     = "web_user_data.sh"

templates/cw_agent_init.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+%{ if prometheus_enabled || metrics_enabled }
+
+echo 'Configuring CloudWatch agent'
+
+sudo mkdir -p /etc/cloudwatch
+echo -n '${cw_agent_config}' > /etc/cloudwatch/cw_agent_config.json
+echo -n '${cw_metrics_config}' > /etc/cloudwatch/cw_metrics_config.json
+echo -n '${cw_prometheus_config}' > /etc/cloudwatch/cw_prometheus_config.json
+
+sudo mkdir -p /etc/prometheus
+echo -n '${prometheus_config}' > /etc/prometheus/config.yml
+
+sudo yum install -y amazon-cloudwatch-agent
+
+/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
+  -a fetch-config \
+  -m ec2 \
+  -c file:/etc/cloudwatch/cw_agent_config.json -s
+
+%{ else }
+
+echo 'CloudWatch agent NOT enabled'
+
+%{ endif }

templates/cw_agent_metrics_init.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+%{ if metrics_enabled }
+
+echo 'Enabling CloudWatch EC2 metrics'
+
+/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
+  -a append-config \
+  -m ec2 \
+  -c file:/etc/cloudwatch/cw_metrics_config.json -s
+
+%{ else }
+
+echo 'CloudWatch EC2 metrics NOT enabled'
+
+%{ endif }

templates/cw_agent_prometheus_init.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+%{ if prometheus_enabled }
+
+echo 'Enabling CloudWatch Prometheus metrics'
+
+/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
+  -a append-config \
+  -m ec2 \
+  -c file:/etc/cloudwatch/cw_prometheus_config.json -s
+
+%{ else }
+
+echo 'CloudWatch Prometheus metrics NOT enabled'
+
+%{ endif }

templates/web_user_data.sh

@@ -7,24 +7,17 @@ exec > >(tee /var/log/user-data.log|logger -t user-data-extra -s 2>/dev/console)
 sudo yum update -y
 sudo yum upgrade -y
 
-%{ if prometheus_enabled || metrics_enabled }
+sudo aws s3 cp s3://${var.user_data_bucket_name}/cw_agent_init.sh /tmp
+sudo chmod +x /tmp/cw_agent_init.sh
+/tmp/cw_agent_init.sh
 
-echo 'Configuring CloudWatch agent'
+sudo aws s3 cp s3://${var.user_data_bucket_name}/cw_agent_metrics_init.sh /tmp
+sudo chmod +x /tmp/cw_agent_metrics_init.sh
+/tmp/cw_agent_metrics_init.sh
 
-sudo mkdir -p /etc/prometheus
-echo -n '${prometheus_config}' > /etc/prometheus/config.yml
-
-sudo mkdir -p /etc/cloudwatch
-echo -n '${cloudwatch_config}' > /etc/cloudwatch/cloudwatch_config.json
-
-sudo yum install -y amazon-cloudwatch-agent
-
-/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
-  -a fetch-config \
-  -m ec2 \
-  -c file:/etc/cloudwatch/cloudwatch_config.json -s
-
-%{ endif }
+sudo aws s3 cp s3://${var.user_data_bucket_name}/cw_agent_prometheus_init.sh /tmp
+sudo chmod +x /tmp/cw_agent_prometheus_init.sh
+/tmp/cw_agent_prometheus_init.sh
 
 echo 'Configuring Concourse'
 

templates/worker_user_data.sh

@@ -7,21 +7,13 @@ exec > >(tee /var/log/user-data.log|logger -t user-data-extra -s 2>/dev/console)
 sudo yum update -y
 sudo yum upgrade -y
 
-%{ if metrics_enabled }
+sudo aws s3 cp s3://${var.user_data_bucket_name}/cw_agent_init.sh /tmp
+sudo chmod +x /tmp/cw_agent_init.sh
+/tmp/cw_agent_init.sh
 
-echo 'Configuring CloudWatch agent'
-
-sudo mkdir -p /etc/cloudwatch
-echo -n '${cloudwatch_config}' > /etc/cloudwatch/cloudwatch_config.json
-
-sudo yum install -y amazon-cloudwatch-agent
-
-/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
-  -a fetch-config \
-  -m ec2 \
-  -c file:/etc/cloudwatch/cloudwatch_config.json -s
-
-%{ endif }
+sudo aws s3 cp s3://${var.user_data_bucket_name}/cw_agent_metrics_init.sh /tmp
+sudo chmod +x /tmp/cw_agent_metrics_init.sh
+/tmp/cw_agent_metrics_init.sh
 
 echo 'Configuring Concourse'