Install/configure new CloudWatch agent

Use new agent for collecting CloudWatch metrics and logs.

aws/cloudformation/cloud_formation_stack.yml.erb

@@ -108,8 +108,7 @@ Resources:
               - 'logs:PutRetentionPolicy'
               - 'logs:PutLogEvents'
             Resource:
-              - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:<%=environment%>-syslog"
-              - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:<%=environment%>-syslog:log-stream:*"
+              - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:<%=environment%>-*"
               - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${AWS::StackName}"
               - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${AWS::StackName}:log-stream:*"
           # Put custom metrics to CloudWatch.

cookbooks/.kitchen.ec2.yml

@@ -11,11 +11,11 @@ driver:
   instance_type: m5.4xlarge
 <% if ENV['SUBNET_ID'] %>
   subnet_id: <%= ENV['SUBNET_ID'] %>
+  associate_public_ip: true
 <% end %>
 <% if ENV['IMAGE_ID'] %>
   image_id: <%= ENV['IMAGE_ID']%>
 <% end %>
-  associate_public_ip: true
   block_device_mappings:
     - device_name: /dev/sda1
       ebs:
@@ -29,3 +29,6 @@ driver:
   iam_profile_name: <%=ENV['IAM_PROFILE_NAME']%>
 provisioner:
   name: chef_zero
+  environments_path: ../cdo-apps/test/environments
+  client_rb:
+    environment: adhoc

cookbooks/Berksfile.lock

@@ -10,10 +10,8 @@ DEPENDENCIES
     path: cdo-authorized-keys
   cdo-awscli
     path: cdo-awscli
-  cdo-cloudwatch-extra-metrics
-    path: cdo-cloudwatch-extra-metrics
-  cdo-cloudwatch-logger
-    path: cdo-cloudwatch-logger
+  cdo-cloudwatch-agent
+    path: cdo-cloudwatch-agent
   cdo-github-access
     path: cdo-github-access
   cdo-home-ubuntu
@@ -40,6 +38,8 @@ DEPENDENCIES
     path: cdo-ruby
   cdo-secrets
     path: cdo-secrets
+  cdo-syslog
+    path: cdo-syslog
   cdo-tippecanoe
     path: cdo-tippecanoe
   cdo-users
@@ -58,6 +58,7 @@ GRAPH
   ark (4.0.0)
     build-essential (>= 0.0.0)
     seven_zip (>= 0.0.0)
+  aws_cloudwatch (1.0.2)
   build-essential (8.2.1)
     mingw (>= 1.1)
     seven_zip (>= 0.0.0)
@@ -68,8 +69,7 @@ GRAPH
     apt (>= 0.0.0)
     build-essential (>= 0.0.0)
     cdo-analytics (>= 0.0.0)
-    cdo-cloudwatch-extra-metrics (>= 0.0.0)
-    cdo-cloudwatch-logger (>= 0.0.0)
+    cdo-cloudwatch-agent (>= 0.0.0)
     cdo-i18n (>= 0.0.0)
     cdo-java-7 (>= 0.0.0)
     cdo-jemalloc (>= 0.0.0)
@@ -81,6 +81,7 @@ GRAPH
     cdo-repository (>= 0.0.0)
     cdo-ruby (>= 0.0.0)
     cdo-secrets (>= 0.0.0)
+    cdo-syslog (>= 0.0.0)
     cdo-tippecanoe (>= 0.0.0)
     cdo-varnish (>= 0.0.0)
     chef_client_updater (>= 0.0.0)
@@ -93,12 +94,8 @@ GRAPH
     apt (~> 2.6.0)
     poise-python (>= 0.0.0)
     sudo-user (>= 0.0.0)
-  cdo-cloudwatch-extra-metrics (0.1.8)
-    ark (>= 0.0.0)
-  cdo-cloudwatch-logger (0.1.14)
-    cdo-awscli (>= 0.0.0)
-    poise-service (>= 0.0.0)
-    sudo-user (>= 0.0.0)
+  cdo-cloudwatch-agent (0.1.1)
+    aws_cloudwatch (>= 0.0.0)
   cdo-github-access (0.1.9)
   cdo-home-ubuntu (0.1.3)
   cdo-i18n (0.0.0)
@@ -108,7 +105,7 @@ GRAPH
     build-essential (>= 0.0.0)
   cdo-jemalloc (0.1.4)
     ark (>= 0.0.0)
-  cdo-mysql (0.1.25)
+  cdo-mysql (0.1.27)
     apt (~> 2.6.0)
   cdo-nginx (0.0.16)
     apt (>= 0.0.0)
@@ -126,6 +123,8 @@ GRAPH
     apt (~> 2.6.0)
     sudo-user (>= 0.0.0)
   cdo-secrets (0.1.8)
+  cdo-syslog (0.1.1)
+    sudo-user (>= 0.0.0)
   cdo-tippecanoe (0.1.4)
     ark (>= 0.0.0)
   cdo-users (0.1.30)

cookbooks/cdo-apps/metadata.rb

@@ -10,7 +10,7 @@
 depends 'build-essential'
 depends 'yarn'
 
-depends 'cdo-cloudwatch-extra-metrics'
+depends 'cdo-cloudwatch-agent'
 depends 'cdo-repository'
 depends 'cdo-secrets'
 depends 'cdo-postfix'
@@ -27,6 +27,6 @@
 depends 'cdo-redis'
 depends 'cdo-i18n'
 depends 'cdo-analytics'
-depends 'cdo-cloudwatch-logger'
+depends 'cdo-syslog'
 depends 'cdo-jemalloc'
 depends 'cdo-tippecanoe'

cookbooks/cdo-apps/recipes/default.rb

@@ -85,8 +85,8 @@
 include_recipe 'cdo-postfix'
 include_recipe 'cdo-varnish'
 
-include_recipe 'cdo-cloudwatch-extra-metrics'
-include_recipe 'cdo-cloudwatch-logger' if node[:ec2]
+include_recipe 'cdo-cloudwatch-agent'
+include_recipe 'cdo-syslog'
 
 include_recipe 'cdo-apps::jemalloc' if node['cdo-apps']['jemalloc']
 include_recipe 'cdo-apps::bundle_bootstrap'

cookbooks/cdo-cloudwatch-extra-metrics/.kitchen.yml → cookbooks/cdo-cloudwatch-agent/.kitchen.yml

@@ -1,19 +1,25 @@
 # Note: EC2 driver with IAM Instance Profile required to run integration test successfully, e.g.:
 # IAM_PROFILE_NAME=[IAMProfileName] KITCHEN_LOCAL_YAML=../.kitchen.ec2.yml bundle exec kitchen verify
 ---
-transport:
-  name: sftp
 driver:
-  name: docker
-  use_sudo: false
+  name: dokken
+  chef_version: 15.2.20
+  privileged: true
+verifier:
+  name: inspec
+transport:
+  name: dokken
 provisioner:
-  name: chef_zero
-  require_chef_omnibus: 15.2.20
+  name: dokken
 platforms:
   - name: ubuntu-18.04
+    driver:
+      image: dokken/ubuntu-18.04
     run_list:
       - recipe[apt]
 suites:
   - name: default
     run_list:
-      - recipe[cdo-cloudwatch-extra-metrics]
+      - recipe[cdo-cloudwatch-agent]
+    attributes:
+      cdo-cloudwatch-agent:

cookbooks/cdo-cloudwatch-agent/README.md

@@ -0,0 +1,5 @@
+cdo-cloudwatch-agent Cookbook
+=====================================
+
+Installs/configures CloudWatch Agent, following setup instructions defined at
+https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Install-CloudWatch-Agent.html

cookbooks/cdo-cloudwatch-agent/attributes/default.rb

@@ -0,0 +1,6 @@
+default['cdo-cloudwatch-agent'] = {
+  log_files: {
+    amazon_cloudwatch_agent: '/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log',
+    syslog: '/var/log/syslog'
+  }
+}

cookbooks/cdo-cloudwatch-agent/metadata.rb

@@ -0,0 +1,9 @@
+name             'cdo-cloudwatch-agent'
+maintainer       'Code.org'
+maintainer_email 'will@code.org'
+license          'All rights reserved'
+description      'Installs/Configures cdo-cloudwatch-agent'
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version          '0.1.1'
+
+depends 'aws_cloudwatch'

cookbooks/cdo-cloudwatch-agent/recipes/default.rb

@@ -0,0 +1,9 @@
+#
+# Cookbook Name:: cdo-cloudwatch-agent
+# Recipe:: default
+#
+
+aws_cloudwatch_agent 'default' do
+  action      [:install, :configure, :restart]
+  json_config 'amazon-cloudwatch-agent.json.erb'
+end

cookbooks/cdo-cloudwatch-agent/templates/default/amazon-cloudwatch-agent.json.erb

@@ -0,0 +1,81 @@
+{
+  "metrics": {
+    "append_dimensions": {
+      "AutoScalingGroupName":"${aws:AutoScalingGroupName}",
+      "InstanceId":"${aws:InstanceId}"
+    },
+    "aggregation_dimensions": [["AutoScalingGroupName"]],
+    "metrics_collected": {
+      "statsd": {},
+      "cpu": {
+        "totalcpu": true,
+        "measurement": [
+          "cpu_usage_idle",
+          "cpu_usage_nice",
+          "cpu_usage_iowait",
+          "cpu_usage_system",
+          "cpu_usage_user"
+        ],
+        "metrics_collection_interval": 1,
+        "append_dimensions": {
+          "Environment": "<%=node.chef_environment%>"
+        }
+      },
+      "disk": {
+        "measurement": [
+          "free",
+          "used_percent"
+        ],
+        "ignore_file_system_types": [
+          "sysfs", "devtmpfs", "squashfs", "tmpfs"
+        ],
+        "append_dimensions": {
+          "Environment": "<%=node.chef_environment%>"
+        }
+      },
+      "swap": {
+        "measurement": [
+          "swap_used",
+          "swap_free"
+        ],
+        "append_dimensions": {
+          "Environment": "<%=node.chef_environment%>"
+        }
+      },
+      "mem": {
+        "measurement": [
+          "mem_used_percent",
+          "mem_cached",
+          "mem_total"
+        ],
+        "metrics_collection_interval": 1,
+        "append_dimensions": {
+          "Environment": "<%=node.chef_environment%>"
+        }
+      },
+      "processes": {
+        "measurement": [
+          "processes_running",
+          "processes_sleeping",
+          "processes_total",
+          "processes_dead",
+          "total_threads"
+        ],
+        "append_dimensions": {
+          "Environment": "<%=node.chef_environment%>"
+        }
+      }
+    },
+    "force_flush_interval" : 30
+  },
+  "logs": {
+    "logs_collected": {
+      "files": {
+        "collect_list": <%= node.default['cdo-cloudwatch-agent']['log_files'].map do |name, path|
+          {log_group_name: "#{node.chef_environment}-#{name}", file_path: path}
+        end.to_json %>
+      }
+    },
+    "force_flush_interval" : 15
+  }
+}

cookbooks/cdo-cloudwatch-agent/test/integration/default/inspec/ruby_spec.rb

@@ -0,0 +1,11 @@
+require 'serverspec'
+set :backend, :exec
+
+def cmd(exec, match)
+  describe command(exec) do
+    its(:stdout) {should match match}
+  end
+end
+
+# Ensure proper version of agent is installed.
+cmd '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent --version', 'AmazonCloudWatchAgent v1.231221.0'