From 8d7b211dd62d9f5ace6e3cad230ca4479866a56f Mon Sep 17 00:00:00 2001
From: Ash Berlin-Taylor <ash_github@firemirror.com>
Date: Fri, 21 Jan 2022 20:15:12 +0000
Subject: [PATCH] Disable shipping logs to CloudWatch

This was useful when we were debugging the "communication lost with
instance" but we haven't seen those in months, and CloudWatch Logs
accounts for 10% of our monthly AWS spend(!) and we just don't need it
anymore

I have included some previously un-pushed changes to the Vector config
to drop a few of the more common and less interesting lines. Useful for
posterity.
---
 cloud-init.yml                             |  5 +++--
 github-runner-ami/packer/files/vector.toml | 12 ++++++++++--
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/cloud-init.yml b/cloud-init.yml
index 183a6f4..bc69e87 100644
--- a/cloud-init.yml
+++ b/cloud-init.yml
@@ -1,5 +1,5 @@
 #cloud-config
-# 2021-05-17 09:21+01:00
+# 2022-01-21 19:31 BMT
 
 mounts:
   - [tmpfs, /tmp, tmpfs, "defaults,noatime,size=10%"]
@@ -15,5 +15,6 @@ runcmd:
       set -eu -o pipefail
       echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/environment
       echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/default/vector
-      systemctl enable --now vector
+      #aws logs create-log-stream --log-group-name GitHubRunners --log-stream-name $(hostname) || true
+      #systemctl enable --now vector
       systemctl enable --now actions.runner
diff --git a/github-runner-ami/packer/files/vector.toml b/github-runner-ami/packer/files/vector.toml
index 81112e6..f9f3308 100644
--- a/github-runner-ami/packer/files/vector.toml
+++ b/github-runner-ami/packer/files/vector.toml
@@ -31,8 +31,8 @@ data_dir = "/var/lib/vector"
   fields = ["_CAP_EFFECTIVE", "_SYSTEMD_SLICE", "_SYSTEMD_CGROUP",
     "_SYSTEMD_INVOCATION_ID", "_SELINUX_CONTEXT", "_COMM", "_BOOT_ID",
     "_MACHINE_ID", "_STREAM_ID", "_PID", "_GID", "_UID","_TRANSPORT",
-    "__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "PRIORITY",
-    "source_type"]
+    "__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "SYSLOG_TIMESTAMP",
+    "PRIORITY", "_EXE", "source_type"]
 
 [sources.runner-logs]
   type = "file"
@@ -64,6 +64,9 @@ data_dir = "/var/lib/vector"
       !match!(.message, r'Try to append \d+ batches web console lines for record')
     } else if .logger == "HostContext" {
       !starts_with!(.message, "Well known directory")
+    } else if .logger == "JobDispatcher" {
+      !starts_with!(.message, "Successfully renew job request")
+    }
     } else {
       true
     }
@@ -76,12 +79,17 @@ data_dir = "/var/lib/vector"
 [transforms.grok-job-logs]
   type = "remap"
   inputs = ["job-logs"]
+  drop_on_abort = true
   source = '''
     structured, err = parse_grok(.message, "%{TIMESTAMP_ISO8601:timestamp} %{GREEDYDATA:message}")
 
     if err == null {
       . = merge(., structured)
       .type = "job-output"
+
+      if length(.message) == 0 ?? true {
+        abort # Don't send empty messages to cloud watch!
+      }
     }
   '''