From 31e8e43c573632ea9f0da987287f3c2ccb00a0f1 Mon Sep 17 00:00:00 2001
From: Nathan Flurry <git@nathanflurry.com>
Date: Fri, 31 May 2024 23:10:38 +0000
Subject: [PATCH] chore(vector): filter unneeded go & prom metrics

---
 infra/tf/vector/vars.tf                       |  4 ++++
 infra/tf/vector/vector.tf                     |  7 ++++---
 .../install_scripts/components/vector.rs      | 19 ++++++++++---------
 3 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/infra/tf/vector/vars.tf b/infra/tf/vector/vars.tf
index a5667809e..19150b5da 100644
--- a/infra/tf/vector/vars.tf
+++ b/infra/tf/vector/vars.tf
@@ -2,6 +2,10 @@ variable "namespace" {
 	type = string
 }
 
+variable "deploy_method_cluster" {
+	type = bool
+}
+
 # MARK: ClickHouse
 variable "clickhouse_enabled" {
 	type = bool
diff --git a/infra/tf/vector/vector.tf b/infra/tf/vector/vector.tf
index ff48a8819..3c10b5ddc 100644
--- a/infra/tf/vector/vector.tf
+++ b/infra/tf/vector/vector.tf
@@ -1,6 +1,6 @@
 locals {
 	service_vector = lookup(var.services, "vector", {
-		count = 1
+		count = var.deploy_method_cluster ? 2 : 1
 		resources = {
 			cpu = 1000
 			memory = 2048
@@ -23,9 +23,10 @@ resource "helm_release" "vector" {
 	namespace = "vector"
 	repository = "https://helm.vector.dev"
 	chart = "vector"
-	version = "0.38.0"
+	version = "0.33.0"
 	values = [yamlencode({
 		role = "Aggregator"
+		replicas = local.service_vector.count
 		podPriorityClassName = "service-priority"
 		resources = var.limit_resources ? {
 			limits = {
@@ -40,7 +41,7 @@ resource "helm_release" "vector" {
 			data_dir = "/vector-data-dir"
 			api = {
 				enabled = true
-				address = "127.0.0.1:8686"
+				address = "0.0.0.0:8686"
 				playground = false
 			}
 			sources = {
diff --git a/svc/pkg/cluster/worker/src/workers/server_install/install_scripts/components/vector.rs b/svc/pkg/cluster/worker/src/workers/server_install/install_scripts/components/vector.rs
index 8e0642763..67ec8a199 100644
--- a/svc/pkg/cluster/worker/src/workers/server_install/install_scripts/components/vector.rs
+++ b/svc/pkg/cluster/worker/src/workers/server_install/install_scripts/components/vector.rs
@@ -38,15 +38,16 @@ pub fn configure(config: &Config, pool_type: backend::cluster::PoolType) -> Stri
 		[api]
 			enabled = true
 
+		[transforms.filter_metrics]
+            type = "filter"
+			inputs = [{sources}]
+            condition = '!starts_with!(.name, "go_") && !starts_with!(.name, "promhttp_")'
+
+
 		[transforms.add_meta]
 			type = "remap"
-			inputs = [{sources}]
+			inputs = ["filter_metrics"]
 			source = '''
-			# Drop go stats
-			if starts_with!(.name, "go_") {{
-				abort
-			}}
-
 			.tags.server_id = "___SERVER_ID___"
 			.tags.datacenter_id = "___DATACENTER_ID___"
 			.tags.cluster_id = "___CLUSTER_ID___"
@@ -62,9 +63,9 @@ pub fn configure(config: &Config, pool_type: backend::cluster::PoolType) -> Stri
 			compression = true
 
 			# Buffer to disk for durability & reduce memory usage
-			buffer.max_events = 500
-			buffer.max_size = 268435488
-			buffer.type = "disk"
+            buffer.type = "disk"
+            buffer.max_size = 268435488  # 256 MB
+            buffer.when_full = "block"
 		"#
 	);