chore(vector): filter unneeded go & prom metrics

rivet-gg · May 31, 2024 · 31e8e43 · 31e8e43
1 parent f97a051
commit 31e8e43
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 12 deletions.
diff --git a/infra/tf/vector/vars.tf b/infra/tf/vector/vars.tf
@@ -2,6 +2,10 @@ variable "namespace" {
 	type = string
 }
 
+variable "deploy_method_cluster" {
+	type = bool
+}
+
 # MARK: ClickHouse
 variable "clickhouse_enabled" {
 	type = bool

diff --git a/infra/tf/vector/vector.tf b/infra/tf/vector/vector.tf
@@ -1,6 +1,6 @@
 locals {
 	service_vector = lookup(var.services, "vector", {
-		count = 1
+		count = var.deploy_method_cluster ? 2 : 1
 		resources = {
 			cpu = 1000
 			memory = 2048
@@ -23,9 +23,10 @@ resource "helm_release" "vector" {
 	namespace = "vector"
 	repository = "https://helm.vector.dev"
 	chart = "vector"
-	version = "0.38.0"
+	version = "0.33.0"
 	values = [yamlencode({
 		role = "Aggregator"
+		replicas = local.service_vector.count
 		podPriorityClassName = "service-priority"
 		resources = var.limit_resources ? {
 			limits = {
@@ -40,7 +41,7 @@ resource "helm_release" "vector" {
 			data_dir = "/vector-data-dir"
 			api = {
 				enabled = true
-				address = "127.0.0.1:8686"
+				address = "0.0.0.0:8686"
 				playground = false
 			}
 			sources = {

diff --git a/svc/pkg/cluster/worker/src/workers/server_install/install_scripts/components/vector.rs b/svc/pkg/cluster/worker/src/workers/server_install/install_scripts/components/vector.rs
@@ -38,15 +38,16 @@ pub fn configure(config: &Config, pool_type: backend::cluster::PoolType) -> Stri
 		[api]
 			enabled = true
 
+		[transforms.filter_metrics]
+            type = "filter"
+			inputs = [{sources}]
+            condition = '!starts_with!(.name, "go_") && !starts_with!(.name, "promhttp_")'
+
+
 		[transforms.add_meta]
 			type = "remap"
-			inputs = [{sources}]
+			inputs = ["filter_metrics"]
 			source = '''
-			# Drop go stats
-			if starts_with!(.name, "go_") {{
-				abort
-			}}
-
 			.tags.server_id = "___SERVER_ID___"
 			.tags.datacenter_id = "___DATACENTER_ID___"
 			.tags.cluster_id = "___CLUSTER_ID___"
@@ -62,9 +63,9 @@ pub fn configure(config: &Config, pool_type: backend::cluster::PoolType) -> Stri
 			compression = true
 
 			# Buffer to disk for durability & reduce memory usage
-			buffer.max_events = 500
-			buffer.max_size = 268435488
-			buffer.type = "disk"
+            buffer.type = "disk"
+            buffer.max_size = 268435488  # 256 MB
+            buffer.when_full = "block"
 		"#
 	);