fix(cluster): gg leaking dns records because of secondary record clobbering the primary

.dockerignore

@@ -4,8 +4,8 @@
 # Dockerfiles to.
 
 # Git
-**/.git/
-**/.gitignore
+# **/.git/
+# **/.gitignore
 
 **/.DS_Store
 **/symbolCache.db

infra/default-builds/dockerfiles/test-mm-lobby-echo/Dockerfile

@@ -1,4 +1,4 @@
-FROM clux/muslrust:1.75.0 AS build
+FROM clux/muslrust:1.77.2 AS build
 RUN cargo new --bin /app
 WORKDIR /app
 COPY Cargo.toml ./

infra/tf/cockroachdb_managed/main.tf

@@ -54,7 +54,10 @@ data "cockroach_cluster_cert" "main" {
 }
 
 resource "kubernetes_config_map" "crdb_ca" {
-	for_each = toset(["rivet-service", "bolt"])
+	for_each = toset(flatten([
+		["rivet-service", "bolt"],
+		var.prometheus_enabled ? ["grafana"] : []
+	]))
 
 	metadata {
 		name = "crdb-ca"

infra/tf/cockroachdb_managed/vars.tf

@@ -17,3 +17,7 @@ variable "cockroachdb_request_unit_limit" {
 variable "cockroachdb_storage_limit" {
 	type = string
 }
+
+variable "prometheus_enabled" {
+	type = bool
+}

infra/tf/grafana/grafana.tf

@@ -0,0 +1,142 @@
+locals {
+	service_grafana = lookup(var.services, "grafana", {
+		count = 1
+		resources = {
+			cpu = 500
+			memory = 512
+		}
+	})
+
+	grafana_dashboards = {
+		for f in fileset("${path.module}/grafana_dashboards/", "*.json"):
+		"${trimsuffix(f, ".json")}" => {
+			body = file("${path.module}/grafana_dashboards/${f}")
+		}
+	}
+
+	crdb_host = "${try(data.terraform_remote_state.cockroachdb_k8s.outputs.host, data.terraform_remote_state.cockroachdb_managed.outputs.host)}:${try(data.terraform_remote_state.cockroachdb_k8s.outputs.port, data.terraform_remote_state.cockroachdb_managed.outputs.port)}"
+}
+
+module "crdb_user_grafana_secrets" {
+	source = "../modules/secrets"
+
+	keys = [ "crdb/user/grafana/username", "crdb/user/grafana/password" ]
+}
+
+resource "helm_release" "grafana" {
+	name = "grafana"
+	namespace = "grafana"
+	repository = "https://grafana.github.io/helm-charts"
+	chart = "grafana"
+	version = "7.3.9"
+	values = [yamlencode({
+		"grafana.ini" = {
+			auth = {
+				disable_login_form = true
+			}
+			"auth.anonymous" = {
+				enabled = true
+				org_role = "Admin"
+			}
+		}	
+
+		resources = var.limit_resources ? {
+			limits = {
+				memory = "${local.service_grafana.resources.memory}Mi"
+				cpu = "${local.service_grafana.resources.cpu}m"
+			}
+		} : null
+
+		datasources = {
+			"datasources.yaml" = {
+				apiVersion = 1
+
+				datasources = [
+					{
+						name = "Prometheus"
+						type = "prometheus"
+						uid = "prometheus"
+						url = "http://prometheus-kube-prometheus-prometheus.prometheus:9090/"
+						access = "proxy"
+						isDefault = true
+						jsonData = {
+							httpMethod = "POST"
+							# prometheus.prometheusSpec.scrapeInterval
+							timeInterval = "30s"
+						}
+					},
+					{
+						name = "Loki"
+						type = "loki"
+						uid = "loki"
+						url = "http://loki-gateway.loki.svc.cluster.local:80/"
+						access = "proxy"
+						jsonData = {}
+					},
+					{
+						name = "CockroachDB"
+						type = "postgres"
+						uid = "crdb"
+						url = local.crdb_host
+						user = module.crdb_user_grafana_secrets.values["crdb/user/grafana/username"]
+						secureJsonData = {
+							password = module.crdb_user_grafana_secrets.values["crdb/user/grafana/password"]
+						}
+						jsonData = {
+							sslmode = "verify-ca"
+							sslRootCertFile = "/local/crdb/ca.crt"
+						}
+						secret = true
+					}
+				]
+			}
+		}
+
+		extraConfigmapMounts = [
+			# TLS Cert for postgres datasource
+			{
+				name = "crdb-ca"
+				configMap = "crdb-ca"
+				mountPath = "/local/crdb/ca.crt"
+				subPath = "ca.crt"
+				readOnly = true
+			}
+		]
+
+		sidecar = {
+			dashboards = {
+				enabled = true
+				searchNamespace = ["grafana", "prometheus"]
+			}
+		}
+
+		serviceMonitor = {
+			enabled = true
+			path = "/metrics"
+			labels = {}
+
+			interval = ""
+			scheme = "http"
+			tlsConfig = {}
+			scrapeTimeout = "15s"
+
+			relabelings = []
+		}
+	})]
+}
+
+resource "kubernetes_config_map" "grafana_dashboard" {
+	for_each = local.grafana_dashboards
+
+	metadata {
+		namespace = "grafana"
+		name = "grafana-rivet-${each.key}"
+		labels = {
+			grafana_dashboard = "1"
+		}
+	}
+
+	data = {
+		"${each.key}.json" = each.value.body
+	}
+}

infra/tf/grafana/providers.tf

@@ -0,0 +1,10 @@
+provider "kubernetes" {
+	config_path = var.kubeconfig_path
+}
+
+provider "helm" {
+	kubernetes {
+		config_path = var.kubeconfig_path
+	}
+}
+

infra/tf/grafana/vars.tf

@@ -0,0 +1,23 @@
+variable "namespace" {
+	type = string
+}
+
+# MARK: Services
+variable "services" {
+	type = map(object({
+		count = number
+		resources = object({
+			cpu = number
+			memory = number
+		})
+	}))
+}
+
+# MARK: K8s
+variable "kubeconfig_path" {
+	type = string
+}
+
+variable "limit_resources" {
+	type = bool
+}

infra/tf/k8s_infra/grafana.tf

@@ -1,20 +1,3 @@
-locals {
-	service_grafana = lookup(var.services, "grafana", {
-		count = 1
-		resources = {
-			cpu = 500
-			memory = 512
-		}
-	})
-
-	grafana_dashboards = {
-		for f in fileset("${path.module}/grafana_dashboards/", "*.json"):
-		"${trimsuffix(f, ".json")}" => {
-			body = file("${path.module}/grafana_dashboards/${f}")
-		}
-	}
-}
-
 resource "kubernetes_namespace" "grafana" {
 	count = var.prometheus_enabled ? 1 : 0
 
@@ -23,122 +6,3 @@ resource "kubernetes_namespace" "grafana" {
 	}
 }
 
-resource "helm_release" "grafana" {
-	count = var.prometheus_enabled ? 1 : 0
-	depends_on = [helm_release.vpa]
-
-	name = "grafana"
-	namespace = kubernetes_namespace.grafana.0.metadata.0.name
-	repository = "https://grafana.github.io/helm-charts"
-	chart = "grafana"
-	version = "7.3.9"
-	values = [yamlencode({
-		"grafana.ini" = {
-			auth = {
-				disable_login_form = true
-			}
-			"auth.anonymous" = {
-				enabled = true
-				org_role = "Admin"
-			}
-		}	
-
-		resources = var.limit_resources ? {
-			limits = {
-				memory = "${local.service_grafana.resources.memory}Mi"
-				cpu = "${local.service_grafana.resources.cpu}m"
-			}
-		} : null
-
-		datasources = {
-			"datasources.yaml" = {
-				apiVersion = 1
-
-				datasources = [
-					{
-						name = "Prometheus"
-						type = "prometheus"
-						uid = "prometheus"
-						url = "http://prometheus-kube-prometheus-prometheus.prometheus:9090/"
-						access = "proxy"
-						isDefault = true
-						jsonData = {
-							httpMethod = "POST"
-							# prometheus.prometheusSpec.scrapeInterval
-							timeInterval = "30s"
-						}
-					},
-					{
-						name = "Loki"
-						type = "loki"
-						uid = "loki"
-						url = "http://loki-gateway.loki.svc.cluster.local:80/"
-						access = "proxy"
-						jsonData = {}
-					},
-					{
-						name = "CockroachDB"
-						type = "postgres"
-						uid = "crdb"
-						url = local.crdb_host
-						user = module.crdb_user_grafana_secrets.values["crdb/user/grafana/username"]
-						secureJsonData = {
-							password = module.crdb_user_grafana_secrets.values["crdb/user/grafana/password"]
-						}
-						jsonData = {
-							sslmode = "verify-ca"
-							sslRootCertFile = "/local/crdb/ca.crt"
-						}
-						secret = true
-					}
-				]
-			}
-		}
-
-		extraConfigmapMounts = [
-			# TLS Cert for postgres datasource
-			{
-				name = kubernetes_config_map.crdb_ca["grafana"].metadata.0.name
-				configMap = "crdb-ca"
-				mountPath = "/local/crdb/ca.crt"
-				subPath = "ca.crt"
-				readOnly = true
-			}
-		]
-
-		sidecar = {
-			dashboards = {
-				enabled = true
-			}
-		}
-
-		serviceMonitor = {
-			enabled = true
-			path = "/metrics"
-			labels = {}
-
-			interval = ""
-			scheme = "http"
-			tlsConfig = {}
-			scrapeTimeout = "15s"
-
-			relabelings = []
-		}
-	})]
-}
-
-resource "kubernetes_config_map" "grafana_dashboard" {
-	for_each = var.prometheus_enabled ? local.grafana_dashboards : {}
-
-	metadata {
-		namespace = kubernetes_namespace.grafana.0.metadata.0.name
-		name = "grafana-rivet-${each.key}"
-		labels = {
-			grafana_dashboard = "1"
-		}
-	}
-
-	data = {
-		"${each.key}.json" = each.value.body
-	}
-}

infra/tf/k8s_infra/prometheus.tf

@@ -63,8 +63,6 @@ locals {
 			]
 		}] : []
 	])
-
-	crdb_host = "${try(data.terraform_remote_state.cockroachdb_k8s.outputs.host, data.terraform_remote_state.cockroachdb_managed.outputs.host)}:${try(data.terraform_remote_state.cockroachdb_k8s.outputs.port, data.terraform_remote_state.cockroachdb_managed.outputs.port)}"
 }
 
 module "alertmanager_secrets" {
@@ -320,6 +318,7 @@ resource "helm_release" "prometheus" {
 		# Configured in grafana.tf
 		grafana = {
 			enabled = false
+			forceDeployDashboards = true
 		}
 
 		extraManifests = flatten([

infra/tf/k8s_infra/traefik.tf

@@ -8,6 +8,7 @@ module "traefik_secrets" {
 	source = "../modules/secrets"
 
 	keys = [
+		"rivet/api_route/token",
 		"rivet/api_traefik_provider/token",
 	]
 }
@@ -64,7 +65,9 @@ resource "helm_release" "traefik" {
 		} : null
 
 		additionalArguments = [
-			"--providers.http.endpoint=http://rivet-api-internal-monolith.rivet-service.svc.cluster.local/traefik-provider/config/core?token=${module.traefik_secrets.values["rivet/api_traefik_provider/token"]}",
+			# "--providers.http.endpoint=http://rivet-api-internal-monolith.rivet-service.svc.cluster.local/traefik-provider/config/core?token=${module.traefik_secrets.values["rivet/api_traefik_provider/token"]}",
+			# LEGACY:
+			"--providers.http.endpoint=http://rivet-api-route.rivet-service.svc.cluster.local/traefik/config/core?token=${module.traefik_secrets.values["rivet/api_route/token"]}",
 			"--providers.http.pollInterval=2.5s",
 			# See docs/infrastructure/TIMEOUTS.md
 			"--entryPoints.web.transport.lifeCycle.graceTimeOut=60s",