Create prometheus configmap for each instance size

base/monitoring/cadvisor/cadvisor.Service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: cadvisor
+  annotations:
+    prometheus.io/port: "48080"
+    sourcegraph.prometheus/scrape: "true"
+  labels:
+    app: cadvisor
+    deploy: sourcegraph
+    sourcegraph-resource-requires: no-cluster-admin
+    app.kubernetes.io/component: cadvisor
+spec:
+  ports:
+    - name: http # For querying metrics.
+      port: 48080
+  selector:
+    app: cadvisor
+  type: ClusterIP

base/monitoring/cadvisor/kustomization.yaml

@@ -3,3 +3,4 @@ kind: Kustomization
 resources:
   - cadvisor.DaemonSet.yaml
   - cadvisor.ServiceAccount.yaml
+  - cadvisor.Service.yaml

base/monitoring/grafana/grafana.ConfigMap.yaml

@@ -1,4 +1,11 @@
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    deploy: sourcegraph
+    sourcegraph-resource-requires: no-cluster-admin
+    app.kubernetes.io/component: grafana
+  name: grafana
 data:
   datasources.yml: |
     apiVersion: 1
@@ -10,10 +17,3 @@ data:
         url: http://prometheus:30090
         isDefault: true
         editable: false
-kind: ConfigMap
-metadata:
-  labels:
-    deploy: sourcegraph
-    sourcegraph-resource-requires: no-cluster-admin
-    app.kubernetes.io/component: grafana
-  name: grafana

base/monitoring/prometheus/prometheus.ConfigMap.yaml

@@ -30,36 +30,62 @@ data:
 
     # Configure targets to scrape
     scrape_configs:
+
       # Scrape prometheus itself for metrics.
       - job_name: 'builtin-prometheus'
         static_configs:
           - targets: ['127.0.0.1:9092']
+
       - job_name: 'builtin-alertmanager'
         metrics_path: /alertmanager/metrics
         static_configs:
           - targets: ['127.0.0.1:9093']
 
-      - job_name: 'sg'
+      - job_name: 'sourcegraph-services'
         relabel_configs:
           - source_labels: [__address__]
             target_label: instance
             regex: (.*)\.(.*)
             replacement: ${1}_${2}
+        metric_relabel_configs:
+          - source_labels: [container_label_io_kubernetes_pod_namespace]
+            regex: kube-system
+            action: drop
         file_sd_configs:
           - files:
               - '/sg_prometheus_add_ons/*_targets.yml'
+
+      - job_name: 'cadvisor'
+        static_configs:
+          - targets: ['cadvisor:48080']
+        relabel_configs:
+          - source_labels: [__address__]
+            target_label: instance
+            regex: (.*)\.(.*)
+            replacement: ${1}_${2}
+          - source_labels: [container_label_io_kubernetes_pod_name]
+            target_label: name
+        metric_relabel_configs:
+          - source_labels: [container_label_io_kubernetes_pod_namespace]
+            regex: kube-system
+            action: drop
+          - source_labels: [container_label_io_kubernetes_container_name, container_label_io_kubernetes_pod_name]
+            regex: (.+)
+            action: replace
+            target_label: name
+            separator: '-'
+
   prometheus_targets.yml: |
     - labels:
         nodename: "sourcegraph-services"
-        job: node
+        job: sourcegraph-frontend
       targets:
-        - cadvisor:8080
-        - sourcegraph-frontend-internal:6060
+        - sourcegraph-frontend:6060
     - labels:
         nodename: "sourcegraph-services"
         job: github-proxy
       targets:
-        - github-proxy:80
+        - github-proxy:6060
     - labels:
         nodename: "sourcegraph-services"
         job: repo-updater
@@ -74,7 +100,7 @@ data:
         nodename: "sourcegraph-services"
         job: worker-executors
       targets:
-        - worker:6996
+        - worker-executors:6996
     - labels:
         nodename: "sourcegraph-services"
         job: node
@@ -85,97 +111,6 @@ data:
         job: precise-code-intel-worker
       targets:
         - precise-code-intel-worker:6060
-        # Add new entries here for every searcher/symbol/gitserver replica.
-    - labels:
-        nodename: "sourcegraph-services"
-        job: zoekt-indexserver
-      targets:
-        - indexed-search-0.indexed-search:6072
-        - indexed-search-1.indexed-search:6072
-        - indexed-search-2.indexed-search:6072
-        - indexed-search-3.indexed-search:6072
-        - indexed-search-4.indexed-search:6072
-        - indexed-search-5.indexed-search:6072
-        - indexed-search-6.indexed-search:6072
-        - indexed-search-7.indexed-search:6072
-        - indexed-search-8.indexed-search:6072
-        - indexed-search-9.indexed-search:6072
-        - indexed-search-10.indexed-search:6072
-    - labels:
-        nodename: "sourcegraph-services"
-        job: zoekt-webserver
-      targets:
-        - indexed-search-0.indexed-search:6070
-        - indexed-search-1.indexed-search:6070
-        - indexed-search-2.indexed-search:6070
-        - indexed-search-3.indexed-search:6070
-        - indexed-search-4.indexed-search:6070
-        - indexed-search-5.indexed-search:6070
-        - indexed-search-6.indexed-search:6070
-        - indexed-search-7.indexed-search:6070
-        - indexed-search-8.indexed-search:6070
-        - indexed-search-9.indexed-search:6070
-        - indexed-search-10.indexed-search:6070
-    - labels:
-        nodename: "sourcegraph-services"
-        job: sourcegraph-frontend
-      targets:
-        - sourcegraph-frontend-0.sourcegraph-frontend:6060
-        - sourcegraph-frontend-1.sourcegraph-frontend:6060
-        - sourcegraph-frontend-2.sourcegraph-frontend:6060
-        - sourcegraph-frontend-3.sourcegraph-frontend:6060
-        - sourcegraph-frontend-4.sourcegraph-frontend:6060
-        - sourcegraph-frontend-5.sourcegraph-frontend:6060
-        - sourcegraph-frontend-6.sourcegraph-frontend:6060
-        - sourcegraph-frontend-7.sourcegraph-frontend:6060
-        - sourcegraph-frontend-8.sourcegraph-frontend:6060
-        - sourcegraph-frontend-9.sourcegraph-frontend:6060
-        - sourcegraph-frontend-10.sourcegraph-frontend:6060
-    - labels:
-        nodename: "sourcegraph-services"
-        job: gitserver
-      targets:
-        - gitserver-0.gitserver:6060
-        - gitserver-1.gitserver:6060
-        - gitserver-2.gitserver:6060
-        - gitserver-3.gitserver:6060
-        - gitserver-4.gitserver:6060
-        - gitserver-5.gitserver:6060
-        - gitserver-6.gitserver:6060
-        - gitserver-7.gitserver:6060
-        - gitserver-8.gitserver:6060
-        - gitserver-9.gitserver:6060
-        - gitserver-10.gitserver:6060
-    - labels:
-        nodename: "sourcegraph-services"
-        job: searcher
-      targets:
-        - searcher-0.searcher:6060
-        - searcher-1.searcher:6060
-        - searcher-2.searcher:6060
-        - searcher-3.searcher:6060
-        - searcher-4.searcher:6060
-        - searcher-5.searcher:6060
-        - searcher-6.searcher:6060
-        - searcher-7.searcher:6060
-        - searcher-8.searcher:6060
-        - searcher-9.searcher:6060
-        - searcher-10.searcher:6060
-    - labels:
-        nodename: "sourcegraph-services"
-        job: symbols
-      targets:
-        - symbols-0.symbols:6060
-        - symbols-1.symbols:6060
-        - symbols-2.symbols:6060
-        - symbols-3.symbols:6060
-        - symbols-4.symbols:6060
-        - symbols-5.symbols:6060
-        - symbols-6.symbols:6060
-        - symbols-7.symbols:6060
-        - symbols-8.symbols:6060
-        - symbols-9.symbols:6060
-        - symbols-10.symbols:6060
     - labels:
         nodename: "sourcegraph-services"
         job: pgsql
@@ -211,9 +146,10 @@ data:
         job: otel-collector
       targets:
         - otel-collector:8888
+
   extra_rules.yml: |
     groups:
-    - name: docker.rules
+    - name: container.rules
       rules:
         - record: container:process_cpu_seconds_total:ratio_rate5m
           expr: sum by (instance) (rate(process_cpu_seconds_total[5m])) / engine_daemon_engine_cpus_cpus
@@ -223,3 +159,36 @@ data:
           expr: max by (instance) (process_resident_memory_bytes)
         - record: container:process_virtual_memory_bytes:max
           expr: max by (instance) (process_virtual_memory_bytes)
+
+  symbols_targets.yml: |
+    - labels:
+        nodename: "sourcegraph-services"
+        job: symbols
+      targets:
+        - symbols-0.symbols:6060
+
+  searcher_targets.yml: |
+    - labels:
+        nodename: "sourcegraph-services"
+        job: searcher
+      targets:
+        - searcher-0.searcher:6060
+
+  gitserver_targets.yml: |
+    - labels:
+        nodename: "sourcegraph-services"
+        job: gitserver
+      targets:
+        - gitserver-0.gitserver:6060
+
+  indexed-search_targets.yml: |
+    - labels:
+        nodename: "sourcegraph-services"
+        job: zoekt-indexserver
+      targets:
+        - indexed-search-0.indexed-search:6072
+    - labels:
+        nodename: "sourcegraph-services"
+        job: zoekt-webserver
+      targets:
+        - indexed-search-0.indexed-search:6070

base/sourcegraph/frontend/sourcegraph-frontend.Service.yaml

@@ -15,6 +15,9 @@ spec:
     - name: http
       port: 30080
       targetPort: http
+    - name: debug
+      port: 6060
+      targetPort: debug
   selector:
     app: sourcegraph-frontend
   type: ClusterIP

base/sourcegraph/redis/redis-cache.Service.yaml

@@ -15,6 +15,9 @@ spec:
     - name: redis
       port: 6379
       targetPort: redis
+    - name: redisexp
+      port: 9121
+      targetPort: redisexp
   selector:
     app: redis-cache
   type: ClusterIP

base/sourcegraph/redis/redis-store.Service.yaml

@@ -15,6 +15,9 @@ spec:
     - name: redis
       port: 6379
       targetPort: redis
+    - name: redisexp
+      port: 9121
+      targetPort: redisexp
   selector:
     app: redis-store
   type: ClusterIP

base/sourcegraph/searcher/searcher.Service.yaml

@@ -11,6 +11,7 @@ metadata:
     sourcegraph-resource-requires: no-cluster-admin
   name: searcher
 spec:
+  clusterIP: None
   ports:
     - name: http
       port: 3181

base/sourcegraph/symbols/symbols.Service.yaml

@@ -11,6 +11,7 @@ metadata:
     sourcegraph-resource-requires: no-cluster-admin
   name: symbols
 spec:
+  clusterIP: None
   ports:
     - name: http
       port: 3184

base/sourcegraph/syntect-server/syntect-server.Deployment.yaml

@@ -44,6 +44,8 @@ spec:
           ports:
             - containerPort: 9238
               name: http
+            - containerPort: 6060
+              name: debug
           readinessProbe:
             tcpSocket:
               port: http

base/sourcegraph/syntect-server/syntect-server.Service.yaml

@@ -1,6 +1,9 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    prometheus.io/port: "6060"
+    sourcegraph.prometheus/scrape: "true"
   labels:
     app: syntect-server
     app.kubernetes.io/component: syntect-server
@@ -12,6 +15,9 @@ spec:
     - name: http
       port: 9238
       targetPort: http
+    - name: debug
+      port: 6060
+      targetPort: debug
   selector:
     app: syntect-server
   type: ClusterIP

components/monitoring/rbacs/prometheus/prometheus.ConfigMap.yaml

@@ -296,3 +296,7 @@ data:
             app: alertmanager
   extra_rules.yml: ""
   prometheus_targets.yml: ""
+  symbols_targets.yml: ""
+  searcher_targets.yml: ""
+  gitserver_targets.yml: ""
+  indexed-search_targets.yml: ""

components/sizes/l/kustomization.yaml

@@ -37,3 +37,8 @@ patchesJson6902:
       group: apps
       version: v1
     path: patches/codeintel-db.yaml
+  - target:
+      kind: ConfigMap
+      name: prometheus
+      version: v1
+    path: patches/prometheus.yaml