diff --git a/base/monitoring/cadvisor/cadvisor.Service.yaml b/base/monitoring/cadvisor/cadvisor.Service.yaml new file mode 100644 index 00000000..fb9eea06 --- /dev/null +++ b/base/monitoring/cadvisor/cadvisor.Service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: cadvisor + annotations: + prometheus.io/port: "48080" + sourcegraph.prometheus/scrape: "true" + labels: + app: cadvisor + deploy: sourcegraph + sourcegraph-resource-requires: no-cluster-admin + app.kubernetes.io/component: cadvisor +spec: + ports: + - name: http # For querying metrics. + port: 48080 + selector: + app: cadvisor + type: ClusterIP diff --git a/base/monitoring/cadvisor/kustomization.yaml b/base/monitoring/cadvisor/kustomization.yaml index a479d147..97d984da 100644 --- a/base/monitoring/cadvisor/kustomization.yaml +++ b/base/monitoring/cadvisor/kustomization.yaml @@ -3,3 +3,4 @@ kind: Kustomization resources: - cadvisor.DaemonSet.yaml - cadvisor.ServiceAccount.yaml + - cadvisor.Service.yaml diff --git a/base/monitoring/grafana/grafana.ConfigMap.yaml b/base/monitoring/grafana/grafana.ConfigMap.yaml index 02f00b0e..c970a779 100644 --- a/base/monitoring/grafana/grafana.ConfigMap.yaml +++ b/base/monitoring/grafana/grafana.ConfigMap.yaml @@ -1,4 +1,11 @@ apiVersion: v1 +kind: ConfigMap +metadata: + labels: + deploy: sourcegraph + sourcegraph-resource-requires: no-cluster-admin + app.kubernetes.io/component: grafana + name: grafana data: datasources.yml: | apiVersion: 1 @@ -10,10 +17,3 @@ data: url: http://prometheus:30090 isDefault: true editable: false -kind: ConfigMap -metadata: - labels: - deploy: sourcegraph - sourcegraph-resource-requires: no-cluster-admin - app.kubernetes.io/component: grafana - name: grafana diff --git a/base/monitoring/prometheus/prometheus.ConfigMap.yaml b/base/monitoring/prometheus/prometheus.ConfigMap.yaml index 90b32d94..cc11c477 100644 --- a/base/monitoring/prometheus/prometheus.ConfigMap.yaml +++ b/base/monitoring/prometheus/prometheus.ConfigMap.yaml @@ -30,36 +30,62 @@ data: # Configure targets to scrape scrape_configs: + # Scrape prometheus itself for metrics. - job_name: 'builtin-prometheus' static_configs: - targets: ['127.0.0.1:9092'] + - job_name: 'builtin-alertmanager' metrics_path: /alertmanager/metrics static_configs: - targets: ['127.0.0.1:9093'] - - job_name: 'sg' + - job_name: 'sourcegraph-services' relabel_configs: - source_labels: [__address__] target_label: instance regex: (.*)\.(.*) replacement: ${1}_${2} + metric_relabel_configs: + - source_labels: [container_label_io_kubernetes_pod_namespace] + regex: kube-system + action: drop file_sd_configs: - files: - '/sg_prometheus_add_ons/*_targets.yml' + + - job_name: 'cadvisor' + static_configs: + - targets: ['cadvisor:48080'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + regex: (.*)\.(.*) + replacement: ${1}_${2} + - source_labels: [container_label_io_kubernetes_pod_name] + target_label: name + metric_relabel_configs: + - source_labels: [container_label_io_kubernetes_pod_namespace] + regex: kube-system + action: drop + - source_labels: [container_label_io_kubernetes_container_name, container_label_io_kubernetes_pod_name] + regex: (.+) + action: replace + target_label: name + separator: '-' + prometheus_targets.yml: | - labels: nodename: "sourcegraph-services" - job: node + job: sourcegraph-frontend targets: - - cadvisor:8080 - - sourcegraph-frontend-internal:6060 + - sourcegraph-frontend:6060 - labels: nodename: "sourcegraph-services" job: github-proxy targets: - - github-proxy:80 + - github-proxy:6060 - labels: nodename: "sourcegraph-services" job: repo-updater @@ -74,7 +100,7 @@ data: nodename: "sourcegraph-services" job: worker-executors targets: - - worker:6996 + - worker-executors:6996 - labels: nodename: "sourcegraph-services" job: node @@ -85,97 +111,6 @@ data: job: precise-code-intel-worker targets: - precise-code-intel-worker:6060 - # Add new entries here for every searcher/symbol/gitserver replica. - - labels: - nodename: "sourcegraph-services" - job: zoekt-indexserver - targets: - - indexed-search-0.indexed-search:6072 - - indexed-search-1.indexed-search:6072 - - indexed-search-2.indexed-search:6072 - - indexed-search-3.indexed-search:6072 - - indexed-search-4.indexed-search:6072 - - indexed-search-5.indexed-search:6072 - - indexed-search-6.indexed-search:6072 - - indexed-search-7.indexed-search:6072 - - indexed-search-8.indexed-search:6072 - - indexed-search-9.indexed-search:6072 - - indexed-search-10.indexed-search:6072 - - labels: - nodename: "sourcegraph-services" - job: zoekt-webserver - targets: - - indexed-search-0.indexed-search:6070 - - indexed-search-1.indexed-search:6070 - - indexed-search-2.indexed-search:6070 - - indexed-search-3.indexed-search:6070 - - indexed-search-4.indexed-search:6070 - - indexed-search-5.indexed-search:6070 - - indexed-search-6.indexed-search:6070 - - indexed-search-7.indexed-search:6070 - - indexed-search-8.indexed-search:6070 - - indexed-search-9.indexed-search:6070 - - indexed-search-10.indexed-search:6070 - - labels: - nodename: "sourcegraph-services" - job: sourcegraph-frontend - targets: - - sourcegraph-frontend-0.sourcegraph-frontend:6060 - - sourcegraph-frontend-1.sourcegraph-frontend:6060 - - sourcegraph-frontend-2.sourcegraph-frontend:6060 - - sourcegraph-frontend-3.sourcegraph-frontend:6060 - - sourcegraph-frontend-4.sourcegraph-frontend:6060 - - sourcegraph-frontend-5.sourcegraph-frontend:6060 - - sourcegraph-frontend-6.sourcegraph-frontend:6060 - - sourcegraph-frontend-7.sourcegraph-frontend:6060 - - sourcegraph-frontend-8.sourcegraph-frontend:6060 - - sourcegraph-frontend-9.sourcegraph-frontend:6060 - - sourcegraph-frontend-10.sourcegraph-frontend:6060 - - labels: - nodename: "sourcegraph-services" - job: gitserver - targets: - - gitserver-0.gitserver:6060 - - gitserver-1.gitserver:6060 - - gitserver-2.gitserver:6060 - - gitserver-3.gitserver:6060 - - gitserver-4.gitserver:6060 - - gitserver-5.gitserver:6060 - - gitserver-6.gitserver:6060 - - gitserver-7.gitserver:6060 - - gitserver-8.gitserver:6060 - - gitserver-9.gitserver:6060 - - gitserver-10.gitserver:6060 - - labels: - nodename: "sourcegraph-services" - job: searcher - targets: - - searcher-0.searcher:6060 - - searcher-1.searcher:6060 - - searcher-2.searcher:6060 - - searcher-3.searcher:6060 - - searcher-4.searcher:6060 - - searcher-5.searcher:6060 - - searcher-6.searcher:6060 - - searcher-7.searcher:6060 - - searcher-8.searcher:6060 - - searcher-9.searcher:6060 - - searcher-10.searcher:6060 - - labels: - nodename: "sourcegraph-services" - job: symbols - targets: - - symbols-0.symbols:6060 - - symbols-1.symbols:6060 - - symbols-2.symbols:6060 - - symbols-3.symbols:6060 - - symbols-4.symbols:6060 - - symbols-5.symbols:6060 - - symbols-6.symbols:6060 - - symbols-7.symbols:6060 - - symbols-8.symbols:6060 - - symbols-9.symbols:6060 - - symbols-10.symbols:6060 - labels: nodename: "sourcegraph-services" job: pgsql @@ -211,9 +146,10 @@ data: job: otel-collector targets: - otel-collector:8888 + extra_rules.yml: | groups: - - name: docker.rules + - name: container.rules rules: - record: container:process_cpu_seconds_total:ratio_rate5m expr: sum by (instance) (rate(process_cpu_seconds_total[5m])) / engine_daemon_engine_cpus_cpus @@ -223,3 +159,36 @@ data: expr: max by (instance) (process_resident_memory_bytes) - record: container:process_virtual_memory_bytes:max expr: max by (instance) (process_virtual_memory_bytes) + + symbols_targets.yml: | + - labels: + nodename: "sourcegraph-services" + job: symbols + targets: + - symbols-0.symbols:6060 + + searcher_targets.yml: | + - labels: + nodename: "sourcegraph-services" + job: searcher + targets: + - searcher-0.searcher:6060 + + gitserver_targets.yml: | + - labels: + nodename: "sourcegraph-services" + job: gitserver + targets: + - gitserver-0.gitserver:6060 + + indexed-search_targets.yml: | + - labels: + nodename: "sourcegraph-services" + job: zoekt-indexserver + targets: + - indexed-search-0.indexed-search:6072 + - labels: + nodename: "sourcegraph-services" + job: zoekt-webserver + targets: + - indexed-search-0.indexed-search:6070 diff --git a/base/sourcegraph/frontend/sourcegraph-frontend.Service.yaml b/base/sourcegraph/frontend/sourcegraph-frontend.Service.yaml index 7ad97ceb..ff9cb2fe 100644 --- a/base/sourcegraph/frontend/sourcegraph-frontend.Service.yaml +++ b/base/sourcegraph/frontend/sourcegraph-frontend.Service.yaml @@ -15,6 +15,9 @@ spec: - name: http port: 30080 targetPort: http + - name: debug + port: 6060 + targetPort: debug selector: app: sourcegraph-frontend type: ClusterIP diff --git a/base/sourcegraph/redis/redis-cache.Service.yaml b/base/sourcegraph/redis/redis-cache.Service.yaml index 035f4e6d..a7c45341 100644 --- a/base/sourcegraph/redis/redis-cache.Service.yaml +++ b/base/sourcegraph/redis/redis-cache.Service.yaml @@ -15,6 +15,9 @@ spec: - name: redis port: 6379 targetPort: redis + - name: redisexp + port: 9121 + targetPort: redisexp selector: app: redis-cache type: ClusterIP diff --git a/base/sourcegraph/redis/redis-store.Service.yaml b/base/sourcegraph/redis/redis-store.Service.yaml index 7af9c5fb..45df245c 100644 --- a/base/sourcegraph/redis/redis-store.Service.yaml +++ b/base/sourcegraph/redis/redis-store.Service.yaml @@ -15,6 +15,9 @@ spec: - name: redis port: 6379 targetPort: redis + - name: redisexp + port: 9121 + targetPort: redisexp selector: app: redis-store type: ClusterIP diff --git a/base/sourcegraph/searcher/searcher.Service.yaml b/base/sourcegraph/searcher/searcher.Service.yaml index bd887bef..1487cfe0 100644 --- a/base/sourcegraph/searcher/searcher.Service.yaml +++ b/base/sourcegraph/searcher/searcher.Service.yaml @@ -11,6 +11,7 @@ metadata: sourcegraph-resource-requires: no-cluster-admin name: searcher spec: + clusterIP: None ports: - name: http port: 3181 diff --git a/base/sourcegraph/symbols/symbols.Service.yaml b/base/sourcegraph/symbols/symbols.Service.yaml index ce88778b..4bc3996a 100644 --- a/base/sourcegraph/symbols/symbols.Service.yaml +++ b/base/sourcegraph/symbols/symbols.Service.yaml @@ -11,6 +11,7 @@ metadata: sourcegraph-resource-requires: no-cluster-admin name: symbols spec: + clusterIP: None ports: - name: http port: 3184 diff --git a/base/sourcegraph/syntect-server/syntect-server.Deployment.yaml b/base/sourcegraph/syntect-server/syntect-server.Deployment.yaml index 619f9f29..483acf9e 100644 --- a/base/sourcegraph/syntect-server/syntect-server.Deployment.yaml +++ b/base/sourcegraph/syntect-server/syntect-server.Deployment.yaml @@ -44,6 +44,8 @@ spec: ports: - containerPort: 9238 name: http + - containerPort: 6060 + name: debug readinessProbe: tcpSocket: port: http diff --git a/base/sourcegraph/syntect-server/syntect-server.Service.yaml b/base/sourcegraph/syntect-server/syntect-server.Service.yaml index 4289230b..1a1e5d2e 100644 --- a/base/sourcegraph/syntect-server/syntect-server.Service.yaml +++ b/base/sourcegraph/syntect-server/syntect-server.Service.yaml @@ -1,6 +1,9 @@ apiVersion: v1 kind: Service metadata: + annotations: + prometheus.io/port: "6060" + sourcegraph.prometheus/scrape: "true" labels: app: syntect-server app.kubernetes.io/component: syntect-server @@ -12,6 +15,9 @@ spec: - name: http port: 9238 targetPort: http + - name: debug + port: 6060 + targetPort: debug selector: app: syntect-server type: ClusterIP diff --git a/components/monitoring/rbacs/prometheus/prometheus.ConfigMap.yaml b/components/monitoring/rbacs/prometheus/prometheus.ConfigMap.yaml index f227280c..07fade2f 100644 --- a/components/monitoring/rbacs/prometheus/prometheus.ConfigMap.yaml +++ b/components/monitoring/rbacs/prometheus/prometheus.ConfigMap.yaml @@ -296,3 +296,7 @@ data: app: alertmanager extra_rules.yml: "" prometheus_targets.yml: "" + symbols_targets.yml: "" + searcher_targets.yml: "" + gitserver_targets.yml: "" + indexed-search_targets.yml: "" diff --git a/components/sizes/l/kustomization.yaml b/components/sizes/l/kustomization.yaml index 76dbb168..351ffc12 100644 --- a/components/sizes/l/kustomization.yaml +++ b/components/sizes/l/kustomization.yaml @@ -37,3 +37,8 @@ patchesJson6902: group: apps version: v1 path: patches/codeintel-db.yaml + - target: + kind: ConfigMap + name: prometheus + version: v1 + path: patches/prometheus.yaml diff --git a/components/sizes/l/patches/prometheus.yaml b/components/sizes/l/patches/prometheus.yaml new file mode 100644 index 00000000..d055bafa --- /dev/null +++ b/components/sizes/l/patches/prometheus.yaml @@ -0,0 +1,39 @@ +- op: replace + path: /data/indexed-search_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: zoekt-indexserver + targets: + - indexed-search-0.indexed-search:6072 + - indexed-search-1.indexed-search:6072 + - labels: + nodename: "sourcegraph-services" + job: zoekt-webserver + targets: + - indexed-search-0.indexed-search:6070 + - indexed-search-1.indexed-search:6070 +- op: replace + path: /data/gitserver_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: gitserver + targets: + - gitserver-0.gitserver:6060 +- op: replace + path: /data/searcher_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: searcher + targets: + - searcher-0.searcher:6060 +- op: replace + path: /data/symbols_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: symbols + targets: + - symbols-0.symbols:6060 diff --git a/components/sizes/m/kustomization.yaml b/components/sizes/m/kustomization.yaml index 76dbb168..351ffc12 100644 --- a/components/sizes/m/kustomization.yaml +++ b/components/sizes/m/kustomization.yaml @@ -37,3 +37,8 @@ patchesJson6902: group: apps version: v1 path: patches/codeintel-db.yaml + - target: + kind: ConfigMap + name: prometheus + version: v1 + path: patches/prometheus.yaml diff --git a/components/sizes/m/patches/prometheus.yaml b/components/sizes/m/patches/prometheus.yaml new file mode 100644 index 00000000..378b4a55 --- /dev/null +++ b/components/sizes/m/patches/prometheus.yaml @@ -0,0 +1,37 @@ +- op: replace + path: /data/indexed-search_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: zoekt-indexserver + targets: + - indexed-search-0.indexed-search:6072 + - labels: + nodename: "sourcegraph-services" + job: zoekt-webserver + targets: + - indexed-search-0.indexed-search:6070 +- op: replace + path: /data/gitserver_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: gitserver + targets: + - gitserver-0.gitserver:6060 +- op: replace + path: /data/searcher_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: searcher + targets: + - searcher-0.searcher:6060 +- op: replace + path: /data/symbols_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: symbols + targets: + - symbols-0.symbols:6060 diff --git a/components/sizes/s/kustomization.yaml b/components/sizes/s/kustomization.yaml index 76dbb168..351ffc12 100644 --- a/components/sizes/s/kustomization.yaml +++ b/components/sizes/s/kustomization.yaml @@ -37,3 +37,8 @@ patchesJson6902: group: apps version: v1 path: patches/codeintel-db.yaml + - target: + kind: ConfigMap + name: prometheus + version: v1 + path: patches/prometheus.yaml diff --git a/components/sizes/s/patches/prometheus.yaml b/components/sizes/s/patches/prometheus.yaml new file mode 100644 index 00000000..378b4a55 --- /dev/null +++ b/components/sizes/s/patches/prometheus.yaml @@ -0,0 +1,37 @@ +- op: replace + path: /data/indexed-search_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: zoekt-indexserver + targets: + - indexed-search-0.indexed-search:6072 + - labels: + nodename: "sourcegraph-services" + job: zoekt-webserver + targets: + - indexed-search-0.indexed-search:6070 +- op: replace + path: /data/gitserver_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: gitserver + targets: + - gitserver-0.gitserver:6060 +- op: replace + path: /data/searcher_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: searcher + targets: + - searcher-0.searcher:6060 +- op: replace + path: /data/symbols_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: symbols + targets: + - symbols-0.symbols:6060 diff --git a/components/sizes/xl/kustomization.yaml b/components/sizes/xl/kustomization.yaml index 7d863540..3686dc08 100644 --- a/components/sizes/xl/kustomization.yaml +++ b/components/sizes/xl/kustomization.yaml @@ -38,3 +38,8 @@ patchesJson6902: group: apps version: v1 path: patches/codeintel-db.yaml + - target: + kind: ConfigMap + name: prometheus + version: v1 + path: patches/prometheus.yaml diff --git a/components/sizes/xl/patches/prometheus.yaml b/components/sizes/xl/patches/prometheus.yaml new file mode 100644 index 00000000..3827a2b8 --- /dev/null +++ b/components/sizes/xl/patches/prometheus.yaml @@ -0,0 +1,43 @@ +- op: replace + path: /data/indexed-search_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: zoekt-indexserver + targets: + - indexed-search-0.indexed-search:6072 + - indexed-search-1.indexed-search:6072 + - indexed-search-2.indexed-search:6072 + - labels: + nodename: "sourcegraph-services" + job: zoekt-webserver + targets: + - indexed-search-0.indexed-search:6070 + - indexed-search-1.indexed-search:6070 + - indexed-search-2.indexed-search:6070 +- op: replace + path: /data/gitserver_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: gitserver + targets: + - gitserver-0.gitserver:6060 + - gitserver-1.gitserver:6060 +- op: replace + path: /data/searcher_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: searcher + targets: + - searcher-0.searcher:6060 + - searcher-1.searcher:6060 +- op: replace + path: /data/symbols_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: symbols + targets: + - symbols-0.symbols:6060 diff --git a/components/sizes/xs/kustomization.yaml b/components/sizes/xs/kustomization.yaml index fd36a155..f836b83c 100644 --- a/components/sizes/xs/kustomization.yaml +++ b/components/sizes/xs/kustomization.yaml @@ -25,3 +25,8 @@ patchesJson6902: group: apps version: v1 path: patches/databases.yaml + - target: + kind: ConfigMap + name: prometheus + version: v1 + path: patches/prometheus.yaml diff --git a/components/sizes/xs/patches/prometheus.yaml b/components/sizes/xs/patches/prometheus.yaml new file mode 100644 index 00000000..378b4a55 --- /dev/null +++ b/components/sizes/xs/patches/prometheus.yaml @@ -0,0 +1,37 @@ +- op: replace + path: /data/indexed-search_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: zoekt-indexserver + targets: + - indexed-search-0.indexed-search:6072 + - labels: + nodename: "sourcegraph-services" + job: zoekt-webserver + targets: + - indexed-search-0.indexed-search:6070 +- op: replace + path: /data/gitserver_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: gitserver + targets: + - gitserver-0.gitserver:6060 +- op: replace + path: /data/searcher_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: searcher + targets: + - searcher-0.searcher:6060 +- op: replace + path: /data/symbols_targets.yml + value: | + - labels: + nodename: "sourcegraph-services" + job: symbols + targets: + - symbols-0.symbols:6060 diff --git a/examples/k3s/l/kustomization.yaml b/examples/k3s/l/kustomization.yaml index 4c4a7f5d..86125910 100644 --- a/examples/k3s/l/kustomization.yaml +++ b/examples/k3s/l/kustomization.yaml @@ -6,8 +6,6 @@ resources: # Sourcegraph Main Stack - ../../../base/sourcegraph components: - # Monitoring Stacks - - ../../../components/monitoring # Use resources for a size-L instance - ../../../components/sizes/l # This configures the Sourcegraph instance and networking to work in a k3s cluster diff --git a/examples/k3s/m/kustomization.yaml b/examples/k3s/m/kustomization.yaml index ed95323e..9c1e5e6e 100644 --- a/examples/k3s/m/kustomization.yaml +++ b/examples/k3s/m/kustomization.yaml @@ -6,8 +6,6 @@ resources: # Sourcegraph Main Stack - ../../../base/sourcegraph components: - # Monitoring Stacks - - ../../../components/monitoring # Use resources for a size-m instance - ../../../components/sizes/m # This configures the Sourcegraph instance and networking to work in a k3s cluster diff --git a/examples/k3s/s/kustomization.yaml b/examples/k3s/s/kustomization.yaml index e382d37a..b15da606 100644 --- a/examples/k3s/s/kustomization.yaml +++ b/examples/k3s/s/kustomization.yaml @@ -6,8 +6,6 @@ resources: # Sourcegraph Main Stack - ../../../base/sourcegraph components: - # Monitoring Stacks - - ../../../components/monitoring # Use resources for a size-S instance - ../../../components/sizes/s # This configures the Sourcegraph instance and networking to work in a k3s cluster diff --git a/examples/k3s/xl/kustomization.yaml b/examples/k3s/xl/kustomization.yaml index c22e2247..6a41c210 100644 --- a/examples/k3s/xl/kustomization.yaml +++ b/examples/k3s/xl/kustomization.yaml @@ -6,8 +6,6 @@ resources: # Sourcegraph Main Stack - ../../../base/sourcegraph components: - # Monitoring Stacks - - ../../../components/monitoring # Use resources for a size-XL instance - ../../../components/sizes/xl # This configures the Sourcegraph instance and networking to work in a k3s cluster