Federate lsif-server metrics

base/lsif-server/lsif-server.Deployment.yaml

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    description: LSIF HTTP server for code intelligence.
+    description: HTTP server for LSIF-based precise code intelligence.
   labels:
     deploy: sourcegraph
   name: lsif-server
@@ -40,23 +40,23 @@ spec:
         name: lsif-server
         livenessProbe:
           httpGet:
-            path: /ping
+            path: /healthz
             port: server
             scheme: HTTP
           initialDelaySeconds: 60
           timeoutSeconds: 5
         readinessProbe:
           httpGet:
-            path: /ping
+            path: /healthz
             port: server
             scheme: HTTP
           periodSeconds: 5
           timeoutSeconds: 5
         ports:
         - containerPort: 3186
           name: server
-        - containerPort: 3187
-          name: worker
+        - containerPort: 9090
+          name: prometheus
         resources:
           limits:
             cpu: "2"

base/lsif-server/lsif-server.Service.yaml

@@ -2,7 +2,8 @@ apiVersion: v1
 kind: Service
 metadata:
   annotations:
-    sourcegraph.prometheus/scrape: "true"
+    prometheus.io/port: "9090"
+    sourcegraph.prometheus/federate: "true"
   labels:
     app: lsif-server
     deploy: sourcegraph
@@ -12,9 +13,9 @@ spec:
   - name: server
     port: 3186
     targetPort: server
-  - name: worker
-    port: 3187
-    targetPort: worker
+  - name: prometheus
+    port: 9090
+    targetPort: prometheus
   selector:
     app: lsif-server
   type: ClusterIP

base/prometheus/prometheus.ConfigMap.yaml

@@ -156,6 +156,65 @@ data:
         action: replace
         target_label: instance
 
+    # Scrape config for federated Prometheus instance endpoints. This is currently only
+    # used to proxy the metrics from multiple processes inside the lsif-server container
+    # to be made available (as we can't have Prometheus scrape a dynamic port range based
+    # on a ConfigMap value).
+    #
+    #
+    # The relabeling allows the actual service scrape endpoint to be configured
+    # via the following annotations:
+    #
+    # * `prometheus.io/federate`: Only scrape services that have a value of `true`
+    # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
+    # to set this to `https` & most likely set the `tls_config` of the scrape config.
+    # * `prometheus.io/port`: The port the Prometheus cluster is exposed on (usually 9090).
+    #
+    # TODO(efritz,uwedeportivo) - see if the value for 'match[]' can be supplied as an 
+    # annotation so that this selector can be generalized if we want to use federation for 
+    # another application in the future.
+    - job_name: 'kubernetes-federate'
+      honor_labels: true
+      metrics_path: '/federate'
+      params:
+        'match[]':
+          - '{__name__=~"lsif_.*"}'
+
+      kubernetes_sd_configs:
+      - role: endpoints
+
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_service_annotation_sourcegraph_prometheus_federate]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+        action: replace
+        target_label: __scheme__
+        regex: (https?)
+      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+        action: replace
+        target_label: __address__
+        regex: (.+)(?::\d+);(\d+)
+        replacement: $1:$2
+      - action: labelmap
+        regex: __meta_kubernetes_service_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        # Sourcegraph specific customization. We want a more convenient to type label.
+        # target_label: kubernetes_namespace
+        target_label: ns
+      - source_labels: [__meta_kubernetes_service_name]
+        action: replace
+        target_label: kubernetes_name
+      # Sourcegraph specific customization. We want a nicer name for job
+      - source_labels: [app]
+        action: replace
+        target_label: job
+      # Sourcegraph specific customization. We want a nicer name for instance
+      - source_labels: [__meta_kubernetes_pod_name]
+        action: replace
+        target_label: instance
+
     # Example scrape config for probing services via the Blackbox Exporter.
     #
     # The relabeling allows the actual service scrape endpoint to be configured