Improve Prometheus setup, add injection scripts

kube/README.md

@@ -1,16 +1,39 @@
-# Kubernetes Test Setup
+# Kubernetes setup
 
-This directory contains files to deploy Prometheus with the sidecar in a Kubernetes
-cluster. Additional manifests deploy the Prometheus node exporter and kube-state-metrics,
-which provide a further variety of metrics.
+This directory contains patch scripts to inject the Prometheus sidecar into
+existing Prometheus installations and to deploy a full example setup.
 
-To deploy all components:
+Required environment variables:
+* `KUBE_NAMESPACE`: namespace to run the script against
+* `KUBE_CLUSTER`: cluster name parameter for the sidecar
+* `GCP_REGION`: GCP region parameter for the sidecar
+* `GCP_PROJECT`: GCP project parameter for the sidecar
 
-`KUBE_NAMESPACE=sidecar-test GCP_REGION=your_region GCP_PROJECT=your_project_id KUBE_CLUSTER=clustername ./deploy.sh`
+## `patch.sh`
 
-Setting `USE_OPERATOR=1` will deploy Prometheus via the [coreos/prometheus-operator](https://github.com/coreos/prometheus-operator).
+Inject sidecar into Deployments or StatefulSets:
 
-To tear down everything:
+```sh
+./patch.sh <deployment|statefulset> <name> 
+```
 
-`kubectl delete namespace "${KUBE_NAMESPACE}"`
+Additional environment variables:
+* `DATA_DIR`: data directory for the sidecar
+* `DATA_VOLUME`: name of the volume that contains Prometheus's data
 
+## `patch-operated.sh`
+
+Injects sidecar into Prometheus deployments controlled by the [prometheus-operator](https://github.com/coreos/prometheus-operator):
+
+```sh
+./patch-operated.sh <prometheus_name>
+```
+
+## `full/deploy.sh`
+
+Deploys a basic Prometheus deployment to monitor Kubernetes components and
+custom services that are annotated with the well-known `prometheus.io/*` annotations.
+
+```sh
+./full/deploy.sh
+```

kube/deploy.sh → kube/full/deploy.sh

@@ -3,29 +3,23 @@
 set -e
 set -u
 
+pushd "$(dirname "$0")"
+
 # Override to use a different Docker image version for the sidecar.
 export SIDECAR_IMAGE_TAG=${SIDECAR_IMAGE_TAG:-'master'}
-export USE_OPERATOR=${USE_OPERATOR:-''}
 export KUBE_NAMESPACE=${KUBE_NAMESPACE:-'default'}
 
-echo "Deploy to namespace ${KUBE_NAMESPACE} for Stackdriver project ${GCP_PROJECT} (location=${GCP_REGION}, cluster=${KUBE_CLUSTER}), operator=${USE_OPERATOR}"
+echo "Deploy to namespace ${KUBE_NAMESPACE} for Stackdriver project ${GCP_PROJECT} (location=${GCP_REGION}, cluster=${KUBE_CLUSTER})"
 
-envsubst < prometheus-base.yaml > _prometheus-base.yaml.tmp
-envsubst < prometheus-meta-operated.yaml > _prometheus-meta-operated.yaml.tmp
-envsubst < prometheus-meta.yaml > _prometheus-meta.yaml.tmp
+envsubst < prometheus.yaml > _prometheus.yaml.tmp
 envsubst < node-exporter.yaml > _node-exporter.yaml.tmp
 envsubst < kube-state-metrics.yaml > _kube-state-metrics.yaml.tmp
 
-kubectl apply -f _prometheus-base.yaml.tmp --as=admin --as-group=system:masters
-
-if [ -n "${USE_OPERATOR}" ]; then
-  kubectl apply -f _prometheus-meta-operated.yaml.tmp
-else
-  kubectl apply -f _prometheus-meta.yaml.tmp
-fi
-
+kubectl apply -f _prometheus.yaml.tmp
 kubectl apply -f _node-exporter.yaml.tmp
 kubectl apply -f _kube-state-metrics.yaml.tmp --as=admin --as-group=system:masters
 
-rm _*.tmp
+DATA_DIR=/data DATA_VOLUME=data-volume ../patch.sh deploy prometheus-meta
 
+rm _*.tmp
+popd

kube/kube-state-metrics.yaml → kube/full/kube-state-metrics.yaml

@@ -124,6 +124,10 @@ kind: Service
 metadata:
   labels:
     k8s-app: kube-state-metrics
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: '8081'
+    prometheus.io/port2: '8082'
   name: kube-state-metrics
   namespace: ${KUBE_NAMESPACE}
 spec:

kube/node-exporter.yaml → kube/full/node-exporter.yaml

@@ -56,6 +56,10 @@ kind: Service
 metadata:
   labels:
     app: node-exporter
+  annotations:
+    a: b
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: '9101'
   name: node-exporter
   namespace: ${KUBE_NAMESPACE}
 spec:

kube/full/prometheus.yaml

@@ -0,0 +1,251 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ${KUBE_NAMESPACE}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - nodes/proxy
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs: ["get", "list", "watch"]
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus
+  namespace: ${KUBE_NAMESPACE}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus
+  namespace: ${KUBE_NAMESPACE}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: prometheus-k8s
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: '9090'
+    prometheus.io/port2: '9091'
+  name: prometheus-k8s
+  namespace: ${KUBE_NAMESPACE}
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+  ports:
+  - name: prometheus
+    nodePort: 32387
+    port: 9090
+    protocol: TCP
+    targetPort: 9090
+  - name: sidecar
+    nodePort: 30182
+    port: 9091
+    protocol: TCP
+    targetPort: 9091
+  selector:
+    app: prometheus
+    prometheus: k8s
+  sessionAffinity: None
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: prometheus-k8s
+  namespace: ${KUBE_NAMESPACE}
+  labels:
+    app: prometheus
+    prometheus: k8s
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+      prometheus: k8s
+  template:
+    metadata:
+      labels:
+        app: prometheus
+        prometheus: k8s
+    spec:
+      serviceAccount: prometheus
+      securityContext:
+        runAsUser: 0
+      containers:
+      - name: prometheus
+        image: quay.io/prometheus/prometheus:v2.4.3
+        imagePullPolicy: Always
+        args:
+        - "--config.file=/etc/prometheus/config/prometheus.yaml"
+        - "--storage.tsdb.path=/data"
+        - "--storage.tsdb.min-block-duration=15m"
+        - "--storage.tsdb.max-block-duration=4h"
+        - "--storage.tsdb.retention=48h"
+        ports:
+        - name: prometheus
+          containerPort: 9090
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/prometheus/config
+        - name: data-volume
+          mountPath: /data
+      volumes:
+      - name: config-volume
+        configMap:
+          name: prometheus-k8s
+      - name: data-volume
+        emptyDir: {}
+      terminationGracePeriodSeconds: 300
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-k8s
+  namespace: ${KUBE_NAMESPACE}
+data:
+  prometheus.yaml: |
+    scrape_configs:
+    - job_name: kubernetes-apiservers
+      kubernetes_sd_configs:
+      - role: endpoints
+      relabel_configs:
+      - action: keep
+        regex: default;kubernetes;https
+        source_labels:
+        - __meta_kubernetes_namespace
+        - __meta_kubernetes_service_name
+        - __meta_kubernetes_endpoint_port_name
+      scheme: https
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+    - job_name: kubernetes-nodes-kubelet
+      kubernetes_sd_configs:
+      - role: node
+      relabel_configs:
+      - target_label: __address__
+        regex: "(.+):10250"
+        source_labels: [__address__]
+        replacement: "${1}:10255"
+
+    - job_name: kubernetes-nodes-cadvisor
+      kubernetes_sd_configs:
+      - role: node
+      relabel_configs:
+      - target_label: __metrics_path__
+        replacement: /metrics/cadvisor
+      - target_label: __address__
+        regex: "(.+):10250"
+        source_labels: [__address__]
+        replacement: "${1}:10255"
+
+    # Configuration for the first port (prometheus.io/port) that service
+    # endpoints are annotated with.
+    - job_name: kubernetes-service-endpoints1
+      kubernetes_sd_configs:
+      - role: endpoints
+      relabel_configs:
+      - action: keep
+        regex: true
+        source_labels:
+        - __meta_kubernetes_service_annotation_prometheus_io_scrape
+      - action: replace
+        regex: (https?)
+        source_labels:
+        - __meta_kubernetes_service_annotation_prometheus_io_scheme
+        target_label: __scheme__
+      - action: replace
+        regex: (.+)
+        source_labels:
+        - __meta_kubernetes_service_annotation_prometheus_io_path
+        target_label: __metrics_path__
+      - action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        source_labels:
+        - __address__
+        - __meta_kubernetes_service_annotation_prometheus_io_port
+        target_label: __address__
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_namespace
+        target_label: k8s_namespace
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_service_name
+        target_label: k8s_service
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_name
+        target_label: k8s_pod
+
+    # Configuration for the seocnd port (prometheus.io/port2) that service
+    # endpoints are annotated with.
+    - job_name: kubernetes-service-endpoints2
+      kubernetes_sd_configs:
+      - role: endpoints
+      relabel_configs:
+      - action: keep
+        regex: true
+        source_labels:
+        - __meta_kubernetes_service_annotation_prometheus_io_scrape
+      - action: replace
+        regex: (https?)
+        source_labels:
+        - __meta_kubernetes_service_annotation_prometheus_io_scheme
+        target_label: __scheme__
+      - action: replace
+        regex: (.+)
+        source_labels:
+        - __meta_kubernetes_service_annotation_prometheus_io_path
+        target_label: __metrics_path__
+      - action: keep
+        source_labels:
+        - __meta_kubernetes_service_annotation_prometheus_io_port2
+        regex: .+
+      - action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        source_labels:
+        - __address__
+        - __meta_kubernetes_service_annotation_prometheus_io_port2
+        target_label: __address__
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_namespace
+        target_label: k8s_namespace
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_service_name
+        target_label: k8s_service
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_name
+        target_label: k8s_pod

kube/patch-operated.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -u
+
+if [  $# -le 1 ]; then
+  echo -e "Usage: $0 <prometheus_name>\n"
+  exit 1
+fi
+
+kubectl -n "${KUBE_NAMESPACE}" patch prometheus "$1" --type merge --patch "
+spec:
+  containers:
+  - name: sidecar
+    image: gcr.io/prometheus-to-sd/stackdriver-prometheus-sidecar:${SIDECAR_IMAGE_TAG}
+    imagePullPolicy: Always
+    args:
+    - \"--stackdriver.project-id=${GCP_PROJECT}\"
+    - \"--prometheus.wal-directory=/data/wal\"
+    - \"--stackdriver.kubernetes.location=${GCP_REGION}\"
+    - \"--stackdriver.kubernetes.cluster-name=${KUBE_CLUSTER}\"
+    ports:
+    - name: sidecar
+      containerPort: 9091
+    volumeMounts:
+    - mountPath: /data
+      name: prometheus-$1-db
+"