diff --git a/kube/README.md b/kube/README.md index 19131f19..f807af61 100644 --- a/kube/README.md +++ b/kube/README.md @@ -1,16 +1,39 @@ -# Kubernetes Test Setup +# Kubernetes setup -This directory contains files to deploy Prometheus with the sidecar in a Kubernetes -cluster. Additional manifests deploy the Prometheus node exporter and kube-state-metrics, -which provide a further variety of metrics. +This directory contains patch scripts to inject the Prometheus sidecar into +existing Prometheus installations and to deploy a full example setup. -To deploy all components: +Required environment variables: +* `KUBE_NAMESPACE`: namespace to run the script against +* `KUBE_CLUSTER`: cluster name parameter for the sidecar +* `GCP_REGION`: GCP region parameter for the sidecar +* `GCP_PROJECT`: GCP project parameter for the sidecar -`KUBE_NAMESPACE=sidecar-test GCP_REGION=your_region GCP_PROJECT=your_project_id KUBE_CLUSTER=clustername ./deploy.sh` +## `patch.sh` -Setting `USE_OPERATOR=1` will deploy Prometheus via the [coreos/prometheus-operator](https://github.com/coreos/prometheus-operator). +Inject sidecar into Deployments or StatefulSets: -To tear down everything: +```sh +./patch.sh +``` -`kubectl delete namespace "${KUBE_NAMESPACE}"` +Additional environment variables: +* `DATA_DIR`: data directory for the sidecar +* `DATA_VOLUME`: name of the volume that contains Prometheus's data +## `patch-operated.sh` + +Injects sidecar into Prometheus deployments controlled by the [prometheus-operator](https://github.com/coreos/prometheus-operator): + +```sh +./patch-operated.sh +``` + +## `full/deploy.sh` + +Deploys a basic Prometheus deployment to monitor Kubernetes components and +custom services that are annotated with the well-known `prometheus.io/*` annotations. + +```sh +./full/deploy.sh +``` diff --git a/kube/deploy.sh b/kube/full/deploy.sh similarity index 54% rename from kube/deploy.sh rename to kube/full/deploy.sh index 44ba868f..ee640fd7 100755 --- a/kube/deploy.sh +++ b/kube/full/deploy.sh @@ -3,29 +3,23 @@ set -e set -u +pushd "$(dirname "$0")" + # Override to use a different Docker image version for the sidecar. export SIDECAR_IMAGE_TAG=${SIDECAR_IMAGE_TAG:-'master'} -export USE_OPERATOR=${USE_OPERATOR:-''} export KUBE_NAMESPACE=${KUBE_NAMESPACE:-'default'} -echo "Deploy to namespace ${KUBE_NAMESPACE} for Stackdriver project ${GCP_PROJECT} (location=${GCP_REGION}, cluster=${KUBE_CLUSTER}), operator=${USE_OPERATOR}" +echo "Deploy to namespace ${KUBE_NAMESPACE} for Stackdriver project ${GCP_PROJECT} (location=${GCP_REGION}, cluster=${KUBE_CLUSTER})" -envsubst < prometheus-base.yaml > _prometheus-base.yaml.tmp -envsubst < prometheus-meta-operated.yaml > _prometheus-meta-operated.yaml.tmp -envsubst < prometheus-meta.yaml > _prometheus-meta.yaml.tmp +envsubst < prometheus.yaml > _prometheus.yaml.tmp envsubst < node-exporter.yaml > _node-exporter.yaml.tmp envsubst < kube-state-metrics.yaml > _kube-state-metrics.yaml.tmp -kubectl apply -f _prometheus-base.yaml.tmp --as=admin --as-group=system:masters - -if [ -n "${USE_OPERATOR}" ]; then - kubectl apply -f _prometheus-meta-operated.yaml.tmp -else - kubectl apply -f _prometheus-meta.yaml.tmp -fi - +kubectl apply -f _prometheus.yaml.tmp kubectl apply -f _node-exporter.yaml.tmp kubectl apply -f _kube-state-metrics.yaml.tmp --as=admin --as-group=system:masters -rm _*.tmp +DATA_DIR=/data DATA_VOLUME=data-volume ../patch.sh deploy prometheus-meta +rm _*.tmp +popd diff --git a/kube/kube-state-metrics.yaml b/kube/full/kube-state-metrics.yaml similarity index 96% rename from kube/kube-state-metrics.yaml rename to kube/full/kube-state-metrics.yaml index 4a205898..2ae7e600 100644 --- a/kube/kube-state-metrics.yaml +++ b/kube/full/kube-state-metrics.yaml @@ -124,6 +124,10 @@ kind: Service metadata: labels: k8s-app: kube-state-metrics + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8081' + prometheus.io/port2: '8082' name: kube-state-metrics namespace: ${KUBE_NAMESPACE} spec: diff --git a/kube/node-exporter.yaml b/kube/full/node-exporter.yaml similarity index 93% rename from kube/node-exporter.yaml rename to kube/full/node-exporter.yaml index 33a6e0d8..bc9814a1 100644 --- a/kube/node-exporter.yaml +++ b/kube/full/node-exporter.yaml @@ -56,6 +56,10 @@ kind: Service metadata: labels: app: node-exporter + annotations: + a: b + prometheus.io/scrape: 'true' + prometheus.io/port: '9101' name: node-exporter namespace: ${KUBE_NAMESPACE} spec: diff --git a/kube/full/prometheus.yaml b/kube/full/prometheus.yaml new file mode 100644 index 00000000..8a1d04e8 --- /dev/null +++ b/kube/full/prometheus.yaml @@ -0,0 +1,251 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ${KUBE_NAMESPACE} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: ${KUBE_NAMESPACE} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: ${KUBE_NAMESPACE} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: prometheus-k8s + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9090' + prometheus.io/port2: '9091' + name: prometheus-k8s + namespace: ${KUBE_NAMESPACE} +spec: + type: LoadBalancer + externalTrafficPolicy: Cluster + ports: + - name: prometheus + nodePort: 32387 + port: 9090 + protocol: TCP + targetPort: 9090 + - name: sidecar + nodePort: 30182 + port: 9091 + protocol: TCP + targetPort: 9091 + selector: + app: prometheus + prometheus: k8s + sessionAffinity: None +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: prometheus-k8s + namespace: ${KUBE_NAMESPACE} + labels: + app: prometheus + prometheus: k8s +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + prometheus: k8s + template: + metadata: + labels: + app: prometheus + prometheus: k8s + spec: + serviceAccount: prometheus + securityContext: + runAsUser: 0 + containers: + - name: prometheus + image: quay.io/prometheus/prometheus:v2.4.3 + imagePullPolicy: Always + args: + - "--config.file=/etc/prometheus/config/prometheus.yaml" + - "--storage.tsdb.path=/data" + - "--storage.tsdb.min-block-duration=15m" + - "--storage.tsdb.max-block-duration=4h" + - "--storage.tsdb.retention=48h" + ports: + - name: prometheus + containerPort: 9090 + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus/config + - name: data-volume + mountPath: /data + volumes: + - name: config-volume + configMap: + name: prometheus-k8s + - name: data-volume + emptyDir: {} + terminationGracePeriodSeconds: 300 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-k8s + namespace: ${KUBE_NAMESPACE} +data: + prometheus.yaml: | + scrape_configs: + - job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + - job_name: kubernetes-nodes-kubelet + kubernetes_sd_configs: + - role: node + relabel_configs: + - target_label: __address__ + regex: "(.+):10250" + source_labels: [__address__] + replacement: "${1}:10255" + + - job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - target_label: __metrics_path__ + replacement: /metrics/cadvisor + - target_label: __address__ + regex: "(.+):10250" + source_labels: [__address__] + replacement: "${1}:10255" + + # Configuration for the first port (prometheus.io/port) that service + # endpoints are annotated with. + - job_name: kubernetes-service-endpoints1 + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: k8s_namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: k8s_service + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: k8s_pod + + # Configuration for the seocnd port (prometheus.io/port2) that service + # endpoints are annotated with. + - job_name: kubernetes-service-endpoints2 + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: keep + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_port2 + regex: .+ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port2 + target_label: __address__ + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: k8s_namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: k8s_service + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: k8s_pod diff --git a/kube/patch-operated.sh b/kube/patch-operated.sh new file mode 100755 index 00000000..43ee35ef --- /dev/null +++ b/kube/patch-operated.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -e +set -u + +if [ $# -le 1 ]; then + echo -e "Usage: $0 \n" + exit 1 +fi + +kubectl -n "${KUBE_NAMESPACE}" patch prometheus "$1" --type merge --patch " +spec: + containers: + - name: sidecar + image: gcr.io/prometheus-to-sd/stackdriver-prometheus-sidecar:${SIDECAR_IMAGE_TAG} + imagePullPolicy: Always + args: + - \"--stackdriver.project-id=${GCP_PROJECT}\" + - \"--prometheus.wal-directory=/data/wal\" + - \"--stackdriver.kubernetes.location=${GCP_REGION}\" + - \"--stackdriver.kubernetes.cluster-name=${KUBE_CLUSTER}\" + ports: + - name: sidecar + containerPort: 9091 + volumeMounts: + - mountPath: /data + name: prometheus-$1-db +" diff --git a/kube/patch.sh b/kube/patch.sh new file mode 100755 index 00000000..3e4a5307 --- /dev/null +++ b/kube/patch.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -e +set -u + +usage() { + echo -e "Usage: $0 \n" +} + +if [ $# -le 1 ]; then + usage + exit 1 +fi + +kubectl -n "${KUBE_NAMESPACE}" patch "$1" "$2" --type strategic --patch " +spec: + template: + spec: + containers: + - name: sidecar + image: gcr.io/prometheus-to-sd/stackdriver-prometheus-sidecar:${SIDECAR_IMAGE_TAG} + imagePullPolicy: Always + args: + - \"--stackdriver.project-id=${GCP_PROJECT}\" + - \"--prometheus.wal-directory=${DATA_DIR}/wal\" + - \"--stackdriver.kubernetes.location=${GCP_REGION}\" + - \"--stackdriver.kubernetes.cluster-name=${KUBE_CLUSTER}\" + #- \"--stackdriver.generic.location=${GCP_REGION}\" + #- \"--stackdriver.generic.namespace=${KUBE_CLUSTER}\" + ports: + - name: sidecar + containerPort: 9091 + volumeMounts: + - name: ${DATA_VOLUME} + mountPath: ${DATA_DIR} +" diff --git a/kube/prometheus-base.yaml b/kube/prometheus-base.yaml deleted file mode 100644 index 52ddaa8b..00000000 --- a/kube/prometheus-base.yaml +++ /dev/null @@ -1,70 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ${KUBE_NAMESPACE} ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: prometheus -rules: -- apiGroups: [""] - resources: - - nodes - - nodes/proxy - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: - - extensions - resources: - - ingresses - verbs: ["get", "list", "watch"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus - namespace: ${KUBE_NAMESPACE} ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: prometheus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus -subjects: -- kind: ServiceAccount - name: prometheus - namespace: ${KUBE_NAMESPACE} ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus-meta - name: prometheus-meta - namespace: ${KUBE_NAMESPACE} -spec: - type: LoadBalancer - externalTrafficPolicy: Cluster - ports: - - name: prometheus - nodePort: 32387 - port: 9090 - protocol: TCP - targetPort: 9090 - - name: sidecar - nodePort: 30182 - port: 9091 - protocol: TCP - targetPort: 9091 - selector: - app: prometheus - prometheus: meta - sessionAffinity: None diff --git a/kube/prometheus-meta-operated.yaml b/kube/prometheus-meta-operated.yaml deleted file mode 100644 index 3749e867..00000000 --- a/kube/prometheus-meta-operated.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: Prometheus -metadata: - labels: - prometheus: meta - name: meta - namespace: ${KUBE_NAMESPACE} -spec: - replicas: 1 - resources: - requests: - memory: 200Mi - serviceAccountName: prometheus - serviceMonitorSelector: - matchExpressions: - - key: app - operator: Exists - version: v2.4.3 - # Inject the sidecar container. - containers: - - name: sidecar - image: gcr.io/prometheus-to-sd/stackdriver-prometheus-sidecar:${SIDECAR_IMAGE_TAG} - imagePullPolicy: Always - args: - - "--stackdriver.project-id=${GCP_PROJECT}" - - "--prometheus.wal-directory=/data/wal" - - "--stackdriver.kubernetes.location=${GCP_REGION}" - - "--stackdriver.kubernetes.cluster-name=${KUBE_CLUSTER}" - ports: - - name: sidecar - containerPort: 9091 - volumeMounts: - - mountPath: /data - name: prometheus-meta-db ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: any - name: meta - namespace: ${KUBE_NAMESPACE} -spec: - endpoints: - - interval: 30s - port: web - - interval: 30s - port: metrics - - interval: 30s - port: metrics-self - selector: - matchExpression: - - key: app - operator: Exists ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: prometheus-meta - name: prometheus-meta-sidecar - namespace: ${KUBE_NAMESPACE} -spec: - type: ClusterIP - clusterIP: None - ports: - - name: web - port: 9091 - protocol: TCP - targetPort: sidecar - selector: - app: prometheus diff --git a/kube/prometheus-meta.yaml b/kube/prometheus-meta.yaml deleted file mode 100644 index 6673f621..00000000 --- a/kube/prometheus-meta.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: prometheus-meta - namespace: ${KUBE_NAMESPACE} - labels: - app: prometheus-meta -spec: - replicas: 1 - selector: - matchLabels: - app: prometheus - prometheus: meta - template: - metadata: - labels: - app: prometheus - prometheus: meta - spec: - serviceAccount: prometheus - securityContext: - runAsUser: 0 - containers: - - name: prometheus - image: quay.io/prometheus/prometheus:v2.4.3 - imagePullPolicy: Always - args: - - "--config.file=/etc/prometheus/config/prometheus.yaml" - - "--storage.tsdb.path=/data" - - "--storage.tsdb.min-block-duration=15m" - - "--storage.tsdb.max-block-duration=4h" - - "--storage.tsdb.retention=48h" - ports: - - name: prometheus - containerPort: 9090 - volumeMounts: - - name: config-volume - mountPath: /etc/prometheus/config - - name: data-volume - mountPath: /data - - name: sidecar - image: gcr.io/prometheus-to-sd/stackdriver-prometheus-sidecar:${SIDECAR_IMAGE_TAG} - imagePullPolicy: Always - args: - - "--stackdriver.project-id=${GCP_PROJECT}" - - "--prometheus.wal-directory=/data/wal" - - "--stackdriver.kubernetes.location=${GCP_REGION}" - - "--stackdriver.kubernetes.cluster-name=${KUBE_CLUSTER}" - #- "--stackdriver.generic.location=${GCP_REGION}" - #- "--stackdriver.generic.namespace=${KUBE_CLUSTER}" - ports: - - name: sidecar - containerPort: 9091 - volumeMounts: - - name: data-volume - mountPath: /data - volumes: - - name: config-volume - configMap: - name: prometheus-meta - - name: data-volume - emptyDir: {} - terminationGracePeriodSeconds: 300 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-meta - namespace: ${KUBE_NAMESPACE} -data: - prometheus.yaml: | - global: - scrape_interval: 10s - scrape_configs: - - job_name: 'prometheus' - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - source_labels: [__meta_kubernetes_pod_name, __address__] - target_label: instance - regex: '(.+);.+:(.+)' - replacement: '${1}:${2}' - - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name] - target_label: job - regex: '(.+);(.+)' - replacement: '${1}/${2}'