Skip to content
This repository has been archived by the owner on Oct 11, 2024. It is now read-only.

Commit

Permalink
Upgrade Prometheus (#1500)
Browse files Browse the repository at this point in the history
* Prometheus annotations

* Prometheus configmap

* Enable config reloading

* Config reloader

* Remove Prometheus Docker Image

* docker-compose for prometheus
  • Loading branch information
gdbelvin authored Mar 24, 2020
1 parent 2d08dc7 commit 769594f
Show file tree
Hide file tree
Showing 15 changed files with 323 additions and 19 deletions.
2 changes: 2 additions & 0 deletions deploy/kubernetes/base/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ resources:
- monitor-deployment.yaml
- monitor-service.yaml
- prometheus-deployment.yaml
- prometheus-configmap.yaml
- prometheus-service.yaml
- prometheus-rbac.yaml
- sequencer-deployment.yaml
- sequencer-service.yaml
- server-deployment.yaml
Expand Down
5 changes: 5 additions & 0 deletions deploy/kubernetes/base/log-server-service.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
prometheus.io/path: '/metrics'
prometheus.io/port: '8091'
prometheus.io/scheme: 'http'
labels:
io.kompose.service: log-server
name: log-server
Expand Down
5 changes: 5 additions & 0 deletions deploy/kubernetes/base/log-signer-service.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
prometheus.io/path: '/metrics'
prometheus.io/port: '8091'
prometheus.io/scheme: 'http'
labels:
io.kompose.service: log-signer
name: log-signer
Expand Down
5 changes: 5 additions & 0 deletions deploy/kubernetes/base/map-server-service.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
prometheus.io/path: '/metrics'
prometheus.io/port: '8091'
prometheus.io/scheme: 'http'
labels:
io.kompose.service: map-server
name: map-server
Expand Down
5 changes: 5 additions & 0 deletions deploy/kubernetes/base/monitor-service.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
prometheus.io/path: '/metrics'
prometheus.io/port: '8071'
prometheus.io/scheme: 'http'
labels:
io.kompose.service: monitor
name: monitor
Expand Down
220 changes: 220 additions & 0 deletions deploy/kubernetes/base/prometheus-configmap.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,220 @@
apiVersion: v1
kind: ConfigMap
apiVersion: v1
data:
prometheus.yml: |
# A scrape configuration for running Prometheus on a Kubernetes cluster.
# This uses separate scrape configs for cluster components (i.e. API server, node)
# and services to allow each to use different authentication configs.
#
# Kubernetes labels will be added as Prometheus labels on metrics via the
# `labelmap` relabeling action.
#
# If you are using Kubernetes 1.7.2 or earlier, please take note of the comments
# for the kubernetes-cadvisor job; you will need to edit or remove this job.
# Scrape config for API servers.
#
# Kubernetes exposes API servers as endpoints to the default/kubernetes
# service so this uses `endpoints` role and uses relabelling to only keep
# the endpoints associated with the default/kubernetes service using the
# default named port `https`. This works for single API server deployments as
# well as HA API server deployments.
scrape_configs:
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
- role: endpoints
# Default to scraping over https. If required, just disable this or change to
# `http`.
scheme: https
# This TLS & bearer token file config is used to connect to the actual scrape
# endpoints for cluster components. This is separate to discovery auth
# configuration because discovery & scraping are two separate concerns in
# Prometheus. The discovery auth config is automatic if Prometheus runs inside
# the cluster. Otherwise, more config options have to be provided within the
# <kubernetes_sd_config>.
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
# If your node certificates are self-signed or use a different CA to the
# master CA, then disable certificate verification below. Note that
# certificate verification is an integral part of a secure infrastructure
# so this should only be disabled in a controlled environment. You can
# disable certificate verification by uncommenting the line below.
#
# insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
# Keep only the default/kubernetes service endpoints for the https port. This
# will add targets for each API server which Kubernetes adds an endpoint to
# the default/kubernetes service.
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
# Scrape config for nodes (kubelet).
#
# Rather than connecting directly to the node, the scrape is proxied though the
# Kubernetes apiserver. This means it will work if Prometheus is running out of
# cluster, or can't connect to nodes for some other reason (e.g. because of
# firewalling).
- job_name: 'kubernetes-nodes'
# Default to scraping over https. If required, just disable this or change to
# `http`.
scheme: https
# This TLS & bearer token file config is used to connect to the actual scrape
# endpoints for cluster components. This is separate to discovery auth
# configuration because discovery & scraping are two separate concerns in
# Prometheus. The discovery auth config is automatic if Prometheus runs inside
# the cluster. Otherwise, more config options have to be provided within the
# <kubernetes_sd_config>.
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics
# Scrape config for Kubelet cAdvisor.
#
# This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics
# (those whose names begin with 'container_') have been removed from the
# Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to
# retrieve those metrics.
#
# In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor
# HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics"
# in that case (and ensure cAdvisor's HTTP server hasn't been disabled with
# the --cadvisor-port=0 Kubelet flag).
#
# This job is not necessary and should be removed in Kubernetes 1.6 and
# earlier versions, or it will cause the metrics to be scraped twice.
- job_name: 'kubernetes-cadvisor'
# Default to scraping over https. If required, just disable this or change to
# `http`.
scheme: https
# This TLS & bearer token file config is used to connect to the actual scrape
# endpoints for cluster components. This is separate to discovery auth
# configuration because discovery & scraping are two separate concerns in
# Prometheus. The discovery auth config is automatic if Prometheus runs inside
# the cluster. Otherwise, more config options have to be provided within the
# <kubernetes_sd_config>.
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
# Example scrape config for service endpoints.
#
# The relabeling allows the actual service scrape endpoint to be configured
# for all or only some endpoints.
- job_name: 'kubernetes-service-endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
# Example relabel to scrape only endpoints that have
# "prometheus.io/scrape = true" annotation.
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
# Example relabel to customize metric path based on endpoints
# "prometheus.io/path = <metric path>" annotation.
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
# Example relabel to scrape only single, desired port for the service based
# on endpoints "prometheus.io/port = <port>" annotation.
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
# Example relabel to configure scrape scheme for all service scrape targets
# based on endpoints "prometheus.io/scheme = <scheme>" annotation.
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
# scrape config for pods
#
# The relabeling allows the actual pod scrape to be configured
# for all the declared ports (or port-free target if none is declared)
# or only some ports.
- job_name: 'kubernetes-pods'
kubernetes_sd_configs:
- role: pod
relabel_configs:
# Example relabel to scrape only pods that have
# "prometheus.io/should_be_scraped = true" annotation.
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
#
# Example relabel to customize metric path based on pod
# "prometheus.io/metric_path = <metric path>" annotation.
# - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_metric_path]
# action: replace
# target_label: __metrics_path__
# regex: (.+)
#
# Example relabel to scrape only single, desired port for the pod
# based on pod "prometheus.io/scrape_port = <port>" annotation.
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
metadata:
name: prometheus
18 changes: 17 additions & 1 deletion deploy/kubernetes/base/prometheus-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,35 @@ spec:
labels:
io.kompose.service: prometheus
spec:
serviceAccountName: prometheus
restartPolicy: Always
containers:
- name: prometheus
image: gcr.io/key-transparency/prometheus:latest
image: prom/prometheus:v2.11.0
imagePullPolicy: Always
args:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/data/"
- "--web.enable-lifecycle"
# - "--log.level=debug" # Uncomment this to enable debug logs. Can be very verbose.
ports:
- containerPort: 9090
volumeMounts:
- name: config-volume
mountPath: /etc/prometheus
- name: prometheus-data-volume
mountPath: /data
# - name: watch
# image: weaveworks/watch:master-5b2a6e5
# imagePullPolicy: IfNotPresent
# args: ["-v", "-t", "-p=/etc/prometheus", "curl", "-X", "POST", "--fail", "-o", "-", "-sS", "http://localhost:9090/-/reload"]
# volumeMounts:
# - name: config-volume
# mountPath: /etc/prometheus
volumes:
- name: config-volume
configMap:
name: prometheus
- name: prometheus-data-volume
emptyDir: {}
status: {}
39 changes: 39 additions & 0 deletions deploy/kubernetes/base/prometheus-rbac.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: prometheus
rules:
- apiGroups: [""]
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups:
- extensions
resources:
- ingresses
verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: default
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: default
5 changes: 5 additions & 0 deletions deploy/kubernetes/base/sequencer-service.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
prometheus.io/path: '/metrics'
prometheus.io/port: '8081'
prometheus.io/scheme: 'http'
labels:
io.kompose.service: sequencer
name: sequencer
Expand Down
5 changes: 5 additions & 0 deletions deploy/kubernetes/base/server-service.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
prometheus.io/path: '/metrics'
prometheus.io/port: '8081'
prometheus.io/scheme: 'http'
labels:
io.kompose.service: server
name: server
Expand Down
5 changes: 0 additions & 5 deletions deploy/prometheus/Dockerfile

This file was deleted.

6 changes: 0 additions & 6 deletions docker-compose.override.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,6 @@ services:
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"

prometheus:
image: gcr.io/key-transparency/prometheus:${TRAVIS_COMMIT}
build:
context: .
dockerfile: deploy/prometheus/Dockerfile

server:
image: gcr.io/key-transparency/keytransparency-server:${TRAVIS_COMMIT}
build:
Expand Down
Loading

0 comments on commit 769594f

Please sign in to comment.