prometheus-config-map.yml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-server-conf
  labels:
    name: prometheus-server-conf
  namespace: monitoring
data:
  prometheus.rules: |-
    groups:
    - name: node.alerts
      rules:
      - alert: KubernetesHostHighCPUUsage
        expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 15m
        labels:
          severity: "warning"
          context: "node"
          cluster:
        annotations:
          summary: "High load on node"
          description: "Node {{ $labels.instance }} has more than 90% CPU load"
      - alert: KubernetesNodeDiskUsagePercentage
        expr: (100 - 100 * sum(node_filesystem_avail_bytes{device!~"tmpfs|by-uuid",fstype=~"xfs|ext"} / node_filesystem_size_bytes{device!~"tmpfs|by-uuid",fstype=~"xfs|ext"}) BY (instance,device)) > 85
        for: 15m
        labels:
          severity: "warning"
          context: "node"
          cluster:
        annotations:
          description: "Node disk usage above 85%"
          summary: "Disk usage on target {{ $labels.instance }} device {{ $labels.device }} at 85%"
      - alert: KubernetesNodeContainerOOMKilled
        expr: sum by (instance) (changes(node_vmstat_oom_kill[2h])) > 3
        labels:
          severity: "warning"
          context: "node"
          cluster:
        annotations:
          description: "More than 3 OOM killed pods on a node within 2h"
          summary: "More than 3 OOM killed pods on node {{ $labels.instance }} within 2h"
      - alert: KubernetesNodeHighLoad15
        expr: node_load15 > 20
        for: 15m
        labels:
          severity: "warning"
          context: "node"
          cluster:
        annotations:
          description: "Load average over 20 in the last 15 minutes"
          summary: "The node {{ $labels.instance }} load average is over 20 in the last 15 minutes"
      - alert: KubernetesNodeDiskPressure
        expr: kube_node_status_condition{app="kube-state-metrics",condition="DiskPressure",status="true"} == 1
        for: 10m
        labels:
          severity: "warning"
          context: "node"
          cluster:
        annotations:
          description: "Insufficient disk space"
          summary: "Node {{ $labels.node }} under pressure due to insufficient available disk space"
      - alert: KubernetesNodeMemoryPressure
        expr: kube_node_status_condition{app="kube-state-metrics",condition="MemoryPressure",status="true"} == 1
        for: 10m
        labels:
          severity: "warning"
          context: "node"
          cluster:
        annotations:
          description: "Insufficient memory"
          summary: "Node {{ $labels.node }} under pressure due to insufficient available memory"
      - alert: KubernetesNodePIDPressure
        expr: kube_node_status_condition{app="kube-state-metrics",condition="PIDPressure",status="true"} == 1
        for: 10m
        labels:
          severity: "warning"
          context: "node"
          cluster:
        annotations:
          description: "Too many processes on the node"
          summary: "Node {{ $labels.node }} under pressure due to too many processes"
      - alert: KubernetesNodeUnschedulable
        expr: sum by(node) (kube_node_spec_unschedulable{app="kube-state-metrics"} == 1)
        for: 15m
        labels:
          severity: warning
          context: node
          cluster:
        annotations:
          description: "Node unschedulable"
          summary: "Node {{ $labels.node }} is unschedulable for more than 15 min"
    - name: pod.alerts
      rules:
      - alert: KubernetesPodWaiting
        expr: (sum(kube_pod_container_status_waiting_reason{app="kube-state-metrics"}) by (container,namespace,reason)) > 0
        for: 15m
        labels:
          severity: "warning"
          context: "pod"
          cluster:
        annotations:
          description: "Pod waiting to start"
          summary: "Pod {{ $labels.container }} in {{ $labels.namespace }} namespace waiting to start for more than 15 min"
      - alert: KubernetesPodHighAvailability
        expr: (sum by(container,namespace) (kube_pod_container_status_running{app="kube-state-metrics",namespace="kube-system",container="coredns"} or kube_pod_container_status_running{app="kube-state-metrics",namespace="kube-system",container="etcd"} or kube_pod_container_status_running{app="kube-state-metrics",namespace="kube-system",container="kube-apiserver"}) < 2)
        for: 15m
        labels:
          severity: "warning"
          context: "pod"
          cluster:
        annotations:
          description: "Pod high availability requirements not met"
          summary: "Pod {{ $labels.container }} in {{ $labels.namespace }} namespace does not meet HA requirements, less than 2 pods running"
      - alert: KubernetesPodPhasePending
        expr: sum by(namespace) (kube_pod_status_phase{app="kube-state-metrics",phase="Pending"} > 0)
        for: 15m
        labels:
          severity: warning
          context: pod
          cluster:
        annotations:
          description: "Pod in pending phase"
          summary: "At least one pod in {{ $labels.namespace }} namespace is pending for more than 15 min"
      - alert: KubernetesPodUnscheduleable
        expr: sum by(namespace) (kube_pod_status_unschedulable > 0)
        for: 15m
        labels:
          severity: warning
          context: pod
          cluster:
        annotations:
          description: "Pod in unschedulable state"
          summary: "At least one pod in {{ $labels.namespace }} namespace is unschedulable for more than 15 min"
    - name: CPU
      rules:
      - expr: sum(rate(container_cpu_usage_seconds_total{container_name!=""}[5m]))
        record: cluster:cpu_usage:rate5m
      - expr: rate(container_cpu_usage_seconds_total{container_name!=""}[5m])
        record: cluster:cpu_usage_nosum:rate5m
      - expr: avg(irate(container_cpu_usage_seconds_total{container_name!="POD", container_name!=""}[5m])) by (container_name,pod_name,namespace)
        record: kubecost_container_cpu_usage_irate
      - expr: sum(container_memory_working_set_bytes{container_name!="POD",container_name!=""}) by (container_name,pod_name,namespace)
        record: kubecost_container_memory_working_set_bytes
      - expr: sum(container_memory_working_set_bytes{container_name!="POD",container_name!=""})
        record: kubecost_cluster_memory_working_set_bytes
    - name: Savings
      rules:
      - expr: sum(avg(kube_pod_owner{owner_kind!="DaemonSet"}) by (pod) * sum(container_cpu_allocation) by (pod))
        record: kubecost_savings_cpu_allocation
        labels:
          daemonset: "false"
      - expr: sum(avg(kube_pod_owner{owner_kind="DaemonSet"}) by (pod) * sum(container_cpu_allocation) by (pod)) / sum(kube_node_info)
        record: kubecost_savings_cpu_allocation
        labels:
          daemonset: "true"
      - expr: sum(avg(kube_pod_owner{owner_kind!="DaemonSet"}) by (pod) * sum(container_memory_allocation_bytes) by (pod))
        record: kubecost_savings_memory_allocation_bytes
        labels:
          daemonset: "false"
      - expr: sum(avg(kube_pod_owner{owner_kind="DaemonSet"}) by (pod) * sum(container_memory_allocation_bytes) by (pod)) / sum(kube_node_info)
        record: kubecost_savings_memory_allocation_bytes
        labels:
          daemonset: "true"
      - expr: label_replace(sum(kube_pod_status_phase{phase="Running",namespace!="kube-system"} > 0) by (pod, namespace), "pod_name", "$1", "pod", "(.+)")
        record: kubecost_savings_running_pods
      - expr: sum(rate(container_cpu_usage_seconds_total{container_name!="",container_name!="POD",instance!=""}[5m])) by (namespace, pod_name, container_name, instance)
        record: kubecost_savings_container_cpu_usage_seconds
      - expr: sum(container_memory_working_set_bytes{container_name!="",container_name!="POD",instance!=""}) by (namespace, pod_name, container_name, instance)
        record: kubecost_savings_container_memory_usage_bytes
      - expr: avg(sum(kube_pod_container_resource_requests{resource="cpu", unit="core", namespace!="kube-system"}) by (pod, namespace, instance)) by (pod, namespace)
        record: kubecost_savings_pod_requests_cpu_cores
      - expr: avg(sum(kube_pod_container_resource_requests{resource="memory", unit="byte", namespace!="kube-system"}) by (pod, namespace, instance)) by (pod, namespace)
        record: kubecost_savings_pod_requests_memory_bytes
  prometheus.yml: |-
    global:
      evaluation_interval: 60s
      scrape_interval: 15s
      scrape_timeout: 10s
      external_labels:
        cluster: ${CLUSTER_NAME}
    rule_files:
      - /etc/prometheus/prometheus.rules

    # Alertmanager configuration
    alerting:
      alert_relabel_configs:
      - source_labels: [cluster]
        action: replace
        regex: (.*)
        replacement: "$1"
        target_label: cluster
      alertmanagers:
      - static_configs:
        - targets:
          - 'alertmanager.monitoring.svc:9093'

    scrape_configs:
      - job_name: 'kubernetes-apiservers'

        kubernetes_sd_configs:
        - role: endpoints
        scheme: https

        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https

      #--------------------------------------------
      # Scrape config for nodes (kubelet).
      #--------------------------------------------
      - job_name: 'kubernetes-nodes'

        kubernetes_sd_configs:
        - role: node

        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)
        - target_label: __address__
          replacement: kubernetes.default.svc:443
        - source_labels: [__meta_kubernetes_node_name]
          regex: (.+)
          target_label: __metrics_path__
          replacement: /api/v1/nodes/${1}/proxy/metrics

        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      #--------------------------------------------
      # Scrape config for service endpoints. This will
      # scrape node-exporter and kube-state-metrics services.
      # You need this for the following Grafana dashboards:
      # - Kubernetes Cluster Summary
      # - Node Exporter Full
      #--------------------------------------------
      - job_name: kubernetes-service-endpoints

        kubernetes_sd_configs:
        - role: endpoints

        relabel_configs:
        - action: keep
          regex: true
          source_labels:
          - __meta_kubernetes_service_annotation_prometheus_io_scrape
        - action: replace
          regex: (https?)
          source_labels:
          - __meta_kubernetes_service_annotation_prometheus_io_scheme
          target_label: __scheme__
        - action: replace
          regex: (.+)
          source_labels:
          - __meta_kubernetes_service_annotation_prometheus_io_path
          target_label: __metrics_path__
        - action: replace
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
          source_labels:
          - __address__
          - __meta_kubernetes_service_annotation_prometheus_io_port
          target_label: __address__
        - action: labelmap
          regex: __meta_kubernetes_service_label_(.+)
        - action: replace
          source_labels:
          - __meta_kubernetes_namespace
          target_label: kubernetes_namespace
        - action: replace
          source_labels:
          - __meta_kubernetes_service_name
          target_label: kubernetes_name
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_node_name
          target_label: kubernetes_node

      #--------------------------------------------
      # Scrape config for pods.
      #--------------------------------------------
      - job_name: 'kubernetes-pods'

        kubernetes_sd_configs:
        - role: pod

        relabel_configs:
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
          action: replace
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
          target_label: __address__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: kubernetes_pod_name
        - source_labels: [__meta_kubernetes_pod_name, __address__]
          action: drop
          regex: "loki-0;.*:(9095|7946)"

      #--------------------------------------------
      # Scrape config for cadvisor.
      #--------------------------------------------
      - job_name: 'kubernetes-cadvisor'

        kubernetes_sd_configs:
        - role: node

        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)
        - target_label: __address__
          replacement: kubernetes.default.svc:443
        - source_labels: [__meta_kubernetes_node_name]
          regex: (.+)
          target_label: __metrics_path__
          replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

      #--------------------------------------------
      # Scrape the cost-model /metrics endpoint.
      #--------------------------------------------
      - job_name: 'kubecost'
        honor_labels: true
        scrape_interval: 1m
        scrape_timeout: 10s
        metrics_path: /metrics

        scheme: http
        dns_sd_configs:
        - names:
          - kubecost-cost-analyzer.kubecost
          type: 'A'
          port: 9003

      #--------------------------------------------
      # Scrape static config for homelab.
      #--------------------------------------------
      - job_name: 'dns-master'
        static_configs:
          - targets: ['10.11.1.2:9153']
            labels:
              alias: admin1

      - job_name: 'dns-slave1'
        static_configs:
          - targets: ['10.11.1.3:9153']
            labels:
              alias: admin2

      - job_name: 'etcd'
        static_configs:
          - targets: ['10.11.1.31:2381','10.11.1.32:2381','10.11.1.33:2381']

      - job_name: 'haproxy'
        static_configs:
          - targets: ['10.11.1.30:9101']

      - job_name: 'admin1'
        static_configs:
          - targets: ['10.11.1.2:9100']
            labels:
              alias: admin1

      - job_name: 'admin2'
        static_configs:
          - targets: ['10.11.1.3:9100']
            labels:
              alias: admin2

      - job_name: 'pxe'
        static_configs:
          - targets: ['10.11.1.20:9100']
            labels:
              alias: pxe

      - job_name: 'kvm1'
        static_configs:
          - targets: ['10.11.1.21:9100']
            labels:
              alias: kvm1

      - job_name: 'kvm2'
        static_configs:
          - targets: ['10.11.1.22:9100']
            labels:
              alias: kvm2

      - job_name: 'kvm3'
        static_configs:
          - targets: ['10.11.1.23:9100']
            labels:
              alias: kvm3

      - job_name: 'media-pc'
        static_configs:
          - targets: ['10.11.1.28:9100']
            labels:
              alias: media-pc

      - job_name: 'k8s-master1'
        static_configs:
          - targets: ['10.11.1.31:9100']
            labels:
              alias: k8s-master1

      - job_name: 'k8s-master2'
        static_configs:
          - targets: ['10.11.1.32:9100']
            labels:
              alias: k8s-master2

      - job_name: 'k8s-master3'
        static_configs:
          - targets: ['10.11.1.33:9100']
            labels:
              alias: k8s-master3

      - job_name: 'k8s-node1'
        static_configs:
          - targets: ['10.11.1.34:9100']
            labels:
              alias: k8s-node1

      - job_name: 'k8s-node2'
        static_configs:
          - targets: ['10.11.1.35:9100']
            labels:
              alias: k8s-node2

      - job_name: 'k8s-node3'
        static_configs:
          - targets: ['10.11.1.36:9100']
            labels:
              alias: k8s-node3

      - job_name: 'openvas'
        static_configs:
          - targets: ['10.11.1.40:9100']
            labels:
              alias: openvas

      - job_name: 'raspberrypi'
        static_configs:
          - targets: ['10.11.1.7:9100']
            labels:
              alias: raspberrypi

      - job_name: 'pihole-exporter'
        static_configs:
          - targets: ['pihole-exporter.monitoring.svc:9617']
            labels:
              alias: pihole-exporter

      - job_name: 'vps'
        static_configs:
          - targets: ['vps.hl.test:9100']
            labels:
              alias: vps

      - job_name: 'mysql'
        static_configs:
          - targets: ['vps.hl.test:9104']
            labels:
              alias: mysql

      - job_name: 'mikrotik-exporter'
        scrape_interval: 30s
        static_configs:
          - targets:
            - 10.11.1.1 # your Mikrotik router IP you wish to monitor
        metrics_path: /metrics
        params:
          module: [my_router]
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: mikrotik-exporter:9436

      - job_name: 'mikrotik-lte-exporter'
        scrape_interval: 30s
        static_configs:
          - targets:
            - 10.11.1.11 # your Mikrotik LTE router IP you wish to monitor
        metrics_path: /metrics
        params:
          module: [my_router]
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: mikrotik-lte-exporter:9437