Adds mysql prometheus exporter

.github/workflows/kustomize-prometheus-mysql-exporter.yaml

@@ -0,0 +1,33 @@
+name: Kustomize GitHub Actions for Prometheus MySql Exporter
+
+on:
+  pull_request:
+    paths:
+      - kustomize/prometheus-mysql-exporter/**
+      - .github/workflows/kustomize-prometheus-mysql-exporter.yaml
+jobs:
+  kustomize:
+    name: Kustomize
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: azure/setup-helm@v3
+        with:
+          version: latest
+          token: "${{ secrets.GITHUB_TOKEN }}"
+        id: helm
+      - name: Kustomize Install
+        working-directory: /usr/local/bin/
+        run: |
+          if [ ! -f /usr/local/bin/kustomize ]; then
+            curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | sudo bash
+          fi
+      - name: Run Kustomize Build
+        run: |
+          kustomize build kustomize/prometheus-mysql-exporter/ --enable-helm --helm-command ${{ steps.helm.outputs.helm-path }} > /tmp/rendered.yaml
+      - name: Return Kustomize Build
+        uses: actions/upload-artifact@v2
+        with:
+          name: kustomize-prometheus-mysql-exporter-artifact
+          path: /tmp/rendered.yaml

docs/index.md

@@ -46,3 +46,7 @@ Building our cloud future has never been this simple.
 ## Upgrades
   * [Running Genestack Upgrade](genestack-upgrade.md)
   * [Running Kubernetes Upgrade](k8s-upgrade.md)
+
+## Monitoring
+  * [Deploy Prometheus](prometheus.md)
+  * [MySQL Exporter](prometheus-mysql-exporter.md)

docs/prometheus-mysql-exporter.md

@@ -0,0 +1,25 @@
+##Mariadb Exporter
+
+Mysql Exporter is used to expose metrics from a running mysql/mariadb server. The type of metrics exposed is controlled
+by the exporter and expressed in values.yaml file.
+
+##Installation
+
+First create secret containing password for monitoring user
+```
+kubectl --namespace openstack \
+        create secret generic mariadb-monitoring \
+        --type Opaque \
+        --from-literal=username="monitoring" \
+        --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-64};echo;)"
+```
+
+Next, install the exporter
+
+```
+cd /opt/genestack/kustomize/prometheus-mysql-exporter
+
+kubectl kustomize --enable-helm . | kubectl create -n openstack -f -
+```
+
+If the installation is succesful, you should see the exporter pod in openstack namespace.

kustomize/prometheus-mysql-exporter/kustomization.yaml

@@ -0,0 +1,11 @@
+resources:
+  - monitoring_user_create.yaml
+  - monitoring_user_grant.yaml
+
+helmCharts:
+  - name: prometheus-mysql-exporter
+    repo: https://prometheus-community.github.io/helm-charts
+    releaseName: prometheus-mysql-exporter
+    namespace: openstack
+    includeCRDs: true
+    valuesFile: values.yaml

kustomize/prometheus-mysql-exporter/monitoring_user_create.yaml

@@ -0,0 +1,15 @@
+apiVersion: mariadb.mmontes.io/v1alpha1
+kind: User
+metadata:
+  name: monitoring
+spec:
+  name: monitoring
+  mariaDbRef:
+    name: mariadb-galera
+  passwordSecretKeyRef:
+    name: mariadb-monitoring
+    key: password
+  maxUserConnections: 15
+  host: "%"
+  requeueInterval: 30s
+  retryInterval: 5s

kustomize/prometheus-mysql-exporter/monitoring_user_grant.yaml

@@ -0,0 +1,18 @@
+apiVersion: mariadb.mmontes.io/v1alpha1
+kind: Grant
+metadata:
+  name: monitoring-grant
+spec:
+  mariaDbRef:
+    name: mariadb-galera
+  privileges:
+    - "SELECT"
+    - "PROCESS"
+    - "REPLICATION CLIENT"
+  database: "*"
+  table: "*"
+  username: monitoring
+  grantOption: false
+  host: "%"
+  requeueInterval: 30s
+  retryInterval: 5s

kustomize/prometheus-mysql-exporter/values.yaml

@@ -0,0 +1,218 @@
+## Default values for prometheus-mysql-exporter.
+## This is a YAML-formatted file.
+## Declare variables to be passed into your templates.
+
+## namespaceOverride overrides the namespace which the resources will be deployed in
+namespaceOverride: ""
+
+## override release name
+fullnameOverride: ""
+
+replicaCount: 1
+
+image:
+  registry: quay.io
+  repository: prometheus/mysqld-exporter
+  ## if not set charts appVersion var is used
+  tag: ""
+  pullPolicy: "IfNotPresent"
+
+# imagePullSecrets:
+# - name: secret-name
+imagePullSecrets: []
+
+service:
+  labels: {}
+  annotations: {}
+  name: mysql-exporter
+  type: ClusterIP
+  externalPort: 9104
+  internalPort: 9104
+
+serviceMonitor:
+  # enabled should be set to true to enable prometheus-operator discovery of this service
+  enabled: true
+  # interval is the interval at which metrics should be scraped
+  interval: 30s
+  # scrapeTimeout is the timeout after which the scrape is ended
+  scrapeTimeout: 10s
+  namespace: openstack
+  # namespaceSelector: []
+  # additionalLabels is the set of additional labels to add to the ServiceMonitor
+  additionalLabels: {}
+  jobLabel: ""
+  targetLabels: []
+  podTargetLabels: []
+  metricRelabelings: []
+  # Set relabel_configs as per https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+  relabelings: []
+  multipleTarget:
+    enabled: false
+    targets:
+      # target connection information with name (required), endpoint (required) and port (optionnal)
+      # if sharedSecret is not enabled the name must match an entry client.{{ name }} existing in the secret
+      # - endpoint: mysql1.dns.local
+      #   name: mysql1
+      #   port: 3307
+      # - endpoint: mysql2.dns.local
+      #   name: mysql2
+    # Enable shared credentials for all targets
+    sharedSecret:
+      enabled: false
+      name: ""
+
+serviceAccount:
+  # Specifies whether a ServiceAccount should be created
+  create: false
+  # The name of the ServiceAccount to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:
+  annotations: {}
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #  cpu: 100m
+  #  memory: 128Mi
+  # requests:
+  #  cpu: 100m
+  #  memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+podLabels: {}
+
+# Extra Volume Mounts for the mysql exporter container
+extraVolumeMounts: []
+# - name: example
+#   mountPath: /example
+
+# Extra Volumes for the pod
+extraVolumes: []
+# - name: example
+#   configMap:
+#     name: example
+
+podSecurityContext: {}
+  # fsGroup: 65534
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 65534
+
+annotations:
+  prometheus.io/scrape: "true"
+  prometheus.io/path: "/metrics"
+  prometheus.io/port: "9104"
+
+config: {}
+  # Allow to set specifc options on the exporter
+  # logLevel: info
+  # logFormat: "logger:stderr"
+
+collectors:
+  # auto_increment.columns: false
+  # binlog_size: false
+  # engine_innodb_status: false
+  # engine_tokudb_status: false
+  # global_status: true
+  # global_variables: true
+  # info_schema.clientstats: false
+  # info_schema.innodb_metrics: false
+  # info_schema.innodb_tablespaces: false
+  # info_schema.innodb_cmp: false
+  # info_schema.innodb_cmpmem: false
+  # info_schema.processlist: false
+  # info_schema.processlist.min_time: 0
+  # info_schema.query_response_time: false
+  # info_schema.tables: true
+  # info_schema.tables.databases: '*'
+  # info_schema.tablestats: false
+  # info_schema.schemastats: false
+  # info_schema.userstats: false
+  # perf_schema.eventsstatements: false
+  # perf_schema.eventsstatements.digest_text_limit: 120
+  # perf_schema.eventsstatements.limit: false
+  # perf_schema.eventsstatements.timelimit: 86400
+  # perf_schema.eventswaits: false
+  # perf_schema.file_events: false
+  # perf_schema.file_instances: false
+  # perf_schema.indexiowaits: false
+  # perf_schema.tableiowaits: false
+  # perf_schema.tablelocks: false
+  # perf_schema.replication_group_member_stats: false
+  # slave_status: true
+  # slave_hosts: false
+  # heartbeat: false
+  # heartbeat.database: heartbeat
+  # heartbeat.table: heartbeat
+
+# mysql connection params which build the my.cnf config
+mysql:
+  db: ""
+  host: "mariadb-galera"
+  # config my.cnf https://dev.mysql.com/doc/c-api/8.0/en/mysql-options.html
+  additionalConfig:
+    # - connect-timeout=5
+    # - debug
+  pass: "password"
+  port: 3306
+  protocol: ""
+  user: "monitoring"
+  # secret with full config my.cnf
+  existingConfigSecret:
+    name: ""
+    key: ""
+  # secret only containing the password
+  existingPasswordSecret:
+    name: "mariadb-monitoring"
+    key: "password"
+
+# cloudsqlproxy https://cloud.google.com/sql/docs/mysql/sql-proxy
+cloudsqlproxy:
+  enabled: false
+  image:
+    repo: "gcr.io/cloud-sql-connectors/cloud-sql-proxy"
+    tag: "2.4.0"
+    pullPolicy: "IfNotPresent"
+  instanceConnectionName: "project:us-central1:dbname"
+  privateIp: false
+  port: "3306"
+  credentialsSecret: ""
+  # service account json
+  credentials: ""
+  workloadIdentity:
+    enabled: false
+    serviceAccountEmail: ""
+  extraArgs: ""
+
+## Custom PrometheusRules to be defined
+## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart
+## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions
+prometheusRule:
+  enabled: false
+  additionalLabels: {}
+  namespace: ""
+  rules:
+    []
+    ## These are just examples rules, please adapt them to your needs.
+    ## Make sure to constraint the rules to the current service.
+    # - alert: MysqlDown
+    #   expr: mysql_up == 0
+    #   for: 5m
+    #   labels:
+    #     severity: critical
+    #   annotations:
+    #     summary: MySQL down (instance {{ $labels.instance }})
+    #     description: "MySQL instance is down on {{ $labels.instance }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

kustomize/prometheus/alerting_rules.yaml

@@ -10,3 +10,39 @@ additionalPrometheusRulesMap:
           severity: critical
         annotations:
           summary: "Rabbit queue size too large (instance {{ `{{ $labels.instance }}` }} )"
+  database-alerts:
+    groups:
+    - name: Mysql Alerts
+      rules:
+      - alert: MysqlDown
+        expr: mysql_up == 0
+        for: 0m
+        labels:
+          severity: critical
+        annotations:
+          summary: MariaDB down (instance {{ $labels.instance }})
+          description: "MariaDB instance is down on {{ $labels.instance }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: MysqlTooManyConnections(>80%)
+        expr: max_over_time(mysql_global_status_threads_connected[1m]) / mysql_global_variables_max_connections * 100 > 90
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Database too many connections (> 90%) (instance {{ $labels.instance }})
+          description: "More than 90% of MySQL connections are in use on {{ $labels.instance }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: MysqlSlowQueries
+        expr: increase(mysql_global_status_slow_queries[1m]) > 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: MySQL slow queries (instance {{ $labels.instance }})
+          description: "MySQL server mysql has some new slow query.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: MysqlRestarted
+        expr: mysql_global_status_uptime < 60
+        for: 0m
+        labels:
+          severity: info
+        annotations:
+          summary: MySQL restarted (instance {{ $labels.instance }})
+          description: "MySQL has just been restarted, less than one minute ago on {{ $labels.instance }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"