feat(services): add database metrics interface (#1739)

* feat(services): add database metrics interface

* remove some metrics

build/charts/laf-server/templates/rumtime-exporter.yaml

@@ -39,6 +39,8 @@ spec:
               value: {{ .Values.default_region.runtime_exporter_secret | quote }}
             - name: NAMESPACE
               value: {{ .Release.Namespace | quote }}
+            - name: DB_NAMESPACE
+              value: {{ .Release.Namespace | quote }}
 --- 
 apiVersion: v1
 kind: Service
@@ -69,6 +71,10 @@ spec:
       path: "/runtime/metrics/{{ .Values.default_region.runtime_exporter_secret}}"
       scrapeTimeout: 10s
       honorLabels: true
+    - interval: 60s
+      path: "/database/metrics/{{ .Values.default_region.runtime_exporter_secret}}"
+      scrapeTimeout: 10s
+      honorLabels: true
   namespaceSelector:
     matchNames:
       - {{ .Release.Namespace }}

services/runtime-exporter/src/config.ts

@@ -37,6 +37,9 @@ export default class Config {
   static get NAMESPACE(): string {
     return process.env.NAMESPACE || ''
   }
+  static get DB_NAMESPACE(): string {
+    return process.env.DB_NAMESPACE || ''
+  }
 
   static get API_SECRET(): string {
     if (!process.env.API_SECRET) {

services/runtime-exporter/src/handler/get-db-metrics.ts

@@ -0,0 +1,58 @@
+import { RequestHandler } from 'express'
+import { ClusterService, Metric } from '../helper/cluster.service'
+import Config from '../config'
+import * as prom from 'prom-client'
+
+const register = new prom.Registry()
+
+const DATABASE_CPU = new prom.Gauge({
+  name: 'laf_mongo_cpu',
+  help: 'the cpu of the mongo',
+  registers: [register],
+  labelNames: ['container', 'pod', 'appid'],
+})
+
+const DATABASE_MEMORY = new prom.Gauge({
+  name: 'laf_mongo_memory',
+  help: 'the memory of the mongo',
+  registers: [register],
+  labelNames: ['container', 'pod', 'appid'],
+})
+
+function updateMetrics(metric: Metric) {
+  DATABASE_CPU.labels(metric.containerName, metric.podName, metric.appid).set(
+    metric.cpu,
+  )
+  DATABASE_MEMORY.labels(
+    metric.containerName,
+    metric.podName,
+    metric.appid,
+  ).set(metric.memory)
+}
+
+const getDatabaseMetrics: RequestHandler = async (req, res) => {
+  const token = req.params.token
+
+  if (!token || Config.API_SECRET !== token) {
+    return res.status(403).send('forbidden')
+  }
+
+  // Clear the metrics data generated by the last request
+  DATABASE_CPU.reset()
+  DATABASE_MEMORY.reset()
+
+  const databaseMetrics = await ClusterService.getPodMetrics(
+    ClusterService.DB_NAMESPACE,
+    ClusterService.LABEL_DATABASE,
+    'DATABASE',
+  )
+
+  for (const metric of databaseMetrics) {
+    updateMetrics(metric)
+  }
+
+  res.set('Content-Type', 'text/plain')
+  res.send(await register.metrics())
+}
+
+export default getDatabaseMetrics

services/runtime-exporter/src/handler/get-runtime-metrics.ts

@@ -62,8 +62,11 @@ const getRuntimeMetrics: RequestHandler = async (req, res) => {
   RUNTIME_MEMORY.reset()
   RUNTIME_MEMORY_LIMIT.reset()
 
-  const runtimeMetrics =
-    await ClusterService.getRuntimePodMetricsForAllNamespaces()
+  const runtimeMetrics = await ClusterService.getPodMetrics(
+    ClusterService.NAMESPACE,
+    ClusterService.LABEL_KEY_APP_ID,
+    'RUNTIME',
+  )
 
   for (const metric of runtimeMetrics) {
     updateMetrics(metric)

services/runtime-exporter/src/helper/cluster.service.ts

@@ -23,7 +23,9 @@ export class ClusterService {
    * - if kubeconfig is not empty, load from string
    */
   static LABEL_KEY_APP_ID = 'laf.dev/appid'
+  static LABEL_DATABASE = 'app.kubernetes.io/managed-by=kubeblocks'
   static NAMESPACE = Config.NAMESPACE
+  static DB_NAMESPACE = Config.DB_NAMESPACE
 
   static loadKubeConfig() {
     const conf = Config.KUBECONF
@@ -55,42 +57,71 @@ export class ClusterService {
     return new k8s.Metrics(kc)
   }
 
-  static async getRuntimePodMetricsForAllNamespaces(): Promise<Metric[]> {
+  static async getPodMetrics(
+    namespace: string,
+    label: string,
+    app: string,
+  ): Promise<Metric[]> {
     const metricsClient = this.getMetricsClient()
     let res: any
-    if (ClusterService.NAMESPACE) {
+    if (namespace) {
       // eslint-disable-next-line @typescript-eslint/ban-ts-comment
       // @ts-ignore
       res = await metricsClient.metricsApiRequest(
-        `/apis/metrics.k8s.io/v1beta1/namespaces/${ClusterService.NAMESPACE}/pods?labelSelector=laf.dev/appid`,
+        `/apis/metrics.k8s.io/v1beta1/namespaces/${namespace}/pods?labelSelector=${label}`,
       )
     } else {
       // eslint-disable-next-line @typescript-eslint/ban-ts-comment
       // @ts-ignore
       res = await metricsClient.metricsApiRequest(
-        '/apis/metrics.k8s.io/v1beta1/pods?labelSelector=laf.dev/appid',
+        `/apis/metrics.k8s.io/v1beta1/pods?labelSelector=${label}`,
       )
     }
 
     const metricsList: Metric[] = []
-    for (const item of res.items) {
-      const appid = item.metadata.labels[ClusterService.LABEL_KEY_APP_ID]
-      const podName = item.metadata.name
-      for (const container of item.containers) {
-        const containerName = container.name
-        // millicores
-        const cpu = Number(quantityToScalar(container.usage.cpu || 0))
-        // bytes
-        const memory = Number(quantityToScalar(container.usage.memory || 0))
 
-        const metric: Metric = {
-          cpu: cpu,
-          memory: memory,
-          appid: appid,
-          containerName: containerName,
-          podName: podName,
+    if (app === 'RUNTIME') {
+      for (const item of res.items) {
+        const appid: string =
+          item.metadata.labels[ClusterService.LABEL_KEY_APP_ID]
+        const podName = item.metadata.name
+        for (const container of item.containers) {
+          // millicores
+          const cpu = Number(quantityToScalar(container.usage.cpu || 0))
+          // bytes
+          const memory = Number(quantityToScalar(container.usage.memory || 0))
+
+          const metric: Metric = {
+            cpu: cpu,
+            memory: memory,
+            appid: appid,
+            containerName: container.name,
+            podName: podName,
+          }
+          metricsList.push(metric)
+        }
+      }
+    } else {
+      for (const item of res.items) {
+        const appid: string = item.metadata.labels['app.kubernetes.io/instance']
+        const podName = item.metadata.name
+        for (const container of item.containers) {
+          if (container.name === 'mongodb') {
+            // millicores
+            const cpu = Number(quantityToScalar(container.usage.cpu || 0))
+            // bytes
+            const memory = Number(quantityToScalar(container.usage.memory || 0))
+
+            const metric: Metric = {
+              cpu: cpu,
+              memory: memory,
+              appid: appid,
+              containerName: container.name,
+              podName: podName,
+            }
+            metricsList.push(metric)
+          }
         }
-        metricsList.push(metric)
       }
     }
 

services/runtime-exporter/src/index.ts

@@ -2,6 +2,7 @@ import express, { Request, Response } from 'express'
 import Config from './config'
 import { logger } from './logger'
 import getRuntimeMetrics from './handler/get-runtime-metrics'
+import getDatabaseMetrics from './handler/get-db-metrics'
 
 require('express-async-errors')
 const app = express()
@@ -14,6 +15,7 @@ process.on('uncaughtException', (err: Error) => {
 })
 
 app.get('/runtime/metrics/:token', getRuntimeMetrics)
+app.get('/database/metrics/:token', getDatabaseMetrics)
 app.get('/healthz', (_, res: Response) => res.send('ok'))
 
 // express error capture middleware