Add Prometheus metrics and monitoring for Redis

package.json

@@ -16,7 +16,8 @@
     "@google-cloud/storage": "^6.9.3",
     "fastify": "^4.9.2",
     "firebase-admin": "^11.2.0",
-    "ioredis": "^5.2.3"
+    "ioredis": "^5.2.3",
+    "prom-client": "^14.2.0"
   },
   "devDependencies": {
     "@types/argparse": "^2.0.10",

src/RedisCache.ts

@@ -2,6 +2,7 @@ import Cache from './Cache';
 
 import Redis, { RedisOptions } from 'ioredis';
 import Selector from './model/Selector';
+import { redisConnectionGauge, redisFailureCounter, redisSuccessCounter } from './metrics';
 
 class RedisCache implements Cache {
   private static DEFAULT_TTL = 60 * 60 * 24 * 7;
@@ -10,30 +11,72 @@ class RedisCache implements Cache {
 
   constructor(options: RedisOptions) {
     this.redis_ = new Redis(options);
+
+    this.redis_.on('connect', () => {
+      console.log('Redis connected');
+      redisConnectionGauge.set(1);
+    });
+
+    this.redis_.on('ready', () => {
+      console.log('Redis ready');
+      redisConnectionGauge.set(1);
+    });
+
+    this.redis_.on('error', (err) => {
+      console.error('Redis error:', err.message);
+      redisConnectionGauge.set(0);
+    });
+
+    this.redis_.on('close', () => {
+      console.warn('Redis connection closed');
+      redisConnectionGauge.set(0);
+    });
+
+    this.redis_.on('end', () => {
+      console.warn('Redis connection ended');
+      redisConnectionGauge.set(0);
+    });
   }
 
   private static key_ = ({ collection, id }: Selector): string => {
     return `${collection}/${id}`;
   };
 
   async get(selector: Selector): Promise<object | null> {
-    const data = await this.redis_.get(RedisCache.key_(selector));
-    if (!data) return null;
-
-    return JSON.parse(data);
+    try {
+      const data = await this.redis_.get(RedisCache.key_(selector));
+      redisSuccessCounter.inc({ operation: 'get' });
+      if (!data) return null;
+      return JSON.parse(data);
+    } catch (err) {
+      redisFailureCounter.inc({ operation: 'get' });
+      throw err;
+    }
   }
 
   async set(selector: Selector, value: object | null): Promise<void> {
-    if (!value) {
-      await this.redis_.del(RedisCache.key_(selector));
-      return;
+    try {
+      if (!value) {
+        await this.redis_.del(RedisCache.key_(selector));
+        redisSuccessCounter.inc({ operation: 'set' });
+        return;
+      }
+      await this.redis_.setex(RedisCache.key_(selector), RedisCache.DEFAULT_TTL, JSON.stringify(value));
+      redisSuccessCounter.inc({ operation: 'set' });
+    } catch (err) {
+      redisFailureCounter.inc({ operation: 'set' });
+      throw err;
     }
-
-    await this.redis_.setex(RedisCache.key_(selector), RedisCache.DEFAULT_TTL, JSON.stringify(value));
   }
 
   async remove(selector: Selector): Promise<void> {
-    await this.redis_.del(RedisCache.key_(selector));
+    try {
+      await this.redis_.del(RedisCache.key_(selector));
+      redisSuccessCounter.inc({ operation: 'remove' });
+    } catch (err) {
+      redisFailureCounter.inc({ operation: 'remove' });
+      throw err;
+    }
   }
 }
 

src/index.ts

@@ -12,6 +12,7 @@ import authorize, { AuthorizeResult } from './authorize';
 import { CHALLENGE_COMPLETION_COLLECTION, USER_COLLECTION } from './model/constants';
 
 import bigStore from './big-store';
+import { register as metricsRegister } from './metrics';
 
 const UNAUTHORIZED_RESULT = { message: 'Unauthorized' };
 const NOT_FOUND_RESULT = { message: 'Not Found' };
@@ -47,6 +48,12 @@ app.get('/', async (request, reply) => {
   reply.send({ database: 'alive' });
 });
 
+// Prometheus metrics endpoint
+app.get('/metrics', async (request, reply) => {
+  reply.header('Content-Type', metricsRegister.contentType);
+  reply.send(await metricsRegister.metrics());
+});
+
 app.get('/:collection/:id', async (request, reply) => {
   const token = await authenticate(request);
 

src/metrics.ts

@@ -0,0 +1,39 @@
+import { Registry, Gauge, Counter } from 'prom-client';
+
+// Create a custom registry
+export const register = new Registry();
+
+// Redis connection status gauge (1 = connected, 0 = disconnected)
+export const redisConnectionGauge = new Gauge({
+  name: 'database_redis_connection_status',
+  help: 'Redis connection status (1 = connected, 0 = disconnected)',
+  registers: [register],
+});
+
+// Redis operation failures counter
+export const redisFailureCounter = new Counter({
+  name: 'database_redis_operation_failures_total',
+  help: 'Total number of failed Redis operations',
+  labelNames: ['operation'], // 'get', 'set', 'remove'
+  registers: [register],
+});
+
+// Redis operation success counter
+export const redisSuccessCounter = new Counter({
+  name: 'database_redis_operation_success_total',
+  help: 'Total number of successful Redis operations',
+  labelNames: ['operation'],
+  registers: [register],
+});
+
+// HTTP request counter
+export const httpRequestCounter = new Counter({
+  name: 'database_http_requests_total',
+  help: 'Total number of HTTP requests',
+  labelNames: ['method', 'route', 'status_code'],
+  registers: [register],
+});
+
+// Initialize Redis status as disconnected
+redisConnectionGauge.set(0);
+