feat(grpc-sdk): add support for logging to Grafana Loki (#249)

libraries/grpc-sdk/package.json

@@ -25,7 +25,8 @@
     "lodash": "^4.17.21",
     "nice-grpc": "^1.2.0",
     "protobufjs": "^6.11.3",
-    "winston": "^3.8.1"
+    "winston": "^3.8.1",
+    "winston-loki": "^6.0.5"
   },
   "directories": {
     "lib": "dist",

libraries/grpc-sdk/src/index.ts

@@ -35,6 +35,7 @@ import { checkModuleHealth } from './classes/HealthCheck';
 import { ConduitLogger } from './utilities/Logger';
 import winston from 'winston';
 import path from 'path';
+import LokiTransport from 'winston-loki';
 
 export default class ConduitGrpcSdk {
   private readonly serverUrl: string;
@@ -85,6 +86,20 @@ export default class ConduitGrpcSdk {
     } else {
       this.name = name;
     }
+    if (process.env.LOKI_URL && process.env.LOKI_URL !== '') {
+      ConduitGrpcSdk.Logger.addTransport(
+        new LokiTransport({
+          level: 'debug',
+          host: process.env.LOKI_URL,
+          batching: false,
+          replaceTimestamp: true,
+          labels: {
+            module: this.name,
+          },
+        }),
+      );
+    }
+
     this.serverUrl = serverUrl;
     this._watchModules = watchModules;
     this._serviceHealthStatusGetter = serviceHealthStatusGetter;
@@ -105,7 +120,7 @@ export default class ConduitGrpcSdk {
     while (true) {
       try {
         const state = await this.core.check();
-        if (((state as unknown) as HealthCheckStatus) === HealthCheckStatus.SERVING) {
+        if ((state as unknown as HealthCheckStatus) === HealthCheckStatus.SERVING) {
           ConduitGrpcSdk.Logger.log('Core connection established');
           this._initialize();
           break;

libraries/grpc-sdk/src/utilities/Logger.ts

@@ -2,25 +2,37 @@ import winston, { format, LogCallback, Logger } from 'winston';
 import { Indexable } from '../interfaces';
 import { isEmpty } from 'lodash';
 
-const Format = format.combine(
-  format.printf(info => {
-    // This will customize the Error Message
-    if (info.stack) {
-      return `[${info.timestamp}] [${info.level.toUpperCase()}]: ${info.message} ${
-        info.stack
-      }`;
-    }
-    return `[${info.timestamp}] [${info.level.toUpperCase()}]: ${info.message} ${
-      !isEmpty(info.meta)
-        ? info.meta.reduce(
-            (message: string, meta: Indexable) =>
-              `${message} \n  ${JSON.stringify(meta, null, 2)}`,
-            '',
-          )
-        : ''
-    }`;
-  }),
-);
+const processMeta = (meta: Indexable) => {
+  if (Array.isArray(meta)) {
+    return meta.reduce(
+      (message: string, meta: Indexable) =>
+        `${message} \n  ${JSON.stringify(meta, null, 2)}`,
+      '',
+    );
+  } else {
+    return JSON.stringify(meta, null, 2);
+  }
+};
+
+const createFormat = (logMeta: boolean = true) => {
+  return format.combine(
+    format.printf(info => {
+      // This will customize the Error Message
+      if (info.stack) {
+        return `[${info.timestamp}] [${info.level.toUpperCase()}]: ${info.message} ${
+          info.stack
+        }`;
+      }
+      if (logMeta) {
+        return `[${info.timestamp}] [${info.level.toUpperCase()}]: ${info.message} ${
+          !isEmpty(info.meta) ? processMeta(info.meta) : ''
+        }`;
+      } else {
+        return `[${info.timestamp}] [${info.level.toUpperCase()}]: ${info.message}`;
+      }
+    }),
+  );
+};
 const defaultTransport = new winston.transports.Console({
   level: 'debug',
   format: format.combine(
@@ -29,7 +41,7 @@ const defaultTransport = new winston.transports.Console({
     format.timestamp({
       format: 'YYYY-MM-DD HH:mm:ss',
     }),
-    Format,
+    createFormat(false),
     format.colorize({
       all: true,
     }),
@@ -38,6 +50,7 @@ const defaultTransport = new winston.transports.Console({
 
 export class ConduitLogger {
   private readonly _winston: winston.Logger;
+
   constructor(transports?: winston.transport[]) {
     this._winston = winston.createLogger({
       format: format.combine(
@@ -46,12 +59,16 @@ export class ConduitLogger {
         format.timestamp({
           format: 'YYYY-MM-DD HH:mm:ss',
         }),
-        Format,
+        createFormat(),
       ),
       transports: transports ? [...transports, defaultTransport] : [defaultTransport],
     });
   }
 
+  addTransport(transport: winston.transport) {
+    this._winston.add(transport);
+  }
+
   log(message: string, level: string = 'info', cb?: LogCallback): Logger {
     return this._winston.log(level, message, cb);
   }
@@ -87,4 +104,8 @@ export class ConduitLogger {
   verbose(message: string, cb?: LogCallback): Logger {
     return this._winston.verbose(message, cb);
   }
+
+  get winston() {
+    return this._winston;
+  }
 }

libraries/hermes/src/index.ts

@@ -3,7 +3,6 @@ import { RestController } from './Rest';
 import { GraphQLController } from './GraphQl/GraphQL';
 import { SocketController } from './Socket/Socket';
 import ConduitGrpcSdk, { ConduitError, Indexable } from '@conduitplatform/grpc-sdk';
-import { ConduitLogger } from './utils/logger';
 import http from 'http';
 import {
   ConduitRequest,
@@ -16,6 +15,7 @@ import cors from 'cors';
 import cookieParser from 'cookie-parser';
 import path from 'path';
 import { ConduitRoute } from './classes';
+import { createRouteMiddleware } from './utils/logger';
 
 const swaggerRouterMetadata: SwaggerRouterMetadata = {
   urlPrefix: '',
@@ -55,12 +55,10 @@ const swaggerRouterMetadata: SwaggerRouterMetadata = {
 };
 
 export class ConduitRoutingController {
-  private readonly _grpcSdk: ConduitGrpcSdk;
   private _restRouter?: RestController;
   private _graphQLRouter?: GraphQLController;
   private _socketRouter?: SocketController;
   private _middlewareRouter: Router;
-  private readonly logger: ConduitLogger;
   private readonly _cleanupTimeoutMs: number;
   private _cleanupTimeout: NodeJS.Timeout | null = null;
   readonly expressApp = express();
@@ -75,7 +73,6 @@ export class ConduitRoutingController {
     private readonly swaggerMetadata?: SwaggerRouterMetadata,
   ) {
     this._cleanupTimeoutMs = cleanupTimeoutMs < 0 ? 0 : Math.round(cleanupTimeoutMs);
-    this.logger = new ConduitLogger();
     this.start();
     this._middlewareRouter = Router();
     this._middlewareRouter.use(
@@ -123,21 +120,21 @@ export class ConduitRoutingController {
   initRest() {
     if (this._restRouter) return;
     this._restRouter = new RestController(
-      this._grpcSdk,
+      this.grpcSdk,
       this.swaggerMetadata ?? swaggerRouterMetadata,
     );
   }
 
   initGraphQL() {
     if (this._graphQLRouter) return;
-    this._graphQLRouter = new GraphQLController(this._grpcSdk);
+    this._graphQLRouter = new GraphQLController(this.grpcSdk);
   }
 
   initSockets(redisHost: string, redisPort: number) {
     if (this._socketRouter) return;
     this._socketRouter = new SocketController(
       this.socketPort,
-      this._grpcSdk,
+      this.grpcSdk,
       this.expressApp,
       {
         host: redisHost,
@@ -290,7 +287,7 @@ export class ConduitRoutingController {
 
   private registerGlobalMiddleware() {
     this.registerMiddleware(cors(), false);
-    this.registerMiddleware(this.logger.middleware, false);
+    this.registerMiddleware(createRouteMiddleware(ConduitGrpcSdk.Logger.winston), false);
     this.registerMiddleware(express.json({ limit: '50mb' }), false);
     this.registerMiddleware(
       express.urlencoded({ limit: '50mb', extended: false }),

libraries/hermes/src/utils/logger.ts

@@ -1,23 +1,19 @@
 import expressWinston from 'express-winston';
 import winston from 'winston';
 
-export class ConduitLogger {
-  get middleware() {
-    return expressWinston.logger({
-      transports: [new winston.transports.Console()],
-      format: winston.format.combine(winston.format.colorize(), winston.format.cli()),
-      meta: false,
-      msg: 'HTTP {{req.method}} {{req.url}}',
-      expressFormat: true,
-      colorize: false,
-    });
-  }
+export function createRouteMiddleware(winstonInstance: winston.Logger) {
+  return expressWinston.logger({
+    winstonInstance,
+    meta: true,
+    msg: 'HTTP {{req.method}} {{req.url}}',
+    expressFormat: true,
+    colorize: false,
+  });
+}
 
-  get errorLogger() {
-    return expressWinston.errorLogger({
-      transports: [new winston.transports.Console()],
-      format: winston.format.combine(winston.format.prettyPrint({ colorize: true })),
-      meta: false,
-    });
-  }
+export function errorLogger(winstonInstance: winston.Logger) {
+  return expressWinston.errorLogger({
+    winstonInstance,
+    meta: true,
+  });
 }