feat(logstash-http): add axios retry module and fix bulk data seriali…

…zation

docs/appenders/logstash-http.md

@@ -18,6 +18,7 @@ npm install --save @tsed/logger-logstash-http
 - `options.logType` - `string` (optional) - used for the `type` field in the logstash data
 - `options.timeout` - `integer` (optional, defaults to 5000ms) - the timeout for the HTTP request.
 - `options.maxBuffer` - `integer` (optional, defaults to 0) - Group bulk request by the maxBuffer number. By Default the buffer is disabled.
+- `options.retriesOptions` - `object` (optional) - Configure retries strategy. See [axios-retry](https://www.google.com/search?client=firefox-b-d&q=axios-retry) options for more details.
 
 This appender will also pick up Logger context values from the events, and add them as `p_` values in the logFaces event. See the example below for more details.
 

jest.config.js

@@ -72,8 +72,7 @@ export default {
         'lazy': false,
         'noInterop': false
       }
-    }
-    ]
+    }]
   },
 
   modulePathIgnorePatterns: ['<rootDir>/packages/*/lib', '<rootDir>/dist'],

packages/logstash-http/package.json

@@ -42,6 +42,7 @@
   "dependencies": {
     "@types/axios": "^0.14.0",
     "axios": "^0.26.1",
+    "axios-retry": "^3.3.1",
     "tslib": "2.3.1"
   },
   "devDependencies": {

packages/logstash-http/src/LogStashHttpAppender.ts

@@ -1,6 +1,7 @@
 import {$log, Appender, BaseAppender, LogEvent} from "@tsed/logger";
 import * as util from "util";
 import axios, {AxiosBasicCredentials} from "axios";
+import axiosRetry, {IAxiosRetryConfig} from "axios-retry";
 
 function wrapErrorsWithInspect(items: any[]) {
   return items.map((item) => {
@@ -30,13 +31,14 @@ export class LogStashHttpOptions {
   timeout?: number;
   params?: Record<string, any>;
   headers?: Record<string, any>;
+  retryOptions?: IAxiosRetryConfig;
 }
 
 @Appender({name: "logstash-http"})
 export class LogStashHttpAppender extends BaseAppender<LogStashHttpOptions> {
   private client: ReturnType<typeof axios.create>;
 
-  #buffer: string[] = [];
+  #buffer: Record<string, any>[] = [];
 
   build() {
     if ($log.level !== "OFF" && this.config.options) {
@@ -51,21 +53,21 @@ export class LogStashHttpAppender extends BaseAppender<LogStashHttpOptions> {
         },
         withCredentials: true
       });
+
+      axiosRetry(this.client, {
+        retries: 3,
+        retryDelay: axiosRetry.exponentialDelay,
+        ...this.config.options.retryOptions
+      });
     }
   }
 
   write(loggingEvent: LogEvent) {
-    const {application, logType, logChannel} = this.config.options;
+    const {logChannel} = this.config.options;
     const level = loggingEvent.level.toString().toLowerCase();
 
     if (level !== "off") {
       const logstashEvent = [
-        {
-          index: {
-            _index: typeof application === "function" ? application() : application,
-            _type: logType
-          }
-        },
         {
           ...loggingEvent.getData(),
           message: format(loggingEvent.getMessage()),
@@ -77,11 +79,11 @@ export class LogStashHttpAppender extends BaseAppender<LogStashHttpOptions> {
         }
       ];
 
-      this.send(`${JSON.stringify(logstashEvent[0])}\n${JSON.stringify(logstashEvent[1])}`);
+      this.send(logstashEvent);
     }
   }
 
-  send(bulk: string) {
+  send(bulk: Record<string, any>) {
     const {bufferMax = 0} = this.config.options;
     this.#buffer.push(bulk);
 
@@ -97,10 +99,24 @@ export class LogStashHttpAppender extends BaseAppender<LogStashHttpOptions> {
     this.#buffer = [];
 
     if (buffer.length) {
-      const bulk = buffer.join("\n");
       const {url} = this.config.options;
+      const {application, logType} = this.config.options;
+
+      const header = JSON.stringify({
+        index: {
+          _index: typeof application === "function" ? application() : application,
+          _type: logType
+        }
+      });
+
+      const bulkData =
+        buffer
+          .flatMap((obj) => {
+            return [header, JSON.stringify(obj)];
+          }, [])
+          .join("\n") + "\n";
 
-      return this.client.post("", bulk + "\n").catch((error) => {
+      return this.client.post("", bulkData).catch((error) => {
         if (error.response) {
           console.error(
             `Ts.ED Logger.logstash-http Appender error posting to ${url}: ${error.response.status} - ${JSON.stringify(error.response.data)}`

yarn.lock

@@ -1516,6 +1516,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/runtime@npm:^7.15.4":
+  version: 7.19.0
+  resolution: "@babel/runtime@npm:7.19.0"
+  dependencies:
+    regenerator-runtime: ^0.13.4
+  checksum: fa69c351bb05e1db3ceb9a02fdcf620c234180af68cdda02152d3561015f6d55277265d3109815992f96d910f3db709458cae4f8df1c3def66f32e0867d82294
+  languageName: node
+  linkType: hard
+
 "@babel/runtime@npm:^7.8.4, @babel/runtime@npm:^7.9.6":
   version: 7.10.4
   resolution: "@babel/runtime@npm:7.10.4"
@@ -3173,6 +3182,7 @@ __metadata:
     "@tsed/logger": 6.2.2
     "@types/axios": ^0.14.0
     axios: ^0.26.1
+    axios-retry: ^3.3.1
     tslib: 2.3.1
   peerDependencies:
     "@tsed/logger": 6.2.2
@@ -5368,6 +5378,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"axios-retry@npm:^3.3.1":
+  version: 3.3.1
+  resolution: "axios-retry@npm:3.3.1"
+  dependencies:
+    "@babel/runtime": ^7.15.4
+    is-retry-allowed: ^2.2.0
+  checksum: 125c75e08048a28de5d0dfa9de9a1924185d863a4323a71472646a1d5a326f727ff59e75e1b557220da1aad7cba01222c47c26194c5940f295378668dd3b9163
+  languageName: node
+  linkType: hard
+
 "axios@npm:*":
   version: 0.21.1
   resolution: "axios@npm:0.21.1"
@@ -11873,6 +11893,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"is-retry-allowed@npm:^2.2.0":
+  version: 2.2.0
+  resolution: "is-retry-allowed@npm:2.2.0"
+  checksum: 3d1103a9290b5d03626756a41054844633eac78bc5d3e3a95b13afeae94fa3cfbcf7f0b5520d83f75f48a25ce7b142fdbac4217dc4b0630f3ea55e866ec3a029
+  languageName: node
+  linkType: hard
+
 "is-stream@npm:^1.1.0":
   version: 1.1.0
   resolution: "is-stream@npm:1.1.0"