Retry on server error

lib/loggly/common.js

@@ -14,7 +14,7 @@ var arrSize = 100,
     timerFunction = null;
 
 // 
-// Variables for buffer array 
+// Variables for buffer array
 // 
 var arrBufferedMsg = [],
     timerFunctionForBufferedLogs = null;
@@ -24,6 +24,15 @@ var arrBufferedMsg = [],
 //
 var isValidToken = true;
 
+//
+// Variables for server retry
+//
+var arrRetryLogs = [],
+    maxRetryAllowed = 5,
+    totalRetries = 0,
+    statusCode,
+    notFailedOnServerError = true;
+
 var https = require('https'),
     util = require('util'),
     request = require('request'),
@@ -44,7 +53,8 @@ var failCodes = common.failCodes = {
   410: 'Gone',
   500: 'Internal Server Error',
   501: 'Not Implemented',
-  503: 'Throttled'
+  503: 'Throttled',
+  504: 'Gateway Timeout'
 };
 
 //
@@ -157,18 +167,25 @@ common.loggly = function () {
   }
   if (requestBody) {
     requestOptions.body = requestBody;
+    arrRetryLogs = arrRetryLogs.concat(requestBody);
   }
   function sendLogs() {
+    if (arrRetryLogs.length && !requestBody) requestOptions.body = arrRetryLogs[0];
     try {
       request(requestOptions, function (err, res, body) {
-        if (err) {
-          return onError(err);
-        }
-        var statusCode = res.statusCode.toString();
+        if (err) return onError(err);
+        statusCode = res.statusCode.toString();
         if(statusCode === '403') isValidToken = false;
+        if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res);
+        if (statusCode === '200') {
+          arrRetryLogs.splice(0, 1);
+          totalRetries = 0;
+        }
         if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
-          return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
-        } 
+          if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') {
+            return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+          }
+        }
         success(res, body);
       });
     }
@@ -177,23 +194,33 @@ common.loggly = function () {
     }
   }
   function sendBulkLogs() {
-    if (arrMsg.length === 0) {
-      return;
+    if (arrMsg.length === 0 && arrRetryLogs.length === 0) return;
+    var retryLogs = [];
+    if (arrRetryLogs.length && !arrMsg.length) {
+      retryLogs = arrRetryLogs.slice(0, arrSize);
+      requestOptions.body = retryLogs.join('\n');
     }
     //
     // Join Array Message with new line ('\n') character
     //
-    requestOptions.body = arrMsg.join('\n');
-    arrMsg.length = 0;
+    if (arrMsg.length) {
+      requestOptions.body = arrMsg.join('\n');
+      arrMsg.length = 0;
+    }
     try {
       request(requestOptions, function (err, res, body) {
-        if (err) {
-          return onError(err);
-        }
+        if (err) return onError(err);
         var statusCode = res.statusCode.toString();
         if(statusCode === '403') isValidToken = false;
+        if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res);
+        if (statusCode === '200') {
+          arrRetryLogs.splice(0, arrSize);
+          totalRetries = 0;
+        }
         if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
-          return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+          if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') {
+            return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+          }
         }
         success(res, body);
       });
@@ -227,6 +254,27 @@ common.loggly = function () {
     sendLogs();
   }
 
+  //
+  //function to retry sending logs maximum 5 times if server error occurs
+  //
+  function retryOnServerError(err) {
+    if (!arrRetryLogs.length) return;
+    else {
+      if (notFailedOnServerError && totalRetries >= maxRetryAllowed) {
+        console.log('Failed after ' + totalRetries + ' retries on error - ' + statusCode, '"'+err.statusMessage+'"');
+        notFailedOnServerError = false;
+        arrRetryLogs.length = 0;
+        totalRetries = 0;
+      }
+      while (isValidToken && totalRetries < maxRetryAllowed) {
+        console.log('Failed on error code ' + statusCode);
+        console.log('Retried ' + (totalRetries + 1) + ' time');
+        totalRetries++;
+        isBulk ? sendBulkLogs() : sendLogs();
+      }
+    }
+  }
+
   //
   // retries to send buffered logs to loggly in every 30 seconds
   //
@@ -251,7 +299,7 @@ common.loggly = function () {
     requestOptionsForBufferedLogs.body = isBulk ? arrayMessage.join('\n') : arrayMessage[0];
     request(requestOptionsForBufferedLogs, function (err, res, body) {
       if(err) return;
-      var statusCode = res.statusCode.toString();
+      statusCode = res.statusCode.toString();
       if(statusCode === "200") {
         arrBufferedMsg.splice(0, logsInBunch);
         sendBufferdLogstoLoggly();