Added code to retry on server error

lib/loggly/common.js

@@ -14,7 +14,7 @@ var arrSize = 100,
     timerFunction = null;
 
 // 
-// Variables for buffer array 
+// Variables for buffer array
 // 
 var arrBufferedMsg = [],
     timerFunctionForBufferedLogs = null;
@@ -24,6 +24,15 @@ var arrBufferedMsg = [],
 //
 var isValidToken = true;
 
+//
+// Variables for server retry
+//
+var arrRetryLogs = [],
+maxRetryAllowed = 5,
+totalRetries = 0,
+statusCode,
+notfailedOnServerError = true;
+
 var https = require('https'),
     util = require('util'),
     request = require('request'),
@@ -44,7 +53,9 @@ var failCodes = common.failCodes = {
   410: 'Gone',
   500: 'Internal Server Error',
   501: 'Not Implemented',
-  503: 'Throttled'
+  503: 'Throttled',
+  503: 'Throttled',
+  504: 'Gateway Timeout'
 };
 
 //
@@ -137,6 +148,7 @@ common.loggly = function () {
       arrayLogs.push(requestOptions.body);
     }
     storeLogs(arrayLogs);
+    //storeLogsforServerRetry(arrayLogs);
 
     if (!responded) {
       responded = true;
@@ -157,18 +169,28 @@ common.loggly = function () {
   }
   if (requestBody) {
     requestOptions.body = requestBody;
+    arrRetryLogs = arrRetryLogs.concat(requestBody);
+  }
+  if (arrRetryLogs.length && !requestBody) {
+    requestOptions.body = arrRetryLogs[0];
   }
   function sendLogs() {
     try {
       request(requestOptions, function (err, res, body) {
-        if (err) {
-          return onError(err);
-        }
-        var statusCode = res.statusCode.toString();
+        if (err) return onError(err);
+
+        statusCode = res.statusCode.toString();
         if(statusCode === '403') isValidToken = false;
+        if (statusCode === '200') {
+          arrRetryLogs.splice(0, 1);
+          totalRetries = 0;
+        }
+        if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res);
         if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
-          return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
-        } 
+          if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') {
+            return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+          }
+        }
         success(res, body);
       });
     }
@@ -177,23 +199,32 @@ common.loggly = function () {
     }
   }
   function sendBulkLogs() {
-    if (arrMsg.length === 0) {
-      return;
+    // if (arrMsg.length === 0) {
+    //   return;
+    // }
+    if (arrRetryLogs.length && !arrMsg.length) {
+      requestOptions.body = arrRetryLogs.join('\n');
     }
     //
     // Join Array Message with new line ('\n') character
     //
-    requestOptions.body = arrMsg.join('\n');
+    if (arrMsg.length) requestOptions.body = arrMsg.join('\n');
     arrMsg.length = 0;
     try {
       request(requestOptions, function (err, res, body) {
-        if (err) {
-          return onError(err);
-        }
+        if (err) return onError(err);
+
         var statusCode = res.statusCode.toString();
         if(statusCode === '403') isValidToken = false;
+        if (statusCode === '200') {
+          arrRetryLogs.splice(0, arrSize);
+          totalRetries = 0;
+        }
+        if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res);
         if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
-          return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+          if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') {
+            return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+          }
         }
         success(res, body);
       });
@@ -227,6 +258,20 @@ common.loggly = function () {
     sendLogs();
   }
 
+  function retryOnServerError(err) {
+    if (!arrRetryLogs) return;
+    if (notfailedOnServerError && totalRetries >= maxRetryAllowed) {
+      console.log('Failed after ' + totalRetries + ' retries on error - ' + statusCode, err.statusMessage);
+      notfailedOnServerError = false;
+    }
+    while (isValidToken && totalRetries < maxRetryAllowed) {
+      console.log('Failed on error code ' + statusCode);
+      console.log('Retried ' + (totalRetries + 1) + ' time');
+      totalRetries++;
+      isBulk ? sendBulkLogs() : sendLogs();
+    }
+  }
+
   //
   // retries to send buffered logs to loggly in every 30 seconds
   //
@@ -269,6 +314,13 @@ common.loggly = function () {
     if (numberOfLogsToBeRemoved > 0) arrBufferedMsg = arrBufferedMsg.splice(numberOfLogsToBeRemoved);
       arrBufferedMsg = arrBufferedMsg.concat(logs);
     }
+
+    //function storeLogsforServerRetry(logs) {
+      //   if (!logs.length || !bufferOptions) return;
+      //   arrRetryLogs = arrRetryLogs.concat(logs);
+      //   isBulk ? sendBulkLogs(arrRetryLogs) : sendLogs(arrRetryLogs);
+      //   //sendLogs(arrInputLogs);
+      // }
 };
 //
 // ### function serialize (obj, key)