Added code to retry on server errors

lib/loggly/common.js

@@ -24,6 +24,15 @@ var arrBufferedMsg = [],
 //
 var isValidToken = true;
 
+//
+// Variables for server retry
+//
+var arrRetryLogs = [],
+  maxRetryAllowed = 5,
+  totalRetries = 0,
+  statusCode,
+  notfailedOnServerError = true;
+
 var https = require('https'),
     util = require('util'),
     request = require('request'),
@@ -44,7 +53,8 @@ var failCodes = common.failCodes = {
   410: 'Gone',
   500: 'Internal Server Error',
   501: 'Not Implemented',
-  503: 'Throttled'
+  503: 'Throttled',
+  504: 'Gateway Timeout'
 };
 
 //
@@ -157,18 +167,28 @@ common.loggly = function () {
   }
   if (requestBody) {
     requestOptions.body = requestBody;
+    arrRetryLogs = arrRetryLogs.concat(requestBody);
   }
   function sendLogs() {
+    if (arrRetryLogs.length && !requestBody) {
+      requestOptions.body = arrRetryLogs[0];
+    }
     try {
       request(requestOptions, function (err, res, body) {
-        if (err) {
-          return onError(err);
-        }
-        var statusCode = res.statusCode.toString();
+        if (err) return onError(err);
+
+        statusCode = res.statusCode.toString();
         if(statusCode === '403') isValidToken = false;
+        if (statusCode === '200') {
+          arrRetryLogs.splice(0, 1);
+          totalRetries = 0;
+        }
+        if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res);
         if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
+          if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') {
           return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
-        } 
+          }
+        }
         success(res, body);
       });
     }
@@ -177,23 +197,29 @@ common.loggly = function () {
     }
   }
   function sendBulkLogs() {
-    if (arrMsg.length === 0) {
-      return;
+    if (arrRetryLogs.length && !arrMsg.length) {
+      requestOptions.body = arrRetryLogs.join('\n');
     }
     //
     // Join Array Message with new line ('\n') character
     //
-    requestOptions.body = arrMsg.join('\n');
+    if (arrMsg.length) requestOptions.body = arrMsg.join('\n');
     arrMsg.length = 0;
     try {
       request(requestOptions, function (err, res, body) {
-        if (err) {
-          return onError(err);
-        }
-        var statusCode = res.statusCode.toString();
+        if (err) return onError(err);
+
+        statusCode = res.statusCode.toString();
         if(statusCode === '403') isValidToken = false;
+        if (statusCode === '200') {
+          arrRetryLogs.splice(0, arrSize);
+          totalRetries = 0;
+        }
+        if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res);
         if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
+          if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') {
           return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+          }
         }
         success(res, body);
       });
@@ -227,6 +253,21 @@ common.loggly = function () {
     sendLogs();
   }
 
+  function retryOnServerError(err) {
+    if (!arrRetryLogs) return;
+    if (notfailedOnServerError && totalRetries >= maxRetryAllowed) {
+      console.log('Failed after ' + totalRetries + ' retries on error - ' + statusCode, err.statusMessage);
+      notfailedOnServerError = false;
+      arrRetryLogs.length = 0;
+    }
+    while (isValidToken && totalRetries < maxRetryAllowed) {
+      console.log('Failed on error code ' + statusCode);
+      console.log('Retried ' + (totalRetries + 1) + ' time');
+      totalRetries++;
+      isBulk ? sendBulkLogs() : sendLogs();
+    }
+  }
+
   //
   // retries to send buffered logs to loggly in every 30 seconds
   //