Added support for bulk mode

.gitignore

@@ -1,3 +1,2 @@
-test/*.json
 node_modules
 npm-debug.log

.travis.yml

@@ -8,15 +8,9 @@ node_js:
 before_install:
   - travis_retry npm install -g npm@2.14.5
   - travis_retry npm install --loglevel=http
-  - >
-    openssl aes-256-cbc \
-      -K $encrypted_4fb8e306b739_key \
-      -iv $encrypted_4fb8e306b739_iv \
-      -in test/config.json.enc \
-      -out test/config.json -d
 
 script:
-  - npm test
+  - npm run test-as-mock
 
 matrix:
   allow_failures:

README.md

@@ -1,4 +1,4 @@
-# node-loggly
+# node-loggly-bulk
 
 [![Version npm](https://img.shields.io/npm/v/loggly.svg?style=flat-square)](https://www.npmjs.com/package/loggly)[![npm Downloads](https://img.shields.io/npm/dm/loggly.svg?style=flat-square)](https://www.npmjs.com/package/loggly)[![Build Status](https://img.shields.io/travis/winstonjs/node-loggly/master.svg?style=flat-square)](https://travis-ci.org/winstonjs/node-loggly)[![Dependencies](https://img.shields.io/david/winstonjs/node-loggly.svg?style=flat-square)](https://david-dm.org/winstonjs/node-loggly)
 
@@ -8,7 +8,7 @@ A client implementation for Loggly in node.js. Check out Loggly's [Node logging
 
 ## Usage
 
-The `node-loggly` library is compliant with the [Loggly API][api]. Using `node-loggly` is easy for a variety of scenarios: logging, working with devices and inputs, searching, and facet searching.
+The `node-loggly-bulk` library is compliant with the [Loggly API][api]. Using `node-loggly-bulk` is easy for a variety of scenarios: logging, working with devices and inputs, searching, and facet searching.
 
 ### Getting Started
 Before we can do anything with Loggly, we have to create a client with valid credentials. We will authenticate for you automatically:
@@ -31,7 +31,7 @@ Before we can do anything with Loggly, we have to create a client with valid cre
 ```
 
 ### Logging
-There are two ways to send log information to Loggly via node-loggly. The first is to simply call client.log with an appropriate input token:
+There are two ways to send log information to Loggly via node-loggly-bulk. The first is to simply call client.log with an appropriate input token:
 
 ``` js
   client.log('127.0.0.1 - Theres no place like home', function (err, result) {
@@ -120,7 +120,7 @@ It is possible to send arrays, which will result in one single request to Loggly
 ```
 
 ### Searching
-[Searching][search-api] with node-loggly is easy. All you have to do is use the search() method defined on each Loggly client:
+[Searching][search-api] with node-loggly-bulk is easy. All you have to do is use the search() method defined on each Loggly client:
 
 ``` js
   var util = require('util');
@@ -152,13 +152,15 @@ See the [Loggly search guide][search] for more details on how to effectively sea
   $ curl http://npmjs.org/install.sh | sh
 ```
 
-### Installing node-loggly
+### Installing node-loggly-bulk
 ``` bash
   $ npm install loggly
 ```
 
 ## Run Tests
-All of the node-loggly tests are written in [vows][vows], and cover all of the use cases described above. You will need to add your Loggly username, password, subdomain, and a two test inputs to test/data/test-config.json before running tests. When configuring the test inputs on Loggly, the first test input should be named 'test' using the HTTP service. The second input should be name 'test_json' using the HTTP service with the JSON logging option enabled:
+
+### Run Tests by sending events to your Loggly Account
+All of the node-loggly-bulk tests are written in [vows][vows], and cover all of the use cases described above. You will need to add your Loggly username, password, subdomain, and your loggly token to test/config.json before running tests.
 
 ``` js
   {
@@ -176,6 +178,11 @@ Once you have valid Loggly credentials you can run tests with [vows][vows]:
 ``` bash
   $ npm test
 ```
+### Run Tests with Mock HTTP Request
+To mock the HTTP requests and run test cases in your local machine you can run the following command
+```bash
+  $ npm run test-as-mock
+```
 
 #### Author: [Charlie Robbins](http://www.github.com/indexzero)
 #### Contributors: [Marak Squires](http://github.com/marak), [hij1nx](http://github.com/hij1nx), [Kord Campbell](http://loggly.com), [Erik Hedenström](http://github.com/ehedenst),

lib/loggly/client.js

@@ -46,6 +46,7 @@ var Loggly = exports.Loggly = function (options) {
   }
 
   events.EventEmitter.call(this);
+
   this.subdomain    = options.subdomain;
   this.token        = options.token;
   this.host         = options.host || 'logs-01.loggly.com';
@@ -54,7 +55,7 @@ var Loggly = exports.Loggly = function (options) {
   this.proxy        = options.proxy || null;
   this.userAgent    = 'node-loggly ' + loggly.version;
   this.useTagHeader = 'useTagHeader' in options ? options.useTagHeader : true;
-
+  this.isBulk       = options.isBulk || false;
   //
   // Set the tags on this instance.
   //
@@ -101,7 +102,6 @@ Loggly.prototype.log = function (msg, tags, callback) {
   // Remark: Have some extra logic for detecting if we want to make a bulk
   // request to loggly
   //
-  var isBulk = Array.isArray(msg);
   function serialize(msg) {
     if (msg instanceof Object) {
       return self.json ? stringify(msg) : common.serialize(msg);
@@ -111,13 +111,14 @@ Loggly.prototype.log = function (msg, tags, callback) {
     }
   }
 
-  msg = isBulk ? msg.map(serialize).join('\n') : serialize(msg);
+  msg = serialize(msg);
 
   logOptions = {
-    uri:     isBulk ? this.urls.bulk : this.urls.log,
+    uri:     this.isBulk ? this.urls.bulk : this.urls.log,
     method:  'POST',
     body:    msg,
     proxy:   this.proxy,
+    isBulk: this.isBulk,
     headers: {
       host:             this.host,
       accept:           '*/*',

lib/loggly/common.js

@@ -6,6 +6,13 @@
  *
  */
 
+//
+// Variables for Bulk
+//
+var arrSize = 100,
+    arrMsg = [],
+    timerFunction = null;
+
 var https = require('https'),
     util = require('util'),
     request = require('request'),
@@ -69,6 +76,7 @@ common.loggly = function () {
       method,
       auth,
       proxy,
+      isBulk,
       uri;
 
   //
@@ -88,6 +96,7 @@ common.loggly = function () {
       uri         = args[0].uri;
       requestBody = args[0].body;
       auth        = args[0].auth;
+      isBulk      = args[0].isBulk;
       headers     = args[0].headers;
       proxy       = args[0].proxy;
     }
@@ -109,41 +118,81 @@ common.loggly = function () {
       if (callback) { callback(err) }
     }
   }
-
   var requestOptions = {
-    uri: uri,
+    uri: isBulk ? uri + '/tag/' + headers['X-LOGGLY-TAG'] : uri,
     method: method,
-    headers: headers || {},
+    headers: isBulk ? {} : headers || {},             // Set headers empty for bulk
     proxy: proxy
   };
 
   if (auth) {
     requestOptions.headers.authorization = 'Basic ' + new Buffer(auth.username + ':' + auth.password).toString('base64');
   }
-
   if (requestBody) {
     requestOptions.body = requestBody;
   }
-
-  try {
-    request(requestOptions, function (err, res, body) {
-      if (err) {
-        return onError(err);
-      }
-
-      var statusCode = res.statusCode.toString();
-      if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
-        return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
-      }
-
-      success(res, body);
-    });
+  function sendLogs() {
+    try {
+      request(requestOptions, function (err, res, body) {
+        if (err) {
+          return onError(err);
+        }
+        var statusCode = res.statusCode.toString();
+        if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
+          return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+        }
+        success(res, body);
+      });
+    }
+    catch (ex) {
+      onError(ex);
+    }
   }
-  catch (ex) {
-    onError(ex);
+  function sendBulkLogs() {
+    if (arrMsg.length === 0) {
+      return;
+    }
+    //
+    // Join Array Message with new line ('\n') character
+    //
+    requestOptions.body = arrMsg.join('\n');
+    try {
+      request(requestOptions, function (err, res, body) {
+        if (err) {
+          return onError(err);
+        }
+        var statusCode = res.statusCode.toString();
+        if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
+          return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+        }
+        success(res, body);
+      });
+    }
+    catch (ex) {
+      onError(ex);
+    }
+    finally {
+      //
+      // Empty the array
+      //
+      arrMsg.length = 0;
+    }
+  }
+  if (isBulk === true) {
+    if (timerFunction === null) {
+      timerFunction = setInterval(function () {
+        sendBulkLogs();
+      },30000);
+    }
+    arrMsg.push(requestBody);
+    if (arrMsg.length === arrSize) {
+      sendBulkLogs();
+    }
+  }
+  else {
+    sendLogs();
   }
 };
-
 //
 // ### function serialize (obj, key)
 // #### @obj {Object|literal} Object to serialize

package.json

@@ -20,12 +20,15 @@
   },
   "devDependencies": {
     "common-style": "^3.1.0",
-    "vows": "0.8.x"
+    "vows": "0.8.x",
+    "nock": "~7.2.2"
   },
   "main": "./lib/loggly",
   "scripts": {
     "pretest": "common lib/**/*.js lib/*.js test/helpers.js",
-    "test": "vows test/*-test.js --spec"
+    "test": "vows test/*-test.js --spec",
+    "pretest-as-mock": "common lib/**/*.js lib/*.js test-as-mock/helpers.js",
+    "test-as-mock": "vows test-as-mock/*-test.js --spec"
   },
   "license": "MIT",
   "engines": {

test-as-mock/common-test.js

@@ -0,0 +1,32 @@
+/*
+ * common-test.js: Tests for Loggly `common` utility module
+ *
+ * (C) 2010 Charlie Robbins
+ * MIT LICENSE
+ *
+ */
+
+var path = require('path'),
+    vows = require('vows'),
+    assert = require('assert'),
+    common = require('../lib/loggly/common');
+
+vows.describe('node-loggly/common').addBatch({
+  "When using the common module": {
+    "the clone() method": {
+      topic: function () {
+        this.obj = {
+          name: 'common',
+          deep: {
+            first: 'first',
+            second: 'second'
+          }
+        };
+        return common.clone(this.obj);
+      },
+      "should return a deep clone of the object": function (clone) {
+        assert.isFalse(this.obj.deep === clone.deep);
+      }
+    }
+  }
+}).export(module);

test-as-mock/config.json

@@ -0,0 +1,8 @@
+{
+    "token": "test",
+    "subdomain": "testSubdomain",
+    "auth": {
+      "username": "test",
+      "password": "test"
+    }
+  }

test-as-mock/customer-test.js

@@ -0,0 +1,43 @@
+/*
+ * search-test.js: Tests for Loggly search requests
+ *
+ * (C) 2010 Charlie Robbins
+ * MIT LICENSE
+ *
+ */
+var path = require('path'),
+  vows = require('vows'),
+  assert = require('assert'),
+  nock = require('nock'),
+  helpers = require('./helpers');
+
+var options = {},
+  testContext = {},
+  config = helpers.loadConfig(),
+  loggly = require('../lib/loggly').createClient(config);
+
+vows.describe('node-loggly/customer').addBatch({
+  "When using the node-loggly client": {
+    "the customer() method": {
+      topic: function() {
+        nock("https://" + config.subdomain + ".loggly.com")
+          .get('/apiv2/customer')
+          .reply(200, {
+            "tokens": ["test", "test2"],
+            "subdomain": config.subdomain,
+            "subscription": {
+              "key1": "value1"
+            }
+          });
+        loggly.customer(this.callback);
+
+      },
+      "should return a valid customer": function(err, customer) {
+        assert.isNull(err);
+        assert.isArray(customer.tokens);
+        assert.isString(customer.subdomain);
+        assert.isObject(customer.subscription);
+      }
+    }
+  }
+}).export(module);