Add option to batch events

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## Unreleased
 
+* Add option to batch events into a single request.
+
 ## 2.2.1 (Aug 13, 2015)
 
 * Fix bug where multi-byte unicode characters were hashed improperly.

README.md

@@ -87,7 +87,9 @@ You can configure Amplitude by passing an object as the third argument to the `i
       // optional configuration options
       saveEvents: true,
       includeUtm: true,
-      includeReferrer: true
+      includeReferrer: true,
+      batchEvents: true,
+      eventUploadThreshold: 50
     })
 
 | option | description | default |
@@ -97,6 +99,9 @@ You can configure Amplitude by passing an object as the third argument to the `i
 | uploadBatchSize | Maximum number of events to send to the server per request. | 100 |
 | includeUtm | If `true`, finds utm parameters in the query string or the __utmz cookie, parses, and includes them as user propeties on all events uploaded. | `false` |
 | includeReferrer | If `true`, includes `referrer` and `referring_domain` as user propeties on all events uploaded. | `false` |
+| batchEvents | If `true`, events are batched together and uploaded only when the number of unsent events is greater than or equal to `eventUploadThreshold` or after `eventUploadPeriodMillis` milliseconds have passed since the first unsent event was logged. | `false` |
+| eventUploadThreshold | Minimum number of events to batch together per request if `batchEvents` is `true`. | 30 |
+| eventUploadPeriodMillis | Amount of time in milliseconds that the SDK waits before uploading events if `batchEvents` is `true`. | 30*1000 |
 
 
 # Advanced #

amplitude.js

@@ -139,7 +139,10 @@ var DEFAULT_OPTIONS = {
   saveEvents: true,
   sessionTimeout: 30 * 60 * 1000,
   unsentKey: 'amplitude_unsent',
-  uploadBatchSize: 100
+  uploadBatchSize: 100,
+  batchEvents: false,
+  eventUploadThreshold: 30,
+  eventUploadPeriodMillis: 30 * 1000 // 30s
 };
 var LocalStorageKeys = {
   LAST_EVENT_ID: 'amplitude_lastEventId',
@@ -188,11 +191,16 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
       if (opt_config.includeReferrer !== undefined) {
         this.options.includeReferrer = !!opt_config.includeReferrer;
       }
+      if (opt_config.batchEvents !== undefined) {
+        this.options.batchEvents = !!opt_config.batchEvents;
+      }
       this.options.platform = opt_config.platform || this.options.platform;
       this.options.language = opt_config.language || this.options.language;
       this.options.sessionTimeout = opt_config.sessionTimeout || this.options.sessionTimeout;
       this.options.uploadBatchSize = opt_config.uploadBatchSize || this.options.uploadBatchSize;
+      this.options.eventUploadThreshold = opt_config.eventUploadThreshold || this.options.eventUploadThreshold;
       this.options.savedMaxCount = opt_config.savedMaxCount || this.options.savedMaxCount;
+      this.options.eventUploadPeriodMillis = opt_config.eventUploadPeriodMillis || this.options.eventUploadPeriodMillis;
     }
 
     Cookie.options({
@@ -222,9 +230,8 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
         }
       }
     }
-    if (this._unsentEvents.length > 0) {
-      this.sendEvents();
-    }
+
+    this._sendEventsIfReady();
 
     if (this.options.includeUtm) {
       this._initUtmData();
@@ -255,6 +262,23 @@ Amplitude.prototype.nextEventId = function() {
   return this._eventId;
 };
 
+Amplitude.prototype._sendEventsIfReady = function() {
+  if (this._unsentEvents.length === 0) {
+    return;
+  }
+
+  if (!this.options.batchEvents) {
+    this.sendEvents();
+    return;
+  }
+
+  if (this._unsentEvents.length >= this.options.eventUploadThreshold) {
+    this.sendEvents();
+  } else {
+    setTimeout(this.sendEvents.bind(this), this.options.eventUploadPeriodMillis);
+  }
+};
+
 var _loadCookieData = function(scope) {
   var cookieData = Cookie.get(scope.options.cookieName);
   if (cookieData) {
@@ -477,7 +501,7 @@ Amplitude.prototype._logEvent = function(eventType, eventProperties, apiProperti
       this.saveEvents();
     }
 
-    this.sendEvents();
+    this._sendEventsIfReady();
 
     return eventId;
   } catch (e) {
@@ -524,7 +548,7 @@ Amplitude.prototype.removeEvents = function (maxEventId) {
 };
 
 Amplitude.prototype.sendEvents = function() {
-  if (!this._sending && !this.options.optOut) {
+  if (!this._sending && !this.options.optOut && this._unsentEvents.length > 0) {
     this._sending = true;
     var url = ('https:' === window.location.protocol ? 'https' : 'http') + '://' +
         this.options.apiEndpoint + '/';
@@ -557,9 +581,8 @@ Amplitude.prototype.sendEvents = function() {
           }
 
           // Send more events if any queued during previous send.
-          if (scope._unsentEvents.length > 0) {
-            scope.sendEvents();
-          }
+          scope._sendEventsIfReady();
+
         } else if (status === 413) {
           //log('request too large');
           // Can't even get this one massive event through. Drop it.

src/amplitude.js

@@ -27,7 +27,10 @@ var DEFAULT_OPTIONS = {
   saveEvents: true,
   sessionTimeout: 30 * 60 * 1000,
   unsentKey: 'amplitude_unsent',
-  uploadBatchSize: 100
+  uploadBatchSize: 100,
+  batchEvents: false,
+  eventUploadThreshold: 30,
+  eventUploadPeriodMillis: 30 * 1000 // 30s
 };
 var LocalStorageKeys = {
   LAST_EVENT_ID: 'amplitude_lastEventId',
@@ -76,11 +79,16 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
       if (opt_config.includeReferrer !== undefined) {
         this.options.includeReferrer = !!opt_config.includeReferrer;
       }
+      if (opt_config.batchEvents !== undefined) {
+        this.options.batchEvents = !!opt_config.batchEvents;
+      }
       this.options.platform = opt_config.platform || this.options.platform;
       this.options.language = opt_config.language || this.options.language;
       this.options.sessionTimeout = opt_config.sessionTimeout || this.options.sessionTimeout;
       this.options.uploadBatchSize = opt_config.uploadBatchSize || this.options.uploadBatchSize;
+      this.options.eventUploadThreshold = opt_config.eventUploadThreshold || this.options.eventUploadThreshold;
       this.options.savedMaxCount = opt_config.savedMaxCount || this.options.savedMaxCount;
+      this.options.eventUploadPeriodMillis = opt_config.eventUploadPeriodMillis || this.options.eventUploadPeriodMillis;
     }
 
     Cookie.options({
@@ -110,9 +118,8 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
         }
       }
     }
-    if (this._unsentEvents.length > 0) {
-      this.sendEvents();
-    }
+
+    this._sendEventsIfReady();
 
     if (this.options.includeUtm) {
       this._initUtmData();
@@ -143,6 +150,23 @@ Amplitude.prototype.nextEventId = function() {
   return this._eventId;
 };
 
+Amplitude.prototype._sendEventsIfReady = function() {
+  if (this._unsentEvents.length === 0) {
+    return;
+  }
+
+  if (!this.options.batchEvents) {
+    this.sendEvents();
+    return;
+  }
+
+  if (this._unsentEvents.length >= this.options.eventUploadThreshold) {
+    this.sendEvents();
+  } else {
+    setTimeout(this.sendEvents.bind(this), this.options.eventUploadPeriodMillis);
+  }
+};
+
 var _loadCookieData = function(scope) {
   var cookieData = Cookie.get(scope.options.cookieName);
   if (cookieData) {
@@ -365,7 +389,7 @@ Amplitude.prototype._logEvent = function(eventType, eventProperties, apiProperti
       this.saveEvents();
     }
 
-    this.sendEvents();
+    this._sendEventsIfReady();
 
     return eventId;
   } catch (e) {
@@ -412,7 +436,7 @@ Amplitude.prototype.removeEvents = function (maxEventId) {
 };
 
 Amplitude.prototype.sendEvents = function() {
-  if (!this._sending && !this.options.optOut) {
+  if (!this._sending && !this.options.optOut && this._unsentEvents.length > 0) {
     this._sending = true;
     var url = ('https:' === window.location.protocol ? 'https' : 'http') + '://' +
         this.options.apiEndpoint + '/';
@@ -445,9 +469,8 @@ Amplitude.prototype.sendEvents = function() {
           }
 
           // Send more events if any queued during previous send.
-          if (scope._unsentEvents.length > 0) {
-            scope.sendEvents();
-          }
+          scope._sendEventsIfReady();
+
         } else if (status === 413) {
           //log('request too large');
           // Can't even get this one massive event through. Drop it.

test/amplitude.js

@@ -130,12 +130,16 @@ describe('Amplitude', function() {
 
   describe('logEvent', function() {
 
+    var clock;
+
     beforeEach(function() {
+      clock = sinon.useFakeTimers();
       amplitude.init(apiKey);
     });
 
     afterEach(function() {
       reset();
+      clock.restore();
     });
 
     it('should send request', function() {
@@ -260,7 +264,7 @@ describe('Amplitude', function() {
       assert.deepEqual(amplitude2._unsentEvents, []);
     });
 
-    it('should batch events sent', function() {
+    it('should limit events sent', function() {
       amplitude.init(apiKey, null, {uploadBatchSize: 10});
 
       amplitude._sending = true;
@@ -287,6 +291,91 @@ describe('Amplitude', function() {
       assert.deepEqual(events[5].event_properties, {index: 100});
     });
 
+    it('should batch events sent', function() {
+      var eventUploadPeriodMillis = 10*1000;
+      amplitude.init(apiKey, null, {
+        batchEvents: true,
+        eventUploadThreshold: 10,
+        eventUploadPeriodMillis: eventUploadPeriodMillis
+      });
+
+      for (var i = 0; i < 15; i++) {
+        amplitude.logEvent('Event', {index: i});
+      }
+
+      assert.lengthOf(server.requests, 1);
+      var events = JSON.parse(querystring.parse(server.requests[0].requestBody).e);
+      assert.lengthOf(events, 10);
+      assert.deepEqual(events[0].event_properties, {index: 0});
+      assert.deepEqual(events[9].event_properties, {index: 9});
+
+      server.respondWith('success');
+      server.respond();
+
+      assert.lengthOf(server.requests, 1);
+      var unsentEvents = amplitude._unsentEvents;
+      assert.lengthOf(unsentEvents, 5);
+      assert.deepEqual(unsentEvents[4].event_properties, {index: 14});
+
+      // remaining 5 events should be sent by the delayed sendEvent call
+      clock.tick(eventUploadPeriodMillis);
+      assert.lengthOf(server.requests, 2);
+      server.respondWith('success');
+      server.respond();
+      assert.lengthOf(amplitude._unsentEvents, 0);
+      var events = JSON.parse(querystring.parse(server.requests[1].requestBody).e);
+      assert.lengthOf(events, 5);
+      assert.deepEqual(events[4].event_properties, {index: 14});
+    });
+
+    it('should send events after a delay', function() {
+      var eventUploadPeriodMillis = 10*1000;
+      amplitude.init(apiKey, null, {
+        batchEvents: true,
+        eventUploadThreshold: 2,
+        eventUploadPeriodMillis: eventUploadPeriodMillis
+      });
+      amplitude.logEvent('Event');
+
+      // saveEvent should not have been called yet
+      assert.lengthOf(amplitude._unsentEvents, 1);
+      assert.lengthOf(server.requests, 0);
+
+      // saveEvent should be called after delay
+      clock.tick(eventUploadPeriodMillis);
+      assert.lengthOf(server.requests, 1);
+      server.respondWith('success');
+      server.respond();
+      var events = JSON.parse(querystring.parse(server.requests[0].requestBody).e);
+      assert.lengthOf(events, 1);
+      assert.deepEqual(events[0].event_type, 'Event');
+    });
+
+    it('should not send events after a delay if no events to send', function() {
+      var eventUploadPeriodMillis = 10*1000;
+      amplitude.init(apiKey, null, {
+        batchEvents: true,
+        eventUploadThreshold: 2,
+        eventUploadPeriodMillis: eventUploadPeriodMillis
+      });
+      amplitude.logEvent('Event1');
+      amplitude.logEvent('Event2');
+
+      // saveEvent triggered by 2 event batch threshold
+      assert.lengthOf(amplitude._unsentEvents, 2);
+      assert.lengthOf(server.requests, 1);
+      server.respondWith('success');
+      server.respond();
+      var events = JSON.parse(querystring.parse(server.requests[0].requestBody).e);
+      assert.lengthOf(events, 2);
+      assert.deepEqual(events[1].event_type, 'Event2');
+
+      // saveEvent should be called after delay, but no request made
+      assert.lengthOf(amplitude._unsentEvents, 0);
+      clock.tick(eventUploadPeriodMillis);
+      assert.lengthOf(server.requests, 1);
+    })
+
     it('should back off on 413 status', function() {
       amplitude.init(apiKey, null, {uploadBatchSize: 10});