Track referrer the right way

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 * Add getSessionId helper method to fetch the current sessionId.
 * Add support for append user property operation.
+* Add tracking of each user's initial_referrer property (which is captured as a set once operation). Referrer property captured once per user session.
 
 ## 2.7.0 (December 1, 2015)
 

README.md

@@ -44,7 +44,11 @@ If your app has its own login system that you want to track users with, you can
 amplitude.setUserId('USER_ID_HERE');
 ```
 
-A user's data will be merged on the backend so that any events up to that point from the same browser will be tracked under the same user.
+A user's data will be merged on the backend so that any events up to that point from the same browser will be tracked under the same user. Note: if a user logs out, or you want to log the events under an anonymous user, you may set the userId to `null` like so:
+
+```javascript
+amplitude.setUserId(null); // not string 'null'
+```
 
 You can also add the user ID as an argument to the `init` call:
 
@@ -62,7 +66,7 @@ eventProperties.key = 'value';
 amplitude.logEvent('EVENT_IDENTIFIER_HERE', eventProperties);
 ```
 
-# User Property Operations #
+# User Properties and User Property Operations #
 
 The SDK supports the operations `set`, `setOnce`, `unset`, and `add` on individual user properties. The operations are declared via a provided `Identify` interface. Multiple operations can be chained together in a single `Identify` object. The `Identify` object is then passed to the Amplitude client to send to the server. The results of the operations will be visible immediately in the dashboard, and take effect for events logged after.
 
@@ -114,6 +118,18 @@ var identify = new amplitude.Identify()
 amplitude.identify(identify);
 ```
 
+### Arrays in User Properties ###
+
+The SDK supports arrays in user properties. Any of the user property operations above (with the exception of `add`) can accept a Javascript array. You can directly `set` arrays, or use `append` to generate an array.
+
+```javascript
+var identify = new amplitude.Identify()
+    .set('colors', ['rose', 'gold'])
+    .append('ab-tests', 'campaign_a')
+    .append('existing_list', [4, 5]);
+amplitude.identify(identify);
+```
+
 ### Setting Multiple Properties with `setUserProperties` ###
 
 You may use `setUserProperties` shorthand to set multiple user properties at once. This method is simply a wrapper around `Identify.set` and `identify`.
@@ -176,7 +192,7 @@ amplitude.init('YOUR_API_KEY_HERE', null, {
 | savedMaxCount | Maximum number of events to save in localStorage. If more events are logged while offline, old events are removed. | 1000 |
 | uploadBatchSize | Maximum number of events to send to the server per request. | 100 |
 | includeUtm | If `true`, finds utm parameters in the query string or the __utmz cookie, parses, and includes them as user propeties on all events uploaded. | `false` |
-| includeReferrer | If `true`, includes `referrer` and `referring_domain` as user propeties on all events uploaded. | `false` |
+| includeReferrer | If `true`, captures the `referrer` and `referring_domain` for each session, as well as the user's `initial_referrer` and `initial_referring_domain` via a set once operation. | `false` |
 | batchEvents | If `true`, events are batched together and uploaded only when the number of unsent events is greater than or equal to `eventUploadThreshold` or after `eventUploadPeriodMillis` milliseconds have passed since the first unsent event was logged. | `false` |
 | eventUploadThreshold | Minimum number of events to batch together per request if `batchEvents` is `true`. | 30 |
 | eventUploadPeriodMillis | Amount of time in milliseconds that the SDK waits before uploading events if `batchEvents` is `true`. | 30\*1000 (30 sec) |

amplitude.js

@@ -149,7 +149,9 @@ var LocalStorageKeys = {
   LAST_SEQUENCE_NUMBER: 'amplitude_lastSequenceNumber',
   LAST_EVENT_TIME: 'amplitude_lastEventTime',
   SESSION_ID: 'amplitude_sessionId',
+  REFERRER: 'amplitude_referrer',
 
+  // Used in cookie as well
   DEVICE_ID: 'amplitude_deviceId',
   USER_ID: 'amplitude_userId',
   OPT_OUT: 'amplitude_optOut'
@@ -256,6 +258,10 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config, callback) {
     if (this.options.includeUtm) {
       this._initUtmData();
     }
+
+    if (this.options.includeReferrer) {
+      this._saveReferrer(this._getReferrer());
+    }
   } catch (e) {
     log(e);
   }
@@ -467,12 +473,39 @@ Amplitude.prototype._getReferrer = function() {
   return document.referrer;
 };
 
-Amplitude.prototype._getReferringDomain = function() {
-  var parts = this._getReferrer().split("/");
+Amplitude.prototype._getReferringDomain = function(referrer) {
+  if (referrer === null || referrer === undefined || referrer === '') {
+    return null;
+  }
+  var parts = referrer.split('/');
   if (parts.length >= 3) {
     return parts[2];
   }
-  return "";
+  return null;
+};
+
+// since user properties are propagated on the server, only send once per session, don't need to send with every event
+Amplitude.prototype._saveReferrer = function(referrer) {
+  if (referrer === null || referrer === undefined || referrer === '') {
+    return;
+  }
+
+  // always setOnce initial referrer
+  var referring_domain = this._getReferringDomain(referrer);
+  var identify = new Identify().setOnce('initial_referrer', referrer);
+  identify.setOnce('initial_referring_domain', referring_domain);
+
+  // only save referrer if not already in session storage or if storage disabled
+  var hasSessionStorage = window.sessionStorage ? true : false;
+  if ((hasSessionStorage && !window.sessionStorage.getItem(LocalStorageKeys.REFERRER)) || !hasSessionStorage) {
+    identify.set('referrer', referrer).set('referring_domain', referring_domain);
+
+    if (hasSessionStorage) {
+      window.sessionStorage.setItem(LocalStorageKeys.REFERRER, referrer);
+    }
+  }
+
+  this.identify(identify);
 };
 
 Amplitude.prototype.saveEvents = function() {
@@ -628,14 +661,6 @@ Amplitude.prototype._logEvent = function(eventType, eventProperties, apiProperti
     // Only add utm properties to user properties for events
     if (eventType !== IDENTIFY_EVENT) {
       object.merge(userProperties, this._utmProperties);
-
-      // Add referral info onto the user properties
-      if (this.options.includeReferrer) {
-        object.merge(userProperties, {
-          'referrer': this._getReferrer(),
-          'referring_domain': this._getReferringDomain()
-        });
-      }
     }
 
     apiProperties = apiProperties || {};
@@ -2545,15 +2570,25 @@ module.exports = function(val){
   if (val !== val) return 'nan';
   if (val && val.nodeType === 1) return 'element';
 
-  if (typeof Buffer != 'undefined' && Buffer.isBuffer(val)) return 'buffer';
+  if (isBuffer(val)) return 'buffer';
 
   val = val.valueOf
     ? val.valueOf()
-    : Object.prototype.valueOf.apply(val)
+    : Object.prototype.valueOf.apply(val);
 
   return typeof val;
 };
 
+// code borrowed from https://github.com/feross/is-buffer/blob/master/index.js
+function isBuffer(obj) {
+  return !!(obj != null &&
+    (obj._isBuffer || // For Safari 5-7 (missing Object.prototype.constructor)
+      (obj.constructor &&
+      typeof obj.constructor.isBuffer === 'function' &&
+      obj.constructor.isBuffer(obj))
+    ))
+}
+
 }, {}],
 10: [function(require, module, exports) {
 /* jshint eqeqeq: false, forin: false */

src/amplitude.js

@@ -43,7 +43,9 @@ var LocalStorageKeys = {
   LAST_SEQUENCE_NUMBER: 'amplitude_lastSequenceNumber',
   LAST_EVENT_TIME: 'amplitude_lastEventTime',
   SESSION_ID: 'amplitude_sessionId',
+  REFERRER: 'amplitude_referrer',
 
+  // Used in cookie as well
   DEVICE_ID: 'amplitude_deviceId',
   USER_ID: 'amplitude_userId',
   OPT_OUT: 'amplitude_optOut'
@@ -150,6 +152,10 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config, callback) {
     if (this.options.includeUtm) {
       this._initUtmData();
     }
+
+    if (this.options.includeReferrer) {
+      this._saveReferrer(this._getReferrer());
+    }
   } catch (e) {
     log(e);
   }
@@ -361,12 +367,39 @@ Amplitude.prototype._getReferrer = function() {
   return document.referrer;
 };
 
-Amplitude.prototype._getReferringDomain = function() {
-  var parts = this._getReferrer().split("/");
+Amplitude.prototype._getReferringDomain = function(referrer) {
+  if (referrer === null || referrer === undefined || referrer === '') {
+    return null;
+  }
+  var parts = referrer.split('/');
   if (parts.length >= 3) {
     return parts[2];
   }
-  return "";
+  return null;
+};
+
+// since user properties are propagated on the server, only send once per session, don't need to send with every event
+Amplitude.prototype._saveReferrer = function(referrer) {
+  if (referrer === null || referrer === undefined || referrer === '') {
+    return;
+  }
+
+  // always setOnce initial referrer
+  var referring_domain = this._getReferringDomain(referrer);
+  var identify = new Identify().setOnce('initial_referrer', referrer);
+  identify.setOnce('initial_referring_domain', referring_domain);
+
+  // only save referrer if not already in session storage or if storage disabled
+  var hasSessionStorage = window.sessionStorage ? true : false;
+  if ((hasSessionStorage && !window.sessionStorage.getItem(LocalStorageKeys.REFERRER)) || !hasSessionStorage) {
+    identify.set('referrer', referrer).set('referring_domain', referring_domain);
+
+    if (hasSessionStorage) {
+      window.sessionStorage.setItem(LocalStorageKeys.REFERRER, referrer);
+    }
+  }
+
+  this.identify(identify);
 };
 
 Amplitude.prototype.saveEvents = function() {
@@ -522,14 +555,6 @@ Amplitude.prototype._logEvent = function(eventType, eventProperties, apiProperti
     // Only add utm properties to user properties for events
     if (eventType !== IDENTIFY_EVENT) {
       object.merge(userProperties, this._utmProperties);
-
-      // Add referral info onto the user properties
-      if (this.options.includeReferrer) {
-        object.merge(userProperties, {
-          'referrer': this._getReferrer(),
-          'referring_domain': this._getReferringDomain()
-        });
-      }
     }
 
     apiProperties = apiProperties || {};

test/amplitude.js

@@ -27,6 +27,7 @@ describe('Amplitude', function() {
 
   function reset() {
     localStorage.clear();
+    sessionStorage.clear();
     cookie.remove(amplitude.options.cookieName);
     cookie.reset();
   }
@@ -1500,41 +1501,92 @@ describe('Amplitude', function() {
       assert.equal(events[0].user_properties.referring_domain, undefined);
     });
 
-    it('should send referrer data when the includeReferrer flag is true', function() {
+    it('should only send referrer via identify call when the includeReferrer flag is true', function() {
       reset();
-      amplitude.init(apiKey, undefined, {includeReferrer: true});
-
+      amplitude.init(apiKey, undefined, {includeReferrer: true, batchEvents: true, eventUploadThreshold: 2});
       amplitude.logEvent('Referrer Test Event', {});
       assert.lengthOf(server.requests, 1);
       var events = JSON.parse(querystring.parse(server.requests[0].requestBody).e);
+      assert.lengthOf(events, 2);
+
+      // first event should be identify with initial_referrer and referrer
+      assert.equal(events[0].event_type, '$identify');
       assert.deepEqual(events[0].user_properties, {
-        referrer: 'https://amplitude.com/contact',
-        referring_domain: 'amplitude.com'
+        '$set': {
+          'referrer': 'https://amplitude.com/contact',
+          'referring_domain': 'amplitude.com'
+        },
+        '$setOnce': {
+          'initial_referrer': 'https://amplitude.com/contact',
+          'initial_referring_domain': 'amplitude.com'
+        }
       });
+
+      // second event should be the test event with no referrer information
+      assert.equal(events[1].event_type, 'Referrer Test Event');
+      assert.deepEqual(events[1].user_properties, {});
+
+      // referrer should be propagated to session storage
+      assert.equal(sessionStorage.getItem('amplitude_referrer'), 'https://amplitude.com/contact');
     });
 
-    it('should add referrer data to the user properties on events only', function() {
+    it('should not set referrer if referrer data already in session storage', function() {
       reset();
-      amplitude.init(apiKey, undefined, {includeReferrer: true});
-
-      amplitude.setUserProperties({user_prop: true});
+      sessionStorage.setItem('amplitude_referrer', 'https://www.google.com/search?');
+      amplitude.init(apiKey, undefined, {includeReferrer: true, batchEvents: true, eventUploadThreshold: 2});
+      amplitude.logEvent('Referrer Test Event', {});
       assert.lengthOf(server.requests, 1);
       var events = JSON.parse(querystring.parse(server.requests[0].requestBody).e);
+      assert.lengthOf(events, 2);
+
+      // first event should be identify with initial_referrer and NO referrer
+      assert.equal(events[0].event_type, '$identify');
       assert.deepEqual(events[0].user_properties, {
-        $set: {
-          'user_prop': true
+        '$setOnce': {
+          'initial_referrer': 'https://amplitude.com/contact',
+          'initial_referring_domain': 'amplitude.com'
         }
       });
-      server.respondWith('success');
-      server.respond();
 
-      amplitude.logEvent('Referrer test event');
-      assert.lengthOf(server.requests, 2);
-      var events = JSON.parse(querystring.parse(server.requests[1].requestBody).e);
+      // second event should be the test event with no referrer information
+      assert.equal(events[1].event_type, 'Referrer Test Event');
+      assert.deepEqual(events[1].user_properties, {});
+    });
+
+    it('should not override any existing initial referrer values in session storage', function() {
+      reset();
+      sessionStorage.setItem('amplitude_referrer', 'https://www.google.com/search?');
+      amplitude.init(apiKey, undefined, {includeReferrer: true, batchEvents: true, eventUploadThreshold: 3});
+      amplitude._saveReferrer('https://facebook.com/contact');
+      amplitude.logEvent('Referrer Test Event', {});
+      assert.lengthOf(server.requests, 1);
+      var events = JSON.parse(querystring.parse(server.requests[0].requestBody).e);
+      assert.lengthOf(events, 3);
+
+      // first event should be identify with initial_referrer and NO referrer
+      assert.equal(events[0].event_type, '$identify');
       assert.deepEqual(events[0].user_properties, {
-        referrer: 'https://amplitude.com/contact',
-        referring_domain: 'amplitude.com'
+        '$setOnce': {
+          'initial_referrer': 'https://amplitude.com/contact',
+          'initial_referring_domain': 'amplitude.com'
+        }
+      });
+
+      // second event should be another identify but with the new referrer
+      assert.equal(events[1].event_type, '$identify');
+      assert.deepEqual(events[1].user_properties, {
+        '$setOnce': {
+          'initial_referrer': 'https://facebook.com/contact',
+          'initial_referring_domain': 'facebook.com'
+        }
       });
+
+      // third event should be the test event with no referrer information
+      assert.equal(events[2].event_type, 'Referrer Test Event');
+      assert.deepEqual(events[2].user_properties, {});
+
+      // existing value persists
+      assert.equal(sessionStorage.getItem('amplitude_referrer'), 'https://www.google.com/search?');
     });
   });
 

test/browser/amplitudejs.html

@@ -70,7 +70,7 @@
     }
 </script>
 <script>
-    amplitude.init('a2dbce0e18dfe5f8e74493843ff5c053', null, null, function() {
+    amplitude.init('a2dbce0e18dfe5f8e74493843ff5c053', null, {includeReferrer: true}, function() {
         alert(amplitude.options.deviceId);
     });
     amplitude.setVersionName('Web');
@@ -96,6 +96,7 @@ <h3>Amplitude JS Test</h3>
     <li><a href="javascript:setPhotoCount();">Set photo count</a></li>
     <li><a href="javascript:setOncePhotoCount();">Set photo count once</a></li>
     <li><a href="javascript:setCity();">Set city via setUserProperties</a></li>
-
+    <br><br>
+    <li><a href="/test/browser/amplitudejs2.html">Go to second page</a></li>
 </body>
 </html>