Skip to content

Commit

Permalink
Add configurable max GET request size in bytes attribute (close #449)
Browse files Browse the repository at this point in the history
  • Loading branch information
@matus-tomlein
matus-tomlein committed Mar 8, 2022
1 parent 62e839d commit 6ea620c
Show file tree
Hide file tree
Showing 4 changed files with 119 additions and 43 deletions.
1 change: 1 addition & 0 deletions libraries/browser-tracker-core/src/tracker/index.ts
View file
Expand Up @@ -259,6 +259,7 @@ export function Tracker(
configPostPath,
trackerConfiguration.bufferSize ?? 1,
trackerConfiguration.maxPostBytes ?? 40000,
trackerConfiguration.maxGetBytes ?? false,
trackerConfiguration.useStm ?? true,
trackerConfiguration.maxLocalStorageQueueSize ?? 1000,
trackerConfiguration.connectionTimeout ?? 5000,
Expand Down
13 changes: 12 additions & 1 deletion libraries/browser-tracker-core/src/tracker/out_queue.ts
View file
Expand Up @@ -53,6 +53,7 @@ export interface OutQueue {
* @param postPath - The path where events are to be posted
* @param bufferSize - How many events to batch in localStorage before sending them all
* @param maxPostBytes - Maximum combined size in bytes of the event JSONs in a POST request
* @param maxGetBytes - Maximum size in bytes of the complete event URL string in a GET request
* @param useStm - Whether to add timestamp to events
* @param maxLocalStorageQueueSize - Maximum number of queued events we will attempt to store in local storage
* @param connectionTimeout - Defines how long to wait before aborting the request
Expand All @@ -69,6 +70,7 @@ export function OutQueueManager(
postPath: string,
bufferSize: number,
maxPostBytes: number,
maxGetBytes: number | boolean,
useStm: boolean,
maxLocalStorageQueueSize: number,
connectionTimeout: number,
Expand Down Expand Up @@ -234,7 +236,16 @@ export function OutQueueManager(
(outQueue as Array<PostEvent>).push(body);
}
} else {
(outQueue as Array<string>).push(getQuerystring(request));
const querystring = getQuerystring(request);
if (maxGetBytes !== false) {
const requestUrl = createGetUrl(querystring);
const bytes = getUTF8Length(requestUrl);
if (bytes >= maxGetBytes) {
LOG.error('Event (' + bytes + 'B) too big, max is ' + maxGetBytes + ', skipping.');
return;
}
}
(outQueue as Array<string>).push(querystring);
}
let savedToLocalStorage = false;
if (useLocalStorage) {
Expand Down
7 changes: 6 additions & 1 deletion libraries/browser-tracker-core/src/tracker/types.ts
View file
Expand Up @@ -143,10 +143,15 @@ export type TrackerConfiguration = {
*/
crossDomainLinker?: (elt: HTMLAnchorElement | HTMLAreaElement) => boolean;
/**
* The max size a request can be before the tracker will force send it
* The max size a POST request can be before the tracker will force send it
* @defaultValue 40000
*/
maxPostBytes?: number;
/**
* The max size a GET request (its complete URL) can be. Requests over this size will be ignored.
* @defaultValue unlimited
*/
maxGetBytes?: number;
/**
* Whether the tracker should attempt to figure out what the root
* domain is to store cookies on
Expand Down
141 changes: 100 additions & 41 deletions libraries/browser-tracker-core/test/out_queue.test.ts
View file
Expand Up @@ -34,53 +34,112 @@ import { SharedState } from '../src/state';
describe('OutQueueManager', () => {
const maxQueueSize = 2;

var outQueue: OutQueue;

beforeEach(() => {
localStorage.clear();

outQueue = OutQueueManager(
'sp',
new SharedState(),
true,
'post',
'/com.snowplowanalytics.snowplow/tp2',
1,
40000,
false,
maxQueueSize,
5000,
false,
{},
true
);
});

it('should add event to outQueue and store event in local storage', () => {
const expected = { e: 'pv', eid: '20269f92-f07c-44a6-87ef-43e171305076' };
outQueue.enqueueRequest(expected, '');
describe('POST requests', () => {
var outQueue: OutQueue;

beforeEach(() => {
outQueue = OutQueueManager(
'sp',
new SharedState(),
true,
'post',
'/com.snowplowanalytics.snowplow/tp2',
1,
40000,
false,
false,
maxQueueSize,
5000,
false,
{},
true
);
});

it('should add event to outQueue and store event in local storage', () => {
const expected = { e: 'pv', eid: '20269f92-f07c-44a6-87ef-43e171305076' };
outQueue.enqueueRequest(expected, '');

const retrievedQueue = JSON.parse(
window.localStorage.getItem('snowplowOutQueue_sp_post2') ?? fail('Unable to find local storage queue')
);
expect(retrievedQueue).toHaveLength(1);
expect(retrievedQueue[0]).toMatchObject({ bytes: 55, evt: expected });
});

it('should add event to outQueue and store only events up to max local storage queue size in local storage', () => {
const expected1 = { e: 'pv', eid: '65cb78de-470c-4764-8c10-02bd79477a3a' };
const expected2 = { e: 'pv', eid: '6000c7bd-08a6-49c2-b61c-9531d3d46200' };
const unexpected = { e: 'pv', eid: '7a3391a8-622b-4ce4-80ed-c941aa05baf5' };

outQueue.enqueueRequest(expected1, '');
outQueue.enqueueRequest(expected2, '');
outQueue.enqueueRequest(unexpected, '');

const retrievedQueue = JSON.parse(
window.localStorage.getItem('snowplowOutQueue_sp_post2') ?? fail('Unable to find local storage queue')
);
expect(retrievedQueue).toHaveLength(1);
expect(retrievedQueue[0]).toMatchObject({ bytes: 55, evt: expected });
const retrievedQueue = JSON.parse(
window.localStorage.getItem('snowplowOutQueue_sp_post2') ?? fail('Unable to find local storage queue')
);
expect(retrievedQueue).toHaveLength(maxQueueSize);
expect(retrievedQueue[0]).toMatchObject({ bytes: 55, evt: expected1 });
expect(retrievedQueue[1]).toMatchObject({ bytes: 55, evt: expected2 });
});
});

it('should add event to outQueue and store only events up to max local storage queue size in local storage', () => {
const expected1 = { e: 'pv', eid: '65cb78de-470c-4764-8c10-02bd79477a3a' };
const expected2 = { e: 'pv', eid: '6000c7bd-08a6-49c2-b61c-9531d3d46200' };
const unexpected = { e: 'pv', eid: '7a3391a8-622b-4ce4-80ed-c941aa05baf5' };

outQueue.enqueueRequest(expected1, '');
outQueue.enqueueRequest(expected2, '');
outQueue.enqueueRequest(unexpected, '');

const retrievedQueue = JSON.parse(
window.localStorage.getItem('snowplowOutQueue_sp_post2') ?? fail('Unable to find local storage queue')
);
expect(retrievedQueue).toHaveLength(maxQueueSize);
expect(retrievedQueue[0]).toMatchObject({ bytes: 55, evt: expected1 });
expect(retrievedQueue[1]).toMatchObject({ bytes: 55, evt: expected2 });
describe('GET requests', () => {
var getOutQueue: (maxGetBytes: number | boolean) => OutQueue;
const getQuerystring = (p: object) =>
'?' +
Object.entries(p)
.map(([k, v]) => k + '=' + encodeURIComponent(v))
.join('&');

beforeEach(() => {
getOutQueue = (maxGetBytes) =>
OutQueueManager(
'sp',
new SharedState(),
true,
'get',
'/com.snowplowanalytics.snowplow/tp2',
1,
40000,
maxGetBytes,
false,
maxQueueSize,
5000,
false,
{},
true
);
});

it('should add large event to out queue without bytes limit', () => {
var outQueue = getOutQueue(false);

const expected = { e: 'pv', eid: '20269f92-f07c-44a6-87ef-43e171305076', aid: 'x'.repeat(1000) };
outQueue.enqueueRequest(expected, '');

const retrievedQueue = JSON.parse(
window.localStorage.getItem('snowplowOutQueue_sp_get') ?? fail('Unable to find local storage queue')
);
expect(retrievedQueue).toHaveLength(1);
expect(retrievedQueue[0]).toEqual(getQuerystring(expected));
});

it('should not add event larger than max bytes limit to queue', () => {
var outQueue = getOutQueue(100);
const consoleError = jest.fn();
global.console.error = consoleError;

const expected = { e: 'pv', eid: '20269f92-f07c-44a6-87ef-43e171305076', aid: 'x'.repeat(1000) };
outQueue.enqueueRequest(expected, '');

expect(window.localStorage.getItem('snowplowOutQueue_sp_get')).toBeNull;
expect(consoleError.mock.calls.length).toEqual(1);
});
});
});

0 comments on commit 6ea620c

Please sign in to comment.