Skip to content

Commit

Permalink
Merge pull request #58 from nature/dev-denormalise
Browse files Browse the repository at this point in the history 
Make normalisation process opt-in
  • Loading branch information
@philbooth
philbooth committed May 20, 2015
2 parents fbf5f85 + f375020 commit 610ab7a
Show file tree
Hide file tree
Showing 15 changed files with 1,246 additions and 1,233 deletions.
41 changes: 20 additions & 21 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,33 +29,32 @@
"performance"
],
"dependencies": {
"check-types": "3.1.x",
"commander": "2.6.x",
"get-off-my-log": "0.1.x",
"qs": "2.3.x",
"useragent": "2.1.x",
"yamlparser": "0.0.x",
"request": "2.53.x",
"node-uuid": "1.4.x",
"toobusy-js": "0.4.x",
"handlebars": "3.0.x",
"ua-parser-js": "0.7.x"
"check-types": "~3.2",
"commander": "~2.8",
"get-off-my-log": "~0.1",
"qs": "~2.4",
"useragent": "~2.1",
"yamlparser": "~0.0",
"request": "~2.55",
"node-uuid": "~1.4",
"toobusy-js": "~0.4",
"handlebars": "~3.0",
"ua-parser-js": "~0.7"
},
"optionalDependencies": {
"ain2": "1.5.x"
"ain2": "~1.5"
},
"devDependencies": {
"jshint": "2.6.x",
"mocha": "2.2.x",
"chai": "2.1.x",
"mockery": "1.4.x",
"spooks": "2.0.x",
"istanbul": "0.3.x",
"jsdom": "1.5.x"
"jshint": "~2.6",
"mocha": "~2.2",
"chai": "~2.3",
"mockery": "~1.4",
"spooks": "~2.0",
"jsdom": "~1.5"
},
"scripts": {
"lint": "./node_modules/.bin/jshint src --config .jshintrc",
"test": "./node_modules/.bin/istanbul test ./node_modules/.bin/_mocha -- --ui tdd --reporter spec --recursive --colors test"
"lint": "jshint src --config .jshintrc",
"test": "mocha --ui tdd --reporter spec --recursive --colors test"
}
}

206 changes: 2 additions & 204 deletions src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ tlsCiphers = [
'!CAMELLIA'
].join(':'),

signals, normalisationMaps;
signals;

/**
* Public function `listen`.
Expand Down Expand Up @@ -653,7 +653,7 @@ function send (log, state, remoteAddress, validator, filter, mapper, forwarder,
throw null;
}

mappedData = mapper(filter(normaliseData(data)), referer, userAgent, remoteAddress);
mappedData = mapper(filter(data), referer, userAgent, remoteAddress);
if (mappedData === '') {
return pass(log, response, 204, 0);
}
Expand Down Expand Up @@ -693,208 +693,6 @@ function parseData (request, state) {
return qs.parse(state.body, { parameterLimit: Infinity });
}

function normaliseData (data) {
// TODO: Add metadata for URL, browser, geolocation
return {
rt: normaliseRtData(data),
navtiming: normaliseNavtimingData(data),
restiming: normaliseRestimingData(data)
};
}

function normaliseRtData (data) {
/*jshint camelcase:false */

var start, timeToFirstByte, timeToLastByte, timeToLoad;

start = getOptionalDatum(data, 'rt.tstart');
timeToFirstByte = getOptionalDatum(data, 't_resp');
timeToLastByte = getOptionalSum(data, 't_resp', 't_page');
timeToLoad = getOptionalDatum(data, 't_done');

check.assert.maybe.positive(start);
check.assert.maybe.positive(timeToFirstByte);
check.assert.maybe.positive(timeToLastByte);
check.assert.maybe.positive(timeToLoad);

if (check.positive(timeToFirstByte) || check.positive(timeToLastByte) || check.positive(timeToLoad)) {
return {
timestamps: {
start: start
},
events: {},
durations: {
firstbyte: timeToFirstByte,
lastbyte: timeToLastByte,
load: timeToLoad
},
url: data.r
};
}
}

function getOptionalDatum (data, key) {
if (data[key]) {
return parseInt(data[key]);
}
}

function getOptionalSum (data, aKey, bKey) {
if (data[aKey] && data[bKey]) {
return parseInt(data[aKey]) + parseInt(data[bKey]);
}
}

function normaliseNavtimingData (data) {
/*jshint camelcase:false */
var result = normaliseCategory(normalisationMaps.navtiming, data, 'nt_nav_st');

if (result) {
result.type = data.nt_nav_type;
}

return result;
}

normalisationMaps = {
navtiming: {
timestamps: [
{ key: 'nt_nav_st', name: 'start' },
{ key: 'nt_fet_st', name: 'fetchStart' },
{ key: 'nt_ssl_st', name: 'sslStart', optional: true },
{ key: 'nt_req_st', name: 'requestStart' },
{ key: 'nt_domint', name: 'domInteractive' }
],
events: [
{ start: 'nt_unload_st', end: 'nt_unload_end', name: 'unload' },
{ start: 'nt_red_st', end: 'nt_red_end', name: 'redirect' },
{ start: 'nt_dns_st', end: 'nt_dns_end', name: 'dns' },
{ start: 'nt_con_st', end: 'nt_con_end', name: 'connect' },
{ start: 'nt_res_st', end: 'nt_res_end', name: 'response' },
{ start: 'nt_domloading', end: 'nt_domcomp', name: 'dom' },
{ start: 'nt_domcontloaded_st', end: 'nt_domcontloaded_end', name: 'domContent' },
{ start: 'nt_load_st', end: 'nt_load_end', name: 'load' }
],
durations: []
},
restiming: {
timestamps: [
{ key: 'rt_st', name: 'start' },
{ key: 'rt_fet_st', name: 'fetchStart' },
{ key: 'rt_scon_st', name: 'sslStart', optional: true },
{ key: 'rt_req_st', name: 'requestStart', optional: true }
],
events: [
{ start: 'rt_red_st', end: 'rt_red_end', name: 'redirect', optional: true },
{ start: 'rt_dns_st', end: 'rt_dns_end', name: 'dns', optional: true },
{ start: 'rt_con_st', end: 'rt_con_end', name: 'connect', optional: true },
{ start: 'rt_res_st', end: 'rt_res_end', name: 'response', optional: true }
],
durations: []
}
};

function normaliseCategory (map, data, startKey) {
try {
return {
timestamps: normaliseTimestamps(map, data),
events: normaliseEvents(map, data),
durations: normaliseDurations(map, data, startKey)
};
} catch (e) {
}
}

function normaliseTimestamps (map, data) {
return map.timestamps.reduce(function (result, timestamp) {
var value, assert;

if (data[timestamp.key]) {
value = parseInt(data[timestamp.key]);
}

assert = timestamp.optional ? check.assert.maybe : check.assert;
assert.positive(value);

if (value) {
result[timestamp.name] = value;
}

return result;
}, {});
}

function normaliseEvents (map, data) {
return map.events.reduce(function (result, event) {
var start, end, assert;

if (data[event.start] && data[event.end]) {
start = parseInt(data[event.start]);
end = parseInt(data[event.end]);
}

assert = event.optional ? check.assert.maybe : check.assert;
assert.number(start);
check.assert.not.negative(start);
assert.number(end);
check.assert.not.negative(end);

if (check.number(start) && check.number(end)) {
result[event.name] = {
start: start,
end: end
};
}

return result;
}, {});
}

function normaliseDurations (map, data, startKey) {
var start = parseInt(data[startKey]);

return map.durations.reduce(function (result, duration) {
var value, assert;

if (data[duration.end]) {
value = parseInt(data[duration.end]) - start;
}

assert = duration.optional ? check.assert.maybe : check.assert;
assert.number(value);
check.assert.not.negative(value);

if (value) {
result[duration.name] = value;
}

return result;
}, {});
}

function normaliseRestimingData (data) {
/*jshint camelcase:false */

var result;

if (data.restiming) {
result = [];

Object.keys(data.restiming).forEach(function (key) {
var datum = normaliseCategory(normalisationMaps.restiming, data.restiming[key], 'rt_st');

if (datum) {
datum.name = data.restiming[key].rt_name;
datum.type = data.restiming[key].rt_in_type;
}

result.push(datum);
});

return result;
}
}

function pass (log, response, status, bytes) {
log.info('sent ' + bytes + ' bytes');

Expand Down
Loading

0 comments on commit 610ab7a

Please sign in to comment.