Skip to content

Adding tests for S3 access log formatter, other minor updates #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 6, 2019
Merged

Adding tests for S3 access log formatter, other minor updates #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ccf757
Renaming file to handlers/formatS3Access.js for consistency. Explicit…
arithmetric Aug 6, 2019
3ccc3df
Adding tests for formatS3Access handler.
arithmetric Aug 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions handlers/formatS3access.js → handlers/formatS3Access.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
exports.process = function(config) {
console.log('formatS3access for S3 hosted application logs');
console.log('formatS3Access for S3 hosted application logs');
if (!config.data ||
(!config.data.length && config.data.length !== 0)) {
return Promise.reject('Received unexpected S3 log format:' +
Expand Down Expand Up @@ -55,7 +55,7 @@ exports.process = function(config) {
}
item.time = item.time.replace(/:/, ' ').replace(/\[/, '');
item.zone = item.zone.replace(/\]/, '');
item[config.dateField] = new Date(item.time + item.zone);
item[config.dateField] = new Date(item.time + item.zone).toISOString();
output.push(item);
}
config.data = output;
Expand Down
142 changes: 142 additions & 0 deletions test/assets/s3access.format.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
[
{
"bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"bucket": "awsexamplebucket",
"time": "06/Feb/2019 00:00:38",
"zone": "+0000",
"remote_ip": "192.0.2.3",
"requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"request_id": "3E57427F3EXAMPLE",
"operation": "REST.GET.VERSIONING",
"key": "-",
"request_uri": "GET /awsexamplebucket?versioning HTTP/1.1",
"http_status": "200",
"error_code": "-",
"bytes_sent": "113",
"object_size": "-",
"total_time": "7",
"turn_around_time": "-",
"referrer": "-",
"user_agent": "S3Console/0.4",
"version_id": "-",
"host_id": "s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234=",
"signature_version": "SigV2",
"cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256",
"authentication_type": "AuthHeader",
"host_header": "awsexamplebucket.s3.amazonaws.com",
"tls_version": "TLSV1.1",
"date": "2019-02-06T00:00:38.000Z"
},
{
"bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"bucket": "awsexamplebucket",
"time": "06/Feb/2019 00:00:38",
"zone": "+0000",
"remote_ip": "192.0.2.3",
"requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"request_id": "891CE47D2EXAMPLE",
"operation": "REST.GET.LOGGING_STATUS",
"key": "-",
"request_uri": "GET /awsexamplebucket?logging HTTP/1.1",
"http_status": "200",
"error_code": "-",
"bytes_sent": "242",
"object_size": "-",
"total_time": "11",
"turn_around_time": "-",
"referrer": "-",
"user_agent": "S3Console/0.4",
"version_id": "-",
"host_id": "9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234=",
"signature_version": "SigV2",
"cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256",
"authentication_type": "AuthHeader",
"host_header": "awsexamplebucket.s3.amazonaws.com",
"tls_version": "TLSV1.1",
"date": "2019-02-06T00:00:38.000Z"
},
{
"bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"bucket": "awsexamplebucket",
"time": "06/Feb/2019 00:00:38",
"zone": "+0000",
"remote_ip": "192.0.2.3",
"requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"request_id": "A1206F460EXAMPLE",
"operation": "REST.GET.BUCKETPOLICY",
"key": "-",
"request_uri": "GET /awsexamplebucket?policy HTTP/1.1",
"http_status": "404",
"error_code": "NoSuchBucketPolicy",
"bytes_sent": "297",
"object_size": "-",
"total_time": "38",
"turn_around_time": "-",
"referrer": "-",
"user_agent": "S3Console/0.4",
"version_id": "-",
"host_id": "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=",
"signature_version": "SigV2",
"cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256",
"authentication_type": "AuthHeader",
"host_header": "awsexamplebucket.s3.amazonaws.com",
"tls_version": "TLSV1.1",
"date": "2019-02-06T00:00:38.000Z"
},
{
"bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"bucket": "awsexamplebucket",
"time": "06/Feb/2019 00:01:00",
"zone": "+0000",
"remote_ip": "192.0.2.3",
"requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"request_id": "7B4A0FABBEXAMPLE",
"operation": "REST.GET.VERSIONING",
"key": "-",
"request_uri": "GET /awsexamplebucket?versioning HTTP/1.1",
"http_status": "200",
"error_code": "-",
"bytes_sent": "113",
"object_size": "-",
"total_time": "33",
"turn_around_time": "-",
"referrer": "-",
"user_agent": "S3Console/0.4",
"version_id": "-",
"host_id": "Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234=",
"signature_version": "SigV2",
"cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256",
"authentication_type": "AuthHeader",
"host_header": "awsexamplebucket.s3.amazonaws.com",
"tls_version": "TLSV1.1",
"date": "2019-02-06T00:01:00.000Z"
},
{
"bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"bucket": "awsexamplebucket",
"time": "06/Feb/2019 00:01:57",
"zone": "+0000",
"remote_ip": "192.0.2.3",
"requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"request_id": "DD6CC733AEXAMPLE",
"operation": "REST.PUT.OBJECT",
"key": "s3-dg.pdf",
"request_uri": "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1",
"http_status": "200",
"error_code": "-",
"bytes_sent": "-",
"object_size": "4406583",
"total_time": "41754",
"turn_around_time": "28",
"referrer": "-",
"user_agent": "S3Console/0.4",
"version_id": "-",
"host_id": "10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234=",
"signature_version": "SigV4",
"cipher_suite": "ECDHE-RSA-AES128-SHA",
"authentication_type": "AuthHeader",
"host_header": "awsexamplebucket.s3.amazonaws.com",
"tls_version": "TLSV1.1",
"date": "2019-02-06T00:01:57.000Z"
}
]
138 changes: 138 additions & 0 deletions test/assets/s3access.parse.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
[
[
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"awsexamplebucket",
"[06/Feb/2019:00:00:38",
"+0000]",
"192.0.2.3",
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"3E57427F3EXAMPLE",
"REST.GET.VERSIONING",
"-",
"GET /awsexamplebucket?versioning HTTP/1.1",
"200",
"-",
"113",
"-",
"7",
"-",
"-",
"S3Console/0.4",
"-",
"s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234=",
"SigV2",
"ECDHE-RSA-AES128-GCM-SHA256",
"AuthHeader",
"awsexamplebucket.s3.amazonaws.com",
"TLSV1.1"
],
[
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"awsexamplebucket",
"[06/Feb/2019:00:00:38",
"+0000]",
"192.0.2.3",
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"891CE47D2EXAMPLE",
"REST.GET.LOGGING_STATUS",
"-",
"GET /awsexamplebucket?logging HTTP/1.1",
"200",
"-",
"242",
"-",
"11",
"-",
"-",
"S3Console/0.4",
"-",
"9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234=",
"SigV2",
"ECDHE-RSA-AES128-GCM-SHA256",
"AuthHeader",
"awsexamplebucket.s3.amazonaws.com",
"TLSV1.1"
],
[
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"awsexamplebucket",
"[06/Feb/2019:00:00:38",
"+0000]",
"192.0.2.3",
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"A1206F460EXAMPLE",
"REST.GET.BUCKETPOLICY",
"-",
"GET /awsexamplebucket?policy HTTP/1.1",
"404",
"NoSuchBucketPolicy",
"297",
"-",
"38",
"-",
"-",
"S3Console/0.4",
"-",
"BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=",
"SigV2",
"ECDHE-RSA-AES128-GCM-SHA256",
"AuthHeader",
"awsexamplebucket.s3.amazonaws.com",
"TLSV1.1"
],
[
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"awsexamplebucket",
"[06/Feb/2019:00:01:00",
"+0000]",
"192.0.2.3",
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"7B4A0FABBEXAMPLE",
"REST.GET.VERSIONING",
"-",
"GET /awsexamplebucket?versioning HTTP/1.1",
"200",
"-",
"113",
"-",
"33",
"-",
"-",
"S3Console/0.4",
"-",
"Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234=",
"SigV2",
"ECDHE-RSA-AES128-GCM-SHA256",
"AuthHeader",
"awsexamplebucket.s3.amazonaws.com",
"TLSV1.1"
],
[
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"awsexamplebucket",
"[06/Feb/2019:00:01:57",
"+0000]",
"192.0.2.3",
"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
"DD6CC733AEXAMPLE",
"REST.PUT.OBJECT",
"s3-dg.pdf",
"PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1",
"200",
"-",
"-",
"4406583",
"41754",
"28",
"-",
"S3Console/0.4",
"-",
"10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234=",
"SigV4",
"ECDHE-RSA-AES128-SHA",
"AuthHeader",
"awsexamplebucket.s3.amazonaws.com",
"TLSV1.1",
"extra field"
]
]
5 changes: 5 additions & 0 deletions test/assets/s3access.source.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 3E57427F3EXAMPLE REST.GET.VERSIONING - "GET /awsexamplebucket?versioning HTTP/1.1" 200 - 113 - 7 - "-" "S3Console/0.4" - s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 891CE47D2EXAMPLE REST.GET.LOGGING_STATUS - "GET /awsexamplebucket?logging HTTP/1.1" 200 - 242 - 11 - "-" "S3Console/0.4" - 9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be A1206F460EXAMPLE REST.GET.BUCKETPOLICY - "GET /awsexamplebucket?policy HTTP/1.1" 404 NoSuchBucketPolicy 297 - 38 - "-" "S3Console/0.4" - BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:00 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 7B4A0FABBEXAMPLE REST.GET.VERSIONING - "GET /awsexamplebucket?versioning HTTP/1.1" 200 - 113 - 33 - "-" "S3Console/0.4" - Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:57 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be DD6CC733AEXAMPLE REST.PUT.OBJECT s3-dg.pdf "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1" 200 - - 4406583 41754 28 "-" "S3Console/0.4" - 10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234= SigV4 ECDHE-RSA-AES128-SHA AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 "extra field"
50 changes: 50 additions & 0 deletions test/formatS3Access.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/* global before, describe, it */

var assert = require("assert");
var fs = require("fs");

var handler = require("../handlers/formatS3Access");

describe('handler/formatS3Access.js', function() {
describe('#process()', function() {
var dataSource;
var dataJson;

before(function() {
dataSource = JSON.parse(fs.readFileSync(
"test/assets/s3access.parse.json"));
dataJson = JSON.parse(fs.readFileSync(
"test/assets/s3access.format.json"));
});

it('should format parsed S3 Access data',
function(done) {
var config = {
data: dataSource,
dateField: 'date',
setting: true
};
handler.process(config)
.then(function(result) {
assert.ok(result.hasOwnProperty('setting'),
'process returns config object');
assert.deepStrictEqual(result.data, dataJson,
'S3 Access data formatted successfully');
done();
});
});

it('should fail if malformed S3 Access data is provided',
function(done) {
var config = {
data: {malformed: 'data'},
setting: true
};
handler.process(config)
.catch(function(err) {
assert.ok(err, 'error is thrown');
done();
});
});
});
});
25 changes: 19 additions & 6 deletions test/parseSpaces.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* global before, describe, it */
/* global describe, it */

var assert = require("assert");
var fs = require("fs");
Expand All @@ -10,13 +10,26 @@ describe('handler/parseSpaces.js', function() {
var dataSource;
var dataJson;

before(function() {
dataSource = fs.readFileSync("test/assets/elbv2.source.txt");
dataJson = JSON.parse(fs.readFileSync("test/assets/elbv2.parse.json"));
});
it('should parse space separated ELBv2 data',
function(done) {
dataSource = fs.readFileSync("test/assets/elbv2.source.txt");
dataJson = JSON.parse(fs.readFileSync("test/assets/elbv2.parse.json"));
var config = {data: dataSource, setting: true};
handler.process(config)
.then(function(result) {
assert.ok(result.hasOwnProperty('setting'),
'process returns config object');
assert.deepStrictEqual(result.data, dataJson,
'space separated data parsed successfully');
done();
});
});

it('should parse space separated data',
it('should parse space separated S3 log data',
function(done) {
dataSource = fs.readFileSync("test/assets/s3access.source.txt");
dataJson =
JSON.parse(fs.readFileSync("test/assets/s3access.parse.json"));
var config = {data: dataSource, setting: true};
handler.process(config)
.then(function(result) {
Expand Down