diff --git a/handlers/formatS3access.js b/handlers/formatS3Access.js similarity index 90% rename from handlers/formatS3access.js rename to handlers/formatS3Access.js index a02f646..db7d447 100644 --- a/handlers/formatS3access.js +++ b/handlers/formatS3Access.js @@ -1,5 +1,5 @@ exports.process = function(config) { - console.log('formatS3access for S3 hosted application logs'); + console.log('formatS3Access for S3 hosted application logs'); if (!config.data || (!config.data.length && config.data.length !== 0)) { return Promise.reject('Received unexpected S3 log format:' + @@ -55,7 +55,7 @@ exports.process = function(config) { } item.time = item.time.replace(/:/, ' ').replace(/\[/, ''); item.zone = item.zone.replace(/\]/, ''); - item[config.dateField] = new Date(item.time + item.zone); + item[config.dateField] = new Date(item.time + item.zone).toISOString(); output.push(item); } config.data = output; diff --git a/test/assets/s3access.format.json b/test/assets/s3access.format.json new file mode 100644 index 0000000..6c1bbe2 --- /dev/null +++ b/test/assets/s3access.format.json @@ -0,0 +1,142 @@ +[ + { + "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "bucket": "awsexamplebucket", + "time": "06/Feb/2019 00:00:38", + "zone": "+0000", + "remote_ip": "192.0.2.3", + "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "request_id": "3E57427F3EXAMPLE", + "operation": "REST.GET.VERSIONING", + "key": "-", + "request_uri": "GET /awsexamplebucket?versioning HTTP/1.1", + "http_status": "200", + "error_code": "-", + "bytes_sent": "113", + "object_size": "-", + "total_time": "7", + "turn_around_time": "-", + "referrer": "-", + "user_agent": "S3Console/0.4", + "version_id": "-", + "host_id": "s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234=", + "signature_version": "SigV2", + "cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", + "authentication_type": "AuthHeader", + "host_header": "awsexamplebucket.s3.amazonaws.com", + "tls_version": "TLSV1.1", + "date": "2019-02-06T00:00:38.000Z" + }, + { + "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "bucket": "awsexamplebucket", + "time": "06/Feb/2019 00:00:38", + "zone": "+0000", + "remote_ip": "192.0.2.3", + "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "request_id": "891CE47D2EXAMPLE", + "operation": "REST.GET.LOGGING_STATUS", + "key": "-", + "request_uri": "GET /awsexamplebucket?logging HTTP/1.1", + "http_status": "200", + "error_code": "-", + "bytes_sent": "242", + "object_size": "-", + "total_time": "11", + "turn_around_time": "-", + "referrer": "-", + "user_agent": "S3Console/0.4", + "version_id": "-", + "host_id": "9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234=", + "signature_version": "SigV2", + "cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", + "authentication_type": "AuthHeader", + "host_header": "awsexamplebucket.s3.amazonaws.com", + "tls_version": "TLSV1.1", + "date": "2019-02-06T00:00:38.000Z" + }, + { + "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "bucket": "awsexamplebucket", + "time": "06/Feb/2019 00:00:38", + "zone": "+0000", + "remote_ip": "192.0.2.3", + "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "request_id": "A1206F460EXAMPLE", + "operation": "REST.GET.BUCKETPOLICY", + "key": "-", + "request_uri": "GET /awsexamplebucket?policy HTTP/1.1", + "http_status": "404", + "error_code": "NoSuchBucketPolicy", + "bytes_sent": "297", + "object_size": "-", + "total_time": "38", + "turn_around_time": "-", + "referrer": "-", + "user_agent": "S3Console/0.4", + "version_id": "-", + "host_id": "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=", + "signature_version": "SigV2", + "cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", + "authentication_type": "AuthHeader", + "host_header": "awsexamplebucket.s3.amazonaws.com", + "tls_version": "TLSV1.1", + "date": "2019-02-06T00:00:38.000Z" + }, + { + "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "bucket": "awsexamplebucket", + "time": "06/Feb/2019 00:01:00", + "zone": "+0000", + "remote_ip": "192.0.2.3", + "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "request_id": "7B4A0FABBEXAMPLE", + "operation": "REST.GET.VERSIONING", + "key": "-", + "request_uri": "GET /awsexamplebucket?versioning HTTP/1.1", + "http_status": "200", + "error_code": "-", + "bytes_sent": "113", + "object_size": "-", + "total_time": "33", + "turn_around_time": "-", + "referrer": "-", + "user_agent": "S3Console/0.4", + "version_id": "-", + "host_id": "Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234=", + "signature_version": "SigV2", + "cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", + "authentication_type": "AuthHeader", + "host_header": "awsexamplebucket.s3.amazonaws.com", + "tls_version": "TLSV1.1", + "date": "2019-02-06T00:01:00.000Z" + }, + { + "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "bucket": "awsexamplebucket", + "time": "06/Feb/2019 00:01:57", + "zone": "+0000", + "remote_ip": "192.0.2.3", + "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "request_id": "DD6CC733AEXAMPLE", + "operation": "REST.PUT.OBJECT", + "key": "s3-dg.pdf", + "request_uri": "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1", + "http_status": "200", + "error_code": "-", + "bytes_sent": "-", + "object_size": "4406583", + "total_time": "41754", + "turn_around_time": "28", + "referrer": "-", + "user_agent": "S3Console/0.4", + "version_id": "-", + "host_id": "10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234=", + "signature_version": "SigV4", + "cipher_suite": "ECDHE-RSA-AES128-SHA", + "authentication_type": "AuthHeader", + "host_header": "awsexamplebucket.s3.amazonaws.com", + "tls_version": "TLSV1.1", + "date": "2019-02-06T00:01:57.000Z" + } +] diff --git a/test/assets/s3access.parse.json b/test/assets/s3access.parse.json new file mode 100644 index 0000000..c64fd32 --- /dev/null +++ b/test/assets/s3access.parse.json @@ -0,0 +1,138 @@ +[ + [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "awsexamplebucket", + "[06/Feb/2019:00:00:38", + "+0000]", + "192.0.2.3", + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "3E57427F3EXAMPLE", + "REST.GET.VERSIONING", + "-", + "GET /awsexamplebucket?versioning HTTP/1.1", + "200", + "-", + "113", + "-", + "7", + "-", + "-", + "S3Console/0.4", + "-", + "s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234=", + "SigV2", + "ECDHE-RSA-AES128-GCM-SHA256", + "AuthHeader", + "awsexamplebucket.s3.amazonaws.com", + "TLSV1.1" + ], + [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "awsexamplebucket", + "[06/Feb/2019:00:00:38", + "+0000]", + "192.0.2.3", + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "891CE47D2EXAMPLE", + "REST.GET.LOGGING_STATUS", + "-", + "GET /awsexamplebucket?logging HTTP/1.1", + "200", + "-", + "242", + "-", + "11", + "-", + "-", + "S3Console/0.4", + "-", + "9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234=", + "SigV2", + "ECDHE-RSA-AES128-GCM-SHA256", + "AuthHeader", + "awsexamplebucket.s3.amazonaws.com", + "TLSV1.1" + ], + [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "awsexamplebucket", + "[06/Feb/2019:00:00:38", + "+0000]", + "192.0.2.3", + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "A1206F460EXAMPLE", + "REST.GET.BUCKETPOLICY", + "-", + "GET /awsexamplebucket?policy HTTP/1.1", + "404", + "NoSuchBucketPolicy", + "297", + "-", + "38", + "-", + "-", + "S3Console/0.4", + "-", + "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=", + "SigV2", + "ECDHE-RSA-AES128-GCM-SHA256", + "AuthHeader", + "awsexamplebucket.s3.amazonaws.com", + "TLSV1.1" + ], + [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "awsexamplebucket", + "[06/Feb/2019:00:01:00", + "+0000]", + "192.0.2.3", + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "7B4A0FABBEXAMPLE", + "REST.GET.VERSIONING", + "-", + "GET /awsexamplebucket?versioning HTTP/1.1", + "200", + "-", + "113", + "-", + "33", + "-", + "-", + "S3Console/0.4", + "-", + "Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234=", + "SigV2", + "ECDHE-RSA-AES128-GCM-SHA256", + "AuthHeader", + "awsexamplebucket.s3.amazonaws.com", + "TLSV1.1" + ], + [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "awsexamplebucket", + "[06/Feb/2019:00:01:57", + "+0000]", + "192.0.2.3", + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "DD6CC733AEXAMPLE", + "REST.PUT.OBJECT", + "s3-dg.pdf", + "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1", + "200", + "-", + "-", + "4406583", + "41754", + "28", + "-", + "S3Console/0.4", + "-", + "10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234=", + "SigV4", + "ECDHE-RSA-AES128-SHA", + "AuthHeader", + "awsexamplebucket.s3.amazonaws.com", + "TLSV1.1", + "extra field" + ] +] diff --git a/test/assets/s3access.source.txt b/test/assets/s3access.source.txt new file mode 100644 index 0000000..8823761 --- /dev/null +++ b/test/assets/s3access.source.txt @@ -0,0 +1,5 @@ +79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 3E57427F3EXAMPLE REST.GET.VERSIONING - "GET /awsexamplebucket?versioning HTTP/1.1" 200 - 113 - 7 - "-" "S3Console/0.4" - s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 +79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 891CE47D2EXAMPLE REST.GET.LOGGING_STATUS - "GET /awsexamplebucket?logging HTTP/1.1" 200 - 242 - 11 - "-" "S3Console/0.4" - 9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 +79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be A1206F460EXAMPLE REST.GET.BUCKETPOLICY - "GET /awsexamplebucket?policy HTTP/1.1" 404 NoSuchBucketPolicy 297 - 38 - "-" "S3Console/0.4" - BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 +79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:00 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 7B4A0FABBEXAMPLE REST.GET.VERSIONING - "GET /awsexamplebucket?versioning HTTP/1.1" 200 - 113 - 33 - "-" "S3Console/0.4" - Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 +79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:57 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be DD6CC733AEXAMPLE REST.PUT.OBJECT s3-dg.pdf "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1" 200 - - 4406583 41754 28 "-" "S3Console/0.4" - 10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234= SigV4 ECDHE-RSA-AES128-SHA AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 "extra field" diff --git a/test/formatS3Access.spec.js b/test/formatS3Access.spec.js new file mode 100644 index 0000000..07a2d16 --- /dev/null +++ b/test/formatS3Access.spec.js @@ -0,0 +1,50 @@ +/* global before, describe, it */ + +var assert = require("assert"); +var fs = require("fs"); + +var handler = require("../handlers/formatS3Access"); + +describe('handler/formatS3Access.js', function() { + describe('#process()', function() { + var dataSource; + var dataJson; + + before(function() { + dataSource = JSON.parse(fs.readFileSync( + "test/assets/s3access.parse.json")); + dataJson = JSON.parse(fs.readFileSync( + "test/assets/s3access.format.json")); + }); + + it('should format parsed S3 Access data', + function(done) { + var config = { + data: dataSource, + dateField: 'date', + setting: true + }; + handler.process(config) + .then(function(result) { + assert.ok(result.hasOwnProperty('setting'), + 'process returns config object'); + assert.deepStrictEqual(result.data, dataJson, + 'S3 Access data formatted successfully'); + done(); + }); + }); + + it('should fail if malformed S3 Access data is provided', + function(done) { + var config = { + data: {malformed: 'data'}, + setting: true + }; + handler.process(config) + .catch(function(err) { + assert.ok(err, 'error is thrown'); + done(); + }); + }); + }); +}); diff --git a/test/parseSpaces.spec.js b/test/parseSpaces.spec.js index 2fd7422..3017679 100644 --- a/test/parseSpaces.spec.js +++ b/test/parseSpaces.spec.js @@ -1,4 +1,4 @@ -/* global before, describe, it */ +/* global describe, it */ var assert = require("assert"); var fs = require("fs"); @@ -10,13 +10,26 @@ describe('handler/parseSpaces.js', function() { var dataSource; var dataJson; - before(function() { - dataSource = fs.readFileSync("test/assets/elbv2.source.txt"); - dataJson = JSON.parse(fs.readFileSync("test/assets/elbv2.parse.json")); - }); + it('should parse space separated ELBv2 data', + function(done) { + dataSource = fs.readFileSync("test/assets/elbv2.source.txt"); + dataJson = JSON.parse(fs.readFileSync("test/assets/elbv2.parse.json")); + var config = {data: dataSource, setting: true}; + handler.process(config) + .then(function(result) { + assert.ok(result.hasOwnProperty('setting'), + 'process returns config object'); + assert.deepStrictEqual(result.data, dataJson, + 'space separated data parsed successfully'); + done(); + }); + }); - it('should parse space separated data', + it('should parse space separated S3 log data', function(done) { + dataSource = fs.readFileSync("test/assets/s3access.source.txt"); + dataJson = + JSON.parse(fs.readFileSync("test/assets/s3access.parse.json")); var config = {data: dataSource, setting: true}; handler.process(config) .then(function(result) {