From 4864abac759440540b6fa5f735a0edf34a13ce7e Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Tue, 21 Jul 2020 17:13:18 -0400 Subject: [PATCH 01/10] initial commit for http change --- azure/activity_logs_monitoring/index.js | 78 ++++++++++++++++++------- 1 file changed, 56 insertions(+), 22 deletions(-) diff --git a/azure/activity_logs_monitoring/index.js b/azure/activity_logs_monitoring/index.js index 40053cae7..f3dcb1e0e 100644 --- a/azure/activity_logs_monitoring/index.js +++ b/azure/activity_logs_monitoring/index.js @@ -3,9 +3,9 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2020 Datadog, Inc. -var tls = require('tls'); +var https = require('https'); -const VERSION = '0.1.1'; +const VERSION = '0.1.2'; const STRING = 'string'; // example: 'some message' const STRING_ARRAY = 'string-array'; // example: ['one message', 'two message', ...] @@ -17,13 +17,15 @@ const INVALID = 'invalid'; const DD_API_KEY = process.env.DD_API_KEY || ''; const DD_SITE = process.env.DD_SITE || 'datadoghq.com'; -const DD_URL = process.env.DD_URL || 'functions-intake.logs.' + DD_SITE; -const DD_PORT = process.env.DD_PORT || DD_SITE === 'datadoghq.eu' ? 443 : 10516; +const DD_URL = process.env.DD_URL || 'http-intake.logs.' + DD_SITE; +const DD_PORT = process.env.DD_PORT || 443; const DD_TAGS = process.env.DD_TAGS || ''; // Replace '' by your comma-separated list of tags const DD_SERVICE = process.env.DD_SERVICE || 'azure'; const DD_SOURCE = process.env.DD_SOURCE || 'azure'; const DD_SOURCE_CATEGORY = process.env.DD_SOURCE_CATEGORY || 'azure'; +const ONE_SEC = 1000; + module.exports = function(context, eventHubMessages) { if (!DD_API_KEY || DD_API_KEY === '') { context.log.error( @@ -32,34 +34,66 @@ module.exports = function(context, eventHubMessages) { return; } - var socket = getSocket(context); var sender = tagger => record => { record = tagger(record, context); - if (!send(socket, record)) { - // Retry once - socket = getSocket(context); - send(socket, record); - } + sendWithRetries(record, context); }; - handleLogs(sender, eventHubMessages, context); - - socket.end(); context.done(); }; -function getSocket(context) { - var socket = tls.connect({ port: DD_PORT, host: DD_URL }); - socket.on('error', err => { - context.log.error(err.toString()); - socket.end(); +function sendWithRetries(record, context) { + var info = send(record, context); + var sent = info[0]; + var error = info[1]; + if (!sent) { + context.log.warn('retrying...'); + info = send(record, context); + sent = info[0]; + error = info[1]; + if (!sent) { + context.log.error('unable to send request', error); + } + } +} + +function send(record, context) { + const options = { + hostname: DD_URL, + port: 443, + path: '/v1/input', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'DD-API-KEY': DD_API_KEY + }, + timeout: ONE_SEC + }; + var sent = true; + var error = ''; + const request = https.request(options, res => { + var message = ''; + res.on('data', chunk => { + message = message + chunk; + }).on('end', () => { + var response = JSON.parse(message); + + if (!response.statusCode === 200) { + sent = false; + error = 'invalid status code ' + response.statusCode; + } + }); }); - return socket; -} + request.on('error', e => { + sent = false; + error = e.message; + }); -function send(socket, record) { - return socket.write(DD_API_KEY + ' ' + JSON.stringify(record) + '\n'); + // Write data to request body + request.write(JSON.stringify(record)); + request.end(); + return [sent, error]; } function handleLogs(sender, logs, context) { From 9f65d3ab73c996d3dcb96a829ad7b0e4cd53b114 Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Fri, 24 Jul 2020 11:51:36 -0400 Subject: [PATCH 02/10] updates --- azure/activity_logs_monitoring/index.js | 59 +++++++------------------ 1 file changed, 15 insertions(+), 44 deletions(-) diff --git a/azure/activity_logs_monitoring/index.js b/azure/activity_logs_monitoring/index.js index f3dcb1e0e..b1985716e 100644 --- a/azure/activity_logs_monitoring/index.js +++ b/azure/activity_logs_monitoring/index.js @@ -34,30 +34,7 @@ module.exports = function(context, eventHubMessages) { return; } - var sender = tagger => record => { - record = tagger(record, context); - sendWithRetries(record, context); - }; - handleLogs(sender, eventHubMessages, context); - context.done(); -}; - -function sendWithRetries(record, context) { - var info = send(record, context); - var sent = info[0]; - var error = info[1]; - if (!sent) { - context.log.warn('retrying...'); - info = send(record, context); - sent = info[0]; - error = info[1]; - if (!sent) { - context.log.error('unable to send request', error); - } - } -} -function send(record, context) { const options = { hostname: DD_URL, port: 443, @@ -69,31 +46,25 @@ function send(record, context) { }, timeout: ONE_SEC }; - var sent = true; - var error = ''; - const request = https.request(options, res => { - var message = ''; - res.on('data', chunk => { - message = message + chunk; - }).on('end', () => { - var response = JSON.parse(message); - - if (!response.statusCode === 200) { - sent = false; - error = 'invalid status code ' + response.statusCode; + var sender = tagger => record => { + record = tagger(record, context); + + const request = https.request(options, res => { + if (res.statusCode < 200 || res.statusCode > 299) { + context.log.error('unable to send message, err code: ' + res.statusCode); } }); - }); - request.on('error', e => { - sent = false; - error = e.message; - }); + request.on('error', (e) => { + context.log.error('unable to send request') + }) - // Write data to request body - request.write(JSON.stringify(record)); - request.end(); - return [sent, error]; + // Write data to request body + request.write(JSON.stringify(record)); + request.end(); + } + handleLogs(sender, eventHubMessages, context); + context.done(); } function handleLogs(sender, logs, context) { From bd34ca296884a458ba9f06dcb9c569eb4a407c08 Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Fri, 24 Jul 2020 11:53:17 -0400 Subject: [PATCH 03/10] pin lodash version --- azure/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure/package.json b/azure/package.json index 023571f80..b69b55198 100644 --- a/azure/package.json +++ b/azure/package.json @@ -24,6 +24,7 @@ "devDependencies": { "mocha": "^6.2.3", "prettier": "^1.16.4", - "sinon": "^9.0.2" + "sinon": "^9.0.2", + "lodash": ">=4.17.19" } } From 99a98a3d2718add3739de31d99ba75e7185c5103 Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Fri, 24 Jul 2020 11:54:40 -0400 Subject: [PATCH 04/10] lint --- azure/activity_logs_monitoring/index.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/azure/activity_logs_monitoring/index.js b/azure/activity_logs_monitoring/index.js index b1985716e..14d69cdd7 100644 --- a/azure/activity_logs_monitoring/index.js +++ b/azure/activity_logs_monitoring/index.js @@ -34,7 +34,6 @@ module.exports = function(context, eventHubMessages) { return; } - const options = { hostname: DD_URL, port: 443, @@ -51,21 +50,23 @@ module.exports = function(context, eventHubMessages) { const request = https.request(options, res => { if (res.statusCode < 200 || res.statusCode > 299) { - context.log.error('unable to send message, err code: ' + res.statusCode); + context.log.error( + 'unable to send message, err code: ' + res.statusCode + ); } }); - request.on('error', (e) => { - context.log.error('unable to send request') - }) + request.on('error', e => { + context.log.error('unable to send request'); + }); // Write data to request body request.write(JSON.stringify(record)); request.end(); - } + }; handleLogs(sender, eventHubMessages, context); context.done(); -} +}; function handleLogs(sender, logs, context) { var logsType = getLogFormat(logs); From fa29b660219732670edd63ba64467395496e0cdb Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Fri, 24 Jul 2020 11:56:00 -0400 Subject: [PATCH 05/10] update lodash version --- azure/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure/package-lock.json b/azure/package-lock.json index 95ca16427..ceca90d58 100644 --- a/azure/package-lock.json +++ b/azure/package-lock.json @@ -447,9 +447,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", "dev": true }, "lodash.get": { From 4b9a9f531060dfd5e5461f6c07ae24daba82e974 Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Tue, 28 Jul 2020 12:25:10 -0400 Subject: [PATCH 06/10] use promises and finally working retries --- azure/activity_logs_monitoring/index.js | 101 ++++++++++++++---------- 1 file changed, 61 insertions(+), 40 deletions(-) diff --git a/azure/activity_logs_monitoring/index.js b/azure/activity_logs_monitoring/index.js index 14d69cdd7..81d57d4a4 100644 --- a/azure/activity_logs_monitoring/index.js +++ b/azure/activity_logs_monitoring/index.js @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2020 Datadog, Inc. -var https = require('https'); +const httpsLib = require('https'); const VERSION = '0.1.2'; @@ -26,63 +26,82 @@ const DD_SOURCE_CATEGORY = process.env.DD_SOURCE_CATEGORY || 'azure'; const ONE_SEC = 1000; -module.exports = function(context, eventHubMessages) { +module.exports = async function(context, eventHubMessages) { if (!DD_API_KEY || DD_API_KEY === '') { context.log.error( 'You must configure your API key before starting this function (see ## Parameters section)' ); return; } + handleLogs(sender, eventHubMessages, context); +}; - const options = { - hostname: DD_URL, - port: 443, - path: '/v1/input', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'DD-API-KEY': DD_API_KEY - }, - timeout: ONE_SEC - }; - var sender = tagger => record => { - record = tagger(record, context); - - const request = https.request(options, res => { - if (res.statusCode < 200 || res.statusCode > 299) { - context.log.error( - 'unable to send message, err code: ' + res.statusCode - ); - } - }); +function sender(tagger, record, context) { + record = tagger(record, context); + // retry once + asyncSend(tagger, record, context).catch( + asyncSend(tagger, record, context).catch(handleFailure(context)) + ); +} - request.on('error', e => { - context.log.error('unable to send request'); - }); +function handleFailure(context) { + context.log.error('Unable to send message'); +} - // Write data to request body - request.write(JSON.stringify(record)); - request.end(); - }; - handleLogs(sender, eventHubMessages, context); - context.done(); -}; +async function asyncSend(tagger, record, context, tries) { + return await send(record, context); +} + +async function send(record, context) { + return new Promise((resolve, reject) => { + const options = { + hostname: DD_URL, + port: 443, + path: '/v1/input', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'DD-API-KEY': DD_API_KEY + }, + timeout: ONE_SEC + }; + const myReq = httpsLib + .request(options, myResponse => { + if ( + myResponse.statusCode < 200 || + myResponse.statusCode > 299 + ) { + reject(`invalid status code ${myResponse.statusCode}`); + } else { + resolve(); + } + }) + .on('error', error => { + reject(error); + }); + + myReq.write(JSON.stringify(record)); + myReq.end(); + }); +} function handleLogs(sender, logs, context) { var logsType = getLogFormat(logs); switch (logsType) { case STRING: - sender(addTagsToStringLog)(logs); + sender(addTagsToStringLog, logs, context); break; case JSON_STRING: logs = JSON.parse(logs); - sender(addTagsToJsonLog)(logs); + sender(addTagsToJsonLog, logs, context); break; case JSON_OBJECT: - sender(addTagsToJsonLog)(logs); + sender(addTagsToJsonLog, logs, context); break; case STRING_ARRAY: - logs.forEach(sender(addTagsToStringLog)); + logs.forEach(log => { + sender(addTagsToStringLog, log, context); + }); break; case JSON_ARRAY: handleJSONArrayLogs(sender, context, logs, JSON_ARRAY); @@ -104,14 +123,16 @@ function handleJSONArrayLogs(sender, context, logs, logsType) { message = JSON.parse(message); } catch (err) { context.log.warn('log is malformed json, sending as string'); - sender(addTagsToStringLog)(message); + sender(addTagsToStringLog, message, context); return; } } if (message.records != undefined) { - message.records.forEach(sender(addTagsToJsonLog)); + message.records.forEach(log => { + sender(addTagsToJsonLog, log, context); + }); } else { - sender(addTagsToJsonLog)(message); + sender(addTagsToJsonLog, message, context); } }); } From 2aa33728884bf347f523b989a200381355527511 Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Wed, 29 Jul 2020 14:10:11 -0400 Subject: [PATCH 07/10] Revert "use promises and finally working retries" This reverts commit 4b9a9f531060dfd5e5461f6c07ae24daba82e974. --- azure/activity_logs_monitoring/index.js | 101 ++++++++++-------------- 1 file changed, 40 insertions(+), 61 deletions(-) diff --git a/azure/activity_logs_monitoring/index.js b/azure/activity_logs_monitoring/index.js index 81d57d4a4..14d69cdd7 100644 --- a/azure/activity_logs_monitoring/index.js +++ b/azure/activity_logs_monitoring/index.js @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2020 Datadog, Inc. -const httpsLib = require('https'); +var https = require('https'); const VERSION = '0.1.2'; @@ -26,82 +26,63 @@ const DD_SOURCE_CATEGORY = process.env.DD_SOURCE_CATEGORY || 'azure'; const ONE_SEC = 1000; -module.exports = async function(context, eventHubMessages) { +module.exports = function(context, eventHubMessages) { if (!DD_API_KEY || DD_API_KEY === '') { context.log.error( 'You must configure your API key before starting this function (see ## Parameters section)' ); return; } - handleLogs(sender, eventHubMessages, context); -}; -function sender(tagger, record, context) { - record = tagger(record, context); - // retry once - asyncSend(tagger, record, context).catch( - asyncSend(tagger, record, context).catch(handleFailure(context)) - ); -} + const options = { + hostname: DD_URL, + port: 443, + path: '/v1/input', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'DD-API-KEY': DD_API_KEY + }, + timeout: ONE_SEC + }; + var sender = tagger => record => { + record = tagger(record, context); + + const request = https.request(options, res => { + if (res.statusCode < 200 || res.statusCode > 299) { + context.log.error( + 'unable to send message, err code: ' + res.statusCode + ); + } + }); -function handleFailure(context) { - context.log.error('Unable to send message'); -} + request.on('error', e => { + context.log.error('unable to send request'); + }); -async function asyncSend(tagger, record, context, tries) { - return await send(record, context); -} - -async function send(record, context) { - return new Promise((resolve, reject) => { - const options = { - hostname: DD_URL, - port: 443, - path: '/v1/input', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'DD-API-KEY': DD_API_KEY - }, - timeout: ONE_SEC - }; - const myReq = httpsLib - .request(options, myResponse => { - if ( - myResponse.statusCode < 200 || - myResponse.statusCode > 299 - ) { - reject(`invalid status code ${myResponse.statusCode}`); - } else { - resolve(); - } - }) - .on('error', error => { - reject(error); - }); - - myReq.write(JSON.stringify(record)); - myReq.end(); - }); -} + // Write data to request body + request.write(JSON.stringify(record)); + request.end(); + }; + handleLogs(sender, eventHubMessages, context); + context.done(); +}; function handleLogs(sender, logs, context) { var logsType = getLogFormat(logs); switch (logsType) { case STRING: - sender(addTagsToStringLog, logs, context); + sender(addTagsToStringLog)(logs); break; case JSON_STRING: logs = JSON.parse(logs); - sender(addTagsToJsonLog, logs, context); + sender(addTagsToJsonLog)(logs); break; case JSON_OBJECT: - sender(addTagsToJsonLog, logs, context); + sender(addTagsToJsonLog)(logs); break; case STRING_ARRAY: - logs.forEach(log => { - sender(addTagsToStringLog, log, context); - }); + logs.forEach(sender(addTagsToStringLog)); break; case JSON_ARRAY: handleJSONArrayLogs(sender, context, logs, JSON_ARRAY); @@ -123,16 +104,14 @@ function handleJSONArrayLogs(sender, context, logs, logsType) { message = JSON.parse(message); } catch (err) { context.log.warn('log is malformed json, sending as string'); - sender(addTagsToStringLog, message, context); + sender(addTagsToStringLog)(message); return; } } if (message.records != undefined) { - message.records.forEach(log => { - sender(addTagsToJsonLog, log, context); - }); + message.records.forEach(sender(addTagsToJsonLog)); } else { - sender(addTagsToJsonLog, message, context); + sender(addTagsToJsonLog)(message); } }); } From 199b52addc73caeb291ac4c9edc2d99a96dc3a33 Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Thu, 30 Jul 2020 13:08:18 -0400 Subject: [PATCH 08/10] fix it all and get it workng --- azure/activity_logs_monitoring/index.js | 365 +++++++++++--------- azure/test/client.test.js | 430 +++++++++++++++--------- 2 files changed, 476 insertions(+), 319 deletions(-) diff --git a/azure/activity_logs_monitoring/index.js b/azure/activity_logs_monitoring/index.js index 14d69cdd7..189edde6e 100644 --- a/azure/activity_logs_monitoring/index.js +++ b/azure/activity_logs_monitoring/index.js @@ -15,6 +15,9 @@ const JSON_STRING = 'json-string'; // example: '{"key": "value"}' const JSON_STRING_ARRAY = 'json-string-array'; // example: ['{"records": [{}, {}]}'] or ['{"key": "value"}'] const INVALID = 'invalid'; +const JSON_TYPE = 'json'; +const STRING_TYPE = 'string'; + const DD_API_KEY = process.env.DD_API_KEY || ''; const DD_SITE = process.env.DD_SITE || 'datadoghq.com'; const DD_URL = process.env.DD_URL || 'http-intake.logs.' + DD_SITE; @@ -24,193 +27,235 @@ const DD_SERVICE = process.env.DD_SERVICE || 'azure'; const DD_SOURCE = process.env.DD_SOURCE || 'azure'; const DD_SOURCE_CATEGORY = process.env.DD_SOURCE_CATEGORY || 'azure'; -const ONE_SEC = 1000; +class EventhubLogForwarder { + constructor(context) { + this.context = context; + this.options = { + hostname: DD_URL, + port: 443, + path: '/v1/input', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'DD-API-KEY': DD_API_KEY + }, + timeout: 2000 + }; + } -module.exports = function(context, eventHubMessages) { - if (!DD_API_KEY || DD_API_KEY === '') { - context.log.error( - 'You must configure your API key before starting this function (see ## Parameters section)' - ); - return; + formatLogAndSend(messageType, record) { + if (messageType == JSON_TYPE) { + record = this.addTagsToJsonLog(record); + } else { + record = this.addTagsToStringLog(record); + } + return this.sendWithRetry(record); } - const options = { - hostname: DD_URL, - port: 443, - path: '/v1/input', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'DD-API-KEY': DD_API_KEY - }, - timeout: ONE_SEC - }; - var sender = tagger => record => { - record = tagger(record, context); - - const request = https.request(options, res => { - if (res.statusCode < 200 || res.statusCode > 299) { - context.log.error( - 'unable to send message, err code: ' + res.statusCode - ); - } + sendWithRetry(record) { + return new Promise((resolve, reject) => { + return this.send(record) + .then(res => { + resolve(); + }) + .catch(err => { + setTimeout(() => { + this.send(record) + .then(resolve) + .catch(err => { + this.context.log.error( + `unable to send request after 2 tries, err: ${err}` + ); + reject(); + }); + }, 1000); + }); }); + } - request.on('error', e => { - context.log.error('unable to send request'); + send(record) { + return new Promise((resolve, reject) => { + const req = https + .request(this.options, resp => { + if (resp.statusCode < 200 || resp.statusCode > 299) { + reject(`invalid status code ${resp.statusCode}`); + } else { + resolve(); + } + }) + .on('error', error => { + reject(error); + }); + req.write(JSON.stringify(record)); + req.end(); }); + } - // Write data to request body - request.write(JSON.stringify(record)); - request.end(); - }; - handleLogs(sender, eventHubMessages, context); - context.done(); -}; + handleLogs(logs) { + var promises = []; + var logsType = this.getLogFormat(logs); + switch (logsType) { + case STRING: + promises.push(this.formatLogAndSend(STRING_TYPE, logs)); + break; + case JSON_STRING: + logs = JSON.parse(logs); + promises.push(this.formatLogAndSend(JSON_TYPE, logs)); + break; + case JSON_OBJECT: + promises.push(this.formatLogAndSend(JSON_TYPE, logs)); + break; + case STRING_ARRAY: + logs.forEach(log => + promises.push(this.formatLogAndSend(STRING_TYPE, log)) + ); + break; + case JSON_ARRAY: + promises = this.handleJSONArrayLogs(logs, JSON_ARRAY); + break; + case JSON_STRING_ARRAY: + promises = this.handleJSONArrayLogs(logs, JSON_STRING_ARRAY); + break; + case INVALID: + default: + this.context.log.warn('logs format is invalid'); + break; + } + return promises; + } -function handleLogs(sender, logs, context) { - var logsType = getLogFormat(logs); - switch (logsType) { - case STRING: - sender(addTagsToStringLog)(logs); - break; - case JSON_STRING: - logs = JSON.parse(logs); - sender(addTagsToJsonLog)(logs); - break; - case JSON_OBJECT: - sender(addTagsToJsonLog)(logs); - break; - case STRING_ARRAY: - logs.forEach(sender(addTagsToStringLog)); - break; - case JSON_ARRAY: - handleJSONArrayLogs(sender, context, logs, JSON_ARRAY); - break; - case JSON_STRING_ARRAY: - handleJSONArrayLogs(sender, context, logs, JSON_STRING_ARRAY); - break; - case INVALID: - default: - context.log.warn('logs format is invalid'); - break; + handleJSONArrayLogs(logs, logsType) { + var promises = []; + logs.forEach(message => { + if (logsType == JSON_STRING_ARRAY) { + try { + message = JSON.parse(message); + } catch (err) { + this.context.log.warn( + 'log is malformed json, sending as string' + ); + promises.push(this.formatLogAndSend(STRING_TYPE, message)); + return; + } + } + if (message.records != undefined) { + message.records.forEach(message => + promises.push(this.formatLogAndSend(JSON_TYPE, message)) + ); + } else { + this.formatLogAndSend(JSON_TYPE, message); + } + }); + return promises; } -} -function handleJSONArrayLogs(sender, context, logs, logsType) { - logs.forEach(message => { - if (logsType == JSON_STRING_ARRAY) { - try { - message = JSON.parse(message); - } catch (err) { - context.log.warn('log is malformed json, sending as string'); - sender(addTagsToStringLog)(message); - return; + getLogFormat(logs) { + if (typeof logs === 'string') { + if (this.isJsonString(logs)) { + return JSON_STRING; } + return STRING; } - if (message.records != undefined) { - message.records.forEach(sender(addTagsToJsonLog)); - } else { - sender(addTagsToJsonLog)(message); + if (!Array.isArray(logs) && typeof logs === 'object' && logs !== null) { + return JSON_OBJECT; } - }); -} - -function getLogFormat(logs) { - if (typeof logs === 'string') { - if (isJsonString(logs)) { - return JSON_STRING; + if (!Array.isArray(logs)) { + return INVALID; + } + if (typeof logs[0] === 'object') { + return JSON_ARRAY; + } + if (typeof logs[0] === 'string') { + if (this.isJsonString(logs[0])) { + return JSON_STRING_ARRAY; + } else { + return STRING_ARRAY; + } } - return STRING; - } - if (!Array.isArray(logs) && typeof logs === 'object' && logs !== null) { - return JSON_OBJECT; - } - if (!Array.isArray(logs)) { return INVALID; } - if (typeof logs[0] === 'object') { - return JSON_ARRAY; - } - if (typeof logs[0] === 'string') { - if (isJsonString(logs[0])) { - return JSON_STRING_ARRAY; - } else { - return STRING_ARRAY; + + isJsonString(record) { + try { + JSON.parse(record); + return true; + } catch (err) { + return false; } } - return INVALID; -} -function isJsonString(record) { - try { - JSON.parse(record); - return true; - } catch (err) { - return false; + addTagsToJsonLog(record) { + var metadata = this.extractResourceId(record); + record['ddsource'] = metadata.source || DD_SOURCE; + record['ddsourcecategory'] = DD_SOURCE_CATEGORY; + record['service'] = DD_SERVICE; + record['ddtags'] = metadata.tags + .concat([ + DD_TAGS, + 'forwardername:' + this.context.executionContext.functionName + ]) + .filter(Boolean) + .join(','); + return record; } -} -function addTagsToJsonLog(record, context) { - metadata = extractResourceId(record); - record['ddsource'] = metadata.source || DD_SOURCE; - record['ddsourcecategory'] = DD_SOURCE_CATEGORY; - record['service'] = DD_SERVICE; - record['ddtags'] = metadata.tags - .concat([ - DD_TAGS, - 'forwardername:' + context.executionContext.functionName - ]) - .filter(Boolean) - .join(','); - return record; -} - -function addTagsToStringLog(stringLog, context) { - jsonLog = { message: stringLog }; - return addTagsToJsonLog(jsonLog, context); -} + addTagsToStringLog(stringLog) { + var jsonLog = { message: stringLog }; + return this.addTagsToJsonLog(jsonLog); + } -function extractResourceId(record) { - metadata = { tags: [], source: '' }; - if ( - record.resourceId === undefined || - typeof record.resourceId !== 'string' - ) { - return metadata; - } else if (record.resourceId.toLowerCase().startsWith('/subscriptions/')) { - var resourceId = record.resourceId.toLowerCase().split('/'); - if (resourceId.length > 2) { - metadata.tags.push('subscription_id:' + resourceId[2]); - } - if (resourceId.length > 4) { - metadata.tags.push('resource_group:' + resourceId[4]); - } - if (resourceId.length > 6 && resourceId[6]) { - metadata.source = resourceId[6].replace('microsoft.', 'azure.'); - } - return metadata; - } else if (record.resourceId.toLowerCase().startsWith('/tenants/')) { - var resourceId = record.resourceId.toLowerCase().split('/'); - if (resourceId.length > 4 && resourceId[4]) { - metadata.tags.push('tenant:' + resourceId[2]); - metadata.source = resourceId[4] - .replace('microsoft.', 'azure.') - .replace('aadiam', 'activedirectory'); + extractResourceId(record) { + var metadata = { tags: [], source: '' }; + if ( + record.resourceId === undefined || + typeof record.resourceId !== 'string' + ) { + return metadata; + } else if ( + record.resourceId.toLowerCase().startsWith('/subscriptions/') + ) { + var resourceId = record.resourceId.toLowerCase().split('/'); + if (resourceId.length > 2) { + metadata.tags.push('subscription_id:' + resourceId[2]); + } + if (resourceId.length > 4) { + metadata.tags.push('resource_group:' + resourceId[4]); + } + if (resourceId.length > 6 && resourceId[6]) { + metadata.source = resourceId[6].replace('microsoft.', 'azure.'); + } + return metadata; + } else if (record.resourceId.toLowerCase().startsWith('/tenants/')) { + var resourceId = record.resourceId.toLowerCase().split('/'); + if (resourceId.length > 4 && resourceId[4]) { + metadata.tags.push('tenant:' + resourceId[2]); + metadata.source = resourceId[4] + .replace('microsoft.', 'azure.') + .replace('aadiam', 'activedirectory'); + } + return metadata; + } else { + return metadata; } - return metadata; - } else { - return metadata; } } +module.exports = async function(context, eventHubMessages) { + if (!DD_API_KEY || DD_API_KEY === '') { + context.log.error( + 'You must configure your API key before starting this function (see ## Parameters section)' + ); + return; + } + var promises = new EventhubLogForwarder(context).handleLogs( + eventHubMessages + ); + + return Promise.allSettled(promises); +}; + module.exports.forTests = { - getLogFormat, - extractResourceId, - handleLogs, - isJsonString, - addTagsToStringLog, - addTagsToJsonLog, + EventhubLogForwarder, constants: { STRING, STRING_ARRAY, diff --git a/azure/test/client.test.js b/azure/test/client.test.js index eb0534775..430f53d78 100644 --- a/azure/test/client.test.js +++ b/azure/test/client.test.js @@ -1,180 +1,292 @@ var assert = require('assert'); var client = require('../activity_logs_monitoring').forTests; -var constants = client.constants -var sinon = require('sinon') +var constants = client.constants; +var sinon = require('sinon'); +function fakeContext() { + // create a fake context object to pass into handleLogs + contextSpy = sinon.spy(); + contextSpy.log = sinon.spy(); + contextSpy.log.error = function(x) {}; // do nothing + contextSpy.log.warn = function(x) {}; // do nothing + + return contextSpy; +} + +var EventhubLogForwarderInstance = new client.EventhubLogForwarder( + fakeContext() +); +EventhubLogForwarderInstance.sendWithRetry = function(record) {}; // do nothing +console.log(EventhubLogForwarderInstance.getLogFormat); +var handleJsonLogsSpy = sinon.spy(); +var handleStringLogsSpy = sinon.spy(); + +EventhubLogForwarderInstance.addTagsToJsonLog = handleJsonLogsSpy; +EventhubLogForwarderInstance.addTagsToStringLog = handleStringLogsSpy; describe('Azure Log Monitoring', function() { - describe('#getLogFormat', function() { - it('should return string', function() { - eventHubMessages = ''; - assert.equal(constants.STRING, client.getLogFormat(eventHubMessages)); - eventHubMessages = 'foobar'; - assert.equal(constants.STRING, client.getLogFormat(eventHubMessages)); - }); - it('should return string array', function() { - eventHubMessages = ['', 'foobar']; - assert.equal(constants.STRING_ARRAY, client.getLogFormat(eventHubMessages)); - }); - it('should return json object', function() { - eventHubMessages = {'key': 'value', 'otherkey':'othervalue'}; - assert.equal(constants.JSON_OBJECT, client.getLogFormat(eventHubMessages)); + describe('#getLogFormat', function() { + it('should return string', function() { + eventHubMessages = ''; + assert.equal( + constants.STRING, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + eventHubMessages = 'foobar'; + assert.equal( + constants.STRING, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + }); + it('should return string array', function() { + eventHubMessages = ['', 'foobar']; + assert.equal( + constants.STRING_ARRAY, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + }); + it('should return json object', function() { + eventHubMessages = { key: 'value', otherkey: 'othervalue' }; + assert.equal( + constants.JSON_OBJECT, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + }); + it('should return json no records', function() { + eventHubMessages = [ + { key: 'value', otherkey: 'othervalue' }, + { key: 'value', otherkey: 'othervalue' } + ]; + assert.equal( + constants.JSON_ARRAY, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + }); + it('should return invalid', function() { + eventHubMessages = 1; + assert.equal( + constants.INVALID, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + eventHubMessages = () => {}; + assert.equal( + constants.INVALID, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + eventHubMessages = true; + assert.equal( + constants.INVALID, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + eventHubMessages = null; + assert.equal( + constants.INVALID, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + eventHubMessages = undefined; + assert.equal( + constants.INVALID, + EventhubLogForwarderInstance.getLogFormat(eventHubMessages) + ); + }); }); - it('should return json no records', function() { - eventHubMessages = [{'key': 'value', 'otherkey':'othervalue'}, {'key': 'value', 'otherkey':'othervalue'}]; - assert.equal(constants.JSON_ARRAY, client.getLogFormat(eventHubMessages)); - }); - it('should return invalid', function() { - eventHubMessages = 1; - assert.equal(constants.INVALID, client.getLogFormat(eventHubMessages)); - eventHubMessages = () => {}; - assert.equal(constants.INVALID, client.getLogFormat(eventHubMessages)); - eventHubMessages = true; - assert.equal(constants.INVALID, client.getLogFormat(eventHubMessages)); - eventHubMessages = null; - assert.equal(constants.INVALID, client.getLogFormat(eventHubMessages)); - eventHubMessages = undefined; - assert.equal(constants.INVALID, client.getLogFormat(eventHubMessages)); - }); - }); - describe('#extractResourceId', function() { - it('should parse a valid record', function() { - record = {'resourceId': '/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP/PROVIDERS/MICROSOFT.COMPUTE/VIRTUALMACHINES/SOME-VM'} - expectedMetadata = {'tags': ["subscription_id:12345678-1234-abcd-1234-1234567890ab","resource_group:some-resource-group"], 'source': 'azure.compute'} - assert.deepEqual(expectedMetadata, client.extractResourceId(record)) - }); - it('should parse a valid record without provider', function() { - record = {'resourceId': '/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP'} - expectedMetadata = {'tags': ["subscription_id:12345678-1234-abcd-1234-1234567890ab","resource_group:some-resource-group"], 'source': ''} - assert.deepEqual(expectedMetadata, client.extractResourceId(record)) - }); - it('should parse a valid record without provider and resource group', function() { - record = {'resourceId': '/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB'} - expectedMetadata = {'tags': ["subscription_id:12345678-1234-abcd-1234-1234567890ab"], 'source': ''} - assert.deepEqual(expectedMetadata, client.extractResourceId(record)) - }); - it('should not fail on record without resourceId', function() { - record = {'key':'value'} - expectedMetadata = {'tags': [], 'source': ''} - assert.deepEqual(expectedMetadata, client.extractResourceId(record)) - }); - it('should not fail on string record', function() { - record = {'key':'value'} - expectedMetadata = {'tags': [], 'source': ''} - assert.deepEqual(expectedMetadata, client.extractResourceId(record)) + describe('#extractResourceId', function() { + it('should parse a valid record', function() { + record = { + resourceId: + '/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP/PROVIDERS/MICROSOFT.COMPUTE/VIRTUALMACHINES/SOME-VM' + }; + expectedMetadata = { + tags: [ + 'subscription_id:12345678-1234-abcd-1234-1234567890ab', + 'resource_group:some-resource-group' + ], + source: 'azure.compute' + }; + assert.deepEqual( + expectedMetadata, + EventhubLogForwarderInstance.extractResourceId(record) + ); + }); + it('should parse a valid record without provider', function() { + record = { + resourceId: + '/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP' + }; + expectedMetadata = { + tags: [ + 'subscription_id:12345678-1234-abcd-1234-1234567890ab', + 'resource_group:some-resource-group' + ], + source: '' + }; + assert.deepEqual( + expectedMetadata, + EventhubLogForwarderInstance.extractResourceId(record) + ); + }); + it('should parse a valid record without provider and resource group', function() { + record = { + resourceId: + '/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB' + }; + expectedMetadata = { + tags: ['subscription_id:12345678-1234-abcd-1234-1234567890ab'], + source: '' + }; + assert.deepEqual( + expectedMetadata, + EventhubLogForwarderInstance.extractResourceId(record) + ); + }); + it('should not fail on record without resourceId', function() { + record = { key: 'value' }; + expectedMetadata = { tags: [], source: '' }; + assert.deepEqual( + expectedMetadata, + EventhubLogForwarderInstance.extractResourceId(record) + ); + }); + it('should not fail on string record', function() { + record = { key: 'value' }; + expectedMetadata = { tags: [], source: '' }; + assert.deepEqual( + expectedMetadata, + EventhubLogForwarderInstance.extractResourceId(record) + ); + }); + it('should not fail on improper resourceId', function() { + record = { resourceId: 'foo/bar' }; + expectedMetadata = { tags: [], source: '' }; + assert.deepEqual( + expectedMetadata, + EventhubLogForwarderInstance.extractResourceId(record) + ); + }); + it('should not fail with an invalid source', function() { + record = { + resourceId: + '/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP/PROVIDERS/////' + }; + expectedMetadata = { + tags: [ + 'subscription_id:12345678-1234-abcd-1234-1234567890ab', + 'resource_group:some-resource-group' + ], + source: '' + }; + assert.deepEqual( + expectedMetadata, + EventhubLogForwarderInstance.extractResourceId(record) + ); + }); }); - it('should not fail on improper resourceId', function() { - record = {'resourceId': 'foo/bar'} - expectedMetadata = {'tags': [], 'source': ''} - assert.deepEqual(expectedMetadata, client.extractResourceId(record)) - }); - it('should not fail with an invalid source', function() { - record = {'resourceId': '/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP/PROVIDERS/////'} - expectedMetadata = {'tags': ["subscription_id:12345678-1234-abcd-1234-1234567890ab","resource_group:some-resource-group"], 'source': ''} - assert.deepEqual(expectedMetadata, client.extractResourceId(record)) - }); - }) - function fakeContext() { - // create a fake context object to pass into handleLogs - contextSpy = sinon.spy() - contextSpy.log = sinon.spy() - contextSpy.log.error = function(x) {} // do nothing - contextSpy.log.warn = function(x) {} // do nothing - - return contextSpy - } - - function testHandleLogs(logs, expected, assertJson) { - // create a spy to mock the sender call - var handleJsonLogsSpy = sinon.spy(); - var handleStringLogsSpy = sinon.spy(); - - sender = function(tagger) { - if (tagger === client.addTagsToJsonLog) { - return handleJsonLogsSpy; - } else { - return handleStringLogsSpy; - } + function testHandleLogs(logs, expected, assertJson) { + EventhubLogForwarderInstance.handleLogs(record); + if (assertJson == true) { + expected.forEach(message => { + sinon.assert.calledWith(handleJsonLogsSpy, message); + }); + } else { + expected.forEach(message => { + sinon.assert.calledWith(handleStringLogsSpy, message); + }); + } } - client.handleLogs(sender, record, fakeContext()); - if (assertJson == true) { - expected.forEach(message => { - sinon.assert.calledWith(handleJsonLogsSpy, message) - }); - } else { - expected.forEach(message => { - sinon.assert.calledWith(handleStringLogsSpy, message) - }) - } - } - - describe('#handleLogs', function() { - it('should handle string properly', function() { - record = 'hello' - expected = ['hello'] - assert.equal(client.getLogFormat(record), constants.STRING) - testHandleLogs(record, expected, false) - }); + describe('#handleLogs', function() { + it('should handle string properly', function() { + record = 'hello'; + expected = ['hello']; + assert.equal( + EventhubLogForwarderInstance.getLogFormat(record), + constants.STRING + ); + testHandleLogs(record, expected, false); + }); - it('should handle json-string properly', function() { - record = '{"hello": "there"}' - expected = [{'hello': 'there'}] - assert.equal(client.getLogFormat(record), constants.JSON_STRING) - testHandleLogs(record, expected, true) - }); + it('should handle json-string properly', function() { + record = '{"hello": "there"}'; + expected = [{ hello: 'there' }]; + assert.equal( + EventhubLogForwarderInstance.getLogFormat(record), + constants.JSON_STRING + ); + testHandleLogs(record, expected, true); + }); - it('should handle json-object properly', function() { - record = {'hello': 'there'} - expected = [{'hello': 'there'}] - assert.equal(client.getLogFormat(record), constants.JSON_OBJECT) - testHandleLogs(record, expected, true) - }); + it('should handle json-object properly', function() { + record = { hello: 'there' }; + expected = [{ hello: 'there' }]; + assert.equal( + EventhubLogForwarderInstance.getLogFormat(record), + constants.JSON_OBJECT + ); + testHandleLogs(record, expected, true); + }); - it('should handle string-array properly', function() { - record = ['one message', 'two message'] - expected = ['one message', 'two message'] - assert.equal(client.getLogFormat(record), constants.STRING_ARRAY) - testHandleLogs(record, expected, false) - }); + it('should handle string-array properly', function() { + record = ['one message', 'two message']; + expected = ['one message', 'two message']; + assert.equal( + EventhubLogForwarderInstance.getLogFormat(record), + constants.STRING_ARRAY + ); + testHandleLogs(record, expected, false); + }); - it('should handle json-records properly', function() { - record = [{"records": [{"hello": "there"}, {"goodbye": "now"}]}] - expected = [{"hello": "there"}, {"goodbye": "now"}] - assert.equal(client.getLogFormat(record), constants.JSON_ARRAY) //JSON_RECORDS - testHandleLogs(record, expected, true) - }); + it('should handle json-records properly', function() { + record = [{ records: [{ hello: 'there' }, { goodbye: 'now' }] }]; + expected = [{ hello: 'there' }, { goodbye: 'now' }]; + assert.equal( + EventhubLogForwarderInstance.getLogFormat(record), + constants.JSON_ARRAY + ); //JSON_RECORDS + testHandleLogs(record, expected, true); + }); - it('should handle json-array properly', function() { - record = [{"hello": "there"}, {"goodbye": "now"}] - expected = [{"hello": "there"}, {"goodbye": "now"}] - assert.equal(client.getLogFormat(record), constants.JSON_ARRAY) - testHandleLogs(record, expected, true) - }); + it('should handle json-array properly', function() { + record = [{ hello: 'there' }, { goodbye: 'now' }]; + expected = [{ hello: 'there' }, { goodbye: 'now' }]; + assert.equal( + EventhubLogForwarderInstance.getLogFormat(record), + constants.JSON_ARRAY + ); + testHandleLogs(record, expected, true); + }); - it('should handle json-string-array properly records', function() { - record = ['{"records": [{ "time": "xyz"}, {"time": "abc"}]}'] - expected = [{"time": "xyz"}] - assert.equal(client.getLogFormat(record), constants.JSON_STRING_ARRAY) - testHandleLogs(record, expected, true) - }); + it('should handle json-string-array properly records', function() { + record = ['{"records": [{ "time": "xyz"}, {"time": "abc"}]}']; + expected = [{ time: 'xyz' }]; + assert.equal( + EventhubLogForwarderInstance.getLogFormat(record), + constants.JSON_STRING_ARRAY + ); + testHandleLogs(record, expected, true); + }); - it('should handle json-string-array properly no records', function() { - record = ['{"time": "xyz"}'] - expected = [{"time": "xyz"}] - assert.equal(client.getLogFormat(record), constants.JSON_STRING_ARRAY) - testHandleLogs(record, expected, true) - }); + it('should handle json-string-array properly no records', function() { + record = ['{"time": "xyz"}']; + expected = [{ time: 'xyz' }]; + assert.equal( + EventhubLogForwarderInstance.getLogFormat(record), + constants.JSON_STRING_ARRAY + ); + testHandleLogs(record, expected, true); + }); - it('should handle json-string-array with malformed string', function() { - record = ['{"time": "xyz"}', '{"time": "xy'] - expected = ['{"time": "xy'] - assert.equal(client.getLogFormat(record), constants.JSON_STRING_ARRAY) - // just assert that the string method is called for the second message, - // we don't care about the first one for this test - testHandleLogs(record, expected, false) + it('should handle json-string-array with malformed string', function() { + record = ['{"time": "xyz"}', '{"time": "xy']; + expected = ['{"time": "xy']; + assert.equal( + EventhubLogForwarderInstance.getLogFormat(record), + constants.JSON_STRING_ARRAY + ); + // just assert that the string method is called for the second message, + // we don't care about the first one for this test + testHandleLogs(record, expected, false); + }); }); - }) }); From 6dbb30aba3ed70b1ff492528b90468418a6110e4 Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Thu, 30 Jul 2020 13:21:36 -0400 Subject: [PATCH 09/10] remove lodash changes for another pr --- azure/package-lock.json | 6 +++--- azure/package.json | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/azure/package-lock.json b/azure/package-lock.json index ceca90d58..95ca16427 100644 --- a/azure/package-lock.json +++ b/azure/package-lock.json @@ -447,9 +447,9 @@ } }, "lodash": { - "version": "4.17.19", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, "lodash.get": { diff --git a/azure/package.json b/azure/package.json index b69b55198..023571f80 100644 --- a/azure/package.json +++ b/azure/package.json @@ -24,7 +24,6 @@ "devDependencies": { "mocha": "^6.2.3", "prettier": "^1.16.4", - "sinon": "^9.0.2", - "lodash": ">=4.17.19" + "sinon": "^9.0.2" } } From 0b32d5f3300af559477ceb0c5babae9b681c078c Mon Sep 17 00:00:00 2001 From: Claudia D'Adamo Date: Mon, 3 Aug 2020 11:27:03 -0400 Subject: [PATCH 10/10] update test --- azure/test/client.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/test/client.test.js b/azure/test/client.test.js index 430f53d78..1ca5b460b 100644 --- a/azure/test/client.test.js +++ b/azure/test/client.test.js @@ -52,7 +52,7 @@ describe('Azure Log Monitoring', function() { EventhubLogForwarderInstance.getLogFormat(eventHubMessages) ); }); - it('should return json no records', function() { + it('should return json array when there are no records', function() { eventHubMessages = [ { key: 'value', otherkey: 'othervalue' }, { key: 'value', otherkey: 'othervalue' }