From 04e640bee536e9350e30e35b879d120024665951 Mon Sep 17 00:00:00 2001 From: Yohei Kitamura Date: Tue, 3 Sep 2019 16:52:28 -0400 Subject: [PATCH 1/4] Add log stream name prefix filtering, expecially for AWS Batch logs --- cloudwatchlogs-with-dlq/Readme.md | 1 + cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js | 8 +++++++- cloudwatchlogs/README.md | 9 +++++---- cloudwatchlogs/cloudwatchlogs_lambda.js | 8 ++++++++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/cloudwatchlogs-with-dlq/Readme.md b/cloudwatchlogs-with-dlq/Readme.md index 80b5840..7c69560 100644 --- a/cloudwatchlogs-with-dlq/Readme.md +++ b/cloudwatchlogs-with-dlq/Readme.md @@ -21,6 +21,7 @@ The following AWS Lambda environment variables are supported in both the lambda * SOURCE_NAME_OVERRIDE (OPTIONAL) - Override _sourceName metadata field within SumoLogic. * INCLUDE_LOG_INFO (OPTIONAL) - Set it to true when loggroup/logstream values needs to be included in logs. Default is false * LOG_FORMAT - Default is Others. One can choose VPC-JSON for VPC flow logs in json format and VPC-RAW for only RAW message line +* LOG_STREAM_PREFIX (OPTIONAL) - Comma separated list of logStream name prefixes to filter by logStream, expecially for AWS Batch logs ### Configuring Lambda for VPC Flow Logs The following AWS Lambda environment variables are supported in both the lambda functions for VPC flow logs. diff --git a/cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js b/cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js index e405408..89c86a0 100644 --- a/cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js +++ b/cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js @@ -82,7 +82,11 @@ function getConfig(env) { "compressData": env.COMPRESS_DATA || true, "vpcCIDRPrefix": env.VPC_CIDR_PREFIX || '', "includeLogInfo": ("INCLUDE_LOG_INFO" in env) ? env.INCLUDE_LOG_INFO === "true" : false, - "includeSecurityGroupInfo": ("INCLUDE_SECURITY_GROUP_INFO" in env) ? env.INCLUDE_SECURITY_GROUP_INFO === "true" : false + "includeSecurityGroupInfo": ("INCLUDE_SECURITY_GROUP_INFO" in env) ? env.INCLUDE_SECURITY_GROUP_INFO === "true" : false, + // Regex to filter by logStream name prefixes + "logStreamPrefixRegex": ("LOG_STREAM_PREFIX" in env) + ? new RegExp('^(' + env.LOG_STREAM_PREFIX.replace(/,/g, '|') + ')', 'i') + : '' }; if (!config.SumoURL) { return new Error('Undefined SUMO_ENDPOINT environment variable'); @@ -134,6 +138,8 @@ exports.processLogs = function (env, eventAwslogsData, callback) { var records = []; if (awslogsData.messageType === 'CONTROL_MESSAGE') { console.log('Skipping Control Message'); + } else if(config.logStreamPrefixRegex && !awslogsData.logStream.match(config.logStreamPrefixRegex)){ + console.log('Skipping Non-Applicable Log Stream'); } else { records = createRecords(config, awslogsData.logEvents, awslogsData); console.log(records.length + " Records Found"); diff --git a/cloudwatchlogs/README.md b/cloudwatchlogs/README.md index 5c6649a..1fdd64a 100644 --- a/cloudwatchlogs/README.md +++ b/cloudwatchlogs/README.md @@ -1,4 +1,4 @@ -# Sumo Logic Functions for AWS CloudWatch Logs +# Sumo Logic Functions for AWS CloudWatch Logs AWS Lambda function to collector logs from CloudWatch Logs and post them to [SumoLogic](http://www.sumologic.com) via a [HTTP collector endpoint](http://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source) @@ -20,7 +20,7 @@ First create an [HTTP collector endpoint](http://help.sumologic.com/Send_Data/So * Copy code from cloudwatchlogs_lambda.js into the Lambda function code. * Add Environment variables (See below) 5. Scroll down to the `Lambda function handle and role` section, make sure you set the right values that match the function. For role, you can just use the basic execution role. Click next. -6. Finally click on "Create function" to create the function. +6. Finally click on "Create function" to create the function. 7. (Optional) Test this new function with sample AWS CloudWatch Logs template provided by AWS ## Create Stream from CloudWatch Logs @@ -41,6 +41,7 @@ The following AWS Lambda environment variables are supported * `SOURCE_CATEGORY_OVERRIDE` (OPTIONAL) - Override _sourceCategory metadata field within SumoLogic. If `none` will not be overridden * `SOURCE_HOST_OVERRIDE` (OPTIONAL) - Override _sourceHost metadata field within SumoLogic. If `none` will not be overridden * `SOURCE_NAME_OVERRIDE` (OPTIONAL) - Override _sourceName metadata field within SumoLogic. If `none` will not be overridden +* `LOG_STREAM_PREFIX` (OPTIONAL) - Comma separated list of logStream name prefixes to filter by logStream, expecially for AWS Batch logs # Dynamic Metadata Fields @@ -52,7 +53,7 @@ For example: ``` exports.handler = (event, context, callback) => { - + var serverIp = '123.123.123.123' console.log(JSON.stringify({ @@ -62,7 +63,7 @@ exports.handler = (event, context, callback) => { 'source': 'other_source', 'host': serverIp } - + })); console.log('some other log message with default sourceCategory'); }; diff --git a/cloudwatchlogs/cloudwatchlogs_lambda.js b/cloudwatchlogs/cloudwatchlogs_lambda.js index 0f90720..a9cef81 100644 --- a/cloudwatchlogs/cloudwatchlogs_lambda.js +++ b/cloudwatchlogs/cloudwatchlogs_lambda.js @@ -21,6 +21,11 @@ var encoding = process.env.ENCODING || 'utf-8'; // default is utf-8 // Include logStream and logGroup as json fields within the message. Required for SumoLogic AWS Lambda App var includeLogInfo = false; // default is false +// Regex to filter by logStream name prefixes +var logStreamPrefixRegex = process.env.LOG_STREAM_PREFIX + ? new RegExp('^(' + process.env.LOG_STREAM_PREFIX.replace(/,/g, '|') + ')', 'i') + : ''; + // Regex used to detect logs coming from lambda functions. // The regex will parse out the requestID and strip the timestamp // Example: 2016-11-10T23:11:54.523Z 108af3bb-a79b-11e6-8bd7-91c363cc05d9 some message @@ -158,6 +163,9 @@ exports.handler = function (event, context, callback) { if (awslogsData.messageType === 'CONTROL_MESSAGE') { console.log('Control message'); callback(null, 'Success'); + } else if(logStreamPrefixRegex && !awslogsData.logStream.match(logStreamPrefixRegex)){ + console.log('Skipping Non-Applicable Log Stream'); + return callback(null, 'Success'); } var lastRequestID = null; From cc67fdbe7401a9cd33cb351a28cee6207bba8b25 Mon Sep 17 00:00:00 2001 From: Yohei Kitamura Date: Wed, 11 Sep 2019 09:59:02 -0400 Subject: [PATCH 2/4] Fix typo --- cloudwatchlogs-with-dlq/Readme.md | 2 +- cloudwatchlogs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudwatchlogs-with-dlq/Readme.md b/cloudwatchlogs-with-dlq/Readme.md index 7c69560..a092ba5 100644 --- a/cloudwatchlogs-with-dlq/Readme.md +++ b/cloudwatchlogs-with-dlq/Readme.md @@ -21,7 +21,7 @@ The following AWS Lambda environment variables are supported in both the lambda * SOURCE_NAME_OVERRIDE (OPTIONAL) - Override _sourceName metadata field within SumoLogic. * INCLUDE_LOG_INFO (OPTIONAL) - Set it to true when loggroup/logstream values needs to be included in logs. Default is false * LOG_FORMAT - Default is Others. One can choose VPC-JSON for VPC flow logs in json format and VPC-RAW for only RAW message line -* LOG_STREAM_PREFIX (OPTIONAL) - Comma separated list of logStream name prefixes to filter by logStream, expecially for AWS Batch logs +* LOG_STREAM_PREFIX (OPTIONAL) - Comma separated list of logStream name prefixes to filter by logStream, especially for AWS Batch logs ### Configuring Lambda for VPC Flow Logs The following AWS Lambda environment variables are supported in both the lambda functions for VPC flow logs. diff --git a/cloudwatchlogs/README.md b/cloudwatchlogs/README.md index 1fdd64a..945e1d3 100644 --- a/cloudwatchlogs/README.md +++ b/cloudwatchlogs/README.md @@ -41,7 +41,7 @@ The following AWS Lambda environment variables are supported * `SOURCE_CATEGORY_OVERRIDE` (OPTIONAL) - Override _sourceCategory metadata field within SumoLogic. If `none` will not be overridden * `SOURCE_HOST_OVERRIDE` (OPTIONAL) - Override _sourceHost metadata field within SumoLogic. If `none` will not be overridden * `SOURCE_NAME_OVERRIDE` (OPTIONAL) - Override _sourceName metadata field within SumoLogic. If `none` will not be overridden -* `LOG_STREAM_PREFIX` (OPTIONAL) - Comma separated list of logStream name prefixes to filter by logStream, expecially for AWS Batch logs +* `LOG_STREAM_PREFIX` (OPTIONAL) - Comma separated list of logStream name prefixes to filter by logStream, especially for AWS Batch logs # Dynamic Metadata Fields From bd4b656ad0e73869075f3d0339e58b526e112ab0 Mon Sep 17 00:00:00 2001 From: Yohei Kitamura Date: Wed, 11 Sep 2019 10:20:17 -0400 Subject: [PATCH 3/4] Escape RegExp special characters --- cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js | 6 +++++- cloudwatchlogs/cloudwatchlogs_lambda.js | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js b/cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js index 89c86a0..b7a4891 100644 --- a/cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js +++ b/cloudwatchlogs-with-dlq/cloudwatchlogs_lambda.js @@ -85,7 +85,7 @@ function getConfig(env) { "includeSecurityGroupInfo": ("INCLUDE_SECURITY_GROUP_INFO" in env) ? env.INCLUDE_SECURITY_GROUP_INFO === "true" : false, // Regex to filter by logStream name prefixes "logStreamPrefixRegex": ("LOG_STREAM_PREFIX" in env) - ? new RegExp('^(' + env.LOG_STREAM_PREFIX.replace(/,/g, '|') + ')', 'i') + ? new RegExp('^(' + escapeRegExp(env.LOG_STREAM_PREFIX).replace(/,/g, '|') + ')', 'i') : '' }; if (!config.SumoURL) { @@ -99,6 +99,10 @@ function getConfig(env) { return config; } +function escapeRegExp(string) { + return string.replace(/[|\\{}()[\]^$+*?.-]/g, '\\$&'); +} + function transformRecords(config, records) { return new Promise(function (resolve, reject) { if (config.LogFormat === "VPC-JSON" && config.includeSecurityGroupInfo) { diff --git a/cloudwatchlogs/cloudwatchlogs_lambda.js b/cloudwatchlogs/cloudwatchlogs_lambda.js index a9cef81..308a047 100644 --- a/cloudwatchlogs/cloudwatchlogs_lambda.js +++ b/cloudwatchlogs/cloudwatchlogs_lambda.js @@ -23,7 +23,7 @@ var includeLogInfo = false; // default is false // Regex to filter by logStream name prefixes var logStreamPrefixRegex = process.env.LOG_STREAM_PREFIX - ? new RegExp('^(' + process.env.LOG_STREAM_PREFIX.replace(/,/g, '|') + ')', 'i') + ? new RegExp('^(' + escapeRegExp(process.env.LOG_STREAM_PREFIX).replace(/,/g, '|') + ')', 'i') : ''; // Regex used to detect logs coming from lambda functions. @@ -38,6 +38,9 @@ var https = require('https'); var zlib = require('zlib'); var url = require('url'); +function escapeRegExp(string) { + return string.replace(/[|\\{}()[\]^$+*?.-]/g, '\\$&'); +} function sumoMetaKey(awslogsData, message) { var sourceCategory = ''; From e64128c001b48e9c5db772a727ffa3190ecb51a7 Mon Sep 17 00:00:00 2001 From: Yohei Kitamura Date: Fri, 13 Sep 2019 15:43:10 -0400 Subject: [PATCH 4/4] Add LOG_STREAM_PREFIX in Cloudformation template --- cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json index 502ded4..4702d97 100644 --- a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json +++ b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json @@ -28,6 +28,10 @@ "Default": "false", "AllowedValues" : ["true" ,"false"], "Description": "Select true to get loggroup/logstream values in logs" + }, + "LogStreamPrefix": { + "Type": "String", + "Description": "Enter comma separated list of logStream name prefixes to filter by logStream" } }, "Mappings" : { @@ -221,7 +225,8 @@ "Variables": { "SUMO_ENDPOINT": {"Ref": "SumoEndPointURL"}, "LOG_FORMAT": {"Ref": "LogFormat"}, - "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"} + "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, + "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} } } @@ -296,7 +301,8 @@ }, "NUM_OF_WORKERS": {"Ref": "NumOfWorkers"}, "LOG_FORMAT": {"Ref": "LogFormat"}, - "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"} + "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, + "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} } } }