From 0e72c85ef9dced47cdcb19dca721b0b86e0fcfca Mon Sep 17 00:00:00 2001 From: Duc Ha Date: Mon, 6 Nov 2017 15:54:57 -0800 Subject: [PATCH 1/2] SUMO-0: first checkin of cloudwatch event function --- cloudwatchevents/README.md | 50 +++++++++++++ cloudwatchevents/cloudwatchevents.js | 106 +++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 cloudwatchevents/README.md create mode 100644 cloudwatchevents/cloudwatchevents.js diff --git a/cloudwatchevents/README.md b/cloudwatchevents/README.md new file mode 100644 index 0000000..4348672 --- /dev/null +++ b/cloudwatchevents/README.md @@ -0,0 +1,50 @@ +# Sumo Logic Functions for AWS CloudWatch Events + +AWS Lambda function to collector logs from CloudWatch Events and post them to [SumoLogic](http://www.sumologic.com) via a [HTTP collector endpoint](http://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source) + + +# Usage + +First create an [HTTP collector endpoint](http://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source) within SumoLogic. You will need the endpoint URL for the lambda function later. + +## Create Lambda Function + +1. Within the AWS Lambda console select create new Lambda function +2. Select `Blank Function` on the select blueprint page +3. Leave triggers empty for now, click next +4. Configure Lambda + * Select Node.js 4.3 as runtime + * Copy code from cloudwatchevents.js into the Lambda function code. + * Add Environment variables (See below) +5. Scroll down to the `Lambda function handle and role` section, make sure you set the right values that match the function. For role, you can just use the basic execution role. Click next. +6. Finally click on "Create function" to create the function. +7. (Optional) Test this new function with sample AWS CloudWatch Events template provided by AWS + + +# Lambda Environment Variables + +The following AWS Lambda environment variables are supported + +* `SUMO_ENDPOINT` (REQUIRED) - SumoLogic HTTP Collector [endpoint URL](http://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source). +* `SOURCE_CATEGORY_OVERRIDE` (OPTIONAL) - Override _sourceCategory metadata field within SumoLogic. If `none` will not be overridden +* `SOURCE_HOST_OVERRIDE` (OPTIONAL) - Override _sourceHost metadata field within SumoLogic. If `none` will not be overridden +* `SOURCE_NAME_OVERRIDE` (OPTIONAL) - Override _sourceName metadata field within SumoLogic. If `none` will not be overridden + +# Including Outer Event Fields +By default, a CloudWatch Event has a format similar to this: + +``` +{ +"version":"0", +"id":"0123456d-7e46-ecb4-f5a2-e59cec50b100", +"detail-type":"AWS API Call via CloudTrail", +"source":"aws.logs", +"account":"012345678908", +"time":"2017-11-06T23:36:59Z", +"region":"us-east-1", +"resources":[ ], +"detail":▶{ … } +} +``` + +This event will be sent as-is to Sumo Logic. If you just want to send the ```defail``` key instead, set the ```removeOuterFields``` to true. diff --git a/cloudwatchevents/cloudwatchevents.js b/cloudwatchevents/cloudwatchevents.js new file mode 100644 index 0000000..529bf40 --- /dev/null +++ b/cloudwatchevents/cloudwatchevents.js @@ -0,0 +1,106 @@ +///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// CloudWatch Events to SumoLogic // +// https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/cloudwatchevents // +// // +// YOU MUST CREATE A SUMO LOGIC ENDPOINT CALLED SUMO_ENDPOINT AND PASTE IN ENVIRONMENTAL VARIABLES BELOW // +// https://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source // +///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// SumoLogic Endpoint to post logs +var SumoURL = process.env.SUMO_ENDPOINT; + +// For some beta AWS services, the default is to remove the outer fields of the received object since they are not useful. +// change this if necessary. +var removeOuterFields = false; + +// The following parameters override the sourceCategoryOverride, sourceHostOverride and sourceNameOverride metadata fields within SumoLogic. +// Not these can also be overridden via json within the message payload. See the README for more information. +var sourceCategoryOverride = process.env.SOURCE_CATEGORY_OVERRIDE || ''; // If empty sourceCategoryOverride will not be overridden +var sourceHostOverride = process.env.SOURCE_HOST_OVERRIDE || ''; // If empty sourceHostOverride will not be set to the name of the logGroup +var sourceNameOverride = process.env.SOURCE_NAME_OVERRIDE || ''; // If empty sourceNameOverride will not be set to the name of the logStream + +var https = require('https'); +var zlib = require('zlib'); +var url = require('url'); + + +function postToSumo(context, messages) { + var messagesTotal = Object.keys(messages).length; + var messagesSent = 0; + var messageErrors = []; + + var urlObject = url.parse(SumoURL); + var options = { + 'hostname': urlObject.hostname, + 'path': urlObject.pathname, + 'method': 'POST' + }; + + var finalizeContext = function () { + var total = messagesSent + messageErrors.length; + if (total == messagesTotal) { + console.log('messagesSent: ' + messagesSent + ' messagesErrors: ' + messageErrors.length); + if (messageErrors.length > 0) { + context.fail('errors: ' + messageErrors); + } else { + context.succeed(); + } + } + }; + + + Object.keys(messages).forEach(function (key, index) { + var headerArray = key.split(':'); + options.headers = { + 'X-Sumo-Name': headerArray[0], + 'X-Sumo-Category': headerArray[1], + 'X-Sumo-Host': headerArray[2] + }; + + var req = https.request(options, function (res) { + res.setEncoding('utf8'); + res.on('data', function (chunk) {}); + res.on('end', function () { + console.log("Got response code: "+ res.statusCode); + if (res.statusCode == 200) { + messagesSent++; + } else { + messageErrors.push('HTTP Return code ' + res.statusCode); + } + finalizeContext(); + }); + }); + + req.on('error', function (e) { + messageErrors.push(e.message); + finalizeContext(); + }); + + for (var i = 0; i < messages[key].length; i++) { + req.write(JSON.stringify(messages[key][i]) + '\n'); + } + req.end(); + }); +} + + +exports.handler = function (event, context) { + + // Used to hold chunks of messages to post to SumoLogic + var messageList = {}; + + // Validate URL has been set + var urlObject = url.parse(SumoURL); + if (urlObject.protocol != 'https:' || urlObject.host === null || urlObject.path === null) { + context.fail('Invalid SUMO_ENDPOINT environment variable: ' + SumoURL); + } + + console.log(event); + if ((event.source==="aws.guardduty") && (removeOuterFields)) { + final_event =event.detail; + } else { + final_event = event; + } + messageList[sourceNameOverride+':'+sourceCategoryOverride+':'+sourceHostOverride]=[final_event]; + postToSumo(context, messageList); +}; From 08bf03f5538220d39be8a47fee9c7f44c99c1e49 Mon Sep 17 00:00:00 2001 From: Duc Ha Date: Mon, 6 Nov 2017 16:02:33 -0800 Subject: [PATCH 2/2] SUMO-0: Update README file --- cloudwatchevents/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudwatchevents/README.md b/cloudwatchevents/README.md index 4348672..2eae114 100644 --- a/cloudwatchevents/README.md +++ b/cloudwatchevents/README.md @@ -1,6 +1,6 @@ -# Sumo Logic Functions for AWS CloudWatch Events +# Sumo Logic Function for AWS CloudWatch Events -AWS Lambda function to collector logs from CloudWatch Events and post them to [SumoLogic](http://www.sumologic.com) via a [HTTP collector endpoint](http://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source) +AWS Lambda function to collect CloudWatch events and post them to [SumoLogic](http://www.sumologic.com) via a [HTTP collector endpoint](http://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source) # Usage @@ -30,7 +30,7 @@ The following AWS Lambda environment variables are supported * `SOURCE_HOST_OVERRIDE` (OPTIONAL) - Override _sourceHost metadata field within SumoLogic. If `none` will not be overridden * `SOURCE_NAME_OVERRIDE` (OPTIONAL) - Override _sourceName metadata field within SumoLogic. If `none` will not be overridden -# Including Outer Event Fields +# Excluding Outer Event Fields By default, a CloudWatch Event has a format similar to this: ``` @@ -47,4 +47,4 @@ By default, a CloudWatch Event has a format similar to this: } ``` -This event will be sent as-is to Sumo Logic. If you just want to send the ```defail``` key instead, set the ```removeOuterFields``` to true. +This event will be sent as-is to Sumo Logic. If you just want to send the ```detail``` key instead, set the ```removeOuterFields``` variable to true.