Any way to honor the JSON Object in val.Message and not stringify it?

[polds]

Confusing title, but our Lambda functions are logging valid JSON objects and not strings like so:

{
    "fields": {
        "MediaPath": {
            "Path": "1466672936684.jpg",
        },
        "RequestID": "a7188dc9-6968-11a6-be87-3326b0d7d49d",
        "app": "purge"
    },
    "level": "info",
    "timestamp": "2016-08-23T19:36:39.382610474Z",
    "message": "success"
}

When it's passed over to cloudwatchlogs_lambda.js Sumo then sees this:

{
   "id":"32826273204296444651588478853452971566044640956554084352",
   "timestamp":1471980999382,
   "message":"{\"fields\":{\"MediaPath\":{\"Path\":\"1466672936684.jpg\"},\"RequestID\":\"a7188dc9-6968-11a6-be87-3326b0d7d49d\",\"app\":\"purge\"},\"level\":\"info\",\"timestamp\":\"2016-08-23T19:36:39.382610474Z\",\"message\":\"success\"}
",
  "requestID":null,
  "logStream":"2016/08/23/[$LATEST]8e1c1bde6beb44df800fb15f6444263a",
  "logGroup":"/aws/lambda/purge"
}

Which doesn't make it ideal to set up search queries in Sumo. Is there anyway to persist the JSON object through and have the "message" be JSON

[thoean]

I ran into a similar problem and decided to parse the message before sending it to SumoLogic. I've the following code, based on what's provided in this repository. This allows to:

• Parse a message into JSON if the whole message is JSON (for example, if you have an application that already logs messages to CloudWatchLogs in JSON format).
• Parse a message into JSON if only parts of the message are JSON (that's the case with Lambda, which includes the request Id and timestamp before the actual log message).
• It also posts the requests in parallel to SumoLogic by using the async library (you'd need to include that when uploading the Lambda function). An alternative would be to replace that with Promises.

var https = require('https');
var zlib = require('zlib');
var async = require('async');

// mapping of log group to additional values:
// * url: The URL of the SumoLogic HTTP collector (without the hostname, which is always the same.
// * parseJson: Whether to parse the message as json or not
// * partialJson: Whether these logs contain messages that only contain JSON partially
var mappings = {
    "/aws/somethingelsethanlambda/fooservice": {
        "url": "very-long-url-from-sumo-logic-http-collector-1",
        "parseJson": true,
        "partialJson": false
    },
    "/aws/lambda/mylambdafunc1": {
        "url": "very-long-url-from-sumo-logic-http-collector-2",
        "parseJson": true,
        "partialJson": true
    }
}

var hostname = "collectors.us2.sumologic.com";
var basepath = "/receiver/v1/http/";

exports.handler = function(event, context) {

    // get the zipped input from a base64 string
    var zippedInput = new Buffer(event.awslogs.data, 'base64');

    // unzip
    zlib.gunzip(zippedInput, function(error, buffer) {

        // handle problems with unzipping
        if (error) {
            context.fail(error);
        }       

        // get the AWS log data
        var awslogsData = JSON.parse(buffer.toString('ascii'));

        // ignore control messages
        if (awslogsData.messageType === "CONTROL_MESSAGE") {
            console.log("Control message");
            context.succeed("Success");
        }

        // send the log events
        sendLogEvents(awslogsData, context);
    });
};

function sendLogEvents(awslogsData, context) {

    // define counter and callback
    var requestsSucceeded = 0;
    var requestsFailed = 0;
    var callback = function(err) {
        if (err) {
            requestsFailed++;
            console.log(err);
        } else {
            requestsSucceeded++;
        }
        var tot = requestsSucceeded + requestsFailed;
        if (tot == awslogsData.logEvents.length) {
            if (requestsFailed > 0) {
                context.fail(requestsFailed + " / " + (requestsFailed + requestsSucceeded) + " events failed.");
            } else {
                console.log(requestsSucceeded + " events sent.")
                context.succeed();
            }
        }
    };

    var httpRequestOptions = {
        hostname: hostname,
        method: "POST",
        path: basepath + mappings[awslogsData.logGroup].url
    };

    var lambdaRequestId = new RegExp(/(?:RequestId:|Z)\s+([\w\d\-]+)/);
    var lastRequestId = "";
    var sendLogEventFunctions = awslogsData.logEvents.map(function(logEvent) {

        // special handling for parsing JSON functions
        if (mappings[awslogsData.logGroup].parseJson) {

            // JSON is either only partially in the log message; let's ensure to extract it
            // to avoid it being serialized a 2nd time when the HTTP request is made of the whole object being serialized
            // so we parse it here to an object
            if (mappings[awslogsData.logGroup].partialJson) {
                var requestIdParsed = lambdaRequestId.exec(logEvent.message);
                logEvent.requestID = requestIdParsed !== null ? requestIdParsed[1] : lastRequestId;
                var startJson = logEvent.message.indexOf('{');
                var endJson = logEvent.message.lastIndexOf('}');
                if (startJson > -1 && endJson > startJson) {
                    try {
                        logEvent.data = JSON.parse(logEvent.message.substring(startJson, endJson + 1));
                    } catch (e) {
                        // invalid json
                    }
                }
            }

            // JSON is the full message, so simply parse it before it's getting serialized again
            else {
                logEvent.message = JSON.parse(logEvent.message);
            }
        }

        // put the stream and log group into the log event
        logEvent.logStream = awslogsData.logStream;
        logEvent.logGroup = awslogsData.logGroup;

        // the return value, which is a function that does the underlying HTTP request
        return function() { 
            // send the log data
            sendLogEvent(logEvent, httpRequestOptions, callback);
        }
    });

    async.parallel(sendLogEventFunctions);
}

function sendLogEvent(logEvent, httpRequestOptions, callback) {

    // serialize the log event
    var serializedLogEvent = JSON.stringify(logEvent);

    // loop through all log events and send them individually to SumoLogic
    var req = https.request(httpRequestOptions, function(res) {

        var body = '';
        res.setEncoding('utf8');
        res.on('data', function(chunk) { body += chunk; });
        res.on('end', function() {
            if (res.statusCode < 200 || res.statusCode > 299) {
                callback("Processed message. HTTP response status was " + res.statusCode + 
                    ", related message '" + body + 
                    "', HTTP request options '" + JSON.stringify(httpRequestOptions) +
                    "', content sent " + serializedLogEvent);
            } else {
                callback();
            }
        });
    });

    req.on('error', function(e) {
        callback(e.message)
    });

    // send the serialized log event
    req.end(serializedLogEvent);
};

[stevezau]

This is fixed in PR #18 and improvements in PR #19