added exclude_regex, filters out lines matching regex from being tran…

…smitted

README.md

@@ -61,6 +61,7 @@ key = <YOUR LOGDNA INGESTION KEY>
 #### Options
 * `logdir`: sets the paths that the agent will monitor for new files, separate multiple paths using `,`, supports glob patterns + specific files. By default this option is set to monitor .log and extensionless files under /var/log/.
 * `exclude`: excludes files that otherwise would've matched `logdir`, separate multiple excludes using `,`, supports glob patterns + specific files
+* `exclude_regex`: filters out any lines matching pattern in any file. Don't include leading and trailing /.
 * `key`: your LogDNA Ingestion Key. You can obtain one by creating an account on LogDNA.com and once logged in to the webapp, click the Gear icon, then Account Profile.
 * `tags`: host tagging to create dynamic groups on the webapp
 * `hostname`: override os hostname

index.js

@@ -29,6 +29,7 @@ program
     .option('-d, --logdir <dir>', 'adds log dir to config, supports glob patterns', fileUtils.appender(), [])
     .option('-f, --logfile <file>', 'adds log file to config', fileUtils.appender(), [])
     .option('-e, --exclude <file>', 'exclude files from logdir', fileUtils.appender(), [])
+    .option('-r, --exclude-regex <pattern>', 'filter out lines matching pattern')
     .option('-n, --hostname <hostname>', 'uses alternate hostname (default: ' + os.hostname().replace('.ec2.internal', '') + ')')
     .option('-t, --tags <tags>', 'set tags for this host (for auto grouping), separate multiple tags by comma')
     .on('--help', function() {
@@ -138,6 +139,15 @@ checkElevated()
             saveMessages.push('Added exclusion ' + program.exclude.join(', ') + ' to config.');
         }
 
+        if (program.excludeRegex) {
+            parsedConfig.exclude_regex = program.excludeRegex;
+            // strip leading and trailing / if exists
+            if (parsedConfig.exclude_regex.substring(0, 1) === '/' && parsedConfig.exclude_regex.substring(parsedConfig.exclude_regex.length - 1) === '/') {
+                parsedConfig.exclude_regex = parsedConfig.exclude_regex.substring(1, parsedConfig.exclude_regex.length - 1);
+            }
+            saveMessages.push('Added exclude pattern /' + parsedConfig.exclude_regex + '/ to config.');
+        }
+
         if (program.hostname) {
             parsedConfig.hostname = program.hostname;
             saveMessages.push('Hostname ' + parsedConfig.hostname + ' saved to config.');

lib/linebuffer.js

@@ -13,6 +13,7 @@ var socket;
 var flushtimeout;
 var dccount = 0;
 var httperr;
+var exclude_regex;
 
 module.exports.addMessage = function(message) {
     if (buf.length > config.BUF_LIMIT) {
@@ -27,6 +28,12 @@ module.exports.addMessage = function(message) {
         message.l = message.l.substring(0, 32000) + ' (cut off, too long...)';
     }
 
+    if (exclude_regex && exclude_regex.test(message.l)) {
+        debug('excluded regex line ' + message.l);
+        config.bufferStats.excludelines++;
+        return;
+    }
+
     buf.push(message);
     if (config.bufferStats) { config.bufferStats.lines++; }
 
@@ -142,12 +149,13 @@ module.exports.setSocket = function(sock) {
     // kick off initial flush
     if (!flushtimeout) {
         exports.resetStats();
+        exclude_regex = config.exclude_regex && new RegExp(config.exclude_regex);
         flushtimeout = setTimeout(exports.flush, config.FLUSH_INTERVAL);
     }
 };
 
 module.exports.resetStats = function() {
-    config.bufferStats = { lines: 0, buflimit: 0, longlines: 0, flushlimit: 0, flushsize: 0, flushcount: 0, ts: Date.now() };
+    config.bufferStats = { lines: 0, buflimit: 0, longlines: 0, flushlimit: 0, flushsize: 0, flushcount: 0, ts: Date.now(), excludelines: 0 };
 };
 
 module.exports.setConfig = function(cfg) {