Permalink
Browse files

Added sample parser for log4j logs

Added pass-through of parser-specific options
Moved parsers into a subdir
Updated docs accordingly
  • Loading branch information...
1 parent 134dc3d commit 6922936fc8ee9f58fc0f493127b632078f414cf6 @mbabineau mbabineau committed May 12, 2011
Showing with 115 additions and 9 deletions.
  1. +1 −1 Makefile
  2. +13 −6 README
  3. +9 −2 logster
  4. +92 −0 parsers/Log4jLogster.py
  5. 0 { → parsers}/SampleLogster.py
View
@@ -3,4 +3,4 @@ install:
/bin/mkdir -p /var/log/logster
/usr/bin/install -m 0755 -t /usr/sbin logster
/usr/bin/install -m 0644 -t /usr/share/logster logster_helper.py
- /usr/bin/install -m 0644 -t /usr/share/logster SampleLogster.py
+ /usr/bin/install -m 0644 -t /usr/share/logster parsers/*
View
19 README
@@ -27,7 +27,7 @@ Once you have logtail installed, then the only other thing you need to do is run
Usage
-You can test logster from the command line. There are two sample parsers (SampleGangliaLogster and SampleGraphiteLogster) that can be used to generate stats from an Apache access log. The --dry-run option will allow you to see the metrics being generated on stdout rather than sending them to either Ganglia or Graphite.
+You can test logster from the command line. There are two sample parsers: SampleLogster, which generates stats from an Apache access log; and Log4jLogster, which generates stats from a log4j log. The --dry-run option will allow you to see the metrics being generated on stdout rather than sending them to either Ganglia or Graphite.
$ sudo /usr/sbin/logster --dry-run --output=ganglia SampleLogster /var/log/httpd/access_log
@@ -41,17 +41,25 @@ usage: logster [options] parser logfile
Tail a log file and filter each line to generate metrics that can be sent to
common monitoring packages.
-usage: logster [options] parser logfile
+Usage: logster [options] parser logfile
-options:
+Tail a log file and filter each line to generate metrics that can be sent to
+common monitoring packages.
+
+Options:
-h, --help show this help message and exit
-p METRIC_PREFIX, --metric-prefix=METRIC_PREFIX
Add prefix to all published metrics. This is for
people that may multiple instances of same service on
same host.
+ --parser-help Print usage and options for the selected parser
+ --parser-options=PARSER_OPTIONS
+ Options to pass to the logster parser such as "-o
+ VALUE --option2 VALUE". These are parser-specific and
+ passed directly to the parser.
--gmetric-options=GMETRIC_OPTIONS
- Options to pass to gmetric such as -d 180 -c
- /etc/ganglia/gmond.conf (default). These are passed
+ Options to pass to gmetric such as "-d 180 -c
+ /etc/ganglia/gmond.conf" (default). These are passed
directly to gmetric.
--graphite-host=GRAPHITE_HOST
Hostname and port for Graphite collector, e.g.
@@ -64,4 +72,3 @@ options:
Choices are 'graphite', 'ganglia', or 'stdout'.
-d, --dry-run Parse the log file but send stats to standard output.
-D, --debug Provide more verbose logging for debugging.
-
View
11 logster
@@ -71,8 +71,12 @@ cmdline = optparse.OptionParser(usage="usage: %prog [options] parser logfile",
cmdline.add_option('--metric-prefix', '-p', action='store',
help='Add prefix to all published metrics. This is for people that may multiple instances of same service on same host.',
default='')
+cmdline.add_option('--parser-help', action='store_true',
+ help='Print usage and options for the selected parser')
+cmdline.add_option('--parser-options', action='store',
+ help='Options to pass to the logster parser such as "-o VALUE --option2 VALUE". These are parser-specific and passed directly to the parser.')
cmdline.add_option('--gmetric-options', action='store',
- help='Options to pass to gmetric such as -d 180 -c /etc/ganglia/gmond.conf (default). These are passed directly to gmetric.',
+ help='Options to pass to gmetric such as "-d 180 -c /etc/ganglia/gmond.conf" (default). These are passed directly to gmetric.',
default='-d 180 -c /etc/ganglia/gmond.conf')
cmdline.add_option('--graphite-host', action='store',
help='Hostname and port for Graphite collector, e.g. graphite.example.com:2003')
@@ -87,6 +91,9 @@ cmdline.add_option('--debug', '-D', action='store_true', default=False,
help='Provide more verbose logging for debugging.')
options, arguments = cmdline.parse_args()
+if options.parser_help:
+ options.parser_options = '-h'
+
if (len(arguments) != 2):
cmdline.print_help()
cmdline.error("Supply at least two arguments: parser and logfile.")
@@ -234,7 +241,7 @@ def main():
try:
#sys.path.append("/usr/local/share/logster")
module = __import__(class_name)
- parser = getattr(module, class_name)()
+ parser = getattr(module, class_name)(option_string=options.parser_options)
except Exception, e:
print "Failed to instantiate parser (line %s): %s" % (lineno(), e)
sys.exit(1)
View
@@ -0,0 +1,92 @@
+### Author: Mike Babineau <michael.babineau@gmail.com>, EA2D <http://ea2d.com>
+###
+### A sample logster parser file that can be used to count the number
+### of events for each log level in a log4j log.
+###
+### Example (note WARN,ERROR,FATAL is default):
+### sudo ./logster --output=stdout Log4jLogster /var/log/example_app/app.log --parser-options '-l WARN,ERROR,FATAL'
+###
+###
+### Logster copyright 2011, Etsy, Inc.
+###
+### This file is part of Logster.
+###
+### Logster is free software: you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation, either version 3 of the License, or
+### (at your option) any later version.
+###
+### Logster is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with Logster. If not, see <http://www.gnu.org/licenses/>.
+###
+
+import time
+import re
+import optparse
+
+from logster_helper import MetricObject, LogsterParser
+from logster_helper import LogsterParsingException
+
+class Log4jLogster(LogsterParser):
+
+ def __init__(self, option_string=None):
+ '''Initialize any data structures or variables needed for keeping track
+ of the tasty bits we find in the log we are parsing.'''
+
+ if option_string:
+ options = option_string.split(' ')
+ else:
+ options = []
+
+ optparser = optparse.OptionParser()
+ optparser.add_option('--log-levels', '-l', dest='levels', default='WARN,ERROR,FATAL',
+ help='Comma-separated list of log levels to track: (default: "WARN,ERROR,FATAL")')
+
+ opts, args = optparser.parse_args(args=options)
+
+ self.levels = opts.levels.split(',')
+
+ for level in self.levels:
+ # Track counts from 0 for each log level
+ setattr(self, level, 0)
+
+ # Regular expression for matching lines we are interested in, and capturing
+ # fields from the line (in this case, a log level such as WARN, ERROR, or FATAL).
+ self.reg = re.compile('[0-9-_:\.]+ (?P<log_level>%s)' % ('|'.join(self.levels)) )
+
+
+ def parse_line(self, line):
+ '''This function should digest the contents of one line at a time, updating
+ object's state variables. Takes a single argument, the line to be parsed.'''
+
+ try:
+ # Apply regular expression to each line and extract interesting bits.
+ regMatch = self.reg.match(line)
+
+ if regMatch:
+ linebits = regMatch.groupdict()
+ log_level = linebits['log_level']
+
+ if log_level in self.levels:
+ current_val = getattr(self, log_level)
+ setattr(self, log_level, current_val+1)
+
+ else:
+ raise LogsterParsingException, "regmatch failed to match"
+
+ except Exception, e:
+ raise LogsterParsingException, "regmatch or contents failed with %s" % e
+
+
+ def get_state(self, duration):
+ '''Run any necessary calculations on the data collected from the logs
+ and return a list of metric objects.'''
+ self.duration = duration
+
+ metrics = [MetricObject(level, (getattr(self, level) / self.duration)) for level in self.levels]
+ return metrics
File renamed without changes.

0 comments on commit 6922936

Please sign in to comment.