Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

We’re showing branches in this repository, but you can also compare across forks.

base fork: etsy/logster
base: 134dc3dd58
...
head fork: etsy/logster
compare: c368072b02
  • 2 commits
  • 5 files changed
  • 0 commit comments
  • 2 contributors
Commits on May 12, 2011
Mike Babineau mbabineau Added sample parser for log4j logs
Added pass-through of parser-specific options
Moved parsers into a subdir
Updated docs accordingly
6922936
Commits on Jun 23, 2011
Mike Brittain mikebrittain Merge pull request #3 from mbabineau/master
Log4jLogster and parser-specific option support
c368072
2  Makefile
View
@@ -3,4 +3,4 @@ install:
/bin/mkdir -p /var/log/logster
/usr/bin/install -m 0755 -t /usr/sbin logster
/usr/bin/install -m 0644 -t /usr/share/logster logster_helper.py
- /usr/bin/install -m 0644 -t /usr/share/logster SampleLogster.py
+ /usr/bin/install -m 0644 -t /usr/share/logster parsers/*
19 README
View
@@ -27,7 +27,7 @@ Once you have logtail installed, then the only other thing you need to do is run
Usage
-You can test logster from the command line. There are two sample parsers (SampleGangliaLogster and SampleGraphiteLogster) that can be used to generate stats from an Apache access log. The --dry-run option will allow you to see the metrics being generated on stdout rather than sending them to either Ganglia or Graphite.
+You can test logster from the command line. There are two sample parsers: SampleLogster, which generates stats from an Apache access log; and Log4jLogster, which generates stats from a log4j log. The --dry-run option will allow you to see the metrics being generated on stdout rather than sending them to either Ganglia or Graphite.
$ sudo /usr/sbin/logster --dry-run --output=ganglia SampleLogster /var/log/httpd/access_log
@@ -41,17 +41,25 @@ usage: logster [options] parser logfile
Tail a log file and filter each line to generate metrics that can be sent to
common monitoring packages.
-usage: logster [options] parser logfile
+Usage: logster [options] parser logfile
-options:
+Tail a log file and filter each line to generate metrics that can be sent to
+common monitoring packages.
+
+Options:
-h, --help show this help message and exit
-p METRIC_PREFIX, --metric-prefix=METRIC_PREFIX
Add prefix to all published metrics. This is for
people that may multiple instances of same service on
same host.
+ --parser-help Print usage and options for the selected parser
+ --parser-options=PARSER_OPTIONS
+ Options to pass to the logster parser such as "-o
+ VALUE --option2 VALUE". These are parser-specific and
+ passed directly to the parser.
--gmetric-options=GMETRIC_OPTIONS
- Options to pass to gmetric such as -d 180 -c
- /etc/ganglia/gmond.conf (default). These are passed
+ Options to pass to gmetric such as "-d 180 -c
+ /etc/ganglia/gmond.conf" (default). These are passed
directly to gmetric.
--graphite-host=GRAPHITE_HOST
Hostname and port for Graphite collector, e.g.
@@ -64,4 +72,3 @@ options:
Choices are 'graphite', 'ganglia', or 'stdout'.
-d, --dry-run Parse the log file but send stats to standard output.
-D, --debug Provide more verbose logging for debugging.
-
11 logster
View
@@ -71,8 +71,12 @@ cmdline = optparse.OptionParser(usage="usage: %prog [options] parser logfile",
cmdline.add_option('--metric-prefix', '-p', action='store',
help='Add prefix to all published metrics. This is for people that may multiple instances of same service on same host.',
default='')
+cmdline.add_option('--parser-help', action='store_true',
+ help='Print usage and options for the selected parser')
+cmdline.add_option('--parser-options', action='store',
+ help='Options to pass to the logster parser such as "-o VALUE --option2 VALUE". These are parser-specific and passed directly to the parser.')
cmdline.add_option('--gmetric-options', action='store',
- help='Options to pass to gmetric such as -d 180 -c /etc/ganglia/gmond.conf (default). These are passed directly to gmetric.',
+ help='Options to pass to gmetric such as "-d 180 -c /etc/ganglia/gmond.conf" (default). These are passed directly to gmetric.',
default='-d 180 -c /etc/ganglia/gmond.conf')
cmdline.add_option('--graphite-host', action='store',
help='Hostname and port for Graphite collector, e.g. graphite.example.com:2003')
@@ -87,6 +91,9 @@ cmdline.add_option('--debug', '-D', action='store_true', default=False,
help='Provide more verbose logging for debugging.')
options, arguments = cmdline.parse_args()
+if options.parser_help:
+ options.parser_options = '-h'
+
if (len(arguments) != 2):
cmdline.print_help()
cmdline.error("Supply at least two arguments: parser and logfile.")
@@ -234,7 +241,7 @@ def main():
try:
#sys.path.append("/usr/local/share/logster")
module = __import__(class_name)
- parser = getattr(module, class_name)()
+ parser = getattr(module, class_name)(option_string=options.parser_options)
except Exception, e:
print "Failed to instantiate parser (line %s): %s" % (lineno(), e)
sys.exit(1)
92 parsers/Log4jLogster.py
View
@@ -0,0 +1,92 @@
+### Author: Mike Babineau <michael.babineau@gmail.com>, EA2D <http://ea2d.com>
+###
+### A sample logster parser file that can be used to count the number
+### of events for each log level in a log4j log.
+###
+### Example (note WARN,ERROR,FATAL is default):
+### sudo ./logster --output=stdout Log4jLogster /var/log/example_app/app.log --parser-options '-l WARN,ERROR,FATAL'
+###
+###
+### Logster copyright 2011, Etsy, Inc.
+###
+### This file is part of Logster.
+###
+### Logster is free software: you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation, either version 3 of the License, or
+### (at your option) any later version.
+###
+### Logster is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with Logster. If not, see <http://www.gnu.org/licenses/>.
+###
+
+import time
+import re
+import optparse
+
+from logster_helper import MetricObject, LogsterParser
+from logster_helper import LogsterParsingException
+
+class Log4jLogster(LogsterParser):
+
+ def __init__(self, option_string=None):
+ '''Initialize any data structures or variables needed for keeping track
+ of the tasty bits we find in the log we are parsing.'''
+
+ if option_string:
+ options = option_string.split(' ')
+ else:
+ options = []
+
+ optparser = optparse.OptionParser()
+ optparser.add_option('--log-levels', '-l', dest='levels', default='WARN,ERROR,FATAL',
+ help='Comma-separated list of log levels to track: (default: "WARN,ERROR,FATAL")')
+
+ opts, args = optparser.parse_args(args=options)
+
+ self.levels = opts.levels.split(',')
+
+ for level in self.levels:
+ # Track counts from 0 for each log level
+ setattr(self, level, 0)
+
+ # Regular expression for matching lines we are interested in, and capturing
+ # fields from the line (in this case, a log level such as WARN, ERROR, or FATAL).
+ self.reg = re.compile('[0-9-_:\.]+ (?P<log_level>%s)' % ('|'.join(self.levels)) )
+
+
+ def parse_line(self, line):
+ '''This function should digest the contents of one line at a time, updating
+ object's state variables. Takes a single argument, the line to be parsed.'''
+
+ try:
+ # Apply regular expression to each line and extract interesting bits.
+ regMatch = self.reg.match(line)
+
+ if regMatch:
+ linebits = regMatch.groupdict()
+ log_level = linebits['log_level']
+
+ if log_level in self.levels:
+ current_val = getattr(self, log_level)
+ setattr(self, log_level, current_val+1)
+
+ else:
+ raise LogsterParsingException, "regmatch failed to match"
+
+ except Exception, e:
+ raise LogsterParsingException, "regmatch or contents failed with %s" % e
+
+
+ def get_state(self, duration):
+ '''Run any necessary calculations on the data collected from the logs
+ and return a list of metric objects.'''
+ self.duration = duration
+
+ metrics = [MetricObject(level, (getattr(self, level) / self.duration)) for level in self.levels]
+ return metrics
0  SampleLogster.py → parsers/SampleLogster.py
View
File renamed without changes

No commit comments for this range

Something went wrong with that request. Please try again.