first commit

.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+*.class
+*.pyc
+build/
+dist/
+klog.egg-info/

README.md

@@ -0,0 +1,65 @@
+# klogd
+
+Klogd is a simple program to stream [Syslog](http://www.syslog.org) messages to a [Kafka](http://incubator.apache.org/kafka) server.
+
+## Getting Started
+
+### 1) Make sure you have Kafka up and running properly
+
+* [Kafka - Quick Start](http://incubator.apache.org/kafka/quickstart.html)
+
+### 2) Install klogd
+
+    $ git clone git@github.com:leandrosilva/klogd.git
+    $ cd klogd
+    $ python setup.py install
+
+[Setuptools](http://pypi.python.org/pypi/setuptools) is going to install klogd and its dependencies as well:
+
+* Twisted
+* PyParsing
+* PyKafka
+
+### 3) Configure Syslog messages routing
+
+On Mac OS X, for example, you have to edit /etc/syslog.conf to include:
+
+    *.info;authpriv,remoteauth,ftp,install,internal.none  @127.0.0.1:1514
+
+### 4) Re-launch Syslog daemon to reflex the new configuration
+
+On Mac OS X, for example, you have to:
+
+    $ launchctl unload /System/Library/LaunchDaemons/com.apple.syslogd.plist
+    $ launchctl load /System/Library/LaunchDaemons/com.apple.syslogd.plist
+
+### 5) Start klogd
+
+    $ KAFKA_HOST=127.0.0.1 KAFKA_PORT=9092 klogd 
+
+If you don't provide Kafka environment variables, klogd is going to use defaults:
+
+* **Host** - 127.0.0.1
+* **Port** - 9092
+
+### 6) Test it now!
+
+At one terminal:
+
+    $ python tests/kafka_consumer.py
+
+And at another:
+
+    $ logger -p local0.info -t test.app "blah blah blah info info info"
+
+## Copyright
+
+Copyright (c) 2012 Leandro Silva <leandrodoze@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+and associated documentation files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND.

klogd

@@ -0,0 +1,15 @@
+#! python
+# -*- coding: utf-8 -*-
+"""
+    klogd
+    ~~~~~
+
+    It's a simple program to stream Syslog messages to a Kafka server.
+
+    :copyright: (c) 2012 by Leandro Silva.
+    :license: see README.
+"""
+
+import klog.server
+
+klog.server.run()

lib/klog/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""
+    klogd
+    ~~~~~
+
+    It's a simple program to stream Syslog messages to a Kafka server.
+
+    :copyright: (c) 2012 by Leandro Silva.
+    :license: see README.
+"""

lib/klog/server.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+"""
+    klogd
+    ~~~~~
+
+    It's a simple program to stream Syslog messages to a Kafka server.
+
+    :copyright: (c) 2012 by Leandro Silva.
+    :license: see README.
+"""
+
+from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex
+from twisted.internet import reactor
+from twisted.internet.protocol import DatagramProtocol, Factory
+from time import strftime
+import sys, os, json, kafka
+
+severity = ["emerg", "alert", "crit", "err", "warn", "notice", "info", "debug"]
+
+facility = ["kern", "user", "mail", "daemon", "auth", "syslog", "lpr", "news",
+            "uucp", "cron", "authpriv", "ftp", "ntp", "audit", "alert", "at", "local0",
+            "local1", "local2", "local3", "local4", "local5", "local6", "local7"]
+
+class Parser(object):
+  def __init__(self):
+    self.__pattern = self.__build_pattern()
+
+  def __build_pattern(self):
+    ints = Word(nums)
+
+    # priority
+    priority = Suppress("<") + ints + Suppress(">")
+
+    # timestamp
+    month = Word(string.uppercase, string.lowercase, exact=3)
+    day   = ints
+    hour  = Combine(ints + ":" + ints + ":" + ints)
+
+    timestamp = month + day + hour
+
+    # hostname
+    hostname = Word(alphas + nums + "_" + "-" + ".")
+
+    # appname
+    appname = Word(alphas + "/" + "-" + "_" + ".") + Optional(Suppress("[") + ints + Suppress("]")) + Suppress(":")
+
+    # message
+    message = Regex(".*")
+
+    # pattern build
+    return priority + timestamp + hostname + appname + message
+
+  def parse(self, line, (host, port)):
+    parsed_line = self.__pattern.parseString(line)
+
+    _priority = parsed_line[0]
+    (_facility, _severity) = self.__get_level(_priority)
+
+    payload = {}
+    payload["priority"]  = _priority
+    payload["facility"]  = _facility
+    payload["severity"]  = _severity
+    payload["timestamp"] = strftime("%Y-%m-%d %H:%M:%S")
+    payload["hostname"]  = parsed_line[4]
+    payload["hostaddr"]  = host
+    payload["hostport"]  = port
+    payload["appname"]   = parsed_line[5]
+    payload["pid"]       = parsed_line[6]
+    payload["message"]   = parsed_line[7]
+
+    return json.dumps(payload)
+
+  def __get_level(self, priority):
+    _priority = int(priority)
+    _facility = _priority / 8
+    _severity = _priority % 8
+
+    return (facility[_facility], severity[_severity])
+
+class Receiver(DatagramProtocol):
+  def __init__(self):
+    self.__parser = Parser()
+    self.__kafka = Kafka()
+
+  def datagramReceived(self, data, (host, port)):
+    payload = self.__parser.parse(data, (host, port))
+    self.__kafka.send(payload)
+
+class Kafka(object):
+  def __init__(self):
+    try:
+      self.__host = os.getenv("KAFKA_HOST")
+      self.__port = int(os.getenv("KAFKA_PORT"))
+    except Exception:
+      self.__host = "127.0.0.1"
+      self.__port = 9092
+
+  def send(self, payload):
+    producer = kafka.producer.Producer('klog', host=self.__host, port=self.__port)
+    message  = kafka.message.Message(payload)
+    producer.send(message)
+
+# Let's kick off it
+
+def run():
+  reactor.listenUDP(1514, Receiver())
+  reactor.run()

setup.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+    klogd
+    ~~~~~
+
+    It"s a simple program to stream Syslog messages to a Kafka server.
+
+    :copyright: (c) 2012 by Leandro Silva.
+    :license: see README.
+"""
+
+from setuptools import setup
+
+setup(
+        name             = "klog",
+        version          = "0.1",
+
+        install_requires = ["twisted", "pyparsing", "pykafka"],
+        packages         = ["klog"],
+        package_dir      = {"": "lib"},
+
+        scripts          = ["klogd"],
+
+        author           = "Leandro Silva",
+        author_email     = "leandrodoze@gmail.com",
+        keywords         = "klog syslog kafka",
+        description      = "Klogd is a simple program to stream Syslog messages to a Kafka server",
+        url              = "https://github.com/leandrosilva/klogd",
+    )

tests/kafka_consumer.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+"""
+    klogd
+    ~~~~~
+
+    It's a simple program to stream Syslog messages to a Kafka server.
+
+    :copyright: (c) 2012 by Leandro Silva.
+    :license: see README.
+"""
+
+#
+# This script consumes klog topic from Kafka, for test purpose.
+#
+
+import kafka
+
+consumer = kafka.consumer.Consumer("klog")
+
+for message in consumer.loop():
+  print "received:", message
+