/
journal2gelf
executable file
·171 lines (154 loc) · 5.69 KB
/
journal2gelf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
#!/usr/bin/python -u
#
# journal2gelf
# ============
#
# Get structured log records from the systemd journal and send them to a
# Graylog2 server as GELF messages.
#
# Tested on Python 2.7, may work on other versions.
#
# journald / systemd compatibility
# --------------------------------
# For systemd < 190, use '-m' switch to enable multi-line JSON parsing.
#
# Dependencies:
# -------------
# - graypy (pip-install graypy)
#
# Usage:
# ------
#
# By default, journal2gelf will look for input on stdin. eg:
#
# - Send all logs and exit:
#
# `journalctl -o json | journal2gelf`
#
# The `-t` flag can be specified and journal2gelf will automatically
# start journalctl in tail mode. This makes it easier to run as a systemd service.
#
# `journal2gelf -t`
#
# This is equivalent to running:
#
# `journalctl -o json -f | journal2gelf`
#
#
# Copyright 2012 Joe Miller <https://github.com/joemiller>
#
# Released under the MIT license, see LICENSE for details.
#
import sys
import json
import zlib
import graypy
from collections import deque
# TODO:
# - [nice to have] rabbitmq handler
class JournalToGelf:
facility_names = {
0: "kern",
1: "user",
2: "mail",
3: "daemon",
4: "auth",
5: "syslog",
6: "lpr",
7: "news",
8: "uucp",
9: "cron",
10: "authpriv",
16: "local0",
17: "local1",
18: "local2",
19: "local3",
20: "local4",
21: "local5",
22: "local6",
23: "local7"
}
def __init__(self, fp, host='localhost', port=12201, multiline_format=False):
self.buf = deque()
self.fp = fp
self.multiline_format = multiline_format
self.gelf = graypy.GELFHandler(host, port)
def _send_gelf(self):
try:
msg = {'version': '1.0'}
record = json.loads(''.join(self.buf))
for key, value in record.iteritems():
# journalctl's JSON exporter will convert unprintable (incl. newlines)
# strings into an array of integers. We convert these integers into
# their ascii representation and concatenate them together
# to reconstitute the string.
if type(value) is list:
value = ''.join([chr(x) for x in value])
if key == '__REALTIME_TIMESTAMP':
# convert from systemd's format of microseconds expressed as
# an integer to graylog's float format, eg: "seconds.microseconds"
msg['timestamp'] = float(value) / (1000 * 1000)
elif key == 'PRIORITY':
msg['level'] = int(value)
elif key == 'SYSLOG_FACILITY':
msg['facility'] = self.facility_names.get(int(value), 'unknown')
elif key == '_HOSTNAME':
msg['host'] = value
elif key == 'MESSAGE':
msg['short_message'] = value
elif key.startswith('.'):
continue
elif key == '__CURSOR':
continue
else:
# prefix additional fields with '_' for graylog.
msg['_' + key] = value
self.gelf.send(zlib.compress(json.dumps(msg)))
except ValueError:
print "Warning: invalid JSON. Skipping this record."
finally:
self.buf.clear()
def run(self):
for line in self.fp:
line = line.strip()
# systemd > 190 switched to a single-line JSON format for each
# journal event that is much similar to parse.
if not self.multiline_format:
self.buf.append(line)
self._send_gelf()
else:
# systemd < 190 used a multi-line JSON format which requires more work to parse.
# the first line of `journalctl -o json -f` will not be valid json,
# it will be "Logs being at ....". Skip it.
if line.startswith('Logs begin at'):
continue
# The second line will be the opening bracket for a list, but we
# don't need that either. Skip it.
if line.startswith('['):
continue
if line == "},":
self.buf.append('}')
self._send_gelf()
else:
self.buf.append(line)
if __name__ == '__main__':
import optparse
import subprocess
opts_parser = optparse.OptionParser()
opts_parser.add_option('-s', '--server', dest='host', default='localhost',
help='Graylog2 server host or IP (default: %default)')
opts_parser.add_option('-p', '--port', dest='port', default=12201, type='int',
help='Graylog2 server port (default: %default)')
opts_parser.add_option('-t', '--tail', dest='tail', default=False, action='store_true',
help='Start journalctl and tail it forever (default: %default)')
opts_parser.add_option('-m', '--multiline-format', dest='multiline_format', default=False, action='store_true',
help='Expect multi-line JSON format (systemd version < 190) (default: %default))')
(opts, args) = opts_parser.parse_args()
read_from_fd = None
if opts.tail:
read_from_fd = subprocess.Popen(['journalctl', '-o', 'json', '-f'],
stdout=subprocess.PIPE).stdout
else:
read_from_fd = sys.stdin
parser = JournalToGelf(fp=read_from_fd, host=opts.host, port=opts.port, multiline_format=opts.multiline_format)
parser.run()