forked from reddit-archive/reddit
-
Notifications
You must be signed in to change notification settings - Fork 0
/
log_q.py
209 lines (162 loc) · 6.67 KB
/
log_q.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
#! /usr/bin/python
from r2.lib import amqp, emailer
from pylons import g
from datetime import datetime
from md5 import md5
from random import shuffle, choice
import pickle
try:
words = file(g.words_file).read().split("\n")
except IOError:
words = []
shuffle(words)
def randword():
try:
return choice(words)
except IndexError:
return '???'
q = 'log_q'
def run(streamfile=None, verbose=False):
if streamfile:
stream_fp = open(streamfile, "a")
else:
stream_fp = None
def streamlog(msg, important=False):
if stream_fp:
stream_fp.write(msg + "\n")
stream_fp.flush()
if important:
print msg
def add_timestamps (d):
d['hms'] = d['time'].strftime("%H:%M:%S")
d['occ'] = "<%s:%s, pid=%-5s, %s>" % (d['host'], d['port'], d['pid'],
d['time'].strftime("%Y-%m-%d %H:%M:%S"))
def limited_append(l, item):
if len(l) >= 25:
l.pop(12)
l.append(item)
def log_exception(d, daystring):
exc_desc = d['exception_desc']
exc_type = d['exception_type']
exc_str = "%s: %s" % (exc_type, exc_desc)
add_timestamps(d)
tb = []
key_material = exc_type
pretty_lines = []
make_lock_seen = False
flaky_db_seen = False
cassandra_seen = False
for tpl in d['traceback']:
tb.append(tpl)
filename, lineno, funcname, text = tpl
if text is None:
pass
elif (text.startswith("with g.make_lock(") or
text.startswith("with make_lock(")):
make_lock_seen = True
elif (text.startswith("(ProgrammingError) server closed the connection")):
flaky_db_seen = True
if '/cassandra/' in filename.lower():
cassandra_seen = True
if '/pycassa/' in filename.lower():
cassandra_seen = True
key_material += "%s %s " % (filename, funcname)
pretty_lines.append ("%s:%s: %s()" % (filename, lineno, funcname))
pretty_lines.append (" %s" % text)
if exc_desc.startswith("QueuePool limit of size"):
fingerprint = "QueuePool_overflow"
elif exc_desc.startswith("error 2 from memcached_get: HOSTNAME "):
fingerprint = "memcache_suckitude"
elif exc_type == "TimeoutExpired" and make_lock_seen:
fingerprint = "make_lock_timeout"
elif exc_desc.startswith("(OperationalError) FATAL: the database " +
"system is in recovery mode"):
fingerprint = "recovering_db"
elif exc_desc.startswith("(OperationalError) could not connect " +
"to server"):
fingerprint = "unconnectable_db"
elif exc_desc.startswith("(OperationalError) server closed the " +
"connection unexpectedly"):
fingerprint = "flaky_db_op"
elif cassandra_seen:
fingerprint = "something's wrong with cassandra"
else:
fingerprint = md5(key_material).hexdigest()
nickname_key = "error_nickname-" + fingerprint
status_key = "error_status-" + fingerprint
nickname = g.hardcache.get(nickname_key)
if nickname is None:
nickname = '"%s" Exception' % randword().capitalize()
news = ("A new kind of thing just happened! " +
"I'm going to call it a %s\n\n" % nickname)
news += "Where and when: %s\n\n" % d['occ']
news += "Traceback:\n"
news += "\n".join(pretty_lines)
news += exc_str
news += "\n"
emailer.nerds_email(news, "Exception Watcher")
g.hardcache.set(nickname_key, nickname, 86400 * 365)
g.hardcache.set(status_key, "new", 86400)
if g.hardcache.get(status_key) == "fixed":
g.hardcache.set(status_key, "new", 86400)
news = "This was marked as fixed: %s\n" % nickname
news += "But it just occurred, so I'm marking it new again."
emailer.nerds_email(news, "Exception Watcher")
err_key = "-".join(["error", daystring, fingerprint])
existing = g.hardcache.get(err_key)
if not existing:
existing = dict(exception=exc_str, traceback=tb, occurrences=[])
existing.setdefault('times_seen', 0)
existing['times_seen'] += 1
limited_append(existing['occurrences'], d['occ'])
g.hardcache.set(err_key, existing, 7 * 86400)
streamlog ("%s [X] %-70s" % (d['hms'], nickname), verbose)
def log_text(d, daystring):
add_timestamps(d)
char = d['level'][0].upper()
streamlog ("%s [%s] %r" % (d['hms'], char, d['text']), verbose)
logclass_key = "logclass-" + d['classification']
if not g.hardcache.get(logclass_key):
g.hardcache.set(logclass_key, True, 86400 * 90)
if d['level'] != 'debug':
news = "The code just generated a [%s] message.\n" % \
d['classification']
news += "I don't remember ever seeing one of those before.\n"
news += "\n"
news += "It happened on: %s\n" % d['occ']
news += "The log level was: %s\n" % d['level']
news += "The complete text was:\n"
news += repr(d['text'])
emailer.nerds_email (news, "reddit secretary")
occ_key = "-".join(["logtext", daystring,
d['level'], d['classification']])
occurrences = g.hardcache.get(occ_key)
if occurrences is None:
occurrences = []
d2 = {}
d2['occ'] = d['occ']
d2['text'] = repr(d['text'])
limited_append(occurrences, d2)
g.hardcache.set(occ_key, occurrences, 86400 * 7)
def myfunc(msg):
daystring = datetime.now(g.display_tz).strftime("%Y/%m/%d")
try:
d = pickle.loads(msg.body)
except TypeError:
streamlog ("wtf is %r" % msg.body, True)
return
if not 'type' in d:
streamlog ("wtf is %r" % d, True)
elif d['type'] == 'exception':
try:
log_exception(d, daystring)
except Exception as e:
print "Error in log_exception(): %r" % e
elif d['type'] == 'text':
try:
log_text(d, daystring)
except Exception as e:
print "Error in log_text(): %r" % e
else:
streamlog ("wtf is %r" % d['type'], True)
amqp.consume_items(q, myfunc, verbose=verbose)