Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 210 lines (162 sloc) 6.834 kB
a402d48 New features:
Mike authored
1 #! /usr/bin/python
2
3 from r2.lib import amqp, emailer
4 from pylons import g
5 from datetime import datetime
6 from md5 import md5
7 from random import shuffle, choice
8
9 import pickle
10
11 try:
12 words = file(g.words_file).read().split("\n")
13 except IOError:
14 words = []
15
16 shuffle(words)
17
18 def randword():
19 try:
20 return choice(words)
21 except IndexError:
22 return '???'
23
24 q = 'log_q'
25
253c473 @jedberg * Add services and scripts to public repos for ease of reddit instal…
jedberg authored
26 def run(streamfile=None, verbose=False):
a402d48 New features:
Mike authored
27 if streamfile:
28 stream_fp = open(streamfile, "a")
29 else:
30 stream_fp = None
31
32 def streamlog(msg, important=False):
33 if stream_fp:
34 stream_fp.write(msg + "\n")
35 stream_fp.flush()
36 if important:
37 print msg
38
39 def add_timestamps (d):
40 d['hms'] = d['time'].strftime("%H:%M:%S")
41
42 d['occ'] = "<%s:%s, pid=%-5s, %s>" % (d['host'], d['port'], d['pid'],
43 d['time'].strftime("%Y-%m-%d %H:%M:%S"))
44
45 def limited_append(l, item):
46 if len(l) >= 25:
47 l.pop(12)
48 l.append(item)
49
50 def log_exception(d, daystring):
51 exc_desc = d['exception_desc']
52 exc_type = d['exception_type']
53
54 exc_str = "%s: %s" % (exc_type, exc_desc)
55
56 add_timestamps(d)
57
58 tb = []
59
e683889 @jedberg New Features
jedberg authored
60 key_material = exc_type
a402d48 New features:
Mike authored
61 pretty_lines = []
62
e683889 @jedberg New Features
jedberg authored
63 make_lock_seen = False
253c473 @jedberg * Add services and scripts to public repos for ease of reddit instal…
jedberg authored
64 flaky_db_seen = False
08c431b @KeyserSosa * Comply with the spec on 304 errors so Chrome won't barf download.g…
KeyserSosa authored
65 cassandra_seen = False
e683889 @jedberg New Features
jedberg authored
66
a402d48 New features:
Mike authored
67 for tpl in d['traceback']:
68 tb.append(tpl)
69 filename, lineno, funcname, text = tpl
e683889 @jedberg New Features
jedberg authored
70 if text is None:
71 pass
72 elif (text.startswith("with g.make_lock(") or
73 text.startswith("with make_lock(")):
74 make_lock_seen = True
253c473 @jedberg * Add services and scripts to public repos for ease of reddit instal…
jedberg authored
75 elif (text.startswith("(ProgrammingError) server closed the connection")):
76 flaky_db_seen = True
68a06c5 @spladug April 2011 Merge
spladug authored
77 if '/cassandra/' in filename.lower():
78 cassandra_seen = True
79 if '/pycassa/' in filename.lower():
08c431b @KeyserSosa * Comply with the spec on 304 errors so Chrome won't barf download.g…
KeyserSosa authored
80 cassandra_seen = True
a402d48 New features:
Mike authored
81 key_material += "%s %s " % (filename, funcname)
82 pretty_lines.append ("%s:%s: %s()" % (filename, lineno, funcname))
83 pretty_lines.append (" %s" % text)
84
e683889 @jedberg New Features
jedberg authored
85 if exc_desc.startswith("QueuePool limit of size"):
86 fingerprint = "QueuePool_overflow"
87 elif exc_desc.startswith("error 2 from memcached_get: HOSTNAME "):
88 fingerprint = "memcache_suckitude"
89 elif exc_type == "TimeoutExpired" and make_lock_seen:
90 fingerprint = "make_lock_timeout"
253c473 @jedberg * Add services and scripts to public repos for ease of reddit instal…
jedberg authored
91 elif exc_desc.startswith("(OperationalError) FATAL: the database " +
92 "system is in recovery mode"):
93 fingerprint = "recovering_db"
94 elif exc_desc.startswith("(OperationalError) could not connect " +
95 "to server"):
96 fingerprint = "unconnectable_db"
97 elif exc_desc.startswith("(OperationalError) server closed the " +
98 "connection unexpectedly"):
99 fingerprint = "flaky_db_op"
68a06c5 @spladug April 2011 Merge
spladug authored
100 elif cassandra_seen:
101 fingerprint = "something's wrong with cassandra"
e683889 @jedberg New Features
jedberg authored
102 else:
103 fingerprint = md5(key_material).hexdigest()
a402d48 New features:
Mike authored
104
105 nickname_key = "error_nickname-" + fingerprint
106 status_key = "error_status-" + fingerprint
107
108 nickname = g.hardcache.get(nickname_key)
109
110 if nickname is None:
111 nickname = '"%s" Exception' % randword().capitalize()
112 news = ("A new kind of thing just happened! " +
113 "I'm going to call it a %s\n\n" % nickname)
114
115 news += "Where and when: %s\n\n" % d['occ']
116 news += "Traceback:\n"
117 news += "\n".join(pretty_lines)
118 news += exc_str
119 news += "\n"
120
121 emailer.nerds_email(news, "Exception Watcher")
122
123 g.hardcache.set(nickname_key, nickname, 86400 * 365)
124 g.hardcache.set(status_key, "new", 86400)
125
126 if g.hardcache.get(status_key) == "fixed":
127 g.hardcache.set(status_key, "new", 86400)
128 news = "This was marked as fixed: %s\n" % nickname
129 news += "But it just occurred, so I'm marking it new again."
130 emailer.nerds_email(news, "Exception Watcher")
131
132 err_key = "-".join(["error", daystring, fingerprint])
133
134 existing = g.hardcache.get(err_key)
135
136 if not existing:
137 existing = dict(exception=exc_str, traceback=tb, occurrences=[])
138
68a06c5 @spladug April 2011 Merge
spladug authored
139 existing.setdefault('times_seen', 0)
140 existing['times_seen'] += 1
141
a402d48 New features:
Mike authored
142 limited_append(existing['occurrences'], d['occ'])
143
144 g.hardcache.set(err_key, existing, 7 * 86400)
145
146 streamlog ("%s [X] %-70s" % (d['hms'], nickname), verbose)
147
148 def log_text(d, daystring):
149 add_timestamps(d)
150 char = d['level'][0].upper()
151 streamlog ("%s [%s] %r" % (d['hms'], char, d['text']), verbose)
152 logclass_key = "logclass-" + d['classification']
153
154 if not g.hardcache.get(logclass_key):
155 g.hardcache.set(logclass_key, True, 86400 * 90)
156
157 if d['level'] != 'debug':
158 news = "The code just generated a [%s] message.\n" % \
159 d['classification']
160 news += "I don't remember ever seeing one of those before.\n"
161 news += "\n"
162 news += "It happened on: %s\n" % d['occ']
163 news += "The log level was: %s\n" % d['level']
164 news += "The complete text was:\n"
165 news += repr(d['text'])
166 emailer.nerds_email (news, "reddit secretary")
167
168 occ_key = "-".join(["logtext", daystring,
169 d['level'], d['classification']])
170
171 occurrences = g.hardcache.get(occ_key)
172
173 if occurrences is None:
174 occurrences = []
175
176 d2 = {}
177
178 d2['occ'] = d['occ']
179 d2['text'] = repr(d['text'])
180
181 limited_append(occurrences, d2)
182 g.hardcache.set(occ_key, occurrences, 86400 * 7)
183
253c473 @jedberg * Add services and scripts to public repos for ease of reddit instal…
jedberg authored
184 def myfunc(msg):
a402d48 New features:
Mike authored
185 daystring = datetime.now(g.display_tz).strftime("%Y/%m/%d")
186
253c473 @jedberg * Add services and scripts to public repos for ease of reddit instal…
jedberg authored
187 try:
188 d = pickle.loads(msg.body)
189 except TypeError:
190 streamlog ("wtf is %r" % msg.body, True)
191 return
192
193 if not 'type' in d:
194 streamlog ("wtf is %r" % d, True)
195 elif d['type'] == 'exception':
196 try:
197 log_exception(d, daystring)
198 except Exception as e:
199 print "Error in log_exception(): %r" % e
200 elif d['type'] == 'text':
a402d48 New features:
Mike authored
201 try:
253c473 @jedberg * Add services and scripts to public repos for ease of reddit instal…
jedberg authored
202 log_text(d, daystring)
203 except Exception as e:
204 print "Error in log_text(): %r" % e
205 else:
206 streamlog ("wtf is %r" % d['type'], True)
207
208 amqp.consume_items(q, myfunc, verbose=verbose)
a402d48 New features:
Mike authored
209
Something went wrong with that request. Please try again.