This repository has been archived by the owner on Jan 31, 2018. It is now read-only.
/
healthchecks.py
214 lines (166 loc) · 5.41 KB
/
healthchecks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
from collections import namedtuple
from datetime import datetime, timedelta
import logging
from django.conf import settings
from django.core.mail import send_mail
from django.db import connection
from django.template.loader import render_to_string
from fjord.heartbeat.models import Answer
from fjord.mailinglist.utils import get_recipients
log = logging.getLogger('i.heartbeat')
SEVERITY_LOW = 1
SEVERITY_MEDIUM = 5
SEVERITY_HIGH = 10
SEVERITY = {
SEVERITY_LOW: 'low',
SEVERITY_MEDIUM: 'medium',
SEVERITY_HIGH: 'high'
}
Result = namedtuple('Result', ['name', 'severity', 'summary', 'output'])
CHECKS = []
def register_check(cls):
CHECKS.append(cls)
return cls
class Check(object):
name = ''
@classmethod
def check(cls):
pass
@register_check
class CheckAnyAnswers(Check):
"""Are there any heartbeat answers? If not, that's very bad."""
name = 'Are there any heartbeat answers?'
@classmethod
def check(cls):
day_ago = datetime.now() - timedelta(days=1)
count = Answer.objects.filter(received_ts__gt=day_ago).count()
if count == 0:
return Result(
cls.name,
SEVERITY_HIGH,
'0 answers in last 24 hours.',
str(count)
)
return Result(
cls.name,
SEVERITY_LOW,
'%s answers in last 24 hours.' % str(count),
str(count)
)
def tableify(table):
"""Takes a list of lists and converts it into a formatted table
:arg table: list (rows) of lists (columns)
:returns: string
.. Note::
This is text formatting--not html formatting.
"""
num_cols = 0
maxes = []
for row in table:
num_cols = max(num_cols, len(row))
if len(maxes) < len(row):
maxes.extend([0] * (len(row) - len(maxes)))
for i, cell in enumerate(row):
maxes[i] = max(maxes[i], len(str(cell)))
def fix_row(maxes, row):
return ' '.join([
str(cell) + (' ' * (maxes[i] - len(str(cell))))
for i, cell in enumerate(row)
])
return '\n'.join(
[
fix_row(maxes, row)
for row in table
]
)
@register_check
class CheckMissingVotes(Check):
"""FIXME: I don't understand this check"""
name = 'Are there votes of 0 for large cells?'
@classmethod
def check(cls):
# Note: This SQL statement comes from Gregg. It's probably
# mysql-specific.
sql = """
SELECT
sum(score is not NULL) as nvoted,
DATE_FORMAT(received_ts, '%Y-%m-%d') as ydm,
version,
channel,
100*sum(flow_began_ts > 0) / count(received_ts) as pct_began,
100*sum(flow_offered_ts >0) / count(received_ts) as pct_offered,
100*sum(flow_voted_ts > 0)/ count(received_ts) as pct_voted,
100*sum(flow_engaged_ts > 0) / count(received_ts) as pct_engaged,
count(received_ts) as N
FROM heartbeat_answer
WHERE
received_ts > DATE_SUB(now(), interval 1 day)
AND is_test=0
AND survey_id="heartbeat-by-user-first-impression"
AND (locale='en-us')
GROUP BY version, channel, ydm
HAVING
N >= 50
and nvoted = 0
ORDER BY
channel,
version,
ydm;
"""
cursor = connection.cursor()
cursor.execute(sql)
data = cursor.fetchall()
if not data:
# If we get nothing back, then we have serious issues.
return Result(
cls.name,
SEVERITY_HIGH,
'No data from query',
repr(data)
)
data = list(data)
# FIXME: What consistutes SEVERITY_HIGH here?
data.insert(
0,
['nvoted', 'ydm', 'version', 'channel', 'pct_began', 'pct_offered',
'pct_voted', 'pct_engaged', 'N']
)
return Result(
cls.name,
SEVERITY_LOW,
'Data looks ok.',
tableify(data)
)
def get_all_healthchecks():
return CHECKS
def run_healthchecks():
return [checker.check() for checker in get_all_healthchecks()]
def email_healthchecks(results):
has_high = any([result.severity == SEVERITY_HIGH for result in results])
# The subject should indicate very very obviously whether the sky is
# falling or not.
subject = '[hb health] %s (%s)' % (
('RED ALERT' if has_high else 'fine'),
datetime.now().strftime('%Y-%m-%d %H:%M')
)
# We do the entire email body in HTML because some output will want to
# preserve whitespace and use a fixed-width font. Further, this lets
# us make it super easy to spot SEVERITY_HIGH situations.
html_body = render_to_string('heartbeat/email/heartbeat_health.html', {
'severity_name': SEVERITY,
'results': results
})
recipients = get_recipients('heartbeat_health')
if recipients:
send_mail(
subject=subject,
message='This email is in HTML.',
from_email=settings.SERVER_EMAIL,
recipient_list=recipients,
html_message=html_body
)
else:
# FIXME: log this? is that a good idea?
log.info('No recipients for "heartbeat_health"' + '\n' +
subject + '\n' +
html_body)