forked from mozilla/home-snippets-server
-
Notifications
You must be signed in to change notification settings - Fork 0
/
models.py
367 lines (304 loc) · 14 KB
/
models.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
"""
homesnippets models
"""
import hashlib
from datetime import datetime
from time import mktime, gmtime
from django.conf import settings
from django.core import urlresolvers
from django.core.cache import cache
from django.db import models
from django.db.models.signals import post_save, post_delete
from django.utils.translation import ugettext_lazy as _
CACHE_TIMEOUT = getattr(settings, 'SNIPPET_MODEL_CACHE_TIMEOUT')
CACHE_RULE_MATCH_PREFIX = 'homesnippets_ClientMatchRule_Matches_'
CACHE_RULE_LASTMOD_PREFIX = 'homesnippets_ClientMatchRule_LastMod_'
CACHE_RULE_ALL_PREFIX = 'homesnippets_ClientMatchRule_All'
CACHE_RULE_ALL_LASTMOD_PREFIX = 'homesnippets_ClientMatchRule_All_LastMod'
CACHE_RULE_NEW_LASTMOD_PREFIX = 'homesnippets_ClientMatchRule_New_LastMod'
CACHE_SNIPPET_LASTMOD_PREFIX = 'homesnippets_Snippet_LastMod_'
CACHE_SNIPPET_LOOKUP_PREFIX = 'homesnippets_Snippet_Lookup_'
def _key_from_client(args):
plain = '|'.join(['%s=%s'%(k,v) for k,v in args.items()])
return hashlib.md5(plain.encode('UTF-8')).hexdigest()
class ClientMatchRuleManager(models.Manager):
"""Manager for client match rules, allows filtering against match logic"""
def find_match_ids_for_request(self, args):
"""
Finds all match rules that affect the given request. Returns two lists
containing the rules that will exclude or include snippets in the
response.
"""
cache_key = '%s%s' % (CACHE_RULE_MATCH_PREFIX, _key_from_client(args))
cache_hit = cache.get(cache_key)
if cache_hit:
# If since caching this hit, any of the rules involved were
# modified or if any new rules were created, invalidate the results.
lastmod_keys = [
'%s%s' % (CACHE_RULE_LASTMOD_PREFIX, item)
for sublist in cache_hit[1] for item in sublist
]
lastmod_keys.append(CACHE_RULE_NEW_LASTMOD_PREFIX)
lastmods = cache.get_many(lastmod_keys).values()
newer_lastmods = [ m for m in lastmods if m > cache_hit[0] ]
if newer_lastmods:
cache_hit = None
if not cache_hit:
# Cache miss, so recalculate the results and cache them.
rules = self._cached_all()
include_ids, exclude_ids = [], []
# Check every rule
for rule in rules:
if rule.is_match(args):
if rule.exclude:
exclude_ids.append(str(rule.id))
else:
include_ids.append(str(rule.id))
elif not rule.exclude:
# Include rule that doesn't match? Add as an exclude rule
# so that snippets can split required matches into multiple
# rules and combine them together.
exclude_ids.append(str(rule.id))
cache_hit = (mktime(gmtime()), (include_ids, exclude_ids))
cache.set(cache_key, cache_hit, CACHE_TIMEOUT)
return cache_hit[1]
def _cached_all(self):
"""Cached version of self.all(), invalidated by change to any rule."""
c_data = cache.get_many([CACHE_RULE_ALL_PREFIX,
CACHE_RULE_ALL_LASTMOD_PREFIX])
lastmod = c_data.get(CACHE_RULE_ALL_LASTMOD_PREFIX, None)
cache_hit = c_data.get(CACHE_RULE_ALL_PREFIX, None)
# Entire cached set gets invalidated if any rule changed.
if cache_hit and lastmod > cache_hit[0]:
cache_hit = None
if not cache_hit:
cache_hit = ( mktime(gmtime()), self.all() )
cache.set(CACHE_RULE_ALL_PREFIX, cache_hit, CACHE_TIMEOUT)
return cache_hit[1]
class ClientMatchRule(models.Model):
class Meta():
ordering = ( '-modified', )
objects = ClientMatchRuleManager()
description = models.CharField( _('description of rule'),
null=False, blank=False, default="None", max_length=200)
exclude = models.BooleanField( _('exclusion rule?'),
default=False)
# browser/components/nsBrowserContentHandler.js:911:
# const SNIPPETS_URL = "http://snippets.mozilla.com/" + STARTPAGE_VERSION +
# "/%NAME%/%VERSION%/%APPBUILDID%/%BUILD_TARGET%/%LOCALE%/%CHANNEL%
# /%OS_VERSION%/%DISTRIBUTION%/%DISTRIBUTION_VERSION%/";
startpage_version = models.CharField( _('start page version'),
null=True, blank=True, max_length=64)
name = models.CharField( _('product name'),
null=True, blank=True, max_length=64)
version = models.CharField( _('product version'),
null=True, blank=True, max_length=64)
appbuildid = models.CharField( _('app build id'),
null=True, blank=True, max_length=64)
build_target = models.CharField( _('build target'),
null=True, blank=True, max_length=64)
locale = models.CharField( _('locale'),
null=True, blank=True, max_length=64)
channel = models.CharField( _('channel'),
null=True, blank=True, max_length=64)
os_version = models.CharField( _('os version'),
null=True, blank=True, max_length=64)
distribution = models.CharField( _('distribution'),
null=True, blank=True, max_length=64)
distribution_version = models.CharField( _('distribution version'),
null=True, blank=True, max_length=64)
created = models.DateTimeField( _('date created'),
auto_now_add=True, blank=False)
modified = models.DateTimeField( _('date last modified'),
auto_now=True, blank=False)
def __unicode__(self):
fields = ( 'startpage_version', 'name', 'version', 'appbuildid',
'build_target', 'locale', 'channel', 'os_version',
'distribution', 'distribution_version', )
vals = [ getattr(self, field) or '*' for field in fields ]
if self.description:
return self.description
else:
return '%s /%s' % (
self.exclude and 'EXCLUDE' or 'INCLUDE',
'/'.join(vals)
)
def is_match(self, args):
is_match = True
for ak,av in args.items():
mv = getattr(self, ak, None)
if not mv:
continue
if mv.startswith('/'):
# Regex match
import re
try:
p = re.compile(mv[1:-1])
if p.match(av) is None:
is_match = False
except re.error:
# TODO: log error? validate regex in form submit?
is_match = False
elif av != mv:
# Exact match
is_match = False
return is_match
def related_snippets(self):
"""HTML link to snippets covered by this client match rule"""
link = '%s?%s' % (
urlresolvers.reverse('admin:homesnippets_snippet_changelist', args=[]),
'client_match_rules__id__exact=%s' % (self.id)
)
count = self.snippet_set.count()
# TODO: Needs l10n? Maybe not a priority for an admin page.
return (
( (count != 1) and
'<a href="%(link)s" title="Click to list snippets matched by this rule"><strong>%(count)d snippets</strong></a>' or
'<a href="%(link)s" title="Click to list snippets matched by this rule"><strong>%(count)d snippet</strong></a>' )
% dict(count=count, link=link)
)
related_snippets.allow_tags = True
def rule_update_lastmods(sender, instance, created=False, **kwargs):
"""On a change to a rule, bump lastmod timestamps for that rule and the set
of all cached rules."""
now = mktime(gmtime())
lastmods = {
# Timestamp for this rule.
'%s%s' % (CACHE_RULE_LASTMOD_PREFIX, instance.id): now,
# Timestamp for set of all rules.
CACHE_RULE_ALL_LASTMOD_PREFIX: now,
}
if created:
# Update timestamp since last new rule created.
lastmods[CACHE_RULE_NEW_LASTMOD_PREFIX] = now
cache.set_many(lastmods, CACHE_TIMEOUT)
post_save.connect(rule_update_lastmods, sender=ClientMatchRule)
post_delete.connect(rule_update_lastmods, sender=ClientMatchRule)
class SnippetManager(models.Manager):
def find_snippets_with_match_rules(self, args, time_now=None):
"""Find snippets data using match rules.
Returned is a list of dicts with id and body of snippets found, rather
than full Snippet model objects. This makes things easier to cache -
if full snippets are required, try using the id's to look them up.
"""
if time_now is None:
time_now = datetime.now()
preview = ( 'preview' in args ) and args['preview']
include_ids, exclude_ids = \
ClientMatchRule.objects.find_match_ids_for_request(args)
snippets = self.find_snippets_for_rule_ids(preview, include_ids, exclude_ids)
# Filter for date ranges here, rather than in SQL.
#
# This is a compromise to make snippet match results more cacheable -
# ie. cached data should only be recalculated in response to content
# changes, not the passage of time.
snippets_data = [ s for s in snippets if (
( not s['pub_start'] or time_now >= s['pub_start'] ) and
( not s['pub_end'] or time_now < s['pub_end'] )
) ]
return snippets_data
def find_snippets_for_rule_ids(self, preview, include_ids, exclude_ids):
"""Given a set of matching inclusion & exclusion rule IDs, look up the
corresponding snippets."""
if not include_ids and not exclude_ids:
return []
# Could base the cache key on the entire text of the SQL query
# constructed below, but we might someday use something other than a DB
# for persistence.
cache_key = '%s%s' % ( CACHE_SNIPPET_LOOKUP_PREFIX, hashlib.md5(
'include:%s;exclude:%s;preview:%s' % (
','.join(include_ids), ','.join(exclude_ids), preview)
).hexdigest() )
cache_hit = cache.get(cache_key)
if cache_hit:
# Invalidate if any of the lastmods of related rules, snippets, or
# new rule creation is newer than the cache
keys = [ '%s%s' % (CACHE_RULE_LASTMOD_PREFIX, item)
for sublist in cache_hit[1] for item in sublist ]
keys.extend([ '%s%s' % (CACHE_SNIPPET_LASTMOD_PREFIX, item['id'])
for item in cache_hit[2] ])
keys.append(CACHE_RULE_NEW_LASTMOD_PREFIX)
lastmods = cache.get_many(keys).values()
newer_lastmods = [ m for m in lastmods if m > cache_hit[0] ]
if newer_lastmods:
cache_hit = None
if not cache_hit:
# No cache hit, look up the snippets associated with rules.
sql_base = """
SELECT homesnippets_snippet.*
FROM homesnippets_snippet
WHERE ( %s )
ORDER BY priority, pub_start, modified
"""
where = [
'( homesnippets_snippet.disabled <> 1 )',
]
if not preview:
where.append('( homesnippets_snippet.preview <> 1 )')
if include_ids:
where.append("""
homesnippets_snippet.id IN (
SELECT snippet_id
FROM homesnippets_snippet_client_match_rules
WHERE clientmatchrule_id IN (%s)
)
""" % ",".join(include_ids))
if exclude_ids:
where.append("""
homesnippets_snippet.id NOT IN (
SELECT snippet_id
FROM homesnippets_snippet_client_match_rules
WHERE clientmatchrule_id IN (%s)
)
""" % ",".join(exclude_ids))
sql = sql_base % (' AND '.join(where))
# Reduce snippet model objects to more cacheable dicts
snippet_objs = self.raw(sql)
snippets = [
dict(
id=snippet.id,
name=snippet.name,
body=snippet.body,
pub_start=snippet.pub_start,
pub_end=snippet.pub_end,
)
for snippet in snippet_objs
]
cache_hit = ( mktime(gmtime()), (include_ids, exclude_ids), snippets, )
cache.set(cache_key, cache_hit, CACHE_TIMEOUT)
return cache_hit[2]
class Snippet(models.Model):
class Meta():
ordering = ( '-modified', '-pub_start', '-priority' )
objects = SnippetManager()
client_match_rules = models.ManyToManyField(
ClientMatchRule, blank=False)
name = models.CharField( _("short name (only shown to admins)"),
blank=False, max_length=255)
body = models.TextField( _("content body"),
blank=False)
priority = models.IntegerField( _('sort order priority'),
default=0, blank=True, null=True)
disabled = models.BooleanField( _('disabled?'),
default=False)
preview = models.BooleanField( _('preview only?'),
default=False)
pub_start = models.DateTimeField( _('start time'),
blank=True, null=True)
pub_end = models.DateTimeField( _('end time'),
blank=True, null=True)
created = models.DateTimeField( _('date created'),
auto_now_add=True, blank=False)
modified = models.DateTimeField( _('date last modified'),
auto_now=True, blank=False)
def snippet_update_lastmod(sender, instance, **kwargs):
"""On a change to a snippet, bump its cached lastmod timestamp"""
now = mktime(gmtime())
lastmods = {
'%s%s' % (CACHE_SNIPPET_LASTMOD_PREFIX, instance.id): now,
}
for rule in instance.client_match_rules.all():
lastmods['%s%s' % (CACHE_RULE_LASTMOD_PREFIX, rule.id)] = now
cache.set_many(lastmods, CACHE_TIMEOUT)
post_save.connect(snippet_update_lastmod, sender=Snippet)
post_delete.connect(snippet_update_lastmod, sender=Snippet)