This repository has been archived by the owner. It is now read-only.
Permalink
Browse files

Reorganize queries.

del_or_ban, unban, clear_reports handle QueryCache queries more clearly.
  • Loading branch information...
1 parent fb198d5 commit 37da8b585c3c6cc579459dba7718024d45766b5c @bsimpson63 bsimpson63 committed with spladug May 30, 2012
Showing with 116 additions and 127 deletions.
  1. +115 −123 r2/r2/lib/db/queries.py
  2. +1 −4 r2/r2/models/admintools.py
View
@@ -690,31 +690,30 @@ def new_comment(comment, inbox_rels):
sr = Subreddit._byID(comment.sr_id)
- if comment._deleted:
- job_key = "delete_items"
- job.append(get_sr_comments(sr))
- job.append(get_all_comments())
- else:
- job_key = "insert_items"
- with CachedQueryMutator() as m:
+ with CachedQueryMutator() as m:
+ if comment._deleted:
+ job_key = "delete_items"
+ job.append(get_sr_comments(sr))
+ job.append(get_all_comments())
+ else:
+ job_key = "insert_items"
if comment._spam:
m.insert(get_spam_comments(sr), [comment])
- if was_spam_filtered(comment)
+ if was_spam_filtered(comment):
m.insert(get_spam_filtered_comments(sr), [comment])
- amqp.add_item('new_comment', comment._fullname)
- if not g.amqp_host:
- add_comment_tree([comment])
+ amqp.add_item('new_comment', comment._fullname)
+ if not g.amqp_host:
+ add_comment_tree([comment])
- job_dict = { job_key: comment }
- add_queries(job, **job_dict)
+ job_dict = { job_key: comment }
+ add_queries(job, **job_dict)
- # note that get_all_comments() is updated by the amqp process
- # r2.lib.db.queries.run_new_comments (to minimise lock contention)
+ # note that get_all_comments() is updated by the amqp process
+ # r2.lib.db.queries.run_new_comments (to minimise lock contention)
- if inbox_rels:
- for inbox_rel in tup(inbox_rels):
- inbox_owner = inbox_rel._thing1
- with CachedQueryMutator() as m:
+ if inbox_rels:
+ for inbox_rel in tup(inbox_rels):
+ inbox_owner = inbox_rel._thing1
if inbox_rel._name == "inbox":
query = get_inbox_comments(inbox_owner)
elif inbox_rel._name == "selfreply":
@@ -868,7 +867,7 @@ def changed(things, boost_only=False):
message_id = thing._fullname,
delivery_mode = amqp.DELIVERY_TRANSIENT)
-def _by_srid(things,srs=True):
+def _by_srid(things, srs=True):
"""Takes a list of things and returns them in a dict separated by
sr_id, in addition to the looked-up subreddits"""
ret = {}
@@ -884,16 +883,26 @@ def _by_srid(things,srs=True):
return ret
-def _by_author(things):
- by_account = collections.defaultdict(list)
+def _by_author(things, authors=True):
+ ret = collections.defaultdict(list)
for thing in tup(things):
author_id = getattr(thing, 'author_id')
if author_id:
- by_account[author_id].append(thing)
+ ret[author_id].append(thing)
- return by_account
+ if authors:
+ _authors = Account._byID(ret.keys(), return_dict=True) if ret else {}
+ return ret, _authors
+ else:
+ return ret
+def was_spam_filtered(thing):
+ if (thing._spam and not thing._deleted and
+ getattr(thing, 'verdict', None) != 'mod-removed'):
+ return True
+ else:
+ return False
def ban(things):
del_or_ban(things, "ban")
@@ -905,59 +914,80 @@ def delete_comments(comments):
del_or_ban(comments, "del")
def del_or_ban(things, why):
+ query_cache_inserts = []
+ query_cache_deletes = []
+
by_srid, srs = _by_srid(things)
if not by_srid:
return
- for sr_id, things in by_srid.iteritems():
+ for sr_id, sr_things in by_srid.iteritems():
sr = srs[sr_id]
- links = [x for x in things if isinstance(x, Link)]
- comments = [x for x in things if isinstance(x, Comment)]
+ links = [x for x in sr_things if isinstance(x, Link)]
+ comments = [x for x in sr_things if isinstance(x, Comment)]
if links:
- if why == "ban":
- with CachedQueryMutator() as m:
- m.insert(get_spam_links(sr), links)
- # rip it out of the listings. bam!
- results = [get_links(sr, 'hot', 'all'),
- get_links(sr, 'new', 'all'),
- ]
-
+ results = [get_links(sr, 'hot', 'all'), get_links(sr, 'new', 'all')]
for sort in time_filtered_sorts:
for time in db_times.keys():
results.append(get_links(sr, sort, time))
+ add_queries(results, delete_items=links)
- add_queries(results, delete_items = links)
-
+ if why == "ban":
+ query_cache_inserts.append([get_spam_links(sr), links])
+ query_cache_inserts.append([get_spam_filtered_links(sr),
+ [l for l in links if was_spam_filtered(l)]])
if why == "del":
- with CachedQueryMutator() as m:
- m.delete(get_spam_links(sr), links)
- m.delete(get_spam_filtered_links(sr), links)
+ query_cache_deletes.append([get_spam_links(sr), links])
+ query_cache_deletes.append([get_spam_filtered_links(sr), links])
+ query_cache_deletes.append([get_reported_links(sr), links])
if comments:
- with CachedQueryMutator() as m:
- m.insert(get_spam_comments(sr), comments)
- add_queries([get_all_comments(),
- get_sr_comments(sr)], delete_items = comments)
+ if why == "ban":
+ query_cache_inserts.append([get_spam_comments(sr), comments])
+ query_cache_inserts.append([get_spam_filtered_comments(sr),
+ [c for c in comments if was_spam_filtered(c)]])
if why == "del":
- with CachedQueryMutator() as m:
- m.delete(get_spam_comments(sr), comments)
- m.delete(get_spam_filtered_comments(sr), comments)
+ query_cache_deletes.append([get_spam_comments(sr), comments])
+ query_cache_deletes.append([get_spam_filtered_comments(sr), comments])
+ query_cache_deletes.append([get_reported_comments(sr), comments])
- if why == "del":
- with CachedQueryMutator() as m:
- for author_id, things in _by_author(things).iteritems():
- links = [x for x in things if isinstance(x, Link)]
- if links:
- m.insert(get_deleted_links(author_id), links)
+ by_author, authors = _by_author(things)
+
+ for author_id, a_things in by_author.iteritems():
+ author = authors[author_id]
+ links = [x for x in a_things if isinstance(x, Link)]
+ comments = [x for x in a_things if isinstance(x, Comment)]
- comments = [x for x in things if isinstance(x, Comment)]
- if comments:
- m.insert(get_deleted_comments(author_id), comments)
+ if links and why == "del":
+ results = [get_submitted(author, 'hot', 'all'),
+ get_submitted(author, 'new', 'all')]
+ for sort in time_filtered_sorts:
+ for time in db_times.keys():
+ results.append(get_submitted(author, sort, time))
+ add_queries(results, delete_items=links)
+ query_cache_inserts.append([get_deleted_links(author_id), links])
+
+ if comments and why == "del":
+ results = [get_comments(author, 'hot', 'all'),
+ get_comments(author, 'new', 'all')]
+ for sort in time_filtered_sorts:
+ for time in db_times.keys():
+ results.append(get_comments(author, sort, time))
+ add_queries(results, delete_items=comments)
+ query_cache_inserts.append([get_deleted_comments(author_id), comments])
+
+ with CachedQueryMutator() as m:
+ for q, inserts in query_cache_inserts:
+ m.insert(q, inserts)
+ for q, deletes in query_cache_deletes:
+ m.delete(q, deletes)
changed(things)
-def unban(things):
+def unban(things, insert=True):
+ query_cache_deletes = []
+
by_srid, srs = _by_srid(things)
if not by_srid:
return
@@ -967,22 +997,18 @@ def unban(things):
links = [x for x in things if isinstance(x, Link)]
comments = [x for x in things if isinstance(x, Comment)]
- if links:
- with CachedQueryMutator() as m:
- m.delete(get_spam_links(sr), links)
+ if insert and links:
# put it back in the listings
results = [get_links(sr, 'hot', 'all'),
get_links(sr, 'top', 'all'),
get_links(sr, 'controversial', 'all'),
]
-
# the time-filtered listings will have to wait for the
# next mr_top run
-
add_queries(results, insert_items=links)
- # Check if link is being unbanned and should be put in 'new' with
- # current time
+ # Check if link is being unbanned and should be put in
+ # 'new' with current time
new_links = []
for l in links:
ban_info = l.ban_info
@@ -994,84 +1020,50 @@ def unban(things):
else:
new_links.append(l)
add_queries([get_links(sr, 'new', 'all')], insert_items=new_links)
+ query_cache_deletes.append([get_spam_links(sr), links])
+
+ if insert and comments:
+ add_queries([get_all_comments(), get_sr_comments(sr)],
+ insert_items=comments)
+ query_cache_deletes.append([get_spam_comments(sr), comments])
+
+ if links:
+ query_cache_deletes.append([get_spam_filtered_links(sr), links])
if comments:
- with CachedQueryMutator() as m:
- m.delete(get_spam_comments(sr), comments)
- add_queries([get_all_comments(),
- get_sr_comments(sr)], insert_items = comments)
+ query_cache_deletes.append([get_spam_filtered_comments(sr), comments])
+
+ with CachedQueryMutator() as m:
+ for q, deletes in query_cache_deletes:
+ m.delete(q, deletes)
changed(things)
def new_report(thing):
with CachedQueryMutator() as m:
if isinstance(thing, Link):
- sr = Subreddit._byID(thing.sr_id)
- m.insert(get_reported_links(sr), [thing])
+ m.insert(get_reported_links(thing.sr_id), [thing])
elif isinstance(thing, Comment):
- sr = Subreddit._byID(thing.sr_id)
- m.insert(get_reported_comments(sr), [thing])
+ m.insert(get_reported_comments(thing.sr_id), [thing])
def clear_reports(things):
- by_srid, srs = _by_srid(things)
- if not by_srid:
- return
+ query_cache_deletes = []
- for sr_id, sr_things in by_srid.iteritems():
- sr = srs[sr_id]
+ by_srid = _by_srid(things, srs=False)
+ for sr_id, sr_things in by_srid.iteritems():
links = [ x for x in sr_things if isinstance(x, Link) ]
comments = [ x for x in sr_things if isinstance(x, Comment) ]
- with CachedQueryMutator() as m:
- if links:
- m.delete(get_reported_links(sr), links)
- if comments:
- m.delete(get_reported_comments(sr), comments)
-
-def was_spam_filtered(thing):
- if thing._spam and not thing._deleted and \
- getattr(thing, 'verdict', None) != 'mod-removed':
- return True
- else:
- return False
-
-def new_spam_filtered(things):
- by_srid, srs = _by_srid(things)
- if not by_srid:
- return
+ if links:
+ query_cache_deletes.append([get_reported_links(sr_id), links])
+ if comments:
+ query_cache_deletes.append([get_reported_comments(sr_id), comments])
with CachedQueryMutator() as m:
- for sr_id, sr_things in by_srid.iteritems():
- sr = srs[sr_id]
- links = [ x for x in sr_things if isinstance(x, Link) ]
- comments = [ x for x in sr_things if isinstance(x, Comment) ]
+ for q, deletes in query_cache_deletes:
+ m.delete(q, deletes)
- insert_links = []
- delete_links = []
- for l in links:
- if was_spam_filtered(l):
- insert_links.append(l)
- else:
- delete_links.append(l)
-
- insert_comments = []
- delete_comments = []
- for c in comments:
- if was_filtered(c):
- insert_comments.append(c)
- else:
- delete_comments.append(c)
-
- if insert_links:
- m.insert(get_spam_filtered_links(sr), insert_links)
- if delete_links:
- m.delete(get_spam_filtered_links(sr), delete_links)
- if insert_comments:
- m.insert(get_spam_filtered_comments(sr), insert_comments)
- if delete_comments:
- m.delete(get_spam_filtered_comments(sr), delete_comments)
-
def add_all_srs():
"""Recalculates every listing query for every subreddit. Very,
very slow."""
@@ -77,7 +77,6 @@ def spam(self, things, auto=True, moderator_banned=False,
self.set_last_sr_ban(new_things)
queries.ban(new_things)
- queries.new_spam_filtered(all_things)
def unspam(self, things, unbanner=None, train_spam=True, insert=True):
from r2.lib.db import queries
@@ -112,9 +111,7 @@ def unspam(self, things, unbanner=None, train_spam=True, insert=True):
self.author_spammer(things, False)
self.set_last_sr_ban(things)
- if insert:
- queries.unban(things)
- queries.new_spam_filtered(things)
+ queries.unban(things, insert)
def author_spammer(self, things, spam):
"""incr/decr the 'spammer' field for the author of every

0 comments on commit 37da8b5

Please sign in to comment.