Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Lists redis clean-up #5886

Merged
merged 2 commits into from
Dec 5, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions app/models/list.rb
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,23 @@ class List < ApplicationRecord
has_many :accounts, through: :list_accounts

validates :title, presence: true

before_destroy :clean_feed_manager

private

def clean_feed_manager
reblog_key = FeedManager.instance.key(:list, id, 'reblogs')
reblogged_id_set = Redis.current.zrange(reblog_key, 0, -1)

Redis.current.pipelined do
Redis.current.del(FeedManager.instance.key(:list, id))
Redis.current.del(reblog_key)

reblogged_id_set.each do |reblogged_id|
reblog_set_key = FeedManager.instance.key(:list, id, "reblogs:#{reblogged_id}")
Redis.current.del(reblog_set_key)
end
end
end
end
40 changes: 30 additions & 10 deletions app/workers/scheduler/feed_cleanup_scheduler.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,56 @@ class Scheduler::FeedCleanupScheduler
include Sidekiq::Worker

def perform
clean_home_feeds!
clean_list_feeds!
end

private

def clean_home_feeds!
clean_feeds!(inactive_account_ids, :home)
end

def clean_list_feeds!
clean_feeds!(inactive_list_ids, :list)
end

def clean_feeds!(ids, type)
reblogged_id_sets = {}
feedmanager = FeedManager.instance

redis.pipelined do
inactive_user_ids.each do |account_id|
redis.del(feedmanager.key(:home, account_id))
reblog_key = feedmanager.key(:home, account_id, 'reblogs')
ids.each do |feed_id|
redis.del(feed_manager.key(type, feed_id))
reblog_key = feed_manager.key(type, feed_id, 'reblogs')
# We collect a future for this: we don't block while getting
# it, but we can iterate over it later.
reblogged_id_sets[account_id] = redis.zrange(reblog_key, 0, -1)
reblogged_id_sets[feed_id] = redis.zrange(reblog_key, 0, -1)
redis.del(reblog_key)
end
end

# Remove all of the reblog tracking keys we just removed the
# references to.
redis.pipelined do
reblogged_id_sets.each do |account_id, future|
reblogged_id_sets.each do |feed_id, future|
future.value.each do |reblogged_id|
reblog_set_key = feedmanager.key(:home, account_id, "reblogs:#{reblogged_id}")
reblog_set_key = feed_manager.key(type, feed_id, "reblogs:#{reblogged_id}")
redis.del(reblog_set_key)
end
end
end
end

private
def inactive_account_ids
@inactive_account_ids ||= User.confirmed.inactive.pluck(:account_id)
end

def inactive_list_ids
List.where(account_id: inactive_account_ids).pluck(:id)
end

def inactive_user_ids
@inactive_user_ids ||= User.confirmed.inactive.pluck(:account_id)
def feed_manager
FeedManager.instance
end

def redis
Expand Down