Permalink
Browse files

Adding logging, feed cleanup to forced fetch feeds, and expirations o…

…n user read stories.
  • Loading branch information...
samuelclay committed Apr 8, 2013
1 parent 0e7bcab commit 48c5d4d53eb20ad833172d4c41e642db98894fcc
View
@@ -667,6 +667,7 @@ def sync_redis(self, r=None):
if self.story_db_id:
all_read_stories_key = 'RS:%s' % (self.user_id)
r.sadd(all_read_stories_key, self.story_db_id)
+ r.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id)
r.sadd(read_story_key, self.story_db_id)
View
@@ -773,8 +773,6 @@ def update(self, **kwargs):
feed.last_update = datetime.datetime.utcnow()
feed.set_next_scheduled_update()
r.zadd('fetched_feeds_last_hour', feed.pk, int(datetime.datetime.now().strftime('%s')))
- if options['force']:
- feed.sync_redis()
if not feed or original_feed_id != feed.pk:
logging.info(" ---> ~FRFeed changed id, removing %s from tasked_feeds queue..." % original_feed_id)
@@ -1574,6 +1572,8 @@ def sync_redis(self, r=None):
if self.id and self.story_date > DAYS_OF_UNREAD:
r.sadd('F:%s' % self.story_feed_id, self.id)
r.zadd('zF:%s' % self.story_feed_id, self.id, time.mktime(self.story_date.timetuple()))
+ r.expire('F:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
+ r.expire('zF:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
def remove_from_redis(self, r=None):
if not r:
View
@@ -26,8 +26,9 @@ def run(self, **kwargs):
queued_feeds = r.zrangebyscore('scheduled_updates', 0, now_timestamp)
r.zremrangebyscore('scheduled_updates', 0, now_timestamp)
r.sadd('queued_feeds', *queued_feeds)
- logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/%s queued/scheduled)" % (
+ logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
len(queued_feeds),
+ r.zcard('tasked_feeds'),
r.scard('queued_feeds'),
r.zcard('scheduled_updates')))
@@ -85,7 +86,7 @@ def run(self, **kwargs):
Feed.task_feeds(refresh_feeds, verbose=False)
Feed.task_feeds(old_feeds, verbose=False)
- logging.debug(" ---> ~SN~FBTasking took ~SB%s~SN seconds (~SB%s~SN/~SB%s~SN/%s tasked/queued/scheduled)" % (
+ logging.debug(" ---> ~SN~FBTasking took ~SB%s~SN seconds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
int((time.time() - start)),
r.zcard('tasked_feeds'),
r.scard('queued_feeds'),
View
@@ -555,5 +555,5 @@ client-output-buffer-limit pubsub 32mb 8mb 60
# to customize a few per-server settings. Include files can include
# other files, so use this wisely.
#
-# include /path/to/local.conf
+include /etc/redis_server.conf
# include /path/to/other.conf
View
No changes.
View
@@ -0,0 +1 @@
+slaveof db10 6379
View
@@ -337,6 +337,8 @@ def setup_db(engine=None, skip_common=False):
setup_mongo()
elif engine == "redis":
setup_redis()
+ elif engine == "redis_slave":
+ setup_redis(slave=True)
setup_gunicorn(supervisor=False)
setup_db_munin()
@@ -456,10 +458,10 @@ def setup_imaging():
def setup_supervisor():
sudo('apt-get -y install supervisor')
- put('config/supervisord.conf', '/etc/supervisor/supervisord.conf')
- sudo('/etc/init.d/supervisord stop')
+ put('config/supervisord.conf', '/etc/supervisor/supervisord.conf', use_sudo=True)
+ sudo('/etc/init.d/supervisor stop')
sudo('sleep 2')
- sudo('/etc/init.d/supervisord start')
+ sudo('/etc/init.d/supervisor start')
# @parallel
def setup_hosts():
@@ -563,7 +565,7 @@ def setup_sudoers(user=None):
sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user))
def setup_nginx():
- NGINX_VERSION = '1.2.2'
+ NGINX_VERSION = '1.2.8'
with cd(env.VENDOR_PATH), settings(warn_only=True):
sudo("groupadd nginx")
sudo("useradd -g nginx -d /var/www/htdocs -s /bin/false nginx")
@@ -834,8 +836,8 @@ def setup_mongo_mms():
sudo('supervisorctl update')
-def setup_redis():
- redis_version = '2.6.11'
+def setup_redis(slave=False):
+ redis_version = '2.6.12'
with cd(env.VENDOR_PATH):
run('wget http://redis.googlecode.com/files/redis-%s.tar.gz' % redis_version)
run('tar -xzf redis-%s.tar.gz' % redis_version)
@@ -845,6 +847,10 @@ def setup_redis():
put('config/redis-init', '/etc/init.d/redis', use_sudo=True)
sudo('chmod u+x /etc/init.d/redis')
put('config/redis.conf', '/etc/redis.conf', use_sudo=True)
+ if slave:
+ put('config/redis_slave.conf', '/etc/redis_server.conf', use_sudo=True)
+ else:
+ put('config/redis_master.conf', '/etc/redis_server.conf', use_sudo=True)
sudo('mkdir -p /var/lib/redis')
sudo('update-rc.d redis defaults')
sudo('/etc/init.d/redis stop')
@@ -77,7 +77,7 @@
<img src="/media/img/logo_512.png" class="logo">
<h1>NewsBlur is in <span class="error404">maintenance mode</span></h1>
<div class="description">
- <p>This will take approximately 5 minutes. I'm upgrading PostgreSQL and re-building indexes for optimum performance. Tonight's a rough night, but hopefully this will be the last of it.</p>
+ <p>This will take approximately 5-10 minutes. I'm upgrading Redis, which is throwing faults due to memory issues.</p>
<p>To pass the time, go surf <a href="http://mlkshk.com/popular">MLKSHK's popular page</a>.</p>
<p></p>
</div>
View
@@ -369,10 +369,13 @@ def process_feed_wrapper(self, feed_queue):
feed.known_good = True
feed.fetched_once = True
feed = feed.save()
- if random.random() <= 0.02:
+ if self.options['force'] or random.random() <= 0.02:
+ logging.debug(' ---> [%-30s] ~FBPerforming feed cleanup...' % (feed.title[:30],))
+ start_cleanup = time.time()
feed.sync_redis()
MUserStory.delete_old_stories(feed_id=feed.pk)
MUserStory.sync_all_redis(feed_id=feed.pk)
+ logging.debug(' ---> [%-30s] ~FBDone with feed cleanup. Took %s sec.' % (feed.title[:30], time.time() - start_cleanup))
try:
self.count_unreads_for_subscribers(feed)
except TimeoutError:

0 comments on commit 48c5d4d

Please sign in to comment.