Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Adding clean stories task. Updating log colors.

  • Loading branch information...
commit c223fa892c5e4b8a773fd4e42b53a199c7a9c465 1 parent 70f2da6
@samuelclay authored
View
1  apps/reader/models.py
@@ -584,6 +584,7 @@ class MUserStory(mongo.Document):
story_date = mongo.DateTimeField()
story = mongo.ReferenceField(MStory, dbref=True)
found_story = mongo.GenericReferenceField()
+ shared = mongo.BooleanField()
meta = {
'collection': 'userstories',
View
21 apps/reader/tasks.py
@@ -3,7 +3,7 @@
from utils import log as logging
from django.contrib.auth.models import User
from django.conf import settings
-from apps.reader.models import UserSubscription
+from apps.reader.models import UserSubscription, MUserStory
from apps.social.models import MSocialSubscription
@@ -46,4 +46,21 @@ def run(self, **kwargs):
})
settings.MONGOANALYTICSDB.nbanalytics.page_loads.remove({
"date": {"$lt": day_ago},
- })
+ })
+
+class CleanStories(Task):
+ name = 'clean-stories'
+
+ def run(self, **kwargs):
+ days_ago = (datetime.datetime.utcnow() -
+ datetime.timedelta(days=settings.DAYS_OF_UNREAD*5))
+ old_stories = MUserStory.objects.filter(read_date__lte=days_ago)
+ logging.debug(" ---> Cleaning stories from %s days ago... %s/%s read stories" % (
+ settings.DAYS_OF_UNREAD*5,
+ MUserStory.objects.count(),
+ old_stories.count()
+ ))
+ for s, story in enumerate(old_stories):
+ if (s+1) % 1000 == 0:
+ logging.debug(" ---> %s stories removed..." % (s+1))
+ story.delete()
View
8 apps/reader/views.py
@@ -249,8 +249,8 @@ def load_feeds(request):
if not user_subs:
categories = MCategory.serialize()
- logging.user(request, "~FBLoading ~SB~FY%s~FB~SN/~SB~FM%s~FB~SN feeds/socials. %s" % (
- len(feeds.keys()), len(social_feeds), '~SBUpdating counts.' if update_counts else ''))
+ logging.user(request, "~FB~SBLoading ~FY%s~FB/~FM%s~FB feeds/socials%s" % (
+ len(feeds.keys()), len(social_feeds), '. ~FCUpdating counts.' if update_counts else ''))
data = {
'feeds': feeds.values() if version == 2 else feeds,
@@ -341,8 +341,8 @@ def make_feeds_folder(items, parent_folder="", depth=0):
if not user_subs:
categories = MCategory.serialize()
- logging.user(request, "~FBLoading ~SB~FY%s~FB~SN/~SB~FM%s~FB~SN feeds/socials ~FMflat~FB. %s" % (
- len(feeds.keys()), len(social_feeds), '~SBUpdating counts.' if update_counts else ''))
+ logging.user(request, "~FB~SBLoading ~FY%s~FB/~FM%s~FB feeds/socials ~FMflat~FB%s" % (
+ len(feeds.keys()), len(social_feeds), '. ~FCUpdating counts.' if update_counts else ''))
data = {
"flat_folders": flat_folders,
View
24 apps/social/views.py
@@ -36,7 +36,6 @@
@json.json_view
def load_social_stories(request, user_id, username=None):
- start = time.time()
user = get_user(request)
social_user_id = int(user_id)
social_user = get_object_or_404(User, pk=social_user_id)
@@ -74,8 +73,6 @@ def load_social_stories(request, user_id, username=None):
if not stories:
return dict(stories=[], message=message)
- checkpoint1 = time.time()
-
stories, user_profiles = MSharedStory.stories_with_comments_and_profiles(stories, user.pk, check_all=True)
story_feed_ids = list(set(s['story_feed_id'] for s in stories))
@@ -99,8 +96,6 @@ def load_social_stories(request, user_id, username=None):
classifier_titles = classifier_titles + list(MClassifierTitle.objects(user_id=user.pk, feed_id__in=story_feed_ids))
classifier_tags = classifier_tags + list(MClassifierTag.objects(user_id=user.pk, feed_id__in=story_feed_ids))
- checkpoint2 = time.time()
-
story_ids = [story['id'] for story in stories]
userstories_db = MUserStory.objects(user_id=user.pk,
feed_id__in=story_feed_ids,
@@ -168,10 +163,8 @@ def load_social_stories(request, user_id, username=None):
socialsub.needs_unread_recalc = True
socialsub.save()
- diff1 = checkpoint1-start
- diff2 = checkpoint2-start
- logging.user(request, "~FYLoading ~FMshared stories~FY: ~SB%s%s ~SN(~SB%.4ss/%.4ss~SN)" % (
- social_profile.title[:22], ('~SN/p%s' % page) if page > 1 else '', diff1, diff2))
+ logging.user(request, "~FYLoading ~FMshared stories~FY: ~SB%s%s" % (
+ social_profile.title[:22], ('~SN/p%s' % page) if page > 1 else ''))
return {
"stories": stories,
@@ -331,7 +324,6 @@ def sort_stories_by_id(a, b):
}
def load_social_page(request, user_id, username=None, **kwargs):
- start = time.time()
user = request.user
social_user_id = int(user_id)
social_user = get_object_or_404(User, pk=social_user_id)
@@ -365,8 +357,6 @@ def load_social_page(request, user_id, username=None, **kwargs):
has_next_page = True
stories = stories[:-1]
- checkpoint1 = time.time()
-
if not stories:
params = {
"user": user,
@@ -394,8 +384,6 @@ def load_social_page(request, user_id, username=None, **kwargs):
stories, profiles = MSharedStory.stories_with_comments_and_profiles(stories, social_user.pk,
check_all=True)
- checkpoint2 = time.time()
-
if user.is_authenticated():
for story in stories:
if user.pk in story['share_user_ids']:
@@ -446,12 +434,8 @@ def load_social_page(request, user_id, username=None, **kwargs):
'active_story' : active_story,
}
- diff1 = checkpoint1-start
- diff2 = checkpoint2-start
- timediff = time.time()-start
- logging.user(request, "~FYLoading ~FMsocial page~FY: ~SB%s%s ~SN(%.4s seconds, ~SB%.4s/%.4s~SN)" % (
- social_profile.title[:22], ('~SN/p%s' % page) if page > 1 else '', timediff,
- diff1, diff2))
+ logging.user(request, "~FYLoading ~FMsocial page~FY: ~SB%s%s" % (
+ social_profile.title[:22], ('~SN/p%s' % page) if page > 1 else ''))
if format == 'html':
template = 'social/social_stories.xhtml'
else:
View
5 fabfile.py
@@ -157,6 +157,11 @@ def deploy_code(copy_assets=False, full=False):
run('curl -s http://%s > /dev/null' % env.host)
run('curl -s http://%s/api/add_site_load_script/ABCDEF > /dev/null' % env.host)
+@parallel
+def kill():
+ sudo('supervisorctl reload')
+ run('pkill -c gunicorn')
+
def deploy_node():
with cd(env.NEWSBLUR_PATH):
run('sudo supervisorctl restart node_unread')
View
5 settings.py
@@ -362,6 +362,11 @@
'schedule': datetime.timedelta(hours=12),
'options': {'queue': 'beat_tasks'},
},
+ 'clean-stories': {
+ 'task': 'clean-stories',
+ 'schedule': datetime.timedelta(hours=24),
+ 'options': {'queue': 'beat_tasks'},
+ },
'premium-expire': {
'task': 'premium-expire',
'schedule': datetime.timedelta(hours=24),
Please sign in to comment.
Something went wrong with that request. Please try again.