Browse files

Adding mongo feed icon model, bootstrapping, and moved the views. Sti…

…ll need the importer and the canonical rep to match new models.
  • Loading branch information...
1 parent 84cf17a commit 66200f1d763610d66ee1bc8187e0e78c4f03c30a @samuelclay committed Apr 22, 2011
Showing with 68 additions and 20 deletions.
  1. +10 −14 apps/reader/views.py
  2. +25 −2 apps/rss_feeds/models.py
  3. +33 −4 utils/bootstrap_mongo.py
View
24 apps/reader/views.py
@@ -23,7 +23,7 @@
from apps.analyzer.models import get_classifiers_for_user
from apps.reader.models import UserSubscription, UserSubscriptionFolders, MUserStory, Feature
from apps.reader.forms import SignupForm, LoginForm, FeatureForm
-from apps.rss_feeds.models import FeedIcon
+from apps.rss_feeds.models import MFeedIcon
try:
from apps.rss_feeds.models import Feed, MFeedPage, DuplicateFeed, MStory, MStarredStory, FeedLoadtime
except:
@@ -180,17 +180,11 @@ def load_feed_favicons(request):
user_subs = user_subs.filter(active=True)
if feed_ids:
user_subs = user_subs.filter(feed__in=feed_ids)
-
- favicons = {}
- for sub in user_subs:
- try:
- feed_icon = FeedIcon.objects.get(feed__pk=sub.feed.pk)
- favicons[sub.feed.pk] = feed_icon.data
- except FeedIcon.DoesNotExist:
- continue
+ feed_ids = [sub['feed__pk'] for sub in user_subs.values('feed__pk')]
+ feed_icons = dict([(i.feed_id, i.data) for i in MFeedIcon.objects(feed_id__in=feed_ids)])
- return favicons
+ return feed_icons
@ajax_login_required
@json.json_view
@@ -256,7 +250,8 @@ def refresh_feeds(request):
user_subs = user_subs.filter(feed__in=feed_ids)
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
favicons_fetching = [int(f) for f in request.POST.getlist('favicons_fetching') if f]
-
+ feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)])
+
for sub in user_subs:
if (sub.needs_unread_recalc or
sub.unread_count_updated < UNREAD_CUTOFF or
@@ -275,9 +270,10 @@ def refresh_feeds(request):
if request.POST.get('check_fetch_status', False):
feeds[sub.feed.pk]['not_yet_fetched'] = not sub.feed.fetched_once
if sub.feed.pk in favicons_fetching:
- feeds[sub.feed.pk]['favicon'] = sub.feed.icon.data
- feeds[sub.feed.pk]['favicon_color'] = sub.feed.icon.color
- feeds[sub.feed.pk]['favicon_fetching'] = bool(not (sub.feed.icon.not_found or sub.feed.icon.data))
+ feeds[sub.feed.pk]['favicon'] = feed_icons[sub.feed.pk].data
+ feeds[sub.feed.pk]['favicon_color'] = feed_icons[sub.feed.pk].color
+ feeds[sub.feed.pk]['favicon_fetching'] = bool(not (feed_icons[sub.feed.pk].not_found or
+ feed_icons[sub.feed.pk].data))
if settings.DEBUG:
diff = datetime.datetime.utcnow()-start
View
27 apps/rss_feeds/models.py
@@ -61,15 +61,16 @@ def __unicode__(self):
return self.feed_title
def canonical(self, full=False):
+ icon = MFeedIcon.objects(feed_id=self.pk)
feed = {
'id': self.pk,
'feed_title': self.feed_title,
'feed_address': self.feed_address,
'feed_link': self.feed_link,
'updated': relative_timesince(self.last_update),
'subs': self.num_subscribers,
- 'favicon_color': self.icon.color,
- 'favicon_fetching': bool(not (self.icon.not_found or self.icon.data))
+ 'favicon_color': icon.color,
+ 'favicon_fetching': bool(not (icon.not_found or icon.data))
}
if not self.fetched_once:
@@ -883,6 +884,28 @@ def save(self, *args, **kwargs):
if hasattr(self, 'id'): self.delete()
+class MFeedIcon(mongo.Document):
+ feed_id = mongo.IntField(primary_key=True)
+ color = mongo.StringField(max_length=6)
+ data = mongo.StringField()
+ icon_url = mongo.StringField()
+ not_found = mongo.BooleanField(default=False)
+
+ meta = {
+ 'collection' : 'feed_icons',
+ 'allow_inheritance' : False,
+ }
+
+ def save(self, *args, **kwargs):
+ if self.icon_url:
+ self.icon_url = unicode(self.icon_url)
+ try:
+ super(MFeedIcon, self).save(*args, **kwargs)
+ except (IntegrityError, OperationError):
+ # print "Error on Icon: %s" % e
+ if hasattr(self, '_id'): self.delete()
+
+
class MFeedPage(mongo.Document):
feed_id = mongo.IntField(primary_key=True)
page_data = mongo.BinaryField()
View
37 utils/bootstrap_mongo.py
@@ -1,9 +1,9 @@
from pprint import pprint
from django.conf import settings
-from apps.reader.models import MUserStory, UserStory
-from apps.rss_feeds.models import Feed, Story, MStory, StoryAuthor, Tag, MFeedPage, FeedPage
+from apps.reader.models import MUserStory
+from apps.rss_feeds.models import Feed, MStory, MFeedPage
+from apps.rss_feeds.models import MFeedIcon, FeedIcon
from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag
-from apps.analyzer.models import ClassifierTitle, ClassifierAuthor, ClassifierFeed, ClassifierTag
import mongoengine, pymongo
import sys
from mongoengine.queryset import OperationError
@@ -138,6 +138,34 @@ def bootstrap_feedpages():
print "\nMongo DB feed_pages: %s" % MFeedPage.objects().count()
+def bootstrap_feedicons():
+ print "Mongo DB feed_icons: %s" % MFeedIcon.objects().count()
+ db.feed_icons.drop()
+ print "Dropped! Mongo DB feed_icons: %s" % MFeedIcon.objects().count()
+
+ print "FeedIcons: %s" % FeedIcon.objects.count()
+ pprint(db.feed_icons.index_information())
+
+ feeds = Feed.objects.all().order_by('-average_stories_per_month')
+ feed_count = feeds.count()
+ i = 0
+ for feed in feeds:
+ i += 1
+ print "%s/%s: %s" % (i, feed_count, feed,)
+ sys.stdout.flush()
+
+ if not MFeedIcon.objects(feed_id=feed.pk):
+ feed_icon = FeedIcon.objects.filter(feed=feed).values()
+ if feed_icon:
+ try:
+ MFeedIcon(**feed_icon[0]).save()
+ except:
+ print '\n\n!\n\n'
+ continue
+
+
+ print "\nMongo DB feed_icons: %s" % MFeedIcon.objects().count()
+
def compress_stories():
count = MStory.objects().count()
print "Mongo DB stories: %s" % count
@@ -195,4 +223,5 @@ def reindex_stories():
# bootstrap_classifiers()
# bootstrap_feedpages()
# compress_stories()
- reindex_stories()
+ # reindex_stories()
+ bootstrap_feedicons()

0 comments on commit 66200f1

Please sign in to comment.