Permalink
Browse files

First half of DynamoDB trial, converting stories from mongo to dynamo…

…db. Still needs to be updated/inserted on feed update, and then processed with all MStory uses.
  • Loading branch information...
1 parent 95326cc commit 0a03154473872f0b5d96e7a3de3d9bddc315c934 @samuelclay committed Sep 4, 2012
View
@@ -10,7 +10,7 @@
from django.contrib.auth.models import User
from mongoengine.queryset import OperationError
from apps.reader.managers import UserSubscriptionManager
-from apps.rss_feeds.models import Feed, MStory, DuplicateFeed
+from apps.rss_feeds.models import Feed, MStory, DStory, DuplicateFeed
from apps.analyzer.models import MClassifierFeed, MClassifierAuthor, MClassifierTag, MClassifierTitle
from apps.analyzer.models import apply_classifier_titles, apply_classifier_feeds, apply_classifier_authors, apply_classifier_tags
from utils.feed_functions import add_object_to_folder
@@ -137,23 +137,43 @@ def get_stories(self, offset=0, limit=6, order='newest', read_filter='all', with
else:
byscorefunc = r.zrevrangebyscore
min_score = current_time
- # +1 for the intersection b/w zF and F, which carries an implicit score of 1.
- max_score = int(time.mktime(self.mark_read_date.timetuple())) + 1
+ if read_filter == 'unread':
+ # +1 for the intersection b/w zF and F, which carries an implicit score of 1.
+ max_score = int(time.mktime(self.mark_read_date.timetuple())) + 1
+ else:
+ max_score = 0
if settings.DEBUG:
- print " ---> Unread all stories: %s" % r.zrevrange(unread_ranked_stories_key, 0, -1)
+ debug_stories = r.zrevrange(unread_ranked_stories_key, 0, -1, withscores=True)
+ print " ---> Unread all stories (%s - %s) %s stories: %s" % (
+ min_score,
+ max_score,
+ len(debug_stories),
+ debug_stories)
story_ids = byscorefunc(unread_ranked_stories_key, min_score,
max_score, start=offset, num=limit,
withscores=withscores)
-
+ print story_ids, ignore_user_stories, order, read_filter
r.expire(unread_ranked_stories_key, 24*60*60)
if not ignore_user_stories:
r.delete(unread_stories_key)
# XXX TODO: Remove below line after combing redis for these None's.
story_ids = [s for s in story_ids if s and s != 'None'] # ugh, hack
- return story_ids
+ if withscores:
+ return story_ids
+ elif story_ids:
+ if self.feed.backed_by_dynamodb:
+ mstories = DStory.get_batch(story_ids, table=settings.DDB)
+ mstories = sorted(mstories, key=lambda s: s.story_date, reverse=bool(order=='newest'))
+ else:
+ story_date_order = "%sstory_date" % ('' if order == 'oldest' else '-')
+ mstories = MStory.objects(id__in=story_ids).order_by(story_date_order)
+ stories = Feed.format_stories(mstories)
+ return stories
+ else:
+ return []
@classmethod
def feed_stories(cls, user_id, feed_ids, offset=0, limit=6, order='newest', read_filter='all'):
@@ -427,11 +427,8 @@ def load_single_feed(request, feed_id):
except UserSubscription.DoesNotExist:
usersub = None
- if usersub and (read_filter == 'unread' or order == 'oldest'):
- story_ids = usersub.get_stories(order=order, read_filter=read_filter, offset=offset, limit=limit)
- story_date_order = "%sstory_date" % ('' if order == 'oldest' else '-')
- mstories = MStory.objects(id__in=story_ids).order_by(story_date_order)
- stories = Feed.format_stories(mstories)
+ if usersub:
+ stories = usersub.get_stories(order=order, read_filter=read_filter, offset=offset, limit=limit)
else:
stories = feed.get_stories(offset, limit)
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ # Adding field 'Feed.backed_by_dynamodb'
+ db.add_column('feeds', 'backed_by_dynamodb',
+ self.gf('django.db.models.fields.BooleanField')(default=False),
+ keep_default=False)
+
+
+ def backwards(self, orm):
+ # Deleting field 'Feed.backed_by_dynamodb'
+ db.delete_column('feeds', 'backed_by_dynamodb')
+
+
+ models = {
+ 'rss_feeds.duplicatefeed': {
+ 'Meta': {'object_name': 'DuplicateFeed'},
+ 'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+ 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}),
+ 'duplicate_link': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}),
+ 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+ },
+ 'rss_feeds.feed': {
+ 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+ 'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+ 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+ 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'backed_by_dynamodb': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
+ 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
+ 'errors_since_good': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+ 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
+ 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255', 'db_index': 'True'}),
+ 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+ 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
+ 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+ 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+ 'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+ 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+ 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+ },
+ 'rss_feeds.feeddata': {
+ 'Meta': {'object_name': 'FeedData'},
+ 'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
+ 'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+ 'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+ 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+ 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+ },
+ 'rss_feeds.feedloadtime': {
+ 'Meta': {'object_name': 'FeedLoadtime'},
+ 'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'loadtime': ('django.db.models.fields.FloatField', [], {})
+ }
+ }
+
+ complete_apps = ['rss_feeds']
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import DataMigration
+from django.db import models
+from apps.rss_feeds.models import DStory
+
+class Migration(DataMigration):
+
+ def forwards(self, orm):
+ from dynamodb_mapper.model import ConnectionBorg
+
+ conn = ConnectionBorg()
+ conn.create_table(DStory, 100, 100, wait_for_active=True)
+
+ def backwards(self, orm):
+ "Write your backwards methods here."
+
+ models = {
+ 'rss_feeds.duplicatefeed': {
+ 'Meta': {'object_name': 'DuplicateFeed'},
+ 'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+ 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}),
+ 'duplicate_link': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}),
+ 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+ },
+ 'rss_feeds.feed': {
+ 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+ 'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+ 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+ 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
+ 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
+ 'errors_since_good': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+ 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
+ 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255', 'db_index': 'True'}),
+ 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+ 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
+ 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+ 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+ 'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+ 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+ 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+ },
+ 'rss_feeds.feeddata': {
+ 'Meta': {'object_name': 'FeedData'},
+ 'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
+ 'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+ 'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+ 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+ 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+ },
+ 'rss_feeds.feedloadtime': {
+ 'Meta': {'object_name': 'FeedLoadtime'},
+ 'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'loadtime': ('django.db.models.fields.FloatField', [], {})
+ }
+ }
+
+ complete_apps = ['rss_feeds']
+ symmetrical = True
Oops, something went wrong. Retry.

0 comments on commit 0a03154

Please sign in to comment.