Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge branch 'master' of github.com:samuelclay/NewsBlur

* 'master' of github.com:samuelclay/NewsBlur:
  Adding logging to PuSH subscription.
  Showing feed pushes in statistics.
  Instrumentation to count feed pushes.
  Migration to fix starred stories with incorrectly double-encoded url strings.
  Adding MongoDB logging for debug.
  • Loading branch information...
commit 1e45b3f5ac759fbcd407485af190e7045cfa9eef 2 parents 320ceb8 + bb5c4f8
@samuelclay authored
View
13 apps/profile/middleware.py
@@ -31,11 +31,14 @@ def process_response(self, request, response):
time = sum([float(q['time']) for q in connection.queries])
queries = connection.queries
for query in queries:
- query['sql'] = re.sub(r'SELECT (.*?) FROM', 'SELECT * FROM', query['sql'])
- query['sql'] = re.sub(r'SELECT', '~FYSELECT', query['sql'])
- query['sql'] = re.sub(r'INSERT', '~FGINSERT', query['sql'])
- query['sql'] = re.sub(r'UPDATE', '~FY~SBUPDATE', query['sql'])
- query['sql'] = re.sub(r'DELETE', '~FR~SBDELETE', query['sql'])
+ if query.get('mongo'):
+ query['sql'] = "%s: %s" % (query['mongo']['collection'], query['mongo']['query'])
+ else:
+ query['sql'] = re.sub(r'SELECT (.*?) FROM', 'SELECT * FROM', query['sql'])
+ query['sql'] = re.sub(r'SELECT', '~FYSELECT', query['sql'])
+ query['sql'] = re.sub(r'INSERT', '~FGINSERT', query['sql'])
+ query['sql'] = re.sub(r'UPDATE', '~FY~SBUPDATE', query['sql'])
+ query['sql'] = re.sub(r'DELETE', '~FR~SBDELETE', query['sql'])
t = Template("{% for sql in sqllog %}{% if not forloop.first %} {% endif %}[{{forloop.counter}}] {{sql.time}}s: {{sql.sql|safe}}{% if not forloop.last %}\n{% endif %}{% endfor %}")
logging.debug(t.render(Context({'sqllog':queries,'count':len(queries),'time':time})))
return response
View
8 apps/push/views.py
@@ -8,6 +8,8 @@
from apps.push.models import PushSubscription
from apps.push.signals import verified
+from apps.rss_feeds.models import MFeedPushHistory
+from utils import log as logging
def push_callback(request, push_id):
if request.method == 'GET':
@@ -28,6 +30,9 @@ def push_callback(request, push_id):
subscription.set_expiration(int(lease_seconds))
subscription.save()
subscription.feed.setup_push()
+
+ logging.debug(' ---> [%-30s] [%s] ~BBVerified PuSH' % (unicode(subscription.feed)[:30], subscription.feed_id))
+
verified.send(sender=subscription)
return HttpResponse(challenge, content_type='text/plain')
@@ -42,6 +47,7 @@ def push_callback(request, push_id):
# Don't give fat ping, just fetch.
# subscription.feed.queue_pushed_feed_xml(request.raw_post_data)
subscription.feed.queue_pushed_feed_xml("Fetch me")
-
+ MFeedPushHistory.objects.create(feed_id=subscription.feed_id)
+
return HttpResponse('')
return Http404
View
98 apps/rss_feeds/migrations/0055_starred_story_permalinks.py
@@ -0,0 +1,98 @@
+# encoding: utf-8
+from south.v2 import DataMigration
+
+from urllib import unquote
+from apps.rss_feeds.models import MStarredStory
+
+
+class Migration(DataMigration):
+
+ def forwards(self, orm):
+ ss_count = MStarredStory.objects.count()
+ print "%s starred stories" % ss_count
+ group_size = ss_count/1000
+ for group in range(1000):
+ offset = group_size*group
+ print "Group offset: %s/%s-%s" % (group, offset, group_size*(group+1))
+ stories = MStarredStory.objects.order_by('pk')[offset:group_size*(group+1)]
+ for i, story in enumerate(stories):
+ original_permalink = story.story_permalink
+ try:
+ story.story_permalink = unquote(story.story_permalink.encode('utf-8', 'ignore'))
+ if original_permalink != story.story_permalink:
+ story.save()
+ print " ---> Fixing %s" % (story.pk)
+ print " Story #%s from: %s" % (i+offset, original_permalink)
+ print " Story #%s to : %s" % (i+offset, story.story_permalink)
+ except UnicodeDecodeError:
+ print " ***> Did not like: %s" % story.story_permalink
+ pass
+
+ def backwards(self, orm):
+ "Write your backwards methods here."
+ pass
+
+ models = {
+ 'rss_feeds.duplicatefeed': {
+ 'Meta': {'object_name': 'DuplicateFeed'},
+ 'duplicate_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'duplicate_feed_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
+ 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_addresses'", 'to': "orm['rss_feeds.Feed']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+ },
+ 'rss_feeds.feed': {
+ 'Meta': {'ordering': "['feed_title']", 'object_name': 'Feed', 'db_table': "'feeds'"},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+ 'active_premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+ 'active_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1', 'db_index': 'True'}),
+ 'average_stories_per_month': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'branch_from_feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']", 'null': 'True', 'blank': 'True'}),
+ 'creation': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'days_to_trim': ('django.db.models.fields.IntegerField', [], {'default': '90'}),
+ 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+ 'exception_code': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'favicon_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True', 'blank': 'True'}),
+ 'favicon_not_found': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'feed_address': ('django.db.models.fields.URLField', [], {'max_length': '255'}),
+ 'feed_address_locked': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+ 'feed_link': ('django.db.models.fields.URLField', [], {'default': "''", 'max_length': '1000', 'null': 'True', 'blank': 'True'}),
+ 'feed_link_locked': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'feed_title': ('django.db.models.fields.CharField', [], {'default': "'[Untitled]'", 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+ 'fetched_once': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'has_feed_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'has_page': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_page_exception': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'hash_address_and_link': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64', 'db_index': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_push': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+ 'known_good': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+ 'last_load_time': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'last_modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'min_to_decay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'next_scheduled_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'num_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+ 'premium_subscribers': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+ 'queued_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'stories_last_month': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+ },
+ 'rss_feeds.feeddata': {
+ 'Meta': {'object_name': 'FeedData'},
+ 'feed': ('utils.fields.AutoOneToOneField', [], {'related_name': "'data'", 'unique': 'True', 'to': "orm['rss_feeds.Feed']"}),
+ 'feed_classifier_counts': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+ 'feed_tagline': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'popular_authors': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True', 'blank': 'True'}),
+ 'popular_tags': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+ 'story_count_history': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+ },
+ 'rss_feeds.feedloadtime': {
+ 'Meta': {'object_name': 'FeedLoadtime'},
+ 'date_accessed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'feed': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['rss_feeds.Feed']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'loadtime': ('django.db.models.fields.FloatField', [], {})
+ }
+ }
+
+ complete_apps = ['rss_feeds']
View
30 apps/rss_feeds/models.py
@@ -1192,7 +1192,6 @@ def save(self, *args, **kwargs):
def get_data(cls, feed_id):
data = None
feed_page = cls.objects(feed_id=feed_id)
-
if feed_page:
data = feed_page[0].page_data and zlib.decompress(feed_page[0].page_data)
@@ -1307,7 +1306,7 @@ def feed_history(cls, feed_id):
for fetch in fetches:
history = {}
history['message'] = fetch.message
- history['fetch_date'] = fetch.fetch_date
+ history['fetch_date'] = fetch.fetch_date.strftime("%Y-%m-%d %H:%M:%S")
history['status_code'] = fetch.status_code
history['exception'] = fetch.exception
fetch_history.append(history)
@@ -1340,12 +1339,35 @@ def feed_history(cls, feed_id):
for fetch in fetches:
history = {}
history['message'] = fetch.message
- history['fetch_date'] = fetch.fetch_date
+ history['fetch_date'] = fetch.fetch_date.strftime("%Y-%m-%d %H:%M:%S")
history['status_code'] = fetch.status_code
history['exception'] = fetch.exception
fetch_history.append(history)
return fetch_history
-
+
+
+class MFeedPushHistory(mongo.Document):
+ feed_id = mongo.IntField()
+ push_date = mongo.DateTimeField(default=datetime.datetime.now)
+
+ meta = {
+ 'collection': 'feed_push_history',
+ 'allow_inheritance': False,
+ 'ordering': ['-push_date'],
+ 'indexes': ['feed_id', '-push_date'],
+ }
+
+ @classmethod
+ def feed_history(cls, feed_id):
+ pushes = cls.objects(feed_id=feed_id).order_by('-push_date')[:5]
+ push_history = []
+ for push in pushes:
+ history = {}
+ history['push_date'] = push.push_date.strftime("%Y-%m-%d %H:%M:%S")
+ push_history.append(history)
+ return push_history
+
+
class FeedLoadtime(models.Model):
feed = models.ForeignKey(Feed)
date_accessed = models.DateTimeField(auto_now=True)
View
3  apps/rss_feeds/views.py
@@ -7,7 +7,7 @@
from django.template import RequestContext
# from django.db import IntegrityError
from apps.rss_feeds.models import Feed, merge_feeds
-from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory
+from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory, MFeedPushHistory
from apps.analyzer.models import get_classifiers_for_user
from apps.reader.models import UserSubscription
from utils.user_functions import ajax_login_required
@@ -125,6 +125,7 @@ def load_feed_statistics(request, feed_id):
# Fetch histories
stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
+ stats['feed_push_history'] = MFeedPushHistory.feed_history(feed_id)
logging.user(request, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,))
View
12 media/css/reader.css
@@ -4588,8 +4588,11 @@ background: transparent;
.NB-modal-statistics .NB-statistics-stat .NB-statistics-fetches-half {
float: left;
- width: 50%;
text-align: center;
+ margin-right: 18px;
+}
+.NB-modal-statistics .NB-statistics-stat .NB-statistics-fetches-half:last-child {
+ margin-right: 0;
}
.NB-modal-statistics .NB-statistics-stat .NB-statistics-history-stat {
@@ -4631,12 +4634,17 @@ background: transparent;
}
.NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-message {
padding-right: 4px;
- margin-left: 120px;
+ margin-left: 110px;
font-weight: bold;
}
.NB-modal-statistics .NB-statistics-history-fetch .NB-statistics-history-fetch-exception {
display: none;
}
+.NB-modal-statistics .NB-statistics-history-empty {
+ color: #C0C0C0;
+ font-size: 10px;
+ padding: 4px 12px;
+}
.NB-modal-statistics .NB-statistics-classifiers {
border: 1px solid #e6e6e6;
clear: both;
View
4 media/js/newsblur/reader.js
@@ -5954,6 +5954,8 @@
// ====================
setup_dashboard_graphs: function() {
+ if (NEWSBLUR.Globals.debug) return;
+
// Reload dashboard graphs every 10 minutes.
clearInterval(this.locks.load_dashboard_graphs);
this.locks.load_dashboard_graphs = setInterval(_.bind(function() {
@@ -5976,6 +5978,8 @@
setup_feedback_table: function() {
+ if (NEWSBLUR.Globals.debug) return;
+
// Reload feedback module every 10 minutes.
clearInterval(this.locks.load_feedback_table);
this.locks.load_feedback_table = setInterval(_.bind(function() {
View
42 media/js/newsblur/reader_statistics.js
@@ -126,12 +126,16 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, {
])),
$.make('div', { className: 'NB-statistics-stat NB-statistics-fetches'}, [
$.make('div', { className: 'NB-statistics-fetches-half'}, [
- $.make('div', { className: 'NB-statistics-label' }, 'Feed'),
- $.make('div', this.make_history(data, 'feed'))
+ $.make('div', { className: 'NB-statistics-label' }, 'Feed Fetch'),
+ $.make('div', this.make_history(data, 'feed_fetch'))
]),
$.make('div', { className: 'NB-statistics-fetches-half'}, [
- $.make('div', { className: 'NB-statistics-label' }, 'Page'),
- $.make('div', this.make_history(data, 'page'))
+ $.make('div', { className: 'NB-statistics-label' }, 'Page Fetch'),
+ $.make('div', this.make_history(data, 'page_fetch'))
+ ]),
+ $.make('div', { className: 'NB-statistics-fetches-half'}, [
+ $.make('div', { className: 'NB-statistics-label' }, 'Feed Push'),
+ $.make('div', this.make_history(data, 'feed_push'))
])
])
]);
@@ -209,20 +213,24 @@ _.extend(NEWSBLUR.ReaderStatistics.prototype, {
},
make_history: function(data, fetch_type) {
- var fetches = data[fetch_type+'_fetch_history'];
- if (!fetches) return;
+ var fetches = data[fetch_type+'_history'];
+ var $history;
- var $history = _.map(fetches, function(fetch) {
- var feed_ok = _.contains([200, 304], fetch.status_code);
- var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode ';
- return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [
- $.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date),
- $.make('div', { className: 'NB-statistics-history-fetch-message' }, [
- fetch.message,
- $.make('div', { className: 'NB-statistics-history-fetch-code' }, ' ('+fetch.status_code+')')
- ])
- ]);
- });
+ if (!fetches || !fetches.length) {
+ $history = $.make('div', { className: 'NB-statistics-history-empty' }, "Nothing recorded.");
+ } else {
+ $history = _.map(fetches, function(fetch) {
+ var feed_ok = _.contains([200, 304], fetch.status_code) || !fetch.status_code;
+ var status_class = feed_ok ? ' NB-ok ' : ' NB-errorcode ';
+ return $.make('div', { className: 'NB-statistics-history-fetch' + status_class, title: feed_ok ? '' : fetch.exception }, [
+ $.make('div', { className: 'NB-statistics-history-fetch-date' }, fetch.fetch_date || fetch.push_date),
+ $.make('div', { className: 'NB-statistics-history-fetch-message' }, [
+ fetch.message,
+ (fetch.status_code && $.make('div', { className: 'NB-statistics-history-fetch-code' }, ' ('+fetch.status_code+')'))
+ ])
+ ]);
+ });
+ }
return $history;
},
View
3  settings.py
@@ -358,3 +358,6 @@ def custom_show_toolbar(request):
REDIS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=0)
JAMMIT = jammit.JammitAssets(NEWSBLUR_DIR)
+
+if DEBUG:
+ MIDDLEWARE_CLASSES += ('utils.mongo_raw_log_middleware.SqldumpMiddleware',)
View
3  templates/base.html
@@ -18,7 +18,8 @@
'username' : "{{ user.username|safe }}",
'email' : "{{ user.email|safe }}",
'google_favicon_url' : 'http://www.google.com/s2/favicons?domain_url=',
- 'MEDIA_URL' : "{{ MEDIA_URL }}"
+ 'MEDIA_URL' : "{{ MEDIA_URL }}",
+ 'debug' : {{ debug|yesno:"true,false" }}
};
NEWSBLUR.Flags = {
'start_import_from_google_reader': {{ start_import_from_google_reader|yesno:"true,false" }}
View
83 utils/mongo_raw_log_middleware.py
@@ -0,0 +1,83 @@
+from django.core.exceptions import MiddlewareNotUsed
+from django.conf import settings
+from django.db import connection
+from pymongo.connection import Connection
+from time import time
+import struct
+import bson
+from bson.errors import InvalidBSON
+
+class SqldumpMiddleware(object):
+ def __init__(self):
+ if not settings.DEBUG:
+ raise MiddlewareNotUsed()
+
+ def process_view(self, request, callback, callback_args, callback_kwargs):
+ self._used_msg_ids = []
+ if settings.DEBUG:
+ # save old methods
+ self.orig_send_message = \
+ Connection._send_message
+ self.orig_send_message_with_response = \
+ Connection._send_message_with_response
+ # instrument methods to record messages
+ Connection._send_message = \
+ self._instrument(Connection._send_message)
+ Connection._send_message_with_response = \
+ self._instrument(Connection._send_message_with_response)
+ return None
+
+ def process_response(self, request, response):
+ if settings.DEBUG:
+ # remove instrumentation from pymongo
+ Connection._send_message = \
+ self.orig_send_message
+ Connection._send_message_with_response = \
+ self.orig_send_message_with_response
+ return response
+
+ def _instrument(self, original_method):
+ def instrumented_method(*args, **kwargs):
+ message = _mongodb_decode_wire_protocol(args[1][1])
+ if message['msg_id'] in self._used_msg_ids:
+ return original_method(*args, **kwargs)
+ self._used_msg_ids.append(message['msg_id'])
+ start = time()
+ result = original_method(*args, **kwargs)
+ stop = time()
+ duration = stop - start
+ connection.queries.append({
+ 'mongo': message,
+ 'time': '%.3f' % duration,
+ })
+ return result
+ return instrumented_method
+
+def _mongodb_decode_wire_protocol(message):
+ """ http://www.mongodb.org/display/DOCS/Mongo+Wire+Protocol """
+ MONGO_OPS = {
+ 2001: 'msg',
+ 2002: 'insert',
+ 2003: 'reserved',
+ 2004: 'query',
+ 2005: 'get_more',
+ 2006: 'delete',
+ 2007: 'kill_cursors',
+ }
+ _, msg_id, _, opcode, _ = struct.unpack('<iiiii', message[:20])
+ op = MONGO_OPS.get(opcode, 'unknown')
+ zidx = 20
+ collection_name_size = message[zidx:].find('\0')
+ collection_name = message[zidx:zidx+collection_name_size]
+ zidx += collection_name_size + 1
+ skip, limit = struct.unpack('<ii', message[zidx:zidx+8])
+ zidx += 8
+ msg = ""
+ try:
+ if message[zidx:]:
+ msg = bson.decode_all(message[zidx:], as_class=dict, tz_aware=False)
+ except Exception, e:
+ msg = 'invalid bson'
+ return { 'op': op, 'collection': collection_name,
+ 'msg_id': msg_id, 'skip': skip, 'limit': limit,
+ 'query': msg }
Please sign in to comment.
Something went wrong with that request. Please try again.