Move Event.data into abstract node storage

.travis.yml

@@ -1,4 +1,10 @@
 language: python
+services:
+  - memcached
+  - riak
+  - mysql
+  - postgresql
+  - redis-server
 python:
   - "2.6"
   - "2.7"

setup.py

@@ -58,6 +58,7 @@
     'nydus',
     'mock>=0.8.0',
     'redis',
+    'riak',
     'unittest2',
 ]
 

src/sentry/app.py

@@ -25,4 +25,6 @@ def get_instance(path, options):
 
 buffer = get_instance(settings.SENTRY_BUFFER, settings.SENTRY_BUFFER_OPTIONS)
 quotas = get_instance(settings.SENTRY_QUOTAS, settings.SENTRY_QUOTA_OPTIONS)
+nodestore = get_instance(
+    settings.SENTRY_NODESTORE, settings.SENTRY_NODESTORE_OPTIONS)
 search = get_instance(settings.SENTRY_SEARCH, settings.SENTRY_SEARCH_OPTIONS)

src/sentry/conf/server.py

@@ -150,6 +150,7 @@
     'kombu.transport.django',
     'raven.contrib.django.raven_compat',
     'sentry',
+    'sentry.nodestore',
     'sentry.search',
     'sentry.plugins.sentry_interface_types',
     'sentry.plugins.sentry_mail',
@@ -508,10 +509,11 @@
 # Redis connection information (see Nydus documentation)
 SENTRY_REDIS_OPTIONS = {}
 
-# Buffer backend to use
+# Buffer backend
 SENTRY_BUFFER = 'sentry.buffer.Buffer'
 SENTRY_BUFFER_OPTIONS = {}
 
+# Quota backend
 SENTRY_QUOTAS = 'sentry.quotas.Quota'
 SENTRY_QUOTA_OPTIONS = {}
 
@@ -521,6 +523,10 @@
 # The maximum number of events per minute the system should accept.
 SENTRY_SYSTEM_MAX_EVENTS_PER_MINUTE = 0
 
+# Node storage backend
+SENTRY_NODESTORE = 'sentry.nodestore.django.DjangoNodeStorage'
+SENTRY_NODESTORE_OPTIONS = {}
+
 # Search backend
 SENTRY_SEARCH = 'sentry.search.django.DjangoSearchBackend'
 SENTRY_SEARCH_OPTIONS = {}

src/sentry/db/models/fields/__init__.py

@@ -10,3 +10,4 @@
 
 from .bounded import *  # NOQA
 from .gzippeddict import *  # NOQA
+from .node import *  # NOQA

src/sentry/db/models/fields/node.py

@@ -0,0 +1,131 @@
+"""
+sentry.db.models.fields.node
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+from __future__ import absolute_import
+
+import collections
+import logging
+import warnings
+
+from django.db import models
+from django.db.models.signals import post_delete
+
+from sentry.utils.cache import memoize
+from sentry.utils.compat import pickle
+from sentry.utils.strings import decompress, compress
+
+from .gzippeddict import GzippedDictField
+
+__all__ = ('NodeField',)
+
+logger = logging.getLogger('sentry.errors')
+
+
+class NodeData(collections.MutableMapping):
+    def __init__(self, id, data=None):
+        self.id = id
+        self._node_data = data
+
+    def __getitem__(self, key):
+        return self.data[key]
+
+    def __setitem__(self, key, value):
+        self.data[key] = value
+
+    def __delitem__(self, key):
+        del self.data[key]
+
+    def __iter__(self):
+        return iter(self.data)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __repr__(self):
+        cls_name = type(self).__name__
+        if self._node_data:
+            return '<%s: id=%s data=%r>' % (
+                cls_name, self.id, repr(self._node_data))
+        return '<%s: id=%s>' % (cls_name, self.id,)
+
+    @memoize
+    def data(self):
+        from sentry import app
+
+        if self._node_data is not None:
+            return self._node_data
+
+        elif self.id:
+            warnings.warn('You should populate node data before accessing it.')
+            return app.nodestore.get(self.id) or {}
+
+        return {}
+
+    def bind_data(self, data):
+        self._node_data = data
+
+
+class NodeField(GzippedDictField):
+    """
+    Similar to the gzippedictfield except that it stores a reference
+    to an external node.
+    """
+    __metaclass__ = models.SubfieldBase
+
+    def contribute_to_class(self, cls, name):
+        super(NodeField, self).contribute_to_class(cls, name)
+        post_delete.connect(
+            self.on_delete,
+            sender=self.model,
+            weak=False)
+
+    def on_delete(self, instance, **kwargs):
+        from sentry import app
+
+        value = getattr(instance, self.name)
+        if not value.id:
+            return
+
+        app.nodestore.delete(value.id)
+
+    def to_python(self, value):
+        if isinstance(value, basestring) and value:
+            try:
+                value = pickle.loads(decompress(value))
+            except Exception, e:
+                logger.exception(e)
+                value = {}
+        elif not value:
+            value = {}
+
+        if 'node_id' in value:
+            node_id = value.pop('node_id')
+            data = None
+        else:
+            node_id = None
+            data = value
+
+        return NodeData(node_id, data)
+
+    def get_prep_value(self, value):
+        from sentry import app
+
+        if not value and self.null:
+            # save ourselves some storage
+            return None
+
+        # TODO(dcramer): we should probably do this more intelligently
+        # and manually
+        if not value.id:
+            value.id = app.nodestore.create(value.data)
+        else:
+            app.nodestore.set(value.id, value.data)
+
+        return compress(pickle.dumps({
+            'node_id': value.id
+        }))

src/sentry/db/models/manager.py

@@ -224,3 +224,19 @@ def get_from_cache(self, **kwargs):
 
     def create_or_update(self, **kwargs):
         return create_or_update(self.model, **kwargs)
+
+    def bind_nodes(self, object_list, *node_names):
+        from sentry import app
+
+        object_node_list = []
+        for name in node_names:
+            object_node_list.extend((getattr(i, name) for i in object_list if getattr(i, name).id))
+
+        node_ids = [n.id for n in object_node_list]
+        if not node_ids:
+            return
+
+        node_results = app.nodestore.get_multi(node_ids)
+
+        for node in object_node_list:
+            node.bind_data(node_results.get(node.id) or {})

src/sentry/models.py

@@ -41,7 +41,7 @@
 )
 from sentry.db.models import (
     Model, GzippedDictField, BoundedIntegerField, BoundedPositiveIntegerField,
-    update, sane_repr
+    NodeField, update, sane_repr
 )
 from sentry.manager import (
     GroupManager, ProjectManager, MetaManager, InstanceMetaManager, BaseManager,
@@ -482,7 +482,6 @@ class EventBase(Model):
         max_length=MAX_CULPRIT_LENGTH, blank=True, null=True,
         db_column='view')
     checksum = models.CharField(max_length=32, db_index=True)
-    data = GzippedDictField(blank=True, null=True)
     num_comments = BoundedPositiveIntegerField(default=0, null=True)
     platform = models.CharField(max_length=64, null=True)
 
@@ -583,6 +582,7 @@ class Group(EventBase):
     time_spent_count = BoundedIntegerField(default=0)
     score = BoundedIntegerField(default=0)
     is_public = models.NullBooleanField(default=False, null=True)
+    data = GzippedDictField(blank=True, null=True)
 
     objects = GroupManager()
 
@@ -726,6 +726,7 @@ class Event(EventBase):
     time_spent = BoundedIntegerField(null=True)
     server_name = models.CharField(max_length=128, db_index=True, null=True)
     site = models.CharField(max_length=128, db_index=True, null=True)
+    data = NodeField(blank=True, null=True)
 
     objects = BaseManager()
 

src/sentry/nodestore/__init__.py

@@ -0,0 +1,9 @@
+"""
+sentry.nodestore
+~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+from __future__ import absolute_import

src/sentry/nodestore/base.py

@@ -0,0 +1,67 @@
+"""
+sentry.nodestore.base
+~~~~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+from __future__ import absolute_import
+
+import uuid
+
+
+class NodeStorage(object):
+    def create(self, data):
+        """
+        >>> key = nodestore.create({'foo': 'bar'})
+        """
+        node_id = self.generate_id()
+        self.set(node_id, data)
+        return node_id
+
+    def delete(self, id):
+        """
+        >>> nodestore.delete('key1')
+        """
+        raise NotImplementedError
+
+    def get(self, id):
+        """
+        >>> data = nodestore.get('key1')
+        >>> print data
+        """
+        raise NotImplementedError
+
+    def get_multi(self, id_list):
+        """
+        >>> data_map = nodestore.get_multi(['key1', 'key2')
+        >>> print 'key1', data_map['key1']
+        >>> print 'key2', data_map['key2']
+        """
+        return dict(
+            (id, self.get(id))
+            for id in id_list
+        )
+
+    def set(self, id, data):
+        """
+        >>> nodestore.set('key1', {'foo': 'bar'})
+        """
+        raise NotImplementedError
+
+    def set_multi(self, values):
+        """
+        >>> nodestore.set_multi({
+        >>>     'key1': {'foo': 'bar'},
+        >>>     'key2': {'foo': 'baz'},
+        >>> })
+        """
+        for id, data in values.iteritems():
+            self.set(id=id, data=data)
+
+    def generate_id(self):
+        return uuid.uuid4().hex
+
+    def cleanup(self, cutoff_timestamp):
+        raise NotImplementedError

src/sentry/nodestore/django/__init__.py

@@ -0,0 +1,9 @@
+"""
+sentry.nodestore.django
+~~~~~~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+from .backend import DjangoNodeStorage  # NOQA

src/sentry/nodestore/django/backend.py

@@ -0,0 +1,46 @@
+"""
+sentry.nodestore.django.backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+from __future__ import absolute_import
+
+from django.utils import timezone
+
+from sentry.db.models import create_or_update
+from sentry.nodestore.base import NodeStorage
+
+from .models import Node
+
+
+class DjangoNodeStorage(NodeStorage):
+    def delete(self, id):
+        Node.objects.filter(id=id).delete()
+
+    def get(self, id):
+        try:
+            return Node.objects.get(id=id).data
+        except Node.DoesNotExist:
+            return None
+
+    def get_multi(self, id_list):
+        return dict(
+            (n.id, n.data)
+            for n in Node.objects.filter(id__in=id_list)
+        )
+
+    def set(self, id, data):
+        create_or_update(
+            Node,
+            id=id,
+            defaults={
+                'data': data,
+                'timestamp': timezone.now(),
+            },
+        )
+
+    def cleanup(self, cutoff_timestamp):
+        Node.objects.filter(timestamp__lte=cutoff_timestamp).delete()