Permalink
Browse files

Initial code seperation for collector/web apps. Initial work on SQLAl…

…chemy backend.
  • Loading branch information...
1 parent 2232d94 commit d47fa7ce0a33b0642dafe4a1921a9e20986baf6b @dcramer committed Aug 11, 2011
View
5 sentry/__init__.py
@@ -136,8 +136,3 @@ def init_threads():
import logging
logging.basicConfig(level=logging.WARN)
-
-# Import views/templatetags to ensure registration
-import sentry.web.api
-import sentry.web.templatetags
-import sentry.web.views
View
0 sentry/collector/scripts/__init__.py
No changes.
View
163 sentry/collector/scripts/runner.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+"""
+sentry.collector.scripts.runner
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+import eventlet
+import os
+import os.path
+import sys
+
+from daemon.daemon import DaemonContext
+from daemon.runner import DaemonRunner, make_pidlockfile
+from eventlet import wsgi
+from optparse import OptionParser
+
+from sentry import VERSION, app
+from sentry.middleware import WSGIErrorMiddleware
+
+class SentryCollector(DaemonRunner):
+ pidfile_timeout = 10
+ start_message = u"started with pid %(pid)d"
+
+ def __init__(self, host=None, port=None, pidfile=None,
+ logfile=None, daemonize=False, debug=False):
+ if not logfile:
+ logfile = app.config['WEB_LOG_FILE']
+
+ logfile = os.path.realpath(logfile)
+ pidfile = os.path.realpath(pidfile or app.config['WEB_PID_FILE'])
+
+ if daemonize:
+ detach_process = True
+ else:
+ detach_process = False
+
+ self.daemon_context = DaemonContext(detach_process=detach_process)
+ self.daemon_context.stdout = open(logfile, 'w+')
+ self.daemon_context.stderr = open(logfile, 'w+', buffering=0)
+
+ self.pidfile = make_pidlockfile(pidfile, self.pidfile_timeout)
+
+ self.daemon_context.pidfile = self.pidfile
+
+ self.host = host or app.config['WEB_HOST']
+ self.port = port or app.config['WEB_PORT']
+
+ self.debug = debug
+
+ # HACK: set app to self so self.app.run() works
+ self.app = self
+
+ def execute(self, action):
+ self.action = action
+ if self.daemon_context.detach_process is False and self.action == 'start':
+ # HACK:
+ self.run()
+ else:
+ self.do_action()
+
+ def run(self):
+ # Import views/templatetags to ensure registration
+ import sentry.collector.views
+
+ upgrade()
+ app.wsgi_app = WSGIErrorMiddleware(app.wsgi_app)
+ if self.debug:
+ app.run(host=self.host, port=self.port, debug=self.debug)
+ else:
+ wsgi.server(eventlet.listen((self.host, self.port)), app)
+
+def cleanup(days=30, tags=None):
+ from sentry.models import Group, Event
+ import datetime
+
+ ts = datetime.datetime.now() - datetime.timedelta(days=days)
+
+ for event in Event.objects.order_by('date'):
+ if event.date > ts:
+ break
+ event.delete()
+
+ for group in Group.objects.order_by('last_seen'):
+ if group.last_seen > ts:
+ break
+ event.delete()
+
+
+def upgrade():
+ pass
+ # from sentry.conf import settings
+ #
+ # call_command('syncdb', database=settings.DATABASE_USING or 'default', interactive=False)
+ #
+ # if 'south' in django_settings.INSTALLED_APPS:
+ # call_command('migrate', database=settings.DATABASE_USING or 'default', interactive=False)
+
+def main():
+ command_list = ('start', 'stop', 'restart', 'cleanup', 'upgrade')
+ args = sys.argv
+ if len(args) < 2 or args[1] not in command_list:
+ print "usage: sentry [command] [options]"
+ print
+ print "Available subcommands:"
+ for cmd in command_list:
+ print " ", cmd
+ sys.exit(1)
+
+ parser = OptionParser(version="%%prog %s" % VERSION)
+ parser.add_option('--config', metavar='CONFIG')
+ if args[1] == 'start':
+ parser.add_option('--debug', action='store_true', default=False, dest='debug')
+ parser.add_option('--host', metavar='HOSTNAME')
+ parser.add_option('--port', type=int, metavar='PORT')
+ parser.add_option('--daemon', action='store_true', default=False, dest='daemonize')
+ parser.add_option('--no-daemon', action='store_false', default=False, dest='daemonize')
+ parser.add_option('--pidfile', dest='pidfile')
+ parser.add_option('--logfile', dest='logfile')
+ elif args[1] == 'stop':
+ parser.add_option('--pidfile', dest='pidfile')
+ parser.add_option('--logfile', dest='logfile')
+ elif args[1] == 'cleanup':
+ parser.add_option('--days', default='30', type=int,
+ help='Numbers of days to truncate on.')
+ parser.add_option('--tags',
+ help='Limit truncation to only entries tagged with key:value.')
+
+ (options, args) = parser.parse_args()
+
+ if options.config:
+ app.config.from_pyfile(options.config)
+ else:
+ config_path = os.path.expanduser(os.path.join('~', '.sentry', 'sentry.conf.py'))
+ if os.path.exists(config_path):
+ app.config.from_pyfile(config_path)
+
+ if args[0] == 'upgrade':
+ upgrade()
+
+ elif args[0] == 'start':
+ web = SentryCollector(host=options.host, port=options.port,
+ pidfile=options.pidfile, logfile=options.logfile,
+ daemonize=options.daemonize, debug=options.debug)
+ web.execute(args[0])
+
+ elif args[0] == 'restart':
+ web = SentryCollector()
+ web.execute(args[0])
+
+ elif args[0] == 'stop':
+ web = SentryCollector(pidfile=options.pidfile, logfile=options.logfile)
+ web.execute(args[0])
+
+ elif args[0] == 'cleanup':
+ cleanup(days=options.days, tags=options.tags)
+
+ sys.exit(0)
+
+if __name__ == '__main__':
+ main()
View
0 sentry/web/api.py → sentry/collector/views.py
File renamed without changes.
View
4 sentry/db/backends/redis.py
@@ -40,6 +40,9 @@ def _get_constraint_key(self, schema, kwargs):
## Hash table lookups
+ def create_model(self, schema):
+ return
+
def add(self, schema, **values):
# generates a pk and sets the values
pk = self.generate_key(schema)
@@ -72,7 +75,6 @@ def get_meta(self, schema, pk):
def get_data(self, schema, pk):
return self.conn.hgetall(self._get_data_key(schema, pk))
-
def count(self, schema, index='default'):
return self.conn.zcard(self._get_index_key(schema, index))
View
1 sentry/db/backends/sqlalchemy/__init__.py
@@ -0,0 +1 @@
+from sentry.db.backends.sqlalchemy.backend import SQLAlchemyBackend
View
126 sentry/db/backends/sqlalchemy/backend.py
@@ -0,0 +1,126 @@
+"""
+sentry.db.backends.sqlalchemy
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+from __future__ import absolute_import
+
+from sentry.db.backends.base import SentryBackend
+
+import datetime
+
+from sqlalchemy import create_engine
+from sqlalchemy.sql import select
+
+from sentry.db.backends.sqlalchemy.models import metadata, model_map
+
+class SQLAlchemyBackend(SentryBackend):
+ def __init__(self, uri, **kwargs):
+ self.engine = create_engine(uri, **kwargs)
+ metadata.bind = self.engine
+
+ def create_model(self, schema):
+ metadata.create(model_map[schema])
+
+ ## Hash table lookups
+
+ def add(self, schema, **values):
+ # generates a pk and sets the values
+ pk = self.generate_key(schema)
+ table = model_map[schema]
+ table.insert().execute(id=pk, **values)
+ return pk
+
+ def delete(self, schema, pk):
+ table = model_map[schema]
+ table.delete(table.c.id==pk).execute()
+
+ def set(self, schema, pk, **values):
+ table = model_map[schema]
+ table.update(table.c.id==pk).execute(**values)
+
+ def get(self, schema, pk):
+ table = model_map[schema]
+ query = select([table], table.c.id==pk)
+ return query.execute().fetchone()
+
+ def incr(self, schema, pk, key, amount=1):
+ table = model_map[schema]
+ table.update(table.c.id==pk).execute(getattr(table.c, key)==getattr(table.c, key) + amount)
+
+ # meta data is stored in a seperate key to avoid collissions and heavy getall pulls
+
+ def set_meta(self, schema, pk, **values):
+ self.conn.hmset(self._get_metadata_key(schema, pk), values)
+
+ def get_meta(self, schema, pk):
+ return self.conn.hgetall(self._get_metadata_key(schema, pk))
+
+ def get_data(self, schema, pk):
+ return self.conn.hgetall(self._get_data_key(schema, pk))
+
+
+ def count(self, schema, index='default'):
+ return self.conn.zcard(self._get_index_key(schema, index))
+
+ def list(self, schema, index='default', offset=0, limit=-1, desc=False):
+ if limit > 0:
+ end = offset+limit
+ else:
+ end = limit
+ pk_set = self.conn.zrange(self._get_index_key(schema, index), start=offset, end=end, desc=desc)
+ return [(pk, self.get_data(schema, pk)) for pk in pk_set]
+
+ ## Indexes using sorted sets
+
+ def add_relation(self, from_schema, from_pk, to_schema, to_pk, score):
+ # adds a relation to a sorted index for base instance
+ if isinstance(score, datetime.datetime):
+ score = score.strftime('%s.%m')
+ self.conn.zadd(self._get_relation_key(from_schema, from_pk, to_schema), to_pk, float(score))
+
+ def remove_relation(self, from_schema, from_pk, to_schema, to_pk=None):
+ if to_pk:
+ self.conn.zrem(self._get_relation_key(from_schema, from_pk, to_schema), to_pk)
+ else:
+ self.conn.delete(self._get_relation_key(from_schema, from_pk, to_schema))
+
+ def list_relations(self, from_schema, from_pk, to_schema, offset=0, limit=-1, desc=False):
+ # lists relations in a sorted index for base instance
+ # XXX: this is O(n)+1, ugh
+ if limit > 0:
+ end = offset+limit
+ else:
+ end = limit
+
+ pk_set = self.conn.zrange(self._get_relation_key(from_schema, from_pk, to_schema), start=offset, end=end, desc=desc)
+
+ return [(pk, self.conn.hgetall(self._get_data_key(to_schema, pk))) for pk in pk_set]
+
+ def add_to_index(self, schema, pk, index, score):
+ # adds an instance to a sorted index
+ if isinstance(score, datetime.datetime):
+ score = score.strftime('%s.%m')
+ self.conn.zadd(self._get_index_key(schema, index), pk, float(score))
+
+ def remove_from_index(self, schema, pk, index):
+ self.conn.zrem(self._get_index_key(schema, index), pk)
+
+ ## Generic indexes
+
+ # TODO: can we combine constraint indexes with sort indexes? (at least the API)
+
+ def add_to_cindex(self, schema, pk, **kwargs):
+ # adds an index to a composite index (for checking uniqueness)
+ self.conn.sadd(self._get_constraint_key(schema, kwargs), pk)
+
+ def remove_from_cindex(self, schema, pk, **kwargs):
+ # adds an index to a composite index (for checking uniqueness)
+ self.conn.srem(self._get_constraint_key(schema, kwargs), pk)
+
+ def list_by_cindex(self, schema, **kwargs):
+ # returns a list of keys associated with a constraint
+ return list(self.conn.smembers(self._get_constraint_key(schema, kwargs)))
View
49 sentry/db/backends/sqlalchemy/models.py
@@ -0,0 +1,49 @@
+import sentry.models
+from sentry.db import models
+
+from sqlalchemy import Table, Column, Integer, Float, String, Text, \
+ DateTime, MetaData
+
+__all__ = ('metadata', 'model_map', 'model_meta')
+
+column_map = {
+ models.String: lambda x: String(255, default=x.default, nullable=True),
+ models.Text: lambda x: Text(default=x.default, nullable=True),
+ models.Integer: lambda x: Integer(default=x.default, nullable=True),
+ models.Float: lambda x: Float(default=x.default, nullable=True),
+ models.DateTime: lambda x: DateTime(default=x.default, nullable=True),
+ models.List: lambda x: Text(default=x.default, nullable=True),
+}
+
+model_map = {}
+model_meta = {}
+
+def create_sqlalchemy_def(metadata, model):
+ columns = [
+ Column('id', String(32), primary_key=True),
+ ]
+ for name, field in model._meta.fields:
+ columns.append(Column(name, column_map[field](field), nullable=True))
+
+ table = Table(model.__name__.lower(), metadata, *columns)
+
+ return table
+
+def create_sqlalchemy_metadata_def(metadata, model):
+ columns = [
+ Column('id', String(32), primary_key=True),
+ Column('key', String(255), primary_key=True),
+ Column('value', Text(nullable=True), primary_key=True),
+ ]
+
+ table = Table('%s_metadata' % (model.__name__.lower(),), metadata, *columns)
+
+ return table
+
+metadata = MetaData()
+# metadata.create_all(engine)
+
+for var in dir(sentry.models):
+ if isinstance(var, models.Model):
+ model_map[var] = create_sqlalchemy_def(metadata, var)
+ model_meta[var] = create_sqlalchemy_metadata_def(metadata, var)
View
3 sentry/db/models.py
@@ -396,6 +396,9 @@ def to_python(self, value=None):
value = u''
return value
+class Text(String):
+ pass
+
class Integer(Field):
def to_python(self, value=None):
if value:
View
2 sentry/models.py
@@ -27,7 +27,7 @@ class Group(models.Model):
type = models.String() # length 32
hash = models.String() # length 32
# one line summary used for rendering
- message = models.String()
+ message = models.Text()
state = models.Integer(default=0)
count = models.Integer(default=0)
score = models.Float(default=0.0)
View
0 sentry/scripts/__init__.py → sentry/web/scripts/__init__.py
File renamed without changes.
View
0 sentry/scripts/data_faker.py → sentry/web/scripts/data_faker.py
File renamed without changes.
View
17 sentry/scripts/runner.py → sentry/web/scripts/runner.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
"""
-sentry.scripts.runner
-~~~~~~~~~~~~~~~~~~~~~
+sentry.web.scripts.runner
+~~~~~~~~~~~~~~~~~~~~~~~~~
:copyright: (c) 2010 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
@@ -20,7 +20,7 @@
from sentry import VERSION, app
from sentry.middleware import WSGIErrorMiddleware
-class SentryServer(DaemonRunner):
+class SentryWeb(DaemonRunner):
pidfile_timeout = 10
start_message = u"started with pid %(pid)d"
@@ -62,8 +62,13 @@ def execute(self, action):
self.do_action()
def run(self):
+ # Import views/templatetags to ensure registration
+ import sentry.web.templatetags
+ import sentry.web.views
+
upgrade()
app.wsgi_app = WSGIErrorMiddleware(app.wsgi_app)
+
if self.debug:
app.run(host=self.host, port=self.port, debug=self.debug)
else:
@@ -138,17 +143,17 @@ def main():
upgrade()
elif args[0] == 'start':
- web = SentryServer(host=options.host, port=options.port,
+ web = SentryWeb(host=options.host, port=options.port,
pidfile=options.pidfile, logfile=options.logfile,
daemonize=options.daemonize, debug=options.debug)
web.execute(args[0])
elif args[0] == 'restart':
- web = SentryServer()
+ web = SentryWeb()
web.execute(args[0])
elif args[0] == 'stop':
- web = SentryServer(pidfile=options.pidfile, logfile=options.logfile)
+ web = SentryWeb(pidfile=options.pidfile, logfile=options.logfile)
web.execute(args[0])
elif args[0] == 'cleanup':
View
3 setup.py
@@ -59,7 +59,8 @@
include_package_data=True,
entry_points = {
'console_scripts': [
- 'sentry = sentry.scripts.runner:main',
+ 'sentry-web = sentry.web.scripts.runner:main',
+ 'sentry-collector = sentry.collector.scripts.runner:main',
],
},
classifiers=[

0 comments on commit d47fa7c

Please sign in to comment.