diff --git a/CHANGES b/CHANGES index 39ab73b5f7922d..bde9e746249458 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ Version 8.15 (Unreleased) ------------------------- +- Refactor usage of ``sentry.app`` to use individual modules. + API Changes ~~~~~~~~~~~ diff --git a/bin/load-mocks b/bin/load-mocks index 2e93e50215f193..c8fc107c58895e 100755 --- a/bin/load-mocks +++ b/bin/load-mocks @@ -14,8 +14,7 @@ from pytz import utc from random import randint from uuid import uuid4 -from sentry import roles -from sentry.app import tsdb, buffer +from sentry import buffer, tsdb, roles from sentry.models import ( Activity, Broadcast, File, GroupMeta, Organization, OrganizationAccessRequest, OrganizationMember, Project, Release, ReleaseFile, Team, User, UserReport, diff --git a/src/sentry/api/base.py b/src/sentry/api/base.py index 5b2277b2dd0e63..f4fb2b70744715 100644 --- a/src/sentry/api/base.py +++ b/src/sentry/api/base.py @@ -1,10 +1,10 @@ from __future__ import absolute_import import logging +import six import time -from datetime import datetime, timedelta -import six +from datetime import datetime, timedelta from django.conf import settings from django.utils.http import urlquote from django.views.decorators.csrf import csrf_exempt @@ -16,7 +16,8 @@ from rest_framework.response import Response from rest_framework.views import APIView -from sentry.app import raven, tsdb +from sentry import tsdb +from sentry.app import raven from sentry.models import ApiKey, AuditLogEntry from sentry.utils.cursors import Cursor from sentry.utils.dates import to_datetime @@ -233,7 +234,7 @@ def _parse_args(self, request): resolution = request.GET.get('resolution') if resolution: resolution = self._parse_resolution(resolution) - assert resolution in tsdb.rollups + assert resolution in tsdb.get_rollups() end = request.GET.get('until') if end: diff --git a/src/sentry/api/endpoints/group_details.py b/src/sentry/api/endpoints/group_details.py index 901b3412540a2a..ddfeeb6460ce34 100644 --- a/src/sentry/api/endpoints/group_details.py +++ b/src/sentry/api/endpoints/group_details.py @@ -8,7 +8,7 @@ from rest_framework import serializers from rest_framework.response import Response -from sentry.app import tsdb +from sentry import tsdb from sentry.api import client from sentry.api.base import DocSection from sentry.api.bases import GroupEndpoint diff --git a/src/sentry/api/endpoints/group_stats.py b/src/sentry/api/endpoints/group_stats.py index 9df557158859b7..7d822740f5bc23 100644 --- a/src/sentry/api/endpoints/group_stats.py +++ b/src/sentry/api/endpoints/group_stats.py @@ -2,7 +2,7 @@ from rest_framework.response import Response -from sentry.app import tsdb +from sentry import tsdb from sentry.api.base import StatsMixin from sentry.api.bases.group import GroupEndpoint diff --git a/src/sentry/api/endpoints/internal_stats.py b/src/sentry/api/endpoints/internal_stats.py index 48349154cf0655..377568bcefdbf4 100644 --- a/src/sentry/api/endpoints/internal_stats.py +++ b/src/sentry/api/endpoints/internal_stats.py @@ -2,7 +2,7 @@ from rest_framework.response import Response -from sentry.app import tsdb +from sentry import tsdb from sentry.api.base import Endpoint, StatsMixin from sentry.api.permissions import SuperuserPermission diff --git a/src/sentry/api/endpoints/organization_stats.py b/src/sentry/api/endpoints/organization_stats.py index 4ec4b8c3593f99..205e80baea4022 100644 --- a/src/sentry/api/endpoints/organization_stats.py +++ b/src/sentry/api/endpoints/organization_stats.py @@ -2,9 +2,9 @@ from rest_framework.response import Response +from sentry import tsdb from sentry.api.base import DocSection, StatsMixin from sentry.api.bases.organization import OrganizationEndpoint -from sentry.app import tsdb from sentry.models import Project, Team from sentry.utils.apidocs import attach_scenarios, scenario diff --git a/src/sentry/api/endpoints/project_details.py b/src/sentry/api/endpoints/project_details.py index 6c7a3efbf23249..ca268df3249023 100644 --- a/src/sentry/api/endpoints/project_details.py +++ b/src/sentry/api/endpoints/project_details.py @@ -14,7 +14,7 @@ from sentry.api.decorators import sudo_required from sentry.api.serializers import serialize from sentry.api.serializers.models.plugin import PluginSerializer -from sentry.app import digests +from sentry.digests import backend as digests from sentry.models import ( AuditLogEntryEvent, Group, GroupStatus, Project, ProjectBookmark, ProjectStatus, UserOption, DEFAULT_SUBJECT_TEMPLATE diff --git a/src/sentry/api/endpoints/project_group_index.py b/src/sentry/api/endpoints/project_group_index.py index 87d9d2a490359b..6706f37143bea9 100644 --- a/src/sentry/api/endpoints/project_group_index.py +++ b/src/sentry/api/endpoints/project_group_index.py @@ -10,6 +10,7 @@ from rest_framework import serializers from rest_framework.response import Response +from sentry import search from sentry.api.base import DocSection from sentry.api.bases.project import ProjectEndpoint, ProjectEventPermission from sentry.api.fields import UserField @@ -17,7 +18,6 @@ from sentry.api.serializers.models.group import ( SUBSCRIPTION_REASON_MAP, StreamGroupSerializer ) -from sentry.app import search from sentry.constants import DEFAULT_SORT_OPTION from sentry.db.models.query import create_or_update from sentry.models import ( diff --git a/src/sentry/api/endpoints/project_stats.py b/src/sentry/api/endpoints/project_stats.py index 77f88c29776297..296afa838b0e55 100644 --- a/src/sentry/api/endpoints/project_stats.py +++ b/src/sentry/api/endpoints/project_stats.py @@ -2,7 +2,7 @@ from rest_framework.response import Response -from sentry.app import tsdb +from sentry import tsdb from sentry.api.base import DocSection, StatsMixin from sentry.api.bases.project import ProjectEndpoint from sentry.utils.apidocs import scenario, attach_scenarios diff --git a/src/sentry/api/endpoints/team_stats.py b/src/sentry/api/endpoints/team_stats.py index 8625eceb29b5b9..9debc357da51c9 100644 --- a/src/sentry/api/endpoints/team_stats.py +++ b/src/sentry/api/endpoints/team_stats.py @@ -3,7 +3,7 @@ from rest_framework.response import Response from six.moves import range -from sentry.app import tsdb +from sentry import tsdb from sentry.api.base import DocSection, StatsMixin from sentry.api.bases.team import TeamEndpoint from sentry.models import Project diff --git a/src/sentry/api/serializers/models/group.py b/src/sentry/api/serializers/models/group.py index 46972261409ce5..9f78f2be1d8f04 100644 --- a/src/sentry/api/serializers/models/group.py +++ b/src/sentry/api/serializers/models/group.py @@ -8,8 +8,8 @@ from django.db.models import Q from django.utils import timezone +from sentry import tsdb from sentry.api.serializers import Serializer, register, serialize -from sentry.app import tsdb from sentry.constants import LOG_LEVELS from sentry.models import ( Group, GroupAssignee, GroupBookmark, GroupMeta, GroupResolution, diff --git a/src/sentry/app.py b/src/sentry/app.py index d36959d30640e3..e52a775db0b945 100644 --- a/src/sentry/app.py +++ b/src/sentry/app.py @@ -9,14 +9,11 @@ from threading import local -from django.conf import settings from raven.contrib.django.models import client from sentry.utils import redis -from sentry.utils.imports import import_string from sentry.utils.locking.backends.redis import RedisLockBackend from sentry.utils.locking.manager import LockManager -from sentry.utils import warnings class State(local): @@ -25,36 +22,16 @@ class State(local): env = State() - -def get_instance(attribute, options, dangerous=()): - value = getattr(settings, attribute) - - cls = import_string(value) - if cls in dangerous: - warnings.warn( - warnings.UnsupportedBackend( - u'The {!r} backend for {} is not recommended ' - 'for production use.'.format(value, attribute) - ) - ) - - return cls(**options) - - -# TODO(dcramer): this is getting heavy, we should find a better way to structure -# this -buffer = get_instance('SENTRY_BUFFER', settings.SENTRY_BUFFER_OPTIONS) - -from sentry.digests.backends.dummy import DummyBackend -digests = get_instance('SENTRY_DIGESTS', settings.SENTRY_DIGESTS_OPTIONS, (DummyBackend,)) -quotas = get_instance('SENTRY_QUOTAS', settings.SENTRY_QUOTA_OPTIONS) -nodestore = get_instance('SENTRY_NODESTORE', settings.SENTRY_NODESTORE_OPTIONS) -ratelimiter = get_instance('SENTRY_RATELIMITER', settings.SENTRY_RATELIMITER_OPTIONS) -search = get_instance('SENTRY_SEARCH', settings.SENTRY_SEARCH_OPTIONS) -newsletter = get_instance('SENTRY_NEWSLETTER', settings.SENTRY_NEWSLETTER_OPTIONS) - -from sentry.tsdb.dummy import DummyTSDB -tsdb = get_instance('SENTRY_TSDB', settings.SENTRY_TSDB_OPTIONS, (DummyTSDB,)) +# COMPAT +from .buffer import backend as buffer # NOQA +from .digests import backend as digests # NOQA +from .newsletter import backend as newsletter # NOQA +from .nodestore import backend as nodestore # NOQA +from .quotas import backend as quotas # NOQA +from .ratelimits import backend as ratelimiter # NOQA +from .search import backend as search # NOQA +from .tsdb import backend as tsdb # NOQA raven = client + locks = LockManager(RedisLockBackend(redis.clusters.get('default'))) diff --git a/src/sentry/buffer/__init__.py b/src/sentry/buffer/__init__.py index 537281339e7d3a..d83e903186bdc5 100644 --- a/src/sentry/buffer/__init__.py +++ b/src/sentry/buffer/__init__.py @@ -1,10 +1,12 @@ -""" -sentry.buffer -~~~~~~~~~~~~~ - -:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details. -:license: BSD, see LICENSE for more details. -""" from __future__ import absolute_import +from django.conf import settings + +from sentry.utils.functional import LazyBackendWrapper + from .base import Buffer # NOQA + + +backend = LazyBackendWrapper(Buffer, settings.SENTRY_BUFFER, + settings.SENTRY_BUFFER_OPTIONS) +backend.expose(locals()) diff --git a/src/sentry/buffer/base.py b/src/sentry/buffer/base.py index 1188400fb649df..07c85e380bfa28 100644 --- a/src/sentry/buffer/base.py +++ b/src/sentry/buffer/base.py @@ -37,6 +37,7 @@ class Buffer(object): This is useful in situations where a single event might be happening so fast that the queue cant keep up with the updates. """ + __all__ = ('incr', 'process', 'process_pending', 'validate') def incr(self, model, columns, filters, extra=None): """ diff --git a/src/sentry/db/models/fields/node.py b/src/sentry/db/models/fields/node.py index 27037c5d7bc27c..0a7dc7b858f121 100644 --- a/src/sentry/db/models/fields/node.py +++ b/src/sentry/db/models/fields/node.py @@ -18,6 +18,7 @@ from django.db.models.signals import post_delete from south.modelsinspector import add_introspection_rules +from sentry import nodestore from sentry.utils.cache import memoize from sentry.utils.compat import pickle from sentry.utils.strings import decompress, compress @@ -81,8 +82,6 @@ def copy(self): @memoize def data(self): - from sentry.app import nodestore - if self._node_data is not None: return self._node_data @@ -131,8 +130,6 @@ def contribute_to_class(self, cls, name): weak=False) def on_delete(self, instance, **kwargs): - from sentry.app import nodestore - value = getattr(instance, self.name) if not value.id: return @@ -159,8 +156,6 @@ def to_python(self, value): return NodeData(self, node_id, data) def get_prep_value(self, value): - from sentry.app import nodestore - if not value and self.null: # save ourselves some storage return None diff --git a/src/sentry/db/models/manager.py b/src/sentry/db/models/manager.py index bf9cf230c14ffc..6f23b15d7bc883 100644 --- a/src/sentry/db/models/manager.py +++ b/src/sentry/db/models/manager.py @@ -20,6 +20,7 @@ post_save, post_delete, post_init, class_prepared) from django.utils.encoding import smart_text +from sentry import nodestore from sentry.utils.cache import cache from sentry.utils.hashlib import md5_text @@ -292,8 +293,6 @@ def create_or_update(self, **kwargs): return create_or_update(self.model, **kwargs) def bind_nodes(self, object_list, *node_names): - from sentry.app import nodestore - object_node_list = [] for name in node_names: object_node_list.extend(( diff --git a/src/sentry/digests/__init__.py b/src/sentry/digests/__init__.py index e3ded00f5d760f..fe3bdddec759d7 100644 --- a/src/sentry/digests/__init__.py +++ b/src/sentry/digests/__init__.py @@ -1,8 +1,19 @@ from __future__ import absolute_import from collections import namedtuple +from django.conf import settings from sentry.utils.dates import to_datetime +from sentry.utils.functional import LazyBackendWrapper + +from .backends.base import Backend # NOQA +from .backends.dummy import DummyBackend # NOQA + + +backend = LazyBackendWrapper(Backend, settings.SENTRY_DIGESTS, + settings.SENTRY_DIGESTS_OPTIONS, + (DummyBackend,)) +backend.expose(locals()) class Record(namedtuple('Record', 'key value timestamp')): diff --git a/src/sentry/digests/backends/base.py b/src/sentry/digests/backends/base.py index c536ed640c9722..1171cad5f7d9a9 100644 --- a/src/sentry/digests/backends/base.py +++ b/src/sentry/digests/backends/base.py @@ -56,6 +56,11 @@ class Backend(object): be preempted by a new record being added to the timeline, requiring it to be transitioned to "waiting" instead.) """ + __all__ = ( + 'add', 'delete', 'digest', 'enabled', 'maintenance', 'schedule', + 'validate' + ) + def __init__(self, **options): # The ``minimum_delay`` option defines the default minimum amount of # time (in seconds) to wait between scheduling digests for delivery diff --git a/src/sentry/event_manager.py b/src/sentry/event_manager.py index 6ad828e75f7b2f..27088168f2f4a7 100644 --- a/src/sentry/event_manager.py +++ b/src/sentry/event_manager.py @@ -21,8 +21,9 @@ from hashlib import md5 from uuid import uuid4 -from sentry import eventtypes, features -from sentry.app import buffer, tsdb +from sentry import eventtypes, features, buffer +# we need a bunch of unexposed functions from tsdb +from sentry.tsdb import backend as tsdb from sentry.constants import ( CLIENT_RESERVED_ATTRS, LOG_LEVELS, DEFAULT_LOGGER_NAME, MAX_CULPRIT_LENGTH ) diff --git a/src/sentry/models/group.py b/src/sentry/models/group.py index d8b0392ab6d26a..37aeb954140b4e 100644 --- a/src/sentry/models/group.py +++ b/src/sentry/models/group.py @@ -21,8 +21,7 @@ from django.utils import timezone from django.utils.translation import ugettext_lazy as _ -from sentry import eventtypes -from sentry.app import buffer +from sentry import buffer, eventtypes from sentry.constants import ( DEFAULT_LOGGER_NAME, EVENT_ORDERING_KEY, LOG_LEVELS, MAX_CULPRIT_LENGTH ) diff --git a/src/sentry/newsletter/__init__.py b/src/sentry/newsletter/__init__.py index c3961685ab8def..ecb8c965eaf52c 100644 --- a/src/sentry/newsletter/__init__.py +++ b/src/sentry/newsletter/__init__.py @@ -1 +1,12 @@ from __future__ import absolute_import + +from django.conf import settings + +from sentry.utils.functional import LazyBackendWrapper + +from .base import Newsletter # NOQA + + +backend = LazyBackendWrapper(Newsletter, settings.SENTRY_NEWSLETTER, + settings.SENTRY_NEWSLETTER_OPTIONS) +backend.expose(locals()) diff --git a/src/sentry/newsletter/base.py b/src/sentry/newsletter/base.py index e45886c0ff242e..f3874034465f55 100644 --- a/src/sentry/newsletter/base.py +++ b/src/sentry/newsletter/base.py @@ -2,6 +2,9 @@ class Newsletter(object): + __all__ = ('is_enabled', 'get_subscriptions', 'update_subscription', + 'create_or_update_subscription') + DEFAULT_LIST_ID = 1 enabled = False diff --git a/src/sentry/nodestore/__init__.py b/src/sentry/nodestore/__init__.py index 1afaf2dbc205f7..a7c13eed30f514 100644 --- a/src/sentry/nodestore/__init__.py +++ b/src/sentry/nodestore/__init__.py @@ -1,9 +1,12 @@ -""" -sentry.nodestore -~~~~~~~~~~~~~~~~ +from __future__ import absolute_import -:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details. -:license: BSD, see LICENSE for more details. -""" +from django.conf import settings -from __future__ import absolute_import +from sentry.utils.functional import LazyBackendWrapper + +from .base import NodeStorage # NOQA + + +backend = LazyBackendWrapper(NodeStorage, settings.SENTRY_NODESTORE, + settings.SENTRY_NODESTORE_OPTIONS) +backend.expose(locals()) diff --git a/src/sentry/nodestore/base.py b/src/sentry/nodestore/base.py index 55d01b189a10fa..0e9927ca06c0ea 100644 --- a/src/sentry/nodestore/base.py +++ b/src/sentry/nodestore/base.py @@ -16,6 +16,9 @@ class NodeStorage(local): + __all__ = ('create', 'delete', 'delete_multi', 'get', 'get_multi', 'set', + 'set_multi', 'generate_id', 'cleanup', 'validate') + def validate(self): """ Validates the settings for this backend (i.e. such as proper connection diff --git a/src/sentry/plugins/base/notifier.py b/src/sentry/plugins/base/notifier.py index a09bc2bb95b296..13f836e8ab4246 100644 --- a/src/sentry/plugins/base/notifier.py +++ b/src/sentry/plugins/base/notifier.py @@ -10,7 +10,7 @@ __all__ = ('Notifier',) -from sentry.app import ratelimiter +from sentry import ratelimits class Notifier(object): @@ -30,7 +30,7 @@ def should_notify(self, group, event): project = group.project - rate_limited = ratelimiter.is_limited( + rate_limited = ratelimits.is_limited( project=project, key=self.get_conf_key(), limit=10, diff --git a/src/sentry/plugins/bases/notify.py b/src/sentry/plugins/bases/notify.py index 44c0d1d597d8a8..15e3deb05ec84e 100644 --- a/src/sentry/plugins/bases/notify.py +++ b/src/sentry/plugins/bases/notify.py @@ -12,10 +12,7 @@ from django import forms -from sentry.app import ( - digests, - ratelimiter, -) +from sentry import digests, ratelimits from sentry.digests import get_option_key as get_digest_option_key from sentry.digests.notifications import ( event_to_record, @@ -150,7 +147,7 @@ def get_sendable_users(self, project): return member_set def __is_rate_limited(self, group, event): - return ratelimiter.is_limited( + return ratelimits.is_limited( project=group.project, key=self.get_conf_key(), limit=10, diff --git a/src/sentry/quotas/__init__.py b/src/sentry/quotas/__init__.py index 0078930d5e7cc8..6aa2280c39f15b 100644 --- a/src/sentry/quotas/__init__.py +++ b/src/sentry/quotas/__init__.py @@ -1,10 +1,12 @@ -""" -sentry.quotas -~~~~~~~~~~~~~ - -:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details. -:license: BSD, see LICENSE for more details. -""" from __future__ import absolute_import +from django.conf import settings + +from sentry.utils.functional import LazyBackendWrapper + from .base import Quota # NOQA + + +backend = LazyBackendWrapper(Quota, settings.SENTRY_QUOTAS, + settings.SENTRY_QUOTA_OPTIONS) +backend.expose(locals()) diff --git a/src/sentry/quotas/base.py b/src/sentry/quotas/base.py index e2bb5cf4d369d1..4285a1e9aaf854 100644 --- a/src/sentry/quotas/base.py +++ b/src/sentry/quotas/base.py @@ -44,6 +44,11 @@ class Quota(object): respond whether or not a project has been configured to throttle incoming events if they go beyond the specified quota. """ + __all__ = ( + 'get_maximum_quota', 'get_organization_quota', 'get_project_quota', + 'is_rate_limited', 'translate_quota', 'validate', + ) + def __init__(self, **options): pass diff --git a/src/sentry/ratelimits/__init__.py b/src/sentry/ratelimits/__init__.py index c3961685ab8def..8785546b2833b6 100644 --- a/src/sentry/ratelimits/__init__.py +++ b/src/sentry/ratelimits/__init__.py @@ -1 +1,12 @@ from __future__ import absolute_import + +from django.conf import settings + +from sentry.utils.functional import LazyBackendWrapper + +from .base import RateLimiter # NOQA + + +backend = LazyBackendWrapper(RateLimiter, settings.SENTRY_RATELIMITER, + settings.SENTRY_RATELIMITER_OPTIONS) +backend.expose(locals()) diff --git a/src/sentry/ratelimits/base.py b/src/sentry/ratelimits/base.py index de52da9c612794..fae90e2b2b02ba 100644 --- a/src/sentry/ratelimits/base.py +++ b/src/sentry/ratelimits/base.py @@ -2,6 +2,8 @@ class RateLimiter(object): + __all__ = ('is_limited', 'validate') + window = 60 def validate(self): diff --git a/src/sentry/receivers/core.py b/src/sentry/receivers/core.py index c07f51c83b17da..33581a18da22aa 100644 --- a/src/sentry/receivers/core.py +++ b/src/sentry/receivers/core.py @@ -10,7 +10,7 @@ from functools import wraps from pkg_resources import parse_version as Version -from sentry import options +from sentry import buffer, options from sentry.models import ( Organization, OrganizationMember, Project, User, Team, ProjectKey, TagKey, TagValue, GroupTagValue, GroupTagKey @@ -142,8 +142,6 @@ def create_keys_for_project(instance, created, **kwargs): @buffer_incr_complete.connect(sender=TagValue, weak=False) def record_project_tag_count(filters, created, **kwargs): - from sentry import app - if not created: return @@ -152,7 +150,7 @@ def record_project_tag_count(filters, created, **kwargs): if not project_id: project_id = filters['project'].id - app.buffer.incr(TagKey, { + buffer.incr(TagKey, { 'values_seen': 1, }, { 'project_id': project_id, @@ -162,8 +160,6 @@ def record_project_tag_count(filters, created, **kwargs): @buffer_incr_complete.connect(sender=GroupTagValue, weak=False) def record_group_tag_count(filters, created, extra, **kwargs): - from sentry import app - if not created: return @@ -176,7 +172,7 @@ def record_group_tag_count(filters, created, extra, **kwargs): if not group_id: group_id = filters['group'].id - app.buffer.incr(GroupTagKey, { + buffer.incr(GroupTagKey, { 'values_seen': 1, }, { 'project_id': project_id, diff --git a/src/sentry/rules/conditions/event_frequency.py b/src/sentry/rules/conditions/event_frequency.py index 99daef38aa584b..2978ef21901194 100644 --- a/src/sentry/rules/conditions/event_frequency.py +++ b/src/sentry/rules/conditions/event_frequency.py @@ -10,8 +10,9 @@ from datetime import timedelta from django import forms - from django.utils import timezone + +from sentry.tsdb import backend as tsdb from sentry.rules.conditions.base import EventCondition @@ -42,8 +43,6 @@ class BaseEventFrequencyCondition(EventCondition): label = NotImplemented # subclass must implement def __init__(self, *args, **kwargs): - from sentry.app import tsdb - self.tsdb = kwargs.pop('tsdb', tsdb) super(BaseEventFrequencyCondition, self).__init__(*args, **kwargs) diff --git a/src/sentry/runner/initializer.py b/src/sentry/runner/initializer.py index 50cacd2986217c..dfd727c9f0faad 100644 --- a/src/sentry/runner/initializer.py +++ b/src/sentry/runner/initializer.py @@ -296,20 +296,32 @@ def initialize_app(config, skip_backend_validation=False): def validate_backends(): - from sentry import app + from sentry import ( + buffer, digests, nodestore, quotas, ratelimits, search, tsdb + ) backends = ( - app.buffer, - app.digests, - app.nodestore, - app.quotas, - app.ratelimiter, - app.search, - app.tsdb, + buffer, + digests, + nodestore, + quotas, + ratelimits, + search, + tsdb, ) for backend in backends: - backend.validate() + try: + backend.validate() + except AttributeError as exc: + from .importer import ConfigurationError + from sentry.utils.settings import reraise_as + reraise_as(ConfigurationError( + '{} service failed to call validate()\n{}'.format( + backend.__name__, + six.text_type(exc), + ) + )) def validate_options(settings): diff --git a/src/sentry/search/__init__.py b/src/sentry/search/__init__.py index 21e38d3dbe2c91..6258bfc138bba3 100644 --- a/src/sentry/search/__init__.py +++ b/src/sentry/search/__init__.py @@ -1,10 +1,12 @@ -""" -sentry.search -~~~~~~~~~~~~~ +from __future__ import absolute_import -:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details. -:license: BSD, see LICENSE for more details. -""" -from __future__ import absolute_import, print_function +from django.conf import settings -from .base import * # NOQA +from sentry.utils.functional import LazyBackendWrapper + +from .base import SearchBackend # NOQA + + +backend = LazyBackendWrapper(SearchBackend, settings.SENTRY_SEARCH, + settings.SENTRY_SEARCH_OPTIONS) +backend.expose(locals()) diff --git a/src/sentry/search/base.py b/src/sentry/search/base.py index 89ab8dc3153da8..79bf87eb5e93cf 100644 --- a/src/sentry/search/base.py +++ b/src/sentry/search/base.py @@ -13,6 +13,8 @@ class SearchBackend(object): + __all__ = ('query', 'validate') + def __init__(self, **options): pass diff --git a/src/sentry/static/sentry/debug/icons/demo.js b/src/sentry/static/sentry/debug/icons/demo.js index 6f45f1c409c032..f2eb834dbf5b71 100755 --- a/src/sentry/static/sentry/debug/icons/demo.js +++ b/src/sentry/static/sentry/debug/icons/demo.js @@ -2,9 +2,9 @@ if (!('boxShadow' in document.body.style)) { document.body.setAttribute('class', 'noBoxShadow'); } -document.body.addEventListener("click", function(e) { +document.body.addEventListener('click', function(e) { var target = e.target; - if (target.tagName === "INPUT" && + if (target.tagName === 'INPUT' && target.getAttribute('class').indexOf('liga') === -1) { target.select(); } diff --git a/src/sentry/tasks/deletion.py b/src/sentry/tasks/deletion.py index 1de3e5400716ff..3921aad098c925 100644 --- a/src/sentry/tasks/deletion.py +++ b/src/sentry/tasks/deletion.py @@ -12,6 +12,7 @@ from django.db.models import get_model +from sentry import nodestore from sentry.constants import ObjectStatus from sentry.exceptions import DeleteAborted from sentry.signals import pending_delete @@ -353,7 +354,6 @@ def generic_delete(app_label, model_name, object_id, transaction_id=None, def delete_events(relation, transaction_id=None, limit=10000, chunk_limit=100, logger=None): - from sentry.app import nodestore from sentry.models import Event, EventTag while limit > 0: diff --git a/src/sentry/tasks/digests.py b/src/sentry/tasks/digests.py index fdaa64405c5104..fe60c9f9c217f6 100644 --- a/src/sentry/tasks/digests.py +++ b/src/sentry/tasks/digests.py @@ -23,7 +23,7 @@ name='sentry.tasks.digests.schedule_digests', queue='digests.scheduling') def schedule_digests(): - from sentry.app import digests + from sentry import digests deadline = time.time() @@ -46,7 +46,7 @@ def schedule_digests(): name='sentry.tasks.digests.deliver_digest', queue='digests.delivery') def deliver_digest(key, schedule_timestamp=None): - from sentry.app import digests + from sentry import digests try: plugin, project = split_key(key) diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index b9d7142804a702..a55dd530692a74 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -158,7 +158,8 @@ def record_affected_user(event, **kwargs): @instrumented_task( name='sentry.tasks.index_event_tags', default_retry_delay=60 * 5, max_retries=None) -def index_event_tags(organization_id, project_id, event_id, tags, group_id=None, **kwargs): +def index_event_tags(organization_id, project_id, event_id, tags, group_id=None, + **kwargs): from sentry.models import EventTag, Project, TagKey, TagValue Raven.tags_context({ diff --git a/src/sentry/tasks/process_buffer.py b/src/sentry/tasks/process_buffer.py index 4fe71639a0d441..dcfd76afe5c989 100644 --- a/src/sentry/tasks/process_buffer.py +++ b/src/sentry/tasks/process_buffer.py @@ -23,11 +23,13 @@ def process_pending(): """ Process pending buffers. """ - from sentry import app - lock = app.locks.get('buffer:process_pending', duration=60) + from sentry import buffer + from sentry.app import locks + + lock = locks.get('buffer:process_pending', duration=60) try: with lock.acquire(): - app.buffer.process_pending() + buffer.process_pending() except UnableToAcquireLock as error: logger.warning('process_pending.fail', extra={'error': error}) @@ -38,6 +40,6 @@ def process_incr(**kwargs): """ Processes a buffer event. """ - from sentry import app + from sentry import buffer - app.buffer.process(**kwargs) + buffer.process(**kwargs) diff --git a/src/sentry/templatetags/sentry_admin_helpers.py b/src/sentry/templatetags/sentry_admin_helpers.py index 9bc1fdf6d0d829..c2770592095fd2 100644 --- a/src/sentry/templatetags/sentry_admin_helpers.py +++ b/src/sentry/templatetags/sentry_admin_helpers.py @@ -12,13 +12,13 @@ from django import template from django.utils import timezone +from sentry import tsdb + register = template.Library() @register.filter def with_event_counts(project_list): - from sentry.app import tsdb - end = timezone.now() start = end - datetime.timedelta(days=1) diff --git a/src/sentry/tsdb/__init__.py b/src/sentry/tsdb/__init__.py index c3961685ab8def..9f5de479f87e53 100644 --- a/src/sentry/tsdb/__init__.py +++ b/src/sentry/tsdb/__init__.py @@ -1 +1,14 @@ from __future__ import absolute_import + +from django.conf import settings + +from sentry.utils.functional import LazyBackendWrapper + +from .base import BaseTSDB # NOQA +from .dummy import DummyTSDB + + +backend = LazyBackendWrapper(BaseTSDB, settings.SENTRY_TSDB, + settings.SENTRY_TSDB_OPTIONS, + (DummyTSDB,)) +backend.expose(locals()) diff --git a/src/sentry/tsdb/base.py b/src/sentry/tsdb/base.py index 3742f7aca8f84e..139db1d32a1bbd 100644 --- a/src/sentry/tsdb/base.py +++ b/src/sentry/tsdb/base.py @@ -7,10 +7,10 @@ """ from __future__ import absolute_import +import six + from collections import OrderedDict from datetime import timedelta - -import six from django.conf import settings from django.utils import timezone from enum import Enum @@ -75,6 +75,11 @@ class TSDBModel(Enum): class BaseTSDB(object): + __all__ = ( + 'models', 'incr', 'incr_multi', 'get_range', 'get_rollups', 'get_sums', + 'rollup', 'validate', + ) + models = TSDBModel def __init__(self, rollups=None, legacy_rollups=None): @@ -100,6 +105,9 @@ def validate(self): Raise ``InvalidConfiguration`` if there is a configuration error. """ + def get_rollups(self): + return self.rollups + def normalize_to_epoch(self, timestamp, seconds): """ Given a ``timestamp`` (datetime object) normalize to an epoch timestamp. diff --git a/src/sentry/utils/functional.py b/src/sentry/utils/functional.py index ab66769d719f46..ce42df6a9db061 100644 --- a/src/sentry/utils/functional.py +++ b/src/sentry/utils/functional.py @@ -1,6 +1,12 @@ from __future__ import absolute_import -from django.utils.functional import empty +import inspect + +from django.utils.functional import empty, LazyObject + +from sentry.utils import warnings + +from .imports import import_string def extract_lazy_object(lo): @@ -39,3 +45,50 @@ def apply_values(function, mapping): function(values), ), ) + + +class LazyBackendWrapper(LazyObject): + """ + Lazyily instantiates a standard Sentry backend class. + + >>> LazyBackendWrapper(BaseClass, 'path.to.import.Backend', {}) + + Provides an ``expose`` method for dumping public APIs to a context, such as + module locals: + + >>> backend = LazyBackendWrapper(...) + >>> backend.expose(locals()) + """ + def __init__(self, backend_base, backend_path, options, dangerous=()): + super(LazyBackendWrapper, self).__init__() + self.__dict__.update({ + '_backend': backend_path, + '_options': options, + '_base': backend_base, + '_dangerous': dangerous, + }) + + def __getattr__(self, name): + if self._wrapped is empty: + self._setup() + return getattr(self._wrapped, name) + + def _setup(self): + backend = import_string(self._backend) + if backend in self._dangerous: + warnings.warn( + warnings.UnsupportedBackend( + u'The {!r} backend for {} is not recommended ' + 'for production use.'.format(self._backend, self._base) + ) + ) + instance = backend(**self._options) + self._wrapped = instance + + def expose(self, context): + base = self._base + for key in base.__all__: + if inspect.ismethod(getattr(base, key)): + context[key] = (lambda f: lambda *a, **k: getattr(self, f)(*a, **k))(key) + else: + context[key] = getattr(base, key) diff --git a/src/sentry/utils/javascript.py b/src/sentry/utils/javascript.py index 81d6f19c6c4ff6..5f93eec41a652c 100644 --- a/src/sentry/utils/javascript.py +++ b/src/sentry/utils/javascript.py @@ -15,7 +15,8 @@ from django.utils import timezone from django.utils.html import escape -from sentry.app import env, tsdb +from sentry import tsdb +from sentry.app import env from sentry.constants import TAG_LABELS from sentry.models import ( Group, GroupBookmark, GroupMeta, GroupTagKey, GroupSeen, GroupStatus, diff --git a/src/sentry/utils/metrics.py b/src/sentry/utils/metrics.py index 206934b546efc4..05ab9a8f6f50a5 100644 --- a/src/sentry/utils/metrics.py +++ b/src/sentry/utils/metrics.py @@ -2,11 +2,12 @@ __all__ = ['timing', 'incr'] +import logging + from contextlib import contextmanager from django.conf import settings from random import random from time import time -import logging def get_default_backend(): @@ -40,7 +41,7 @@ def _sampled_value(value): def _incr_internal(key, instance=None, tags=None, amount=1): - from sentry.app import tsdb + from sentry import tsdb if _should_sample(): amount = _sampled_value(amount) diff --git a/src/sentry/utils/pytest/sentry.py b/src/sentry/utils/pytest/sentry.py index 8fa5b91d6cc3d5..749a7d50bb2806 100644 --- a/src/sentry/utils/pytest/sentry.py +++ b/src/sentry/utils/pytest/sentry.py @@ -156,8 +156,9 @@ def pytest_configure(config): def pytest_runtest_teardown(item): - from sentry.app import tsdb - tsdb.flush() + from sentry import tsdb + # TODO(dcramer): this only works if this is the correct tsdb backend + tsdb.backend.flush() from sentry.utils.redis import clusters diff --git a/src/sentry/utils/raven.py b/src/sentry/utils/raven.py index 76e178cd39d993..ce09155df1e77b 100644 --- a/src/sentry/utils/raven.py +++ b/src/sentry/utils/raven.py @@ -63,7 +63,7 @@ def send(self, **kwargs): if not is_current_event_safe(): return - from sentry.app import tsdb + from sentry import tsdb from sentry.coreapi import ClientApiHelper from sentry.event_manager import EventManager from sentry.models import Project diff --git a/src/sentry/web/api.py b/src/sentry/web/api.py index 2b5757ffecd818..9a54da75bfb66d 100644 --- a/src/sentry/web/api.py +++ b/src/sentry/web/api.py @@ -17,7 +17,7 @@ from functools import wraps from raven.contrib.django.models import client as Raven -from sentry import app +from sentry import quotas, tsdb from sentry.coreapi import ( APIError, APIForbidden, APIRateLimited, ClientApiHelper, CspApiHelper, LazyData @@ -335,11 +335,11 @@ def process(self, request, project, auth, helper, data, **kwargs): ) if helper.should_filter(project, data, ip_address=remote_addr): - app.tsdb.incr_multi([ - (app.tsdb.models.project_total_received, project.id), - (app.tsdb.models.project_total_blacklisted, project.id), - (app.tsdb.models.organization_total_received, project.organization_id), - (app.tsdb.models.organization_total_blacklisted, project.organization_id), + tsdb.incr_multi([ + (tsdb.models.project_total_received, project.id), + (tsdb.models.project_total_blacklisted, project.id), + (tsdb.models.organization_total_received, project.organization_id), + (tsdb.models.organization_total_blacklisted, project.organization_id), ]) metrics.incr('events.blacklisted') event_filtered.send_robust( @@ -350,7 +350,7 @@ def process(self, request, project, auth, helper, data, **kwargs): raise APIForbidden('Event dropped due to filter') # TODO: improve this API (e.g. make RateLimit act on __ne__) - rate_limit = safe_execute(app.quotas.is_rate_limited, project=project, + rate_limit = safe_execute(quotas.is_rate_limited, project=project, _with_transaction=False) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) @@ -360,11 +360,11 @@ def process(self, request, project, auth, helper, data, **kwargs): if rate_limit is None or rate_limit.is_limited: if rate_limit is None: helper.log.debug('Dropped event due to error with rate limiter') - app.tsdb.incr_multi([ - (app.tsdb.models.project_total_received, project.id), - (app.tsdb.models.project_total_rejected, project.id), - (app.tsdb.models.organization_total_received, project.organization_id), - (app.tsdb.models.organization_total_rejected, project.organization_id), + tsdb.incr_multi([ + (tsdb.models.project_total_received, project.id), + (tsdb.models.project_total_rejected, project.id), + (tsdb.models.organization_total_received, project.organization_id), + (tsdb.models.organization_total_rejected, project.organization_id), ]) metrics.incr('events.dropped', tags={ 'reason': rate_limit.reason_code if rate_limit else 'unknown', @@ -378,9 +378,9 @@ def process(self, request, project, auth, helper, data, **kwargs): if rate_limit is not None: raise APIRateLimited(rate_limit.retry_after) else: - app.tsdb.incr_multi([ - (app.tsdb.models.project_total_received, project.id), - (app.tsdb.models.organization_total_received, project.organization_id), + tsdb.incr_multi([ + (tsdb.models.project_total_received, project.id), + (tsdb.models.organization_total_received, project.organization_id), ]) org_options = OrganizationOption.objects.get_all_values(project.organization_id) diff --git a/src/sentry/web/frontend/accounts.py b/src/sentry/web/frontend/accounts.py index 584a39f573e6f0..cd87a2c6704744 100644 --- a/src/sentry/web/frontend/accounts.py +++ b/src/sentry/web/frontend/accounts.py @@ -27,7 +27,7 @@ from social_auth.models import UserSocialAuth from sudo.decorators import sudo_required -from sentry.app import newsletter +from sentry import newsletter from sentry.models import ( UserEmail, LostPasswordHash, Project, UserOption, Authenticator ) diff --git a/src/social_auth/__init__.py b/src/social_auth/__init__.py index b6c3a334443203..1172d377f6bd4c 100644 --- a/src/social_auth/__init__.py +++ b/src/social_auth/__init__.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +import six + version = (0, 7, 28) -__version__ = '.'.join(map(str, version)) +__version__ = '.'.join(map(six.text_type, version)) diff --git a/src/social_auth/backends/pipeline/user.py b/src/social_auth/backends/pipeline/user.py index ba3258181d12bc..2a2dc856835d14 100644 --- a/src/social_auth/backends/pipeline/user.py +++ b/src/social_auth/backends/pipeline/user.py @@ -28,7 +28,7 @@ def get_username(details, user=None, if email_as_username and details.get('email'): username = details['email'] elif details.get('username'): - username = unicode(details['username']) + username = six.text_type(details['username']) else: username = uuid4().get_hex() diff --git a/src/social_auth/fields.py b/src/social_auth/fields.py index f66092f764bb7c..9016707c4f4be1 100644 --- a/src/social_auth/fields.py +++ b/src/social_auth/fields.py @@ -24,7 +24,7 @@ def to_python(self, value): try: return simplejson.loads(value) except Exception as e: - raise ValidationError(str(e)) + raise ValidationError(six.text_type(e)) else: return value @@ -36,14 +36,14 @@ def validate(self, value, model_instance): try: simplejson.loads(value) except Exception as e: - raise ValidationError(str(e)) + raise ValidationError(six.text_type(e)) def get_prep_value(self, value): """Convert value to JSON string before save""" try: return simplejson.dumps(value) except Exception as e: - raise ValidationError(str(e)) + raise ValidationError(six.text_type(e)) def value_to_string(self, obj): """Return value from object converted to string properly""" @@ -57,5 +57,5 @@ def value_from_object(self, obj): try: from south.modelsinspector import add_introspection_rules add_introspection_rules([], ["^social_auth\.fields\.JSONField"]) -except: +except Exception: pass diff --git a/tests/sentry/api/endpoints/test_group_stats.py b/tests/sentry/api/endpoints/test_group_stats.py index fc69f1101711c4..d23f26c773cc64 100644 --- a/tests/sentry/api/endpoints/test_group_stats.py +++ b/tests/sentry/api/endpoints/test_group_stats.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from sentry.app import tsdb +from sentry import tsdb from sentry.testutils import APITestCase diff --git a/tests/sentry/api/endpoints/test_organization_stats.py b/tests/sentry/api/endpoints/test_organization_stats.py index dea69cb81c0f4a..802966ebdafd8b 100644 --- a/tests/sentry/api/endpoints/test_organization_stats.py +++ b/tests/sentry/api/endpoints/test_organization_stats.py @@ -5,7 +5,7 @@ from django.core.urlresolvers import reverse -from sentry.app import tsdb +from sentry import tsdb from sentry.testutils import APITestCase @@ -26,6 +26,20 @@ def test_simple(self): assert point[1] == 0 assert len(response.data) == 24 + def test_resolution(self): + self.login_as(user=self.user) + + org = self.create_organization(owner=self.user, name='baz') + + tsdb.incr(tsdb.models.organization_total_received, org.id, count=3) + + url = reverse('sentry-api-0-organization-stats', args=[org.slug]) + response = self.client.get('{}?resolution=1d'.format(url), format='json') + + assert response.status_code == 200, response.content + assert response.data[-1][1] == 3, response.data + assert len(response.data) == 1 + def test_id_filtering(self): self.login_as(user=self.user) diff --git a/tests/sentry/api/endpoints/test_project_group_index.py b/tests/sentry/api/endpoints/test_project_group_index.py index ad92cbeeb893af..dfe756ee4efd0c 100644 --- a/tests/sentry/api/endpoints/test_project_group_index.py +++ b/tests/sentry/api/endpoints/test_project_group_index.py @@ -83,7 +83,6 @@ def test_simple_pagination(self): assert links['previous']['results'] == 'false' assert links['next']['results'] == 'true' - print(links['next']['cursor']) response = self.client.get(links['next']['href'], format='json') assert response.status_code == 200 assert len(response.data) == 1 diff --git a/tests/sentry/api/endpoints/test_project_stats.py b/tests/sentry/api/endpoints/test_project_stats.py index d3a7f933f2604f..7bab07a547f268 100644 --- a/tests/sentry/api/endpoints/test_project_stats.py +++ b/tests/sentry/api/endpoints/test_project_stats.py @@ -2,7 +2,7 @@ from django.core.urlresolvers import reverse -from sentry.app import tsdb +from sentry import tsdb from sentry.testutils import APITestCase diff --git a/tests/sentry/api/endpoints/test_team_stats.py b/tests/sentry/api/endpoints/test_team_stats.py index 08239e70bc0640..ad027f98c7d19b 100644 --- a/tests/sentry/api/endpoints/test_team_stats.py +++ b/tests/sentry/api/endpoints/test_team_stats.py @@ -2,7 +2,7 @@ from django.core.urlresolvers import reverse -from sentry.app import tsdb +from sentry import tsdb from sentry.testutils import APITestCase diff --git a/tests/sentry/app/__init__.py b/tests/sentry/app/__init__.py deleted file mode 100644 index c3961685ab8def..00000000000000 --- a/tests/sentry/app/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import absolute_import diff --git a/tests/sentry/app/tests.py b/tests/sentry/app/tests.py deleted file mode 100644 index 99caa74b4570c0..00000000000000 --- a/tests/sentry/app/tests.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from sentry import app -from sentry.testutils import TestCase - - -class AppTest(TestCase): - def test_buffer_is_a_buffer(self): - from sentry.buffer.base import Buffer - self.assertEquals(type(app.buffer), Buffer) diff --git a/tests/sentry/interfaces/test_exception.py b/tests/sentry/interfaces/test_exception.py index a5d8f60696d5f0..923371b781c658 100644 --- a/tests/sentry/interfaces/test_exception.py +++ b/tests/sentry/interfaces/test_exception.py @@ -319,7 +319,6 @@ def test_over_max(self): assert len(value.stacktrace.frames) == 5 for f_num, frame in enumerate(value.stacktrace.frames): assert frame.filename == 'exc %d frame %d' % (e_num, f_num) - print(frame.filename) if e_num in (0, 4): assert frame.vars is not None assert frame.pre_context is not None diff --git a/tests/sentry/lang/native/test_plugin.py b/tests/sentry/lang/native/test_plugin.py index 5e24eb67dcccb8..1f1e5bf9dbdb43 100644 --- a/tests/sentry/lang/native/test_plugin.py +++ b/tests/sentry/lang/native/test_plugin.py @@ -365,7 +365,6 @@ def test_frame_resolution_no_sdk_info(self): assert resp.status_code == 200 event = Event.objects.get() - print event.data['errors'] bt = event.interfaces['sentry.interfaces.Exception'].values[0].stacktrace frames = bt.frames diff --git a/tests/sentry/rules/conditions/test_event_frequency.py b/tests/sentry/rules/conditions/test_event_frequency.py index 796218d6730536..eb2a588de4c4cd 100644 --- a/tests/sentry/rules/conditions/test_event_frequency.py +++ b/tests/sentry/rules/conditions/test_event_frequency.py @@ -7,7 +7,7 @@ import mock import six -from sentry.app import tsdb +from sentry.tsdb import backend as tsdb from sentry.rules.conditions.event_frequency import ( EventFrequencyCondition, EventUniqueUserFrequencyCondition ) diff --git a/tests/sentry/tasks/process_buffer/tests.py b/tests/sentry/tasks/process_buffer/tests.py index b42eaa0999b4a1..21c911945b8923 100644 --- a/tests/sentry/tasks/process_buffer/tests.py +++ b/tests/sentry/tasks/process_buffer/tests.py @@ -4,15 +4,23 @@ import mock -from sentry.tasks.process_buffer import process_incr +from sentry.tasks.process_buffer import process_incr, process_pending from sentry.testutils import TestCase class ProcessIncrTest(TestCase): - @mock.patch('sentry.app.buffer.process') + @mock.patch('sentry.buffer.backend.process') def test_calls_process(self, process): model = mock.Mock() columns = {'times_seen': 1} filters = {'pk': 1} process_incr(model=model, columns=columns, filters=filters) process.assert_called_once_with(model=model, columns=columns, filters=filters) + + +class ProcessPendingTest(TestCase): + @mock.patch('sentry.buffer.backend.process_pending') + def test_nothing(self, mock_process_pending): + # this effectively just says "does the code run" + process_pending() + mock_process_pending.assert_called_once_with()