feat: Add native retention concept

- Add option `system.event-retention-days`.
- Add `Quota.get_event_retention(organization)` API.
- Limit search and event endpoint queries to retention days.

src/sentry/api/endpoints/group_events.py

@@ -2,17 +2,20 @@
 
 import six
 
-from sentry import tagstore
+from datetime import timedelta
+from django.db.models import Q
+from django.utils import timezone
+from rest_framework.response import Response
+
+from sentry import quotas, tagstore
 from sentry.api.base import DocSection, EnvironmentMixin
 from sentry.api.bases import GroupEndpoint
 from sentry.api.serializers import serialize
 from sentry.api.paginator import DateTimePaginator
 from sentry.models import Environment, Event, Group
 from sentry.search.utils import parse_query
 from sentry.utils.apidocs import scenario, attach_scenarios
-from rest_framework.response import Response
 from sentry.search.utils import InvalidQuery
-from django.db.models import Q
 
 
 @scenario('ListAvailableSamples')
@@ -101,4 +104,11 @@ def respond(queryset):
 
             events = events.filter(id__in=event_ids)
 
+        # filter out events which are beyond the retention period
+        retention = quotas.get_event_retention(organization=group.project.organization)
+        if retention:
+            events = events.filter(
+                datetime__gte=timezone.now() - timedelta(days=retention)
+            )
+
         return respond(events)

src/sentry/api/endpoints/project_events.py

@@ -1,5 +1,9 @@
 from __future__ import absolute_import
 
+from datetime import timedelta
+from django.utils import timezone
+
+from sentry import quotas
 from sentry.api.base import DocSection
 from sentry.api.bases.project import ProjectEndpoint
 from sentry.api.serializers import serialize
@@ -45,6 +49,13 @@ def get(self, request, project):
                 message__icontains=query,
             )
 
+        # filter out events which are beyond the retention period
+        retention = quotas.get_event_retention(organization=project.organization)
+        if retention:
+            events = events.filter(
+                datetime__gte=timezone.now() - timedelta(days=retention)
+            )
+
         return self.paginate(
             request=request,
             queryset=events,

src/sentry/options/defaults.py

@@ -29,6 +29,7 @@
 register('system.databases', type=Dict, flags=FLAG_NOSTORE)
 # register('system.debug', default=False, flags=FLAG_NOSTORE)
 register('system.rate-limit', default=0, flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
+register('system.event-retention-days', default=0, flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK)
 register('system.secret-key', flags=FLAG_NOSTORE)
 # Absolute URL to the sentry root directory. Should not include a trailing slash.
 register('system.url-prefix', ttl=60, grace=3600, flags=FLAG_REQUIRED | FLAG_PRIORITIZE_DISK)

src/sentry/quotas/base.py

@@ -46,7 +46,7 @@ class Quota(Service):
     """
     __all__ = (
         'get_maximum_quota', 'get_organization_quota', 'get_project_quota', 'is_rate_limited',
-        'translate_quota', 'validate', 'refund',
+        'translate_quota', 'validate', 'refund', 'get_event_retention',
     )
 
     def __init__(self, **options):
@@ -137,3 +137,6 @@ def get_maximum_quota(self, organization):
         Return the maximum capable rate for an organization.
         """
         return (options.get('system.rate-limit'), 60)
+
+    def get_event_retention(self, organization):
+        return options.get('system.event-retention-days') or None

src/sentry/search/django/backend.py

@@ -8,10 +8,12 @@
 
 from __future__ import absolute_import
 
+from datetime import timedelta
 from django.db import router
 from django.db.models import Q
+from django.utils import timezone
 
-from sentry import tagstore
+from sentry import quotas, tagstore
 from sentry.api.paginator import DateTimePaginator, Paginator
 from sentry.search.base import EMPTY, SearchBackend
 from sentry.search.django.constants import (
@@ -256,6 +258,13 @@ def _build_queryset(
         else:
             score_clause = SORT_CLAUSES[sort_by]
 
+        # filter out groups which are beyond the retention period
+        retention = quotas.get_event_retention(organization=project.organization)
+        if retention:
+            queryset = queryset.filter(
+                last_seen__gte=timezone.now() - timedelta(days=retention)
+            )
+
         queryset = queryset.extra(
             select={'sort_value': score_clause},
         )

tests/sentry/api/endpoints/test_group_events.py

@@ -2,6 +2,9 @@
 
 import six
 
+from datetime import timedelta
+from django.utils import timezone
+
 from sentry import tagstore
 from sentry.models import Environment
 from sentry.testutils import APITestCase
@@ -228,3 +231,26 @@ def test_environment(self):
 
         assert response.status_code == 200, response.content
         assert response.data == []
+
+    def test_filters_based_on_retention(self):
+        self.login_as(user=self.user)
+
+        project = self.create_project()
+        group = self.create_group(project=project)
+        self.create_event(
+            'a' * 32,
+            group=group,
+            datetime=timezone.now() - timedelta(days=2),
+        )
+        event_2 = self.create_event('b' * 32, group=group)
+
+        with self.options({'system.event-retention-days': 1}):
+            response = self.client.get('/api/0/issues/{}/events/'.format(group.id))
+
+        assert response.status_code == 200, response.content
+        assert len(response.data) == 1
+        assert sorted(map(lambda x: x['id'], response.data)) == sorted(
+            [
+                six.text_type(event_2.id),
+            ]
+        )

tests/sentry/api/endpoints/test_project_events.py

@@ -2,6 +2,8 @@
 
 import six
 
+from datetime import timedelta
+from django.utils import timezone
 from django.core.urlresolvers import reverse
 
 from sentry.testutils import APITestCase
@@ -33,3 +35,34 @@ def test_simple(self):
                 six.text_type(event_2.id),
             ]
         )
+
+    def test_filters_based_on_retention(self):
+        self.login_as(user=self.user)
+
+        project = self.create_project()
+        group = self.create_group(project=project)
+        self.create_event(
+            'a' *
+            32,
+            group=group,
+            datetime=timezone.now() - timedelta(days=2),
+        )
+        event_2 = self.create_event('b' * 32, group=group)
+
+        with self.options({'system.event-retention-days': 1}):
+            url = reverse(
+                'sentry-api-0-project-events',
+                kwargs={
+                    'organization_slug': project.organization.slug,
+                    'project_slug': project.slug,
+                }
+            )
+            response = self.client.get(url, format='json')
+
+        assert response.status_code == 200, response.content
+        assert len(response.data) == 1
+        assert sorted(map(lambda x: x['id'], response.data)) == sorted(
+            [
+                six.text_type(event_2.id),
+            ]
+        )

tests/sentry/api/endpoints/test_project_group_index.py

@@ -298,6 +298,17 @@ def test_pending_delete_pending_merge_excluded(self):
         assert len(response.data) == 1
         assert response.data[0]['id'] == six.text_type(group.id)
 
+    def test_filters_based_on_retention(self):
+        self.login_as(user=self.user)
+
+        self.create_group(last_seen=timezone.now() - timedelta(days=2))
+
+        with self.options({'system.event-retention-days': 1}):
+            response = self.client.get(self.path)
+
+        assert response.status_code == 200, response.content
+        assert len(response.data) == 0
+
 
 class GroupUpdateTest(APITestCase):
     @fixture