diff --git a/conftest.py b/conftest.py index dc72a1dfa35f3e..f79b0be26179c5 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,4 @@ from django.conf import settings -import base64 import os import os.path @@ -57,7 +56,8 @@ def __contains__(self, search): settings.INSTALLED_APPS = tuple(settings.INSTALLED_APPS) + ( 'tests', ) - settings.SENTRY_KEY = base64.b64encode(os.urandom(40)) + # Need a predictable key for tests that involve checking signatures + settings.SENTRY_KEY = 'abc123' settings.SENTRY_PUBLIC = False # This speeds up the tests considerably, pbkdf2 is by design, slow. settings.PASSWORD_HASHERS = [ @@ -67,3 +67,4 @@ def __contains__(self, search): # enable draft features settings.SENTRY_ENABLE_EXPLORE_CODE = True settings.SENTRY_ENABLE_EXPLORE_USERS = True + settings.SENTRY_ENABLE_EMAIL_REPLIES = True diff --git a/docs/config/index.rst b/docs/config/index.rst index fbf8c7332cf2f1..f877b4aaac1961 100644 --- a/docs/config/index.rst +++ b/docs/config/index.rst @@ -176,6 +176,47 @@ The following settings are available for the built-in UDP API server: SENTRY_UDP_PORT = 9001 +.. _config-smtp-server: + +SMTP Server +~~~~~~~~~~~ + +The following settings are available for the built-in SMTP mail server: + +.. data:: SENTRY_SMTP_HOST + :noindex: + + The hostname which the smtp server should bind to. + + Defaults to ``localhost``. + + :: + + SENTRY_SMTP_HOST = '0.0.0.0' # bind to all addresses + +.. data:: SENTRY_SMTP_PORT + :noindex: + + The port which the smtp server should listen on. + + Defaults to ``1025``. + + :: + + SENTRY_SMTP_PORT = 1025 + +.. data:: SENTRY_SMTP_HOSTNAME + :noindex: + + The hostname which matches the server's MX record. + + Defaults to ``localhost``. + + :: + + SENTRY_SMTP_HOSTNAME = 'reply.getsentry.com' + + Data Sampling ------------- diff --git a/docs/quickstart/nginx.rst b/docs/quickstart/nginx.rst index b1f430e9f1068c..969856137e3746 100644 --- a/docs/quickstart/nginx.rst +++ b/docs/quickstart/nginx.rst @@ -65,3 +65,40 @@ as well the ``sentry.wsgi`` module: ; allow longer headers for raven.js if applicable ; default: 4096 buffer-size = 32768 + + +Proxying Incoming Email +~~~~~~~~~~~~~~~~~~~~~~~ + +Nginx is recommended for handling incoming emails in front of the Sentry smtp server. + +Below is a sample configuration for Nginx: + +:: + http { + # Bind an http server to localhost only just for the smtp auth + server { + listen 127.0.0.1:80; + + # Return back the address and port for the listening + # Sentry smtp server. Default is 127.0.0.1:1025. + location = /smtp { + add_header Auth-Server 127.0.0.1; + add_header Auth-Port 1025; + return 200; + } + } + } + + mail { + auth_http localhost/smtp; + + server { + listen 25; + + protocol smtp; + proxy on; + smtp_auth none; + xclient off; + } + } diff --git a/setup.py b/setup.py index dcd829aa559d54..e5a186a8ab8814 100755 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ 'django-social-auth>=0.7.28,<0.8.0', 'django-static-compiler>=0.3.0,<0.4.0', 'django-templatetag-sugar>=0.1.0,<0.2.0', + 'email-reply-parser>=0.2.0,<0.3.0', 'gunicorn>=0.17.2,<0.18.0', 'httpagentparser>=1.2.1,<1.3.0', 'logan>=0.5.8.2,<0.6.0', diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 71129f4ddfdefe..54736da49af7a1 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -464,6 +464,12 @@ SENTRY_UDP_HOST = 'localhost' SENTRY_UDP_PORT = 9001 +# SMTP Service +SENTRY_ENABLE_EMAIL_REPLIES = False +SENTRY_SMTP_HOSTNAME = 'localhost' +SENTRY_SMTP_HOST = 'localhost' +SENTRT_SMTP_PORT = 25 + SENTRY_ALLOWED_INTERFACES = set([ 'sentry.interfaces.Exception', 'sentry.interfaces.Message', diff --git a/src/sentry/management/commands/start.py b/src/sentry/management/commands/start.py index ca2cb3173cd52f..3ccdae9c4e330b 100644 --- a/src/sentry/management/commands/start.py +++ b/src/sentry/management/commands/start.py @@ -32,7 +32,7 @@ class Command(BaseCommand): ) def handle(self, service_name='http', address=None, upgrade=True, **options): - from sentry.services import http, udp + from sentry.services import http, udp, smtp if address: if ':' in address: @@ -47,6 +47,7 @@ def handle(self, service_name='http', address=None, upgrade=True, **options): services = { 'http': http.SentryHTTPServer, 'udp': udp.SentryUDPServer, + 'smtp': smtp.SentrySMTPServer, } if service_name == 'worker': diff --git a/src/sentry/models.py b/src/sentry/models.py index 6ed31d4070f2e9..12eebb412020c7 100644 --- a/src/sentry/models.py +++ b/src/sentry/models.py @@ -1099,7 +1099,7 @@ def save(self, *args, **kwargs): self.event.update(num_comments=F('num_comments') + 1) def send_notification(self): - from sentry.utils.email import MessageBuilder + from sentry.utils.email import MessageBuilder, group_id_to_email if self.type != Activity.NOTE or not self.group: return @@ -1156,11 +1156,16 @@ def send_notification(self): 'link': self.group.get_absolute_url(), } + headers = { + 'X-Sentry-Reply-To': group_id_to_email(self.group.pk), + } + msg = MessageBuilder( subject=subject, context=context, template='sentry/emails/new_note.txt', html_template='sentry/emails/new_note.html', + headers=headers, ) try: diff --git a/src/sentry/plugins/sentry_mail/models.py b/src/sentry/plugins/sentry_mail/models.py index e4159b03dce302..5137626a49a13d 100644 --- a/src/sentry/plugins/sentry_mail/models.py +++ b/src/sentry/plugins/sentry_mail/models.py @@ -9,14 +9,13 @@ from django.conf import settings from django.core.urlresolvers import reverse -from django.template.loader import render_to_string from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ from sentry.models import User, UserOption from sentry.plugins import register from sentry.plugins.bases.notify import NotificationPlugin from sentry.utils.cache import cache -from sentry.utils.email import MessageBuilder +from sentry.utils.email import MessageBuilder, group_id_to_email from sentry.utils.http import absolute_uri NOTSET = object() @@ -33,8 +32,8 @@ class MailPlugin(NotificationPlugin): project_conf_form = None subject_prefix = settings.EMAIL_SUBJECT_PREFIX - def _send_mail(self, subject, body, html_body=None, project=None, - headers=None, fail_silently=False): + def _send_mail(self, subject, template=None, html_template=None, body=None, + project=None, headers=None, context=None, fail_silently=False): send_to = self.get_send_to(project) if not send_to: return @@ -43,9 +42,11 @@ def _send_mail(self, subject, body, html_body=None, project=None, msg = MessageBuilder( subject='%s%s' % (subject_prefix, subject), + template=template, + html_template=html_template, body=body, - html_body=html_body, headers=headers, + context=context, ) msg.send(send_to, fail_silently=fail_silently) @@ -66,8 +67,14 @@ def on_alert(self, alert): project.name.encode('utf-8'), alert.message.encode('utf-8'), ) - body = self.get_alert_plaintext_body(alert) - html_body = self.get_alert_html_body(alert) + template = 'sentry/emails/alert.txt' + html_template = 'sentry/emails/alert.html' + + context = { + 'alert': alert, + 'link': alert.get_absolute_url(), + 'settings_link': self.get_notification_settings_url(), + } headers = { 'X-Sentry-Project': project.name, @@ -75,26 +82,14 @@ def on_alert(self, alert): self._send_mail( subject=subject, - body=body, - html_body=html_body, + template=template, + html_template=html_template, project=project, fail_silently=False, headers=headers, + context=context, ) - def get_alert_plaintext_body(self, alert): - return render_to_string('sentry/emails/alert.txt', { - 'alert': alert, - 'link': alert.get_absolute_url(), - }) - - def get_alert_html_body(self, alert): - return render_to_string('sentry/emails/alert.html', { - 'alert': alert, - 'link': alert.get_absolute_url(), - 'settings_link': self.get_notification_settings_url(), - }) - def get_emails_for_users(self, user_ids, project=None): email_list = set() user_ids = set(user_ids) @@ -172,43 +167,35 @@ def notify_users(self, group, event, fail_silently=False): link = group.get_absolute_url() - body = self.get_plaintext_body(group, event, link, interface_list) + template = 'sentry/emails/error.txt' + html_template = 'sentry/emails/error.html' - html_body = self.get_html_body(group, event, link, interface_list) + context = { + 'group': group, + 'event': event, + 'link': link, + 'interfaces': interface_list, + 'settings_link': self.get_notification_settings_url(), + } headers = { 'X-Sentry-Logger': event.logger, 'X-Sentry-Logger-Level': event.get_level_display(), 'X-Sentry-Project': project.name, 'X-Sentry-Server': event.server_name, + 'X-Sentry-Reply-To': group_id_to_email(group.pk), } self._send_mail( subject=subject, - body=body, - html_body=html_body, + template=template, + html_template=html_template, project=project, fail_silently=fail_silently, headers=headers, + context=context, ) - def get_plaintext_body(self, group, event, link, interface_list): - return render_to_string('sentry/emails/error.txt', { - 'group': group, - 'event': event, - 'link': link, - 'interfaces': interface_list, - }) - - def get_html_body(self, group, event, link, interface_list): - return render_to_string('sentry/emails/error.html', { - 'group': group, - 'event': event, - 'link': link, - 'interfaces': interface_list, - 'settings_link': self.get_notification_settings_url(), - }) - # Legacy compatibility MailProcessor = MailPlugin diff --git a/src/sentry/services/smtp.py b/src/sentry/services/smtp.py new file mode 100644 index 00000000000000..45f35ef874e5aa --- /dev/null +++ b/src/sentry/services/smtp.py @@ -0,0 +1,97 @@ +""" +sentry.services.smtp +~~~~~~~~~~~~~~~~~~~~ + +:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details. +:license: BSD, see LICENSE for more details. +""" + +import asyncore +import email +import logging +from smtpd import SMTPServer, SMTPChannel + +from email_reply_parser import EmailReplyParser + +from sentry.services.base import Service +from sentry.tasks.email import process_inbound_email +from sentry.utils.email import email_to_group_id + +logger = logging.getLogger(__name__) + + +# HACK(mattrobenolt): literally no idea what I'm doing. Mostly made this up. +# SMTPChannel doesn't support EHLO response, but nginx requires an EHLO. +# EHLO is available in python 3, so this is backported somewhat +def smtp_EHLO(self, arg): + if not arg: + self.push('501 Syntax: EHLO hostname') + return + if self._SMTPChannel__greeting: + self.push('503 Duplicate HELO/EHLO') + else: + self._SMTPChannel__greeting = arg + self.push('250 %s' % self._SMTPChannel__fqdn) + +SMTPChannel.smtp_EHLO = smtp_EHLO + + +STATUS = { + 200: '200 Ok', + 550: '550 Not found', + 552: '552 Message too long', +} + + +class SentrySMTPServer(Service, SMTPServer): + name = 'smtp' + max_message_length = 20000 # This might be too conservative + + def __init__(self, host=None, port=None, debug=False, workers=None): + from django.conf import settings + + self.host = host or getattr(settings, 'SENTRY_SMTP_HOST', '0.0.0.0') + self.port = port or getattr(settings, 'SENTRY_SMTP_PORT', 1025) + + def process_message(self, peer, mailfrom, rcpttos, raw_message): + if not len(rcpttos): + logger.info('Incoming email had no recipients. Ignoring.') + return STATUS[550] + + if len(raw_message) > self.max_message_length: + logger.info('Inbound email message was too long: %d', len(raw_message)) + return STATUS[552] + + try: + group_id = email_to_group_id(rcpttos[0]) + except Exception: + logger.info('%r is not a valid email address', rcpttos) + return STATUS[550] + + message = email.message_from_string(raw_message) + payload = None + if message.is_multipart(): + for msg in message.walk(): + if msg.get_content_type() == 'text/plain': + payload = msg.get_payload() + break + if payload is None: + # No text/plain part, bailing + return STATUS[200] + else: + payload = message.get_payload() + + payload = EmailReplyParser.parse_reply(payload).strip() + if not payload: + # If there's no body, we don't need to go any further + return STATUS[200] + + process_inbound_email.delay(mailfrom, group_id, payload) + return STATUS[200] + + def run(self): + SMTPServer.__init__(self, (self.host, self.port), None) + try: + asyncore.loop() + except KeyboardInterrupt: + pass diff --git a/src/sentry/tasks/email.py b/src/sentry/tasks/email.py new file mode 100644 index 00000000000000..7317f868caa047 --- /dev/null +++ b/src/sentry/tasks/email.py @@ -0,0 +1,51 @@ +""" +sentry.tasks.email +~~~~~~~~~~~~~~~~~~ + +:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details. +:license: BSD, see LICENSE for more details. +""" +import logging +from celery.task import task + +logger = logging.getLogger(__name__) + + +@task(name='sentry.tasks.email.process_inbound_email', queue='email') +def process_inbound_email(mailfrom, group_id, payload): + """ + """ + from django.contrib.auth.models import User + from sentry.models import Event, Group, Project + from sentry.web.forms import NewNoteForm + + try: + user = User.objects.get(email__iexact=mailfrom) + except User.DoesNotExist: + logger.warning('Inbound email from unknown address: %s', mailfrom) + return + except User.MultipleObjectsReturned: + logger.warning('Inbound email address matches multiple accounts: %s', mailfrom) + return + + try: + group = Group.objects.select_related('project', 'team').get(pk=group_id) + except Group.DoesNotExist: + logger.warning('Group does not exist: %d', group_id) + return + + # Make sure that the user actually has access to this project + if group.project not in Project.objects.get_for_user( + user, team=group.team, superuser=False): + logger.warning('User %r does not have access to group %r', (user, group)) + return + + event = group.get_latest_event() or Event() + + Event.objects.bind_nodes([event], 'data') + event.group = group + event.project = group.project + + form = NewNoteForm({'text': payload}) + if form.is_valid(): + form.save(event, user) diff --git a/src/sentry/utils/email.py b/src/sentry/utils/email.py index f82210731fae30..2f06d8d3d54de6 100644 --- a/src/sentry/utils/email.py +++ b/src/sentry/utils/email.py @@ -8,11 +8,33 @@ from django.conf import settings from django.core.mail import EmailMultiAlternatives +from django.core.signing import Signer +from django.utils.encoding import force_bytes from pynliner import Pynliner from sentry.web.helpers import render_to_string +signer = Signer() + +SMTP_HOSTNAME = getattr(settings, 'SENTRY_SMTP_HOSTNAME', 'localhost') +ENABLE_EMAIL_REPLIES = getattr(settings, 'SENTRY_ENABLE_EMAIL_REPLIES', False) + + +def email_to_group_id(address): + """ + Email address should be in the form of: + {group_id}+{signature}@example.com + """ + address = address.split('@', 1)[0] + signed_data = address.replace('+', ':') + return int(force_bytes(signer.unsign(signed_data))) + + +def group_id_to_email(group_id): + signed_data = signer.sign(str(group_id)) + return '@'.join((signed_data.replace(':', '+'), SMTP_HOSTNAME)) + class MessageBuilder(object): def __init__(self, subject, context=None, template=None, html_template=None, @@ -22,7 +44,7 @@ def __init__(self, subject, context=None, template=None, html_template=None, assert context or not (template or html_template) self.subject = subject - self.context = context + self.context = context or {} self.template = template self.html_template = html_template self.body = body @@ -35,7 +57,12 @@ def build(self, to): else: headers = self.headers.copy() - headers.setdefault('Reply-To', ', '.join(to)) + if ENABLE_EMAIL_REPLIES and 'X-Sentry-Reply-To' in headers: + reply_to = headers['X-Sentry-Reply-To'] + else: + reply_to = ', '.join(to) + + headers.setdefault('Reply-To', reply_to) if self.template: txt_body = render_to_string(self.template, self.context) diff --git a/tests/sentry/smtp/__init__.py b/tests/sentry/smtp/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/tests/sentry/smtp/email.txt b/tests/sentry/smtp/email.txt new file mode 100644 index 00000000000000..aead2702afb34a --- /dev/null +++ b/tests/sentry/smtp/email.txt @@ -0,0 +1,187 @@ +Received: by mail-oa0-f41.google.com with SMTP id g12so4828939oah.0 + for ; Fri, 15 Nov 2013 16:30:57 -0800 (PST) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=ydekproductions.com; s=google; + h=date:from:to:message-id:in-reply-to:references:subject:mime-version + :content-type; + bh=qhYwoiBivmkSmdOPk9ah6956TijbVC5V6fTKFTrCdOw=; + b=D9EUdfjlI558eq8zjSvu5+Br31ISaVQPXl/UQ8oLgRn2GHvPzwjt3GErixDtUJjp3m + KCGJkXZkCJ4nAqhHQF9FigaHfbXNGVG2ut+ZrHCFTlE/UoTWy9ti8LpgHUlRl4DwGl/J + TNyjJtb2LjEfQKzAsjGG0djk9RWLUx2nmIb0g= +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=1e100.net; s=20130820; + h=x-gm-message-state:date:from:to:message-id:in-reply-to:references + :subject:mime-version:content-type; + bh=qhYwoiBivmkSmdOPk9ah6956TijbVC5V6fTKFTrCdOw=; + b=F9C+DZ/izZizGaD6D2iEjxgFpZPBbCi/iRquJVwzbIRsV1H6JPjW0rlrGNgQAg4Vof + H/YdYvhJNv674UCx1k+PMH2LsjHqQM1BudeGRfiD7qyi5hR+miy5U1rNFK4RymMATn2k + 6w8Qw/n+4PfJLgVuNt2hHHfQTfwvevUgRzHu+3n/7f9W5v2swD0wumQyEMfYDRcW+zpP + SW4ntcs1hRdWCtNqavk79Ihy5SRTywzm3b0yje5KINVEuEt3IHhtnXZjAwHQa25Ooqre + jxSmw+o2LFU0bXTkfcgqIYfDoCDY9e+NE1ALlMumzmS7ercjX8BtlWUody9iE10R5IF3 + QCOw== +X-Gm-Message-State: ALoCoQlCqIRM9jNqyf0cupGYZ3Dn44p0nIIXOmSe92yFYV80NgOsXFMRKucCO+XAbDqZY37tEH+m +X-Received: by 10.60.44.141 with SMTP id e13mr3616811oem.84.1384561857799; + Fri, 15 Nov 2013 16:30:57 -0800 (PST) +Return-Path: +Received: from [192.168.144.132] ([38.104.194.126]) + by mx.google.com with ESMTPSA id qe2sm2776990obc.1.2013.11.15.16.30.56 + for + (version=TLSv1 cipher=RC4-SHA bits=128/128); + Fri, 15 Nov 2013 16:30:56 -0800 (PST) +Date: Fri, 15 Nov 2013 16:30:55 -0800 +From: Matt Robenolt +To: MToyOjE+dm7z_hYgCWwsSvl10T_37LKg7U8@localhost +Message-ID: <38169D8BB05D49D0B7F21E0C60F14FD0@ydekproductions.com> +In-Reply-To: <20131115052218.31819.48616@worker-4> +References: <20131115052218.31819.48616@worker-4> +Subject: Re: [Runscope Check] ERROR: runscope check +X-Mailer: sparrow 1.6.4 (build 1176) +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="5286bcbf_4f97e3e4_140" +X-Peer: 209.85.219.41 + +--5286bcbf_4f97e3e4_140 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +sup + +-- +Matt Robenolt +@mattrobenolt + + +On Thursday, November 14, 2013 at 9:22 PM, Sentry wrote: + +> Sentry New Event +> +> runscope check +> +> https://app.getsentry.com/sentry/runscope-check/group/10146301/ +> +> +> +> +> +> +> Logger: +> root +> Level: +> error +> +> Server: +> None +> First Seen: +> Nov. 15, 2013 +> +> +> +> +> Notification Settings (https://app.getsentry.com/account/settings/notifications/) +> +> +> +> +> +> +> +> +> + + + +--5286bcbf_4f97e3e4_140 +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: quoted-printable +Content-Disposition: inline + + +
sup +
+ =20 +

On Thursday, November = +14, 2013 at 9:22 PM, Sentry wrote:

+
+
+ + + + + + + + + +
+
+
+

Sen= +try + =20 + New Event + =20 +

+
+
+
runscope check
+

https://app.getsentry.com/sentry/runscope-check/group/1= +0146301/

+ ++++++ + + + + + + + + + + + + +
Logger:rootLevel:error
Server:None=46irst Seen:Nov. 15, 2013
+

Notification Settings

+
+
+
+ + +
 + =20 + =20 + =20 + =20 +
+ =20 +
+
+
+ +--5286bcbf_4f97e3e4_140-- diff --git a/tests/sentry/smtp/tests.py b/tests/sentry/smtp/tests.py new file mode 100644 index 00000000000000..d23ee356696457 --- /dev/null +++ b/tests/sentry/smtp/tests.py @@ -0,0 +1,32 @@ +import os.path +from sentry.models import Activity +from sentry.services.smtp import SentrySMTPServer, STATUS +from sentry.testutils import TestCase +from sentry.utils.email import group_id_to_email, email_to_group_id + +fixture = open(os.path.dirname(os.path.realpath(__file__)) + '/email.txt').read() + + +class SentrySMTPTest(TestCase): + def setUp(self): + self.address = ('0.0.0.0', 0) + self.server = SentrySMTPServer(*self.address) + self.mailto = group_id_to_email(self.group.pk) + self.event # side effect of generating an event + + def test_decode_email_address(self): + self.assertEqual(email_to_group_id(self.mailto), self.group.pk) + + def test_process_message(self): + self.assertEqual(self.server.process_message('', self.user.email, [self.mailto], fixture), STATUS[200]) + self.assertEqual(Activity.objects.filter(type=Activity.NOTE)[0].data, {'text': 'sup'}) + + def test_process_message_no_recipients(self): + self.assertEqual(self.server.process_message('', self.user.email, [], fixture), STATUS[550]) + + def test_process_message_too_long(self): + self.assertEqual(self.server.process_message('', self.user.email, [self.mailto], fixture * 100), STATUS[552]) + self.assertEqual(Activity.objects.count(), 0) + + def test_process_message_invalid_email(self): + self.assertEqual(self.server.process_message('', self.user.email, ['lol@localhost'], fixture), STATUS[550]) diff --git a/tests/sentry/utils/email/tests.py b/tests/sentry/utils/email/tests.py index 5249a7699f00f8..34571db7cf8072 100644 --- a/tests/sentry/utils/email/tests.py +++ b/tests/sentry/utils/email/tests.py @@ -27,3 +27,25 @@ def test_raw_content(self): 'hello world', 'text/html', ) + + def test_explicit_reply_to(self): + msg = MessageBuilder( + subject='Test', + body='hello world', + html_body='hello world', + headers={'X-Sentry-Reply-To': 'bar@example.com'}, + ) + msg.send(['foo@example.com']) + + assert len(mail.outbox) == 1 + + out = mail.outbox[0] + assert out.to == ['foo@example.com'] + assert out.subject == 'Test' + assert out.extra_headers['Reply-To'] == 'bar@example.com' + assert out.body == 'hello world' + assert len(out.alternatives) == 1 + assert out.alternatives[0] == ( + 'hello world', + 'text/html', + )