Add mailer worker for sending emails

Add an h/mailer.py module with a send() function for user code to send emails
asynchronously off the main thread, and a hypothesis-worker process that sends
the emails.

Change the account activation and forgot password emails to use this module.

Remove unused imports from accounts/views.py since it no longer handles sending
emails itself.

Remove accounts/worker.py since it's no longer used.

Update tests.

Procfile

@@ -1,6 +1,6 @@
 web: gunicorn -w ${WEB_CONCURRENCY:-1} --paster conf/${HYP_ENV:-production}.ini
 notification: hypothesis-worker conf/${HYP_ENV:-production}.ini notification
 nipsa: hypothesis-worker conf/${HYP_ENV:-production}.ini nipsa
-activation: hypothesis-worker conf/${HYP_ENV:-production}.ini activation
+email: hypothesis-worker conf/${HYP_ENV:-production}.ini email
 assets: hypothesis assets conf/${HYP_ENV:-production}.ini
 initdb: hypothesis initdb conf/${HYP_ENV:-production}.ini

h/accounts/test/views_test.py

@@ -81,13 +81,6 @@ def invalid_form(errors=None):
     return form
 
 
-def mock_get_queue_writer():
-    """Return a mock object whose API matches request.get_queue_writer()."""
-    def get_queue_writer():
-        pass
-    return mock.create_autospec(get_queue_writer)
-
-
 @pytest.mark.usefixtures('routes_mapper')
 def test_login_redirects_when_logged_in(authn_policy):
     request = DummyRequest()
@@ -382,6 +375,8 @@ def test_forgot_password_generates_mail(reset_link,
     controller = ForgotPasswordController(request)
     controller.form = form_validating_to({"user": user})
     reset_link.return_value = "http://example.com"
+    reset_mail.return_value = {
+        'recipients': [], 'subject': '', 'body': ''}
 
     controller.forgot_password()
 
@@ -398,10 +393,16 @@ def test_forgot_password_sends_mail(reset_mail, authn_policy, mailer):
     controller = ForgotPasswordController(request)
     controller.form = form_validating_to({"user": user})
     message = reset_mail.return_value
+    reset_mail.return_value = {
+        'recipients': ['giraffe@thezoo.org'],
+        'subject': 'subject',
+        'body': 'body'}
 
     controller.forgot_password()
 
-    assert message in mailer.outbox
+    mailer.send.assert_called_once_with(
+        request, recipients=['giraffe@thezoo.org'], subject='subject',
+        body='body')
 
 
 @forgot_password_fixtures
@@ -482,6 +483,7 @@ def test_reset_password_redirects_on_success():
 
 
 register_fixtures = pytest.mark.usefixtures('activation_model',
+                                            'mailer',
                                             'notify',
                                             'routes_mapper',
                                             'user_model')
@@ -499,9 +501,7 @@ def test_register_returns_errors_when_validation_fails():
 
 @register_fixtures
 def test_register_creates_user_from_form_data(user_model):
-    request = DummyRequest(method="POST",
-                           authenticated_userid=None,
-                           get_queue_writer=mock_get_queue_writer())
+    request = DummyRequest(method="POST", authenticated_userid=None)
     controller = RegisterController(request)
     controller.form = form_validating_to({
         "username": "bob",
@@ -519,9 +519,7 @@ def test_register_creates_user_from_form_data(user_model):
 
 @register_fixtures
 def test_register_adds_new_user_to_session(user_model):
-    request = DummyRequest(method="POST",
-                           authenticated_userid=None,
-                           get_queue_writer=mock_get_queue_writer())
+    request = DummyRequest(method="POST", authenticated_userid=None)
     request.db.add = mock.create_autospec(request.db.add)
     controller = RegisterController(request)
     controller.form = form_validating_to({
@@ -538,9 +536,7 @@ def test_register_adds_new_user_to_session(user_model):
 @register_fixtures
 def test_register_creates_new_activation(activation_model,
                                          user_model):
-    request = DummyRequest(method="POST",
-                           authenticated_userid=None,
-                           get_queue_writer=mock_get_queue_writer())
+    request = DummyRequest(method="POST", authenticated_userid=None)
     controller = RegisterController(request)
     controller.form = form_validating_to({
         "username": "bob",
@@ -558,39 +554,41 @@ def test_register_creates_new_activation(activation_model,
 @register_fixtures
 def test_register_generates_activation_email_from_user(activation_email,
                                                        user_model):
-    request = DummyRequest(method="POST",
-                           authenticated_userid=None,
-                           get_queue_writer=mock_get_queue_writer())
+    request = DummyRequest(method="POST", authenticated_userid=None)
     controller = RegisterController(request)
     controller.form = form_validating_to({
         "username": "bob",
         "email": "bob@example.com",
         "password": "s3crets",
     })
     new_user = user_model.return_value
+    activation_email.return_value = {
+        'recipients': [], 'subject': '', 'body': ''}
 
     controller.register()
 
-    activation_email.assert_called_with(request, new_user)
+    activation_email.assert_called_once_with(request, new_user)
 
 
 @patch('h.accounts.views.activation_email')
 @register_fixtures
-def test_register_publishes_activation_message_to_nsq(activation_email):
-    request = DummyRequest(method="POST",
-                           authenticated_userid=None,
-                           get_queue_writer=mock_get_queue_writer())
+def test_register_sends_email(activation_email, mailer):
+    request = DummyRequest(method="POST", authenticated_userid=None)
     controller = RegisterController(request)
     controller.form = form_validating_to({
         "username": "bob",
         "email": "bob@example.com",
         "password": "s3crets",
     })
+    activation_email.return_value = {
+        'recipients': ['bob@example.com'], 'subject': 'subject',
+        'body': 'body'}
 
     controller.register()
 
-    request.get_queue_writer.return_value.publish.assert_called_once_with(
-        'activations', activation_email.return_value)
+    mailer.send.assert_called_once_with(
+        request, recipients=['bob@example.com'], subject='subject',
+        body='body')
 
 
 @patch('h.accounts.views.RegistrationEvent')
@@ -609,9 +607,7 @@ def test_register_no_event_when_validation_fails(event, notify):
 @patch('h.accounts.views.RegistrationEvent')
 @register_fixtures
 def test_register_event_when_validation_succeeds(event, user_model):
-    request = DummyRequest(method="POST",
-                           authenticated_userid=None,
-                           get_queue_writer=mock_get_queue_writer())
+    request = DummyRequest(method="POST", authenticated_userid=None)
     controller = RegisterController(request)
     controller.form = form_validating_to({
         "username": "bob",
@@ -628,9 +624,7 @@ def test_register_event_when_validation_succeeds(event, user_model):
 
 @register_fixtures
 def test_register_event_redirects_on_success():
-    request = DummyRequest(method="POST",
-                           authenticated_userid=None,
-                           get_queue_writer=mock_get_queue_writer())
+    request = DummyRequest(method="POST", authenticated_userid=None)
     controller = RegisterController(request)
     controller.form = form_validating_to({
         "username": "bob",
@@ -1053,3 +1047,11 @@ def activation_model(config, request):
     model = patcher.start()
     request.addfinalizer(patcher.stop)
     return model
+
+
+@pytest.fixture
+def mailer(request):
+    patcher = patch('h.accounts.views.mailer', autospec=True)
+    module = patcher.start()
+    request.addfinalizer(patcher.stop)
+    return module

h/accounts/views.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 import datetime
-import json
 
 import colander
 import deform
@@ -10,14 +9,13 @@
 from pyramid.exceptions import BadCSRFToken
 from pyramid.view import view_config, view_defaults
 from pyramid.security import forget, remember
-from pyramid_mailer import get_mailer
-from pyramid_mailer.message import Message
 
 from h import i18n
 from h import models
 from h import session
 from h import util
 from h import accounts
+from h import mailer
 from h.accounts import schemas
 from h.accounts.models import User
 from h.accounts.models import Activation
@@ -230,8 +228,7 @@ def _send_forgot_password_email(self, user):
 
         link = reset_password_link(self.request, code)
         message = reset_password_email(user, code, link)
-        mailer = get_mailer(self.request)
-        mailer.send(message)
+        mailer.send(self.request, **message)
 
 
 @view_defaults(route_name='reset_password',
@@ -421,7 +418,7 @@ def _register(self, username, email, password):
 
         # Send the activation email
         message = activation_email(self.request, user)
-        self.request.get_queue_writer().publish('activations', message)
+        mailer.send(self.request, **message)
 
         self.request.session.flash(jinja2.Markup(_(
             'Thank you for creating an account! '
@@ -558,10 +555,10 @@ def activation_email(request, user):
 
 
 def reset_password_email(user, reset_code, reset_link):
-    """
-    Generate a 'reset your password' email for the specified user.
+    """Return the data for a 'reset your password' email for the given user.
+
+    :rtype: dict
 
-    :rtype: pyramid_mailer.message.Message
     """
     emailtext = ("Hello, {username}!\n\n"
                  "Someone requested resetting your password. If it was "
@@ -575,10 +572,11 @@ def reset_password_email(user, reset_code, reset_link):
     body = emailtext.format(code=reset_code,
                             link=reset_link,
                             username=user.username)
-    msg = Message(subject=_("Reset your password"),
-                  recipients=[user.email],
-                  body=body)
-    return msg
+    return {
+        "recipients": [user.email],
+        "subject": _("Reset your password"),
+        "body": body
+    }
 
 
 def reset_password_link(request, reset_code):

h/mailer.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""A module for sending emails asynchronously.
+
+Most code that sends emails should do so through this module.
+
+User code should call the send() function to cause an email to be sent
+asynchronously.
+
+"""
+import json
+
+import pyramid_mailer
+import pyramid_mailer.message
+
+
+def send(request, recipients, subject, body):
+    """Cause an email to be sent asynchronously.
+
+    :param request: the request object for the request that wants to send the
+        email
+    :type request: pyramid.request.Request
+
+    :param recipients: the list of email addresses to send the email to
+    :type recipients: list of unicode strings
+
+    :param subject: the subject of the email
+    :type subject: unicode
+
+    :param body: the body of the email
+    :type body: unicode
+
+    """
+    request.get_queue_writer().publish(
+        "email", {"recipients": recipients, "subject": subject, "body": body})
+
+
+def worker(request):
+    """A hypothesis-worker function for sending emails.
+
+    Connects to the "email" topic that send() above publishes to and
+    sends an email for each message received.
+
+    """
+    def handle_message(_, message):
+        """Receive a message from nsq and send it as an email."""
+        body = json.loads(message.body)
+        email = pyramid_mailer.message.Message(
+            subject=body["subject"], recipients=body["recipients"],
+            body=body["body"])
+        pyramid_mailer.get_mailer(request).send_immediately(email)
+
+    reader = request.get_queue_reader("email", "mailer")
+    reader.on_message.connect(handle_message)
+    reader.start(block=True)

setup.py

@@ -125,7 +125,7 @@ def run_tests(self):
             'hypothesis-worker=h.worker:main',
         ],
         'h.worker': [
-            'activation=h.accounts.worker:worker',
+            'email=h.email:worker',
             'nipsa=h.api.nipsa.worker:worker',
             'notification=h.notification.worker:run',
         ],