Feature/send email

brasilio/settings.py

@@ -242,7 +242,8 @@
 }
 
 # django-rq config
-RQ_QUEUES = {"default": {"URL": REDIS_URL, "DEFAULT_TIMEOUT": 500,}}
+DEFAULT_QUEUE_NAME = "default"
+RQ_QUEUES = {DEFAULT_QUEUE_NAME: {"URL": REDIS_URL, "DEFAULT_TIMEOUT": 500,}}
 RQ = {
     "DEFAULT_RESULT_TTL": 60 * 60 * 24,  # 24-hours
 }

brasilio/test_settings.py

@@ -14,3 +14,4 @@
 TEMPLATES[0]["OPTIONS"]["string_if_invalid"] = TEMPLATE_STRING_IF_INVALID  # noqa
 ENABLE_API_AUTH = True
 DISABLE_RECAPTCHA = False
+EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"

brasilio/worker.py

@@ -1,8 +1,7 @@
-from rq import Worker
-from rq.contrib.sentry import register_sentry
-
 from raven import Client
 from raven.transport import HTTPTransport
+from rq import Worker
+from rq.contrib.sentry import register_sentry
 
 
 class SentryAwareWorker(Worker):

brasilio_auth/management/commands/send_bulk_emails.py

@@ -0,0 +1,57 @@
+from datetime import timedelta
+
+import rows
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.template import Context, Template
+from tqdm import tqdm
+
+from core.email import send_email
+from core.queue import get_redis_queue
+
+
+class Command(BaseCommand):
+    def load_email_template(self, email_template):
+        with open(email_template, "r") as fobj:
+            return Template(fobj.read())
+
+    def print_email_metadata(self, metadata):
+        for key, value in metadata.items():
+            print(f"{key}: {value}")
+
+        print("-" * 80)
+
+    def add_arguments(self, parser):
+        parser.add_argument("--sender", default=settings.DEFAULT_FROM_EMAIL)
+        parser.add_argument("--dry-run", default=False, action="store_true")
+        parser.add_argument("--wait-time", default=15, type=int)
+        parser.add_argument("input_filename")
+        parser.add_argument("template_filename")
+
+    def handle(self, *args, **kwargs):
+        queue = get_redis_queue(settings.DEFAULT_QUEUE_NAME, settings.RQ_QUEUES[settings.DEFAULT_QUEUE_NAME]["URL"])
+
+        input_filename = kwargs["input_filename"]
+        table = rows.import_from_csv(input_filename)
+        error_msg = "Arquivo CSV deve conter campos 'to_email' e 'subject'"
+        assert {"to_email", "subject"}.issubset(set(table.field_names)), error_msg
+
+        template_obj = self.load_email_template(kwargs["template_filename"])
+        wait_time = kwargs["wait_time"]
+        from_email = kwargs["sender"]
+
+        time_offset = 0
+        for row in tqdm(table):
+            context = Context(row._asdict())
+            rendered_template = template_obj.render(context=context)
+            email_kwargs = {
+                "subject": row.subject,
+                "body": rendered_template,
+                "from_email": from_email,
+                "to": [row.to_email],
+            }
+            if not kwargs["dry_run"]:
+                queue.enqueue_in(timedelta(seconds=time_offset), send_email, **email_kwargs)
+                time_offset += wait_time
+            else:
+                self.print_email_metadata(email_kwargs)

brasilio_auth/tests/test_commands.py

@@ -0,0 +1,102 @@
+from datetime import timedelta
+from tempfile import NamedTemporaryFile
+from unittest import mock
+
+import pytest
+from django.conf import settings
+from django.core.management import call_command
+from django.test import TestCase
+
+from core.email import send_email
+
+
+class TestSendBulkEmails(TestCase):
+    def setUp(self):
+        self.p_redis_queue = mock.patch("brasilio_auth.management.commands.send_bulk_emails.get_redis_queue")
+        self.m_redis_queue = self.p_redis_queue.start()
+        self.m_queue = mock.Mock()
+        self.m_redis_queue.return_value = self.m_queue
+
+        self.input_file = NamedTemporaryFile(suffix=".csv")
+        self.email_template = NamedTemporaryFile(suffix=".txt")
+
+        with open(self.input_file.name, "w") as fobj:
+            fobj.write(
+                "nome,data,to_email,subject\nnome_1,data_1,email_1,subject_1\n" "nome_2,data_2,email_2,subject_2"
+            )
+
+        with open(self.email_template.name, "w") as fobj:
+            fobj.write("Enviado para {{ nome }} em {{ data }}")
+
+        self.input_file.seek(0)
+        self.email_template.seek(0)
+
+        self.expexted_send_email = [
+            {
+                "body": "Enviado para nome_1 em data_1",
+                "subject": "subject_1",
+                "to": ["email_1"],
+                "from_email": settings.DEFAULT_FROM_EMAIL,
+            },
+            {
+                "body": "Enviado para nome_2 em data_2",
+                "subject": "subject_2",
+                "to": ["email_2"],
+                "from_email": settings.DEFAULT_FROM_EMAIL,
+            },
+        ]
+
+    def tearDown(self):
+        self.p_redis_queue.stop()
+
+    def test_send_bulk_emails(self):
+        call_command("send_bulk_emails", self.input_file.name, self.email_template.name)
+
+        self.m_redis_queue.assert_called_once_with(
+            settings.DEFAULT_QUEUE_NAME,
+            settings.RQ_QUEUES[settings.DEFAULT_QUEUE_NAME]["URL"]
+        )
+        self.m_queue.enqueue_in.assert_has_calls(
+            [
+                mock.call(timedelta(seconds=0), send_email, **self.expexted_send_email[0]),
+                mock.call(timedelta(seconds=15), send_email, **self.expexted_send_email[1]),
+            ]
+        )
+
+    def test_send_email_custom_from_email(self):
+        kwargs = {"sender": "Example Email <email@example.com>"}
+        call_command("send_bulk_emails", self.input_file.name, self.email_template.name, **kwargs)
+
+        self.expexted_send_email[0]["from_email"] = kwargs["sender"]
+        self.expexted_send_email[1]["from_email"] = kwargs["sender"]
+
+        self.m_queue.enqueue_in.assert_has_calls(
+            [
+                mock.call(timedelta(seconds=0), send_email, **self.expexted_send_email[0]),
+                mock.call(timedelta(seconds=15), send_email, **self.expexted_send_email[1]),
+            ]
+        )
+
+    def test_assert_mandatory_fields(self):
+        with open(self.input_file.name, "w") as fobj:
+            fobj.write("nome,data\nnome_1,data_1\nnome_2,data_2")
+
+        self.input_file.seek(0)
+        with pytest.raises(AssertionError):
+            call_command("send_bulk_emails", self.input_file.name, self.email_template.name)
+
+    def test_do_not_send_mail(self):
+        call_command("send_bulk_emails", self.input_file.name, self.email_template.name, "--dry-run")
+
+        self.m_queue.assert_not_called()
+
+    def test_send_bulk_emails_schedule_with_wait_time(self):
+        kwargs = {"wait_time": 30}
+        call_command("send_bulk_emails", self.input_file.name, self.email_template.name, **kwargs)
+
+        self.m_queue.enqueue_in.assert_has_calls(
+            [
+                mock.call(timedelta(seconds=0), send_email, **self.expexted_send_email[0]),
+                mock.call(timedelta(seconds=30), send_email, **self.expexted_send_email[1]),
+            ]
+        )

core/email.py

@@ -0,0 +1,6 @@
+from django.core.mail import EmailMessage
+
+
+def send_email(subject, body, from_email, to, **kwargs):
+    email = EmailMessage(subject=subject, body=body, from_email=from_email, to=to, **kwargs)
+    email.send()

core/queue.py

@@ -0,0 +1,7 @@
+from rq import Queue
+from redis import Redis
+
+
+def get_redis_queue(name: str, url: str) -> Queue:
+    redis_conn = Redis.from_url(url)
+    return Queue(name=name, connection=redis_conn)

core/tests/test_email.py

@@ -0,0 +1,23 @@
+from django.core import mail
+from django.test import TestCase
+
+from core.email import send_email
+
+
+class TestSendEmail(TestCase):
+    def test_send_emnail(self):
+        subject = "Subject"
+        body = "Body"
+        from_email = "from@example.com"
+        to = ["to@example.com"]
+        reply_to = ["Reply To <replyto@example.com"]
+
+        send_email(
+            subject=subject, body=body, from_email=from_email, to=to, reply_to=reply_to,
+        )
+        assert len(mail.outbox) == 1
+        assert body == mail.outbox[0].body
+        assert subject == mail.outbox[0].subject
+        assert from_email == mail.outbox[0].from_email
+        assert to == mail.outbox[0].to
+        assert reply_to == mail.outbox[0].reply_to

core/tests/test_queue.py

@@ -0,0 +1,31 @@
+from unittest import mock
+
+from django.test import TestCase
+from redis import Redis
+from rq import Queue
+
+from core.queue import get_redis_queue
+
+
+class TestGetRedisQueue(TestCase):
+    def setUp(self):
+        self.m_redis_conn = mock.Mock()
+        self.p_redis_from_url = mock.patch.object(Redis, "from_url")
+        self.m_redis_from_url = self.p_redis_from_url.start()
+        self.m_redis_from_url.return_value = self.m_redis_conn
+
+        self.m_queue = mock.Mock(spec=Queue)
+        self.p_queue_cls = mock.patch("core.queue.Queue")
+        self.m_queue_cls = self.p_queue_cls.start()
+        self.m_queue_cls.return_value = self.m_queue
+
+    def tearDown(self):
+        self.p_queue_cls.stop()
+        self.m_redis_from_url.stop()
+
+    def test_get_redis_queue_with_custom_arguments(self):
+        queue = get_redis_queue(name="name", url="redis://url:6379")
+
+        assert isinstance(queue, Queue)
+        self.m_redis_from_url.assert_called_once_with("redis://url:6379")
+        self.m_queue_cls.assert_called_once_with(name="name", connection=self.m_redis_conn)

core/views.py

@@ -3,23 +3,27 @@
 
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
-from django.core.mail import EmailMessage
 from django.core.paginator import Paginator
 from django.db.models import Q
 from django.http import StreamingHttpResponse
 from django.shortcuts import get_object_or_404, redirect, render
 from django.urls import reverse
 
+from core.email import send_email
 from core.filters import parse_querystring
 from core.forms import ContactForm, DatasetSearchForm, get_table_dynamic_form
 from core.middlewares import disable_non_logged_user_cache
 from core.models import Dataset, Table
+from core.queue import get_redis_queue
 from core.templatetags.utils import obfuscate
 from core.util import cached_http_get_json
 from data_activities_log.activites import recent_activities
 from traffic_control.logging import log_blocked_request
 
 
+QUEUE = get_redis_queue(settings.DEFAULT_QUEUE_NAME, settings.RQ_QUEUES[settings.DEFAULT_QUEUE_NAME]["URL"])
+
+
 class Echo:
     def write(self, value):
         return value
@@ -41,14 +45,15 @@ def contact(request):
 
         if form.is_valid():
             data = form.cleaned_data
-            email = EmailMessage(
+            QUEUE.enqueue(
+                send_email,
                 subject=f"Contato no Brasil.IO: {data['name']}",
                 body=data["message"],
                 from_email=f'{data["name"]} (via Brasil.IO) <{settings.DEFAULT_FROM_EMAIL}>',
                 to=[settings.DEFAULT_FROM_EMAIL],
                 reply_to=[f'{data["name"]} <{data["email"]}>'],
             )
-            email.send()
+
             return redirect(reverse("core:contact") + "?sent=true")
 
     else:

covid19/models.py

@@ -74,8 +74,8 @@ def most_recent_deployed(self, state, date=None):
 class StateSpreadsheetManager(models.Manager):
     def get_state_data(self, state):
         """Return all state cases, grouped by date"""
-        from covid19.spreadsheet_validator import TOTAL_LINE_DISPLAY
         from brazil_data.cities import get_city_info
+        from covid19.spreadsheet_validator import TOTAL_LINE_DISPLAY
 
         cases, reports = defaultdict(dict), {}
         qs = self.get_queryset()

requirements.txt

@@ -31,6 +31,7 @@ openpyxl==3.0.5
 psycopg2-binary==2.8.5
 pytest-django==3.2.1
 pytz==2020.1
+redis==3.5.3
 requests-cache==0.5.2
 requests==2.24.0
 retry==0.9.2