Mandrill: support esp_extra

* Merge esp_extra with Mandrill send payload
* Handle pythonic forms of `recipient_metadata`
  and `template_content` in esp_extra
* DeprecationWarning for Mandrill EmailMessage
  attributes inherited from Djrill

anymail/backends/mandrill.py

@@ -1,6 +1,7 @@
+import warnings
 from datetime import datetime
 
-from ..exceptions import AnymailRequestsAPIError
+from ..exceptions import AnymailRequestsAPIError, AnymailWarning
 from ..message import AnymailRecipientStatus, ANYMAIL_STATUSES
 from ..utils import last, combine, get_anymail_setting
 
@@ -43,14 +44,8 @@ def parse_recipient_status(self, response, payload, message):
         return recipient_status
 
 
-def _expand_merge_vars(vardict):
-    """Convert a Python dict to an array of name-content used by Mandrill.
-
-    { name: value, ... } --> [ {'name': name, 'content': value }, ... ]
-    """
-    # For testing reproducibility, we sort the keys
-    return [{'name': name, 'content': vardict[name]}
-            for name in sorted(vardict.keys())]
+class DjrillDeprecationWarning(AnymailWarning, DeprecationWarning):
+    """Warning for features carried over from Djrill that will be removed soon"""
 
 
 def encode_date_for_mandrill(dt):
@@ -69,13 +64,18 @@ def encode_date_for_mandrill(dt):
 
 class MandrillPayload(RequestsPayload):
 
+    def __init__(self, *args, **kwargs):
+        self.esp_extra = {}  # late-bound in serialize_data
+        super(MandrillPayload, self).__init__(*args, **kwargs)
+
     def get_api_endpoint(self):
         if 'template_name' in self.data:
             return "messages/send-template.json"
         else:
             return "messages/send.json"
 
     def serialize_data(self):
+        self.process_esp_extra()
         return self.serialize_json(self.data)
 
     #
@@ -89,7 +89,9 @@ def init_payload(self):
         }
 
     def set_from_email(self, email):
-        if not getattr(self.message, "use_template_from", False):  # Djrill compat!
+        if getattr(self.message, "use_template_from", False):
+            self.deprecation_warning('message.use_template_from', 'message.from_email = None')
+        else:
             self.data["message"]["from_email"] = email.email
             if email.name:
                 self.data["message"]["from_name"] = email.name
@@ -100,7 +102,9 @@ def add_recipient(self, recipient_type, email):
         to_list.append({"email": email.email, "name": email.name, "type": recipient_type})
 
     def set_subject(self, subject):
-        if not getattr(self.message, "use_template_subject", False):  # Djrill compat!
+        if getattr(self.message, "use_template_subject", False):
+            self.deprecation_warning('message.use_template_subject', 'message.subject = None')
+        else:
             self.data["message"]["subject"] = subject
 
     def set_reply_to(self, emails):
@@ -166,9 +170,59 @@ def set_merge_global_data(self, merge_global_data):
         ]
 
     def set_esp_extra(self, extra):
-        pass
+        # late bind in serialize_data, so that obsolete Djrill attrs can contribute
+        self.esp_extra = extra
 
-    # Djrill leftovers
+    def process_esp_extra(self):
+        if self.esp_extra is not None and len(self.esp_extra) > 0:
+            esp_extra = self.esp_extra
+            # Convert pythonic template_content dict to Mandrill name/content list
+            try:
+                template_content = esp_extra['template_content']
+            except KeyError:
+                pass
+            else:
+                if hasattr(template_content, 'items'):  # if it's dict-like
+                    if esp_extra is self.esp_extra:
+                        esp_extra = self.esp_extra.copy()  # don't modify caller's value
+                    esp_extra['template_content'] = [
+                        {'name': var, 'content': value}
+                        for var, value in template_content.items()]
+            # Convert pythonic recipient_metadata dict to Mandrill rcpt/values list
+            try:
+                recipient_metadata = esp_extra['message']['recipient_metadata']
+            except KeyError:
+                pass
+            else:
+                if hasattr(recipient_metadata, 'keys'):  # if it's dict-like
+                    if esp_extra['message'] is self.esp_extra['message']:
+                        esp_extra['message'] = self.esp_extra['message'].copy()  # don't modify caller's value
+                    # For testing reproducibility, we sort the recipients
+                    esp_extra['message']['recipient_metadata'] = [
+                        {'rcpt': rcpt, 'values': recipient_metadata[rcpt]}
+                        for rcpt in sorted(recipient_metadata.keys())]
+            # Merge esp_extra with payload data: shallow merge within ['message'] and top-level keys
+            self.data.update({k:v for k,v in esp_extra.items() if k != 'message'})
+            try:
+                self.data['message'].update(esp_extra['message'])
+            except KeyError:
+                pass
+
+    # Djrill deprecated message attrs
+
+    def deprecation_warning(self, feature, replacement=None):
+        msg = "Djrill's `%s` will be removed in an upcoming Anymail release." % feature
+        if replacement:
+            msg += " Use `%s` instead." % replacement
+        warnings.warn(msg, DjrillDeprecationWarning)
+
+    def deprecated_to_esp_extra(self, attr, in_message_dict=False):
+        feature = "message.%s" % attr
+        if in_message_dict:
+            replacement = "message.esp_extra = {'message': {'%s': <value>}}" % attr
+        else:
+            replacement = "message.esp_extra = {'%s': <value>}" % attr
+        self.deprecation_warning(feature, replacement)
 
     esp_message_attrs = (
         ('async', last, None),
@@ -188,25 +242,40 @@ def set_esp_extra(self, extra):
         ('subaccount', last, None),
         ('google_analytics_domains', last, None),
         ('google_analytics_campaign', last, None),
+        ('global_merge_vars', combine, None),
+        ('merge_vars', combine, None),
         ('recipient_metadata', combine, None),
-        ('template_content', combine, _expand_merge_vars),
+        ('template_name', last, None),
+        ('template_content', combine, None),
     )
 
     def set_async(self, async):
-        self.data["async"] = async
+        self.deprecated_to_esp_extra('async')
+        self.esp_extra['async'] = async
 
     def set_ip_pool(self, ip_pool):
-        self.data["ip_pool"] = ip_pool
+        self.deprecated_to_esp_extra('ip_pool')
+        self.esp_extra['ip_pool'] = ip_pool
+
+    def set_global_merge_vars(self, global_merge_vars):
+        self.deprecation_warning('message.global_merge_vars', 'message.merge_global_data')
+        self.set_merge_global_data(global_merge_vars)
+
+    def set_merge_vars(self, merge_vars):
+        self.deprecation_warning('message.merge_vars', 'message.merge_data')
+        self.set_merge_data(merge_vars)
+
+    def set_template_name(self, template_name):
+        self.deprecation_warning('message.template_name', 'message.template_id')
+        self.set_template_id(template_name)
 
     def set_template_content(self, template_content):
-        self.data["template_content"] = template_content
+        self.deprecated_to_esp_extra('template_content')
+        self.esp_extra['template_content'] = template_content
 
     def set_recipient_metadata(self, recipient_metadata):
-        # For testing reproducibility, we sort the recipients
-        self.data['message']['recipient_metadata'] = [
-            {'rcpt': rcpt, 'values': recipient_metadata[rcpt]}
-            for rcpt in sorted(recipient_metadata.keys())
-        ]
+        self.deprecated_to_esp_extra('recipient_metadata', in_message_dict=True)
+        self.esp_extra.setdefault('message', {})['recipient_metadata'] = recipient_metadata
 
     # Set up simple set_<attr> functions for any missing esp_message_attrs attrs
     # (avoids dozens of simple `self.data["message"][<attr>] = value` functions)
@@ -225,7 +294,8 @@ def define_message_attr_setters(cls):
     def make_setter(attr, setter_name):
         # sure wish we could use functools.partial to create instance methods (descriptors)
         def setter(self, value):
-            self.data["message"][attr] = value
+            self.deprecated_to_esp_extra(attr, in_message_dict=True)
+            self.esp_extra.setdefault('message', {})[attr] = value
         setter.__name__ = setter_name
         return setter
 

docs/esps/mandrill.rst

@@ -97,9 +97,38 @@ which is the secure, production version of Mandrill's 1.0 API.
 esp_extra support
 -----------------
 
-Anymail's Mandrill backend does not yet implement the
-:attr:`~anymail.message.AnymailMessage.esp_extra` feature.
+To use Mandrill features not directly supported by Anymail, you can
+set a message's :attr:`~anymail.message.AnymailMessage.esp_extra` to
+a `dict` of parameters to merge into Mandrill's `messages/send API`_ call.
+Note that a few parameters go at the top level, but Mandrill expects
+most options within a `'message'` sub-dict---be sure to check their
+API docs:
 
+  .. code-block:: python
+
+      message.esp_extra = {
+          # Mandrill expects 'ip_pool' at top level...
+          'ip_pool': 'Bulk Pool',
+          # ... but 'subaccount' must be within a 'message' dict:
+          'message': {
+              'subaccount': 'Marketing Dept.'
+          }
+      }
+
+Anymail has special handling that lets you specify Mandrill's
+`'recipient_metadata'` as a simple, pythonic `dict` (similar in form
+to Anymail's :attr:`~anymail.message.AnymailMessage.merge_data`),
+rather than Mandrill's more complex list of rcpt/values dicts.
+You can use whichever style you prefer (but either way,
+recipient_metadata must be in `esp_extra['message']`).
+
+Similary, Anymail allows Mandrill's `'template_content'` in esp_extra
+(top level) either as a pythonic `dict` (similar to Anymail's
+:attr:`~anymail.message.AnymailMessage.merge_global_data`) or
+as Mandrill's more complex list of name/content dicts.
+
+.. _messages/send API:
+    https://mandrillapp.com/api/docs/messages.JSON.html#method=send
 
 .. _mandrill-templates:
 
@@ -222,14 +251,19 @@ Changes to settings
   the values from :setting:`ANYMAIL_SEND_DEFAULTS`.
 
 ``MANDRILL_SUBACCOUNT``
-  Use :setting:`ANYMAIL_MANDRILL_SEND_DEFAULTS`:
+  Set :ref:`esp_extra <mandrill-esp-extra>`
+  globally in :setting:`ANYMAIL_SEND_DEFAULTS`:
 
     .. code-block:: python
 
         ANYMAIL = {
             ...
             "MANDRILL_SEND_DEFAULTS": {
-                "subaccount": "<your subaccount>"
+                "esp_extra": {
+                    "message": {
+                        "subaccount": "<your subaccount>"
+                    }
+                }
             }
         }
 
@@ -290,13 +324,30 @@ Changes to EmailMessage attributes
   to use the values from the stored template.
 
 **Other Mandrill-specific attributes**
-  Are currently still supported by Anymail's Mandrill backend,
-  but will be ignored by other Anymail backends.
+  Djrill allowed nearly all Mandrill API parameters to be set
+  as attributes directly on an EmailMessage. With Anymail, you
+  should instead set these in the message's
+  :ref:`esp_extra <mandrill-esp-extra>` dict as described above.
+
+  Although the Djrill style attributes are still supported (for now),
+  Anymail will issue a :exc:`DeprecationWarning` if you try to use them.
+  These warnings are visible during tests (with Django's default test
+  runner), and will explain how to update your code.
+
+  You can also use the following git grep expression to find potential
+  problems:
+
+    .. code-block:: console
+
+        git grep -w \
+          -e 'async' -e 'auto_html' -e 'auto_text' -e 'from_name' -e 'global_merge_vars' \
+          -e 'google_analytics_campaign' -e 'google_analytics_domains' -e 'important' \
+          -e 'inline_css' -e 'ip_pool' -e 'merge_language' -e 'merge_vars' \
+          -e 'preserve_recipients' -e 'recipient_metadata' -e 'return_path_domain' \
+          -e 'signing_domain' -e 'subaccount' -e 'template_content' -e 'template_name' \
+          -e 'tracking_domain' -e 'url_strip_qs' -e 'use_template_from' -e 'use_template_subject' \
+          -e 'view_content_link'
 
-  It's best to eliminate them if they're not essential
-  to your code. In the future, the Mandrill-only attributes
-  will be moved into the
-  :attr:`~anymail.message.AnymailMessage.esp_extra` dict.
 
 **Inline images**
   Djrill (incorrectly) used the presence of a :mailheader:`Content-ID`

tests/test_mandrill_backend.py

@@ -401,6 +401,69 @@ def test_missing_subject(self):
         data = self.get_api_call_json()
         self.assertNotIn('subject', data['message'])
 
+    def test_esp_extra(self):
+        self.message.esp_extra = {
+            'ip_pool': 'Bulk Pool',  # Mandrill send param that goes at top level of API payload
+            'message': {
+                'subaccount': 'Marketing Dept.'  # param that goes within message dict
+            }
+        }
+        self.message.tags = ['test-tag']  # make sure non-esp_extra params are merged
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertEqual(data['ip_pool'], 'Bulk Pool')
+        self.assertEqual(data['message']['subaccount'], 'Marketing Dept.')
+        self.assertEqual(data['message']['tags'], ['test-tag'])
+
+    def test_esp_extra_recipient_metadata(self):
+        """Anymail allows pythonic recipient_metadata dict"""
+        self.message.esp_extra = {'message': {'recipient_metadata': {
+            # Anymail expands simple python dicts into the more-verbose
+            # rcpt/values lists the Mandrill API uses
+            "customer@example.com": {'cust_id': "67890", 'order_id': "54321"},
+            "guest@example.com": {'cust_id': "94107", 'order_id': "43215"} ,
+        }}}
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertCountEqual(data['message']['recipient_metadata'], [
+            {'rcpt': "customer@example.com", 'values': {'cust_id': "67890", 'order_id': "54321"}},
+            {'rcpt': "guest@example.com", 'values': {'cust_id': "94107", 'order_id': "43215"}}])
+
+        # You can also just supply it in Mandrill's native form
+        self.message.esp_extra = {'message': {'recipient_metadata': [
+            {'rcpt': "customer@example.com", 'values': {'cust_id': "80806", 'order_id': "70701"}},
+            {'rcpt': "guest@example.com", 'values': {'cust_id': "21212", 'order_id': "10305"}}]}}
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertCountEqual(data['message']['recipient_metadata'], [
+            {'rcpt': "customer@example.com", 'values': {'cust_id': "80806", 'order_id': "70701"}},
+            {'rcpt': "guest@example.com", 'values': {'cust_id': "21212", 'order_id': "10305"}}])
+
+    def test_esp_extra_template_content(self):
+        """Anymail allows pythonic template_content dict"""
+        self.message.template_id = "welcome_template"  # forces send-template API and default template_content
+        self.message.esp_extra = {'template_content': {
+            # Anymail expands simple python dicts into the more-verbose name/content
+            # structures the Mandrill API uses
+            'HEADLINE': "<h1>Specials Just For *|FNAME|*</h1>",
+            'OFFER_BLOCK': "<p><em>Half off</em> all fruit</p>",
+        }}
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertCountEqual(data['template_content'], [
+            {'name': "HEADLINE", 'content': "<h1>Specials Just For *|FNAME|*</h1>"},
+            {'name': "OFFER_BLOCK", 'content': "<p><em>Half off</em> all fruit</p>"}])
+
+        # You can also just supply it in Mandrill's native form
+        self.message.esp_extra = {'template_content': [
+            {'name': "HEADLINE", 'content': "<h1>Exciting offers for *|FNAME|*</h1>"},
+            {'name': "OFFER_BLOCK", 'content': "<p><em>25% off</em> all fruit</p>"}]}
+        self.message.send()
+        data = self.get_api_call_json()
+        self.assertCountEqual(data['template_content'], [
+            {'name': "HEADLINE", 'content': "<h1>Exciting offers for *|FNAME|*</h1>"},
+            {'name': "OFFER_BLOCK", 'content': "<p><em>25% off</em> all fruit</p>"}])
+
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.
 
@@ -411,11 +474,15 @@ def test_default_omits_options(self):
         self.message.send()
         self.assert_esp_called("/messages/send.json")
         data = self.get_api_call_json()
+        self.assertNotIn('global_merge_vars', data['message'])
+        self.assertNotIn('merge_vars', data['message'])
         self.assertNotIn('metadata', data['message'])
         self.assertNotIn('send_at', data)
         self.assertNotIn('tags', data['message'])
-        self.assertNotIn('track_opens', data['message'])
+        self.assertNotIn('template_content', data['message'])
+        self.assertNotIn('template_name', data['message'])
         self.assertNotIn('track_clicks', data['message'])
+        self.assertNotIn('track_opens', data['message'])
 
     # noinspection PyUnresolvedReferences
     def test_send_attaches_anymail_status(self):