diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e92cd50c..393fcfc3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,19 @@ Release history ^^^^^^^^^^^^^^^ .. This extra heading level keeps the ToC from becoming unmanageably long +vNext +----- + +*Not yet released* + +Features +~~~~~~~~ + +* **Mailgun:** Support Mailgun's new (ESP stored) handlebars templates via `template_id`. + See `docs `__. + (Thanks `@anstosa`_.) + + v6.1 ---- @@ -964,6 +977,7 @@ Features .. _#153: https://github.com/anymail/issues/153 .. _@ailionx: https://github.com/ailionx +.. _@anstosa: https://github.com/anstosa .. _@calvin: https://github.com/calvin .. _@costela: https://github.com/costela .. _@decibyte: https://github.com/decibyte diff --git a/anymail/backends/mailgun.py b/anymail/backends/mailgun.py index cd33acd2..68c2f858 100644 --- a/anymail/backends/mailgun.py +++ b/anymail/backends/mailgun.py @@ -119,58 +119,122 @@ def get_request_params(self, api_url): return params def serialize_data(self): - if self.is_batch() or self.merge_global_data: - self.populate_recipient_variables() + self.populate_recipient_variables() return self.data + # A not-so-brief digression about Mailgun's batch sending, template personalization, + # and metadata tracking capabilities... + # + # Mailgun has two kinds of templates: + # * ESP-stored templates (handlebars syntax), referenced by template name in the + # send API, with substitution data supplied as "custom data" variables. + # Anymail's `template_id` maps to this feature. + # * On-the-fly templating (`%recipient.KEY%` syntax), with template variables + # appearing directly in the message headers and/or body, and data supplied + # as "recipient variables" per-recipient personalizations. Mailgun docs also + # sometimes refer to this data as "template variables," but it's distinct from + # the substitution data used for stored handelbars templates. + # + # Mailgun has two mechanisms for supplying additional data with a message: + # * "Custom data" is supplied via `v:KEY` and/or `h:X-Mailgun-Variables` fields. + # Custom data is passed to tracking webhooks (as 'user-variables') and is + # available for `{{substitutions}}` in ESP-stored handlebars templates. + # Normally, the same custom data is applied to every recipient of a message. + # * "Recipient variables" are supplied via the `recipient-variables` field, and + # provide per-recipient data for batch sending. The recipient specific values + # are available as `%recipient.KEY%` virtually anywhere in the message + # (including header fields and other parameters). + # + # Anymail needs both mechanisms to map its normalized metadata and template merge_data + # features to Mailgun: + # (1) Anymail's `metadata` maps directly to Mailgun's custom data, where it can be + # accessed from webhooks. + # (2) Anymail's `merge_metadata` (per-recipient metadata for batch sends) maps + # *indirectly* through recipient-variables to Mailgun's custom data. To avoid + # conflicts, the recipient-variables mapping prepends 'v:' to merge_metadata keys. + # (E.g., Mailgun's custom-data "user" is set to "%recipient.v:user", which picks + # up its per-recipient value from Mailgun's `recipient-variables[to_email]["v:user"]`.) + # (3) Anymail's `merge_data` (per-recipient template substitutions) maps directly to + # Mailgun's `recipient-variables`, where it can be referenced in on-the-fly templates. + # (4) Anymail's `merge_global_data` (global template substitutions) is copied to + # Mailgun's `recipient-variables` for every recipient, as the default for missing + # `merge_data` keys. + # (5) Only if a stored template is used, `merge_data` and `merge_global_data` are + # *also* mapped *indirectly* through recipient-variables to Mailgun's custom data, + # where they can be referenced in handlebars {{substitutions}}. + # (E.g., Mailgun's custom-data "name" is set to "%recipient.name%", which picks + # up its per-recipient value from Mailgun's `recipient-variables[to_email]["name"]`.) + # + # If Anymail's `merge_data`, `template_id` (stored templates) and `metadata` (or + # `merge_metadata`) are used together, there's a possibility of conflicting keys in + # Mailgun's custom data. Anymail treats that conflict as an unsupported feature error. + def populate_recipient_variables(self): - """Populate Mailgun recipient-variables from merge data and metadata""" - merge_metadata_keys = set() # all keys used in any recipient's merge_metadata - for recipient_metadata in self.merge_metadata.values(): - merge_metadata_keys.update(recipient_metadata.keys()) - metadata_vars = {key: "v:%s" % key for key in merge_metadata_keys} # custom-var for key - - # Set up custom-var substitutions for merge metadata - # data['v:SomeMergeMetadataKey'] = '%recipient.v:SomeMergeMetadataKey%' - for var in metadata_vars.values(): - self.data[var] = "%recipient.{var}%".format(var=var) - - # Any (toplevel) metadata that is also in (any) merge_metadata must be be moved - # into recipient-variables; and all merge_metadata vars must have defaults - # (else they'll get the '%recipient.v:SomeMergeMetadataKey%' literal string). - base_metadata = {metadata_vars[key]: self.metadata.get(key, '') - for key in merge_metadata_keys} - - recipient_vars = {} - for addr in self.to_emails: - # For each recipient, Mailgun recipient-variables[addr] is merger of: - # 1. metadata, for any keys that appear in merge_metadata - recipient_data = base_metadata.copy() - - # 2. merge_metadata[addr], with keys prefixed with 'v:' - if addr in self.merge_metadata: - recipient_data.update({ - metadata_vars[key]: value for key, value in self.merge_metadata[addr].items() - }) - - # 3. merge_global_data (because Mailgun doesn't support global variables) - recipient_data.update(self.merge_global_data) - - # 4. merge_data[addr] - if addr in self.merge_data: - recipient_data.update(self.merge_data[addr]) - - if recipient_data: - recipient_vars[addr] = recipient_data - - self.data['recipient-variables'] = self.serialize_json(recipient_vars) + """Populate Mailgun recipient-variables and custom data from merge data and metadata""" + # (numbers refer to detailed explanation above) + # Mailgun parameters to construct: + recipient_variables = {} + custom_data = {} + + # (1) metadata --> Mailgun custom_data + custom_data.update(self.metadata) + + # (2) merge_metadata --> Mailgun custom_data via recipient_variables + if self.merge_metadata: + def vkey(key): # 'v:key' + return 'v:{}'.format(key) + + merge_metadata_keys = flatset( # all keys used in any recipient's merge_metadata + recipient_data.keys() for recipient_data in self.merge_metadata.values()) + custom_data.update({ # custom_data['key'] = '%recipient.v:key%' indirection + key: '%recipient.{}%'.format(vkey(key)) + for key in merge_metadata_keys}) + base_recipient_data = { # defaults for each recipient must cover all keys + vkey(key): self.metadata.get(key, '') + for key in merge_metadata_keys} + for email in self.to_emails: + this_recipient_data = base_recipient_data.copy() + this_recipient_data.update({ + vkey(key): value + for key, value in self.merge_metadata.get(email, {}).items()}) + recipient_variables.setdefault(email, {}).update(this_recipient_data) + + # (3) and (4) merge_data, merge_global_data --> Mailgun recipient_variables + if self.merge_data or self.merge_global_data: + merge_data_keys = flatset( # all keys used in any recipient's merge_data + recipient_data.keys() for recipient_data in self.merge_data.values()) + merge_data_keys = merge_data_keys.union(self.merge_global_data.keys()) + base_recipient_data = { # defaults for each recipient must cover all keys + key: self.merge_global_data.get(key, '') + for key in merge_data_keys} + for email in self.to_emails: + this_recipient_data = base_recipient_data.copy() + this_recipient_data.update(self.merge_data.get(email, {})) + recipient_variables.setdefault(email, {}).update(this_recipient_data) + + # (5) if template, also map Mailgun custom_data to per-recipient_variables + if self.data.get('template') is not None: + conflicts = merge_data_keys.intersection(custom_data.keys()) + if conflicts: + self.unsupported_feature( + "conflicting merge_data and metadata keys (%s) when using template_id" + % ', '.join("'%s'" % key for key in conflicts)) + custom_data.update({ # custom_data['key'] = '%recipient.key%' indirection + key: '%recipient.{}%'.format(key) + for key in merge_data_keys}) + + # populate Mailgun params + self.data.update({'v:%s' % key: value + for key, value in custom_data.items()}) + if recipient_variables or self.is_batch(): + self.data['recipient-variables'] = self.serialize_json(recipient_variables) # # Payload construction # def init_payload(self): - self.data = {} # {field: [multiple, values]} + self.data = {} # {field: [multiple, values]} self.files = [] # [(field, multiple), (field, values)] self.headers = {} @@ -285,3 +349,12 @@ def isascii(s): except UnicodeEncodeError: return False return True + + +def flatset(iterables): + """Return a set of the items in a single-level flattening of iterables + + >>> flatset([1, 2], [2, 3]) + set(1, 2, 3) + """ + return set(item for iterable in iterables for item in iterable) diff --git a/docs/esps/index.rst b/docs/esps/index.rst index 44805348..3862be66 100644 --- a/docs/esps/index.rst +++ b/docs/esps/index.rst @@ -46,7 +46,7 @@ Email Service Provider |Amazon SES| |Mailgun| |Mailje .. rubric:: :ref:`templates-and-merge` --------------------------------------------------------------------------------------------------------------------------------------------------- -:attr:`~AnymailMessage.template_id` Yes No Yes Yes Yes Yes Yes Yes +:attr:`~AnymailMessage.template_id` Yes Yes Yes Yes Yes Yes Yes Yes :attr:`~AnymailMessage.merge_data` Yes Yes Yes Yes No Yes No Yes :attr:`~AnymailMessage.merge_global_data` Yes (emulated) Yes Yes Yes Yes Yes Yes diff --git a/docs/esps/mailgun.rst b/docs/esps/mailgun.rst index 71a20154..059e3678 100644 --- a/docs/esps/mailgun.rst +++ b/docs/esps/mailgun.rst @@ -217,6 +217,13 @@ Limitations and quirks the message to send, so it won't be present in your Mailgun API logs or the metadata that is sent to tracking webhooks.) +**Additional limitations on merge_data with template_id** + If you are using Mailgun's stored handlebars templates (Anymail's + :attr:`~anymail.message.AnymailMessage.template_id`), :attr:`~anymail.message.AnymailMessage.merge_data` + cannot contain complex types or have any keys that conflict with + :attr:`~anymail.message.AnymailMessage.metadata`. See :ref:`mailgun-template-limitations` + below for more details. + **merge_metadata values default to empty string** If you use Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata` feature, and you supply metadata keys for some recipients but not others, Anymail will first @@ -233,20 +240,43 @@ Limitations and quirks .. _mailgun-templates: -Batch sending/merge +Batch sending/merge and ESP templates ------------------------------------- -Mailgun supports :ref:`batch sending ` with per-recipient -merge data. You can refer to Mailgun "recipient variables" in your -message subject and body, and supply the values with Anymail's -normalized :attr:`~anymail.message.AnymailMessage.merge_data` -and :attr:`~anymail.message.AnymailMessage.merge_global_data` -message attributes: +Mailgun supports :ref:`ESP stored templates `, on-the-fly +templating, and :ref:`batch sending ` with per-recipient merge data. + +.. versionchanged:: 6.2 + + Added support for Mailgun's stored (handlebars) templates. + +Mailgun has two different syntaxes for substituting data into templates: + +* "Recipient variables" look like ``%recipient.name%``, and are used with on-the-fly + templates. You can refer to a recipient variable inside a message's body, subject, + or other message attributes defined in your Django code. See `Mailgun batch sending`_ + for more information. (Note that Mailgun's docs also sometimes refer to recipient + variables as "template *variables*," and there are some additional predefined ones + described in their docs.) + +* "Template *substitutions*" look like ``{{ name }}``, and can *only* be used in + handlebars templates that are defined and stored in your Mailgun account (via + the Mailgun dashboard or API). You refer to a stored template using Anymail's + :attr:`~anymail.message.AnymailMessage.template_id` in your Django code. + See `Mailgun templates`_ for more information. + +With either type of template, you supply the substitution data using Anymail's +normalized :attr:`~anymail.message.AnymailMessage.merge_data` and +:attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes. Anymail +will figure out the correct Mailgun API parameters to use. + +Here's an example defining an on-the-fly template that uses Mailgun recipient variables: .. code-block:: python message = EmailMessage( - ... + from_email="shipping@example.com", + # Use %recipient.___% syntax in subject and body: subject="Your order %recipient.order_no% has shipped", body="""Hi %recipient.name%, We shipped your order %recipient.order_no% @@ -262,16 +292,98 @@ message attributes: 'ship_date': "May 15" # Anymail maps globals to all recipients } +And here's an example that uses the same data with a stored template, which could refer +to ``{{ name }}``, ``{{ order_no }}``, and ``{{ ship_date }}`` in its definition: + + .. code-block:: python + + message = EmailMessage( + from_email="shipping@example.com", + # The message body and html_body come from from the stored template. + # (You can still use %recipient.___% fields in the subject:) + subject="Your order %recipient.order_no% has shipped", + to=["alice@example.com", "Bob "] + ) + message.template_id = 'shipping-notification' # name of template in our account + # The substitution data is exactly the same as in the previous example: + message.merge_data = { + 'alice@example.com': {'name': "Alice", 'order_no': "12345"}, + 'bob@example.com': {'name': "Bob", 'order_no': "54321"}, + } + message.merge_global_data = { + 'ship_date': "May 15" # Anymail maps globals to all recipients + } + +When you supply per-recipient :attr:`~anymail.message.AnymailMessage.merge_data`, +Anymail supplies Mailgun's ``recipient-variables`` parameter, which puts Mailgun +in batch sending mode so that each "to" recipient sees only their own email address. +(Any cc's or bcc's will be duplicated for *every* to-recipient.) + +If you want to use batch sending with a regular message (without a template), set +merge data to an empty dict: `message.merge_data = {}`. + Mailgun does not natively support global merge data. Anymail emulates -the capability by copying any `merge_global_data` values to each -recipient's section in Mailgun's "recipient-variables" API parameter. +the capability by copying any :attr:`~anymail.message.AnymailMessage.merge_global_data` +values to every recipient. + +.. _mailgun-template-limitations: + +Limitations with stored handlebars templates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Although Anymail tries to insulate you from Mailgun's relatively complicated API +parameters for template substitutions in batch sends, there are two cases it can't +handle. These *only* apply to stored handlebars templates (when you've set Anymail's +:attr:`~anymail.message.AnymailMessage.template_id` attribute). + +First, metadata and template merge data substitutions use the same underlying +"custom data" API parameters when a handlebars template is used. If you have any +duplicate keys between your tracking metadata +(:attr:`~anymail.message.AnymailMessage.metadata`/:attr:`~anymail.message.AnymailMessage.merge_metadata`) +and your template merge data +(:attr:`~anymail.message.AnymailMessage.merge_data`/:attr:`~anymail.message.AnymailMessage.merge_global_data`), +Anymail will raise an :exc:`~anymail.exceptions.AnymailUnsupportedFeature` error. + +Second, Mailgun's API does not allow complex data types like lists or dicts to be +passed as template substitutions for a batch send (confirmed with Mailgun support +8/2019). Your Anymail :attr:`~anymail.message.AnymailMessage.merge_data` and +:attr:`~anymail.message.AnymailMessage.merge_global_data` should only use simple +types like string or number. This means you cannot use the handlebars ``{{#each item}}`` +block helper or dotted field notation like ``{{object.field}}`` with data passed +through Anymail's normalized merge data attributes. + +Most ESPs do not support complex merge data types, so trying to do that is not recommended +anyway, for portability reasons. But if you *do* want to pass complex types to Mailgun +handlebars templates, and you're only sending to one recipient at a time, here's a +(non-portable!) workaround: -See the `Mailgun batch sending`_ docs for more information. + .. code-block:: python -.. _Mailgun batch sending: - https://documentation.mailgun.com/user_manual.html#batch-sending + # Using complex substitutions with Mailgun handlebars templates. + # This works only for a single recipient, and is not at all portable between ESPs. + message = EmailMessage( + from_email="shipping@example.com", + to=["alice@example.com"] # single recipient *only* (no batch send) + subject="Your order has shipped", # recipient variables *not* available + ) + message.template_id = 'shipping-notification' # name of template in our account + substitutions = { + 'items': [ # complex substitution data + {'product': "Anvil", 'quantity': 1}, + {'product': "Tacks", 'quantity': 100}, + ], + 'ship_date': "May 15", + } + # Do *not* set Anymail's message.merge_data, merge_global_data, or merge_metadata. + # Instead add Mailgun custom variables directly: + message.extra_headers['X-Mailgun-Variables'] = json.dumps(substitutions) +.. _Mailgun batch sending: + https://documentation.mailgun.com/en/latest/user_manual.html#batch-sending +.. _Mailgun templates: + https://documentation.mailgun.com/en/latest/user_manual.html#templates + .. _mailgun-webhooks: Status tracking webhooks diff --git a/tests/test_mailgun_backend.py b/tests/test_mailgun_backend.py index f213ec96..db19a689 100644 --- a/tests/test_mailgun_backend.py +++ b/tests/test_mailgun_backend.py @@ -490,7 +490,39 @@ def test_only_merge_global_data(self): 'bob@example.com': {'test': "value"}, }) + def test_merge_data_with_template(self): + # Mailgun *stored* (handlebars) templates get their variable substitutions + # from Mailgun's custom-data (not recipient-variables). To support batch sends + # with stored templates, Anymail sets up custom-data to pull values from + # recipient-variables. (Note this same Mailgun custom-data is also used for + # webhook metadata tracking.) + self.message.to = ['alice@example.com', 'Bob '] + self.message.template_id = 'welcome_template' + self.message.merge_data = { + 'alice@example.com': {'name': "Alice", 'group': "Developers"}, + 'bob@example.com': {'name': "Bob"}, # and leave group undefined + } + self.message.merge_global_data = { + 'group': "Users", # default + 'site': "ExampleCo", + } + self.message.send() + data = self.get_api_call_data() + # custom-data variables for merge_data refer to recipient-variables: + self.assertEqual(data['v:name'], '%recipient.name%') + self.assertEqual(data['v:group'], '%recipient.group%') + self.assertEqual(data['v:site'], '%recipient.site%') + # recipient-variables populates them: + self.assertJSONEqual(data['recipient-variables'], { + 'alice@example.com': {'name': "Alice", 'group': "Developers", 'site': "ExampleCo"}, + 'bob@example.com': {'name': "Bob", 'group': "Users", 'site': "ExampleCo"}, + }) + def test_merge_metadata(self): + # Per-recipient custom-data uses the same recipient-variables mechanism + # as above, but prepends 'v:' to the recipient-data keys for metadata to + # keep them separate. + # (For on-the-fly templates -- not stored handlebars templates.) self.message.to = ['alice@example.com', 'Bob '] self.message.merge_metadata = { 'alice@example.com': {'order_id': 123, 'tier': 'premium'}, @@ -528,10 +560,54 @@ def test_merge_data_with_merge_metadata(self): self.assertJSONEqual(data['recipient-variables'], { 'alice@example.com': {'name': "Alice", 'group': "Developers", 'v:order_id': 123, 'v:tier': 'premium'}, - 'bob@example.com': {'name': "Bob", # undefined merge_data --> omitted + 'bob@example.com': {'name': "Bob", 'group': '', # undefined merge_data --> empty string + 'v:order_id': 678, 'v:tier': ''}, # undefined metadata --> empty string + }) + + def test_merge_data_with_merge_metadata_and_template(self): + # This case gets tricky, because when a stored template is used, the per-recipient + # merge_metadata and merge_data both end up in the same Mailgun custom-data keys. + self.message.to = ['alice@example.com', 'Bob '] + self.message.template_id = 'order_notification' + self.message.merge_data = { + 'alice@example.com': {'name': "Alice", 'group': "Developers"}, + 'bob@example.com': {'name': "Bob"}, # and leave group undefined + } + self.message.merge_metadata = { + 'alice@example.com': {'order_id': 123, 'tier': 'premium'}, + 'bob@example.com': {'order_id': 678}, # and leave tier undefined + } + self.message.send() + + data = self.get_api_call_data() + # custom-data covers both merge_data and merge_metadata: + self.assertEqual(data['v:name'], '%recipient.name%') # from merge_data + self.assertEqual(data['v:group'], '%recipient.group%') # from merge_data + self.assertEqual(data['v:order_id'], '%recipient.v:order_id%') # from merge_metadata + self.assertEqual(data['v:tier'], '%recipient.v:tier%') # from merge_metadata + self.assertJSONEqual(data['recipient-variables'], { + 'alice@example.com': {'name': "Alice", 'group': "Developers", + 'v:order_id': 123, 'v:tier': 'premium'}, + 'bob@example.com': {'name': "Bob", 'group': '', # undefined merge_data --> empty string 'v:order_id': 678, 'v:tier': ''}, # undefined metadata --> empty string }) + def test_conflicting_merge_data_with_merge_metadata_and_template(self): + # When a stored template is used, the same Mailgun custom-data must hold both + # per-recipient merge_data and metadata, so there's potential for conflict. + self.message.to = ['alice@example.com', 'Bob '] + self.message.template_id = 'order_notification' + self.message.merge_data = { + 'alice@example.com': {'name': "Alice", 'group': "Developers"}, + 'bob@example.com': {'name': "Bob"}, + } + self.message.metadata = {'group': "Order processing subsystem"} + with self.assertRaisesMessage( + AnymailUnsupportedFeature, + "conflicting merge_data and metadata keys ('group') when using template_id" + ): + self.message.send() + def test_force_batch(self): # Mailgun uses presence of recipient-variables to indicate batch send self.message.to = ['alice@example.com', 'Bob '] diff --git a/tests/test_mailgun_integration.py b/tests/test_mailgun_integration.py index 1647cbd6..a1e4e7a8 100644 --- a/tests/test_mailgun_integration.py +++ b/tests/test_mailgun_integration.py @@ -161,6 +161,27 @@ def test_all_options(self): # (We could try fetching the message from event["storage"]["url"] # to verify content and other headers.) + def test_stored_template(self): + message = AnymailMessage( + template_id='test-template', # name of a real template named in Anymail's Mailgun test account + subject='Your order %recipient.order%', # Mailgun templates don't define subject + from_email='Test From ', # Mailgun templates don't define sender + to=["test+to1@anymail.info"], + # metadata and merge_data must not have any conflicting keys when using template_id + metadata={"meta1": "simple string", "meta2": 2}, + merge_data={ + 'test+to1@anymail.info': { + 'name': "Test Recipient", + } + }, + merge_global_data={ + 'order': '12345', + }, + ) + message.send() + recipient_status = message.anymail_status.recipients + self.assertEqual(recipient_status['test+to1@anymail.info'].status, 'queued') + # As of Anymail 0.10, this test is no longer possible, because # Anymail now raises AnymailInvalidAddress without even calling Mailgun # def test_invalid_from(self):