diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 15f17b7f..38c21ba5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -39,6 +39,17 @@ Breaking changes code is doing something like `message.anymail_status.recipients[email.lower()]`, you should remove the `.lower()` +Features +~~~~~~~~ + +* Add new `merge_metadata` option for providing per-recipient metadata in batch + sends. Available for all supported ESPs *except* Amazon SES and SendinBlue. + See `docs `_. + (Thanks `@janneThoft`_ for the idea and SendGrid implementation.) + +* **Mailjet:** Remove limitation on using `cc` or `bcc` together with `merge_data`. + + Fixes ~~~~~ @@ -908,6 +919,7 @@ Features .. _@calvin: https://github.com/calvin .. _@costela: https://github.com/costela .. _@decibyte: https://github.com/decibyte +.. _@janneThoft: https://github.com/janneThoft .. _@joshkersey: https://github.com/joshkersey .. _@Lekensteyn: https://github.com/Lekensteyn .. _@lewistaylor: https://github.com/lewistaylor diff --git a/anymail/backends/base.py b/anymail/backends/base.py index 2bcbd258..ddac3a78 100644 --- a/anymail/backends/base.py +++ b/anymail/backends/base.py @@ -250,11 +250,16 @@ class BasePayload(object): ) esp_message_attrs = () # subclasses can override + # If any of these attrs are set on a message, treat the message + # as a batch send (separate message for each `to` recipient): + batch_attrs = ('merge_data', 'merge_metadata') + def __init__(self, message, defaults, backend): self.message = message self.defaults = defaults self.backend = backend self.esp_name = backend.esp_name + self._batch_attrs_used = {attr: UNSET for attr in self.batch_attrs} self.init_payload() @@ -287,6 +292,20 @@ def __init__(self, message, defaults, backend): # AttributeError here? Your Payload subclass is missing a set_ implementation setter = getattr(self, 'set_%s' % attr) setter(value) + if attr in self.batch_attrs: + self._batch_attrs_used[attr] = (value is not UNSET) + + def is_batch(self): + """ + Return True if the message should be treated as a batch send. + + Intended to be used inside serialize_data or similar, after all relevant + attributes have been processed. Will error if called before that (e.g., + inside a set_ method or during __init__). + """ + batch_attrs_used = self._batch_attrs_used.values() + assert UNSET not in batch_attrs_used, "Cannot call is_batch before all attributes processed" + return any(batch_attrs_used) def unsupported_feature(self, feature): if not self.backend.ignore_unsupported_features: diff --git a/anymail/backends/mailgun.py b/anymail/backends/mailgun.py index 831df480..49536c4a 100644 --- a/anymail/backends/mailgun.py +++ b/anymail/backends/mailgun.py @@ -68,8 +68,10 @@ def __init__(self, message, defaults, backend, *args, **kwargs): self.all_recipients = [] # used for backend.parse_recipient_status # late-binding of recipient-variables: - self.merge_data = None - self.merge_global_data = None + self.merge_data = {} + self.merge_global_data = {} + self.metadata = {} + self.merge_metadata = {} self.to_emails = [] super(MailgunPayload, self).__init__(message, defaults, backend, auth=auth, *args, **kwargs) @@ -117,32 +119,51 @@ def get_request_params(self, api_url): return params def serialize_data(self): - self.populate_recipient_variables() + if self.is_batch() or self.merge_global_data: + self.populate_recipient_variables() return self.data def populate_recipient_variables(self): - """Populate Mailgun recipient-variables header from merge data""" - merge_data = self.merge_data - - if self.merge_global_data is not None: - # Mailgun doesn't support global variables. - # We emulate them by populating recipient-variables for all recipients. - if merge_data is not None: - merge_data = merge_data.copy() # don't modify the original, which doesn't belong to us - else: - merge_data = {} - for email in self.to_emails: - try: - recipient_data = merge_data[email] - except KeyError: - merge_data[email] = self.merge_global_data - else: - # Merge globals (recipient_data wins in conflict) - merge_data[email] = self.merge_global_data.copy() - merge_data[email].update(recipient_data) - - if merge_data is not None: - self.data['recipient-variables'] = self.serialize_json(merge_data) + """Populate Mailgun recipient-variables from merge data and metadata""" + merge_metadata_keys = set() # all keys used in any recipient's merge_metadata + for recipient_metadata in self.merge_metadata.values(): + merge_metadata_keys.update(recipient_metadata.keys()) + metadata_vars = {key: "v:%s" % key for key in merge_metadata_keys} # custom-var for key + + # Set up custom-var substitutions for merge metadata + # data['v:SomeMergeMetadataKey'] = '%recipient.v:SomeMergeMetadataKey%' + for var in metadata_vars.values(): + self.data[var] = "%recipient.{var}%".format(var=var) + + # Any (toplevel) metadata that is also in (any) merge_metadata must be be moved + # into recipient-variables; and all merge_metadata vars must have defaults + # (else they'll get the '%recipient.v:SomeMergeMetadataKey%' literal string). + base_metadata = {metadata_vars[key]: self.metadata.get(key, '') + for key in merge_metadata_keys} + + recipient_vars = {} + for addr in self.to_emails: + # For each recipient, Mailgun recipient-variables[addr] is merger of: + # 1. metadata, for any keys that appear in merge_metadata + recipient_data = base_metadata.copy() + + # 2. merge_metadata[addr], with keys prefixed with 'v:' + if addr in self.merge_metadata: + recipient_data.update({ + metadata_vars[key]: value for key, value in self.merge_metadata[addr].items() + }) + + # 3. merge_global_data (because Mailgun doesn't support global variables) + recipient_data.update(self.merge_global_data) + + # 4. merge_data[addr] + if addr in self.merge_data: + recipient_data.update(self.merge_data[addr]) + + if recipient_data: + recipient_vars[addr] = recipient_data + + self.data['recipient-variables'] = self.serialize_json(recipient_vars) # # Payload construction @@ -210,6 +231,7 @@ def set_envelope_sender(self, email): self.sender_domain = email.domain def set_metadata(self, metadata): + self.metadata = metadata # save for handling merge_metadata later for key, value in metadata.items(): self.data["v:%s" % key] = value @@ -242,6 +264,10 @@ def set_merge_global_data(self, merge_global_data): # Processed at serialization time (to allow merging global data) self.merge_global_data = merge_global_data + def set_merge_metadata(self, merge_metadata): + # Processed at serialization time (to allow combining with merge_data) + self.merge_metadata = merge_metadata + def set_esp_extra(self, extra): self.data.update(extra) # Allow override of sender_domain via esp_extra diff --git a/anymail/backends/mailjet.py b/anymail/backends/mailjet.py index 2fe45d42..0e873fbc 100644 --- a/anymail/backends/mailjet.py +++ b/anymail/backends/mailjet.py @@ -80,8 +80,10 @@ def __init__(self, message, defaults, backend, *args, **kwargs): 'Content-Type': 'application/json', } # Late binding of recipients and their variables - self.recipients = {} - self.merge_data = None + self.recipients = {'to': []} + self.metadata = None + self.merge_data = {} + self.merge_metadata = {} super(MailjetPayload, self).__init__(message, defaults, backend, auth=auth, headers=http_headers, *args, **kwargs) @@ -89,22 +91,37 @@ def get_api_endpoint(self): return "send" def serialize_data(self): - self._finish_recipients() self._populate_sender_from_template() + if self.is_batch(): + self.data = {'Messages': [ + self._data_for_recipient(to_addr) + for to_addr in self.recipients['to'] + ]} return self.serialize_json(self.data) - # - # Payload construction - # - - def _finish_recipients(self): - # NOTE do not set both To and Recipients, it behaves specially: each - # recipient receives a separate mail but the To address receives one - # listing all recipients. - if "cc" in self.recipients or "bcc" in self.recipients: - self._finish_recipients_single() - else: - self._finish_recipients_with_vars() + def _data_for_recipient(self, email): + # Return send data for single recipient, without modifying self.data + data = self.data.copy() + data['To'] = self._format_email_for_mailjet(email) + + if email.addr_spec in self.merge_data: + recipient_merge_data = self.merge_data[email.addr_spec] + if 'Vars' in data: + data['Vars'] = data['Vars'].copy() # clone merge_global_data + data['Vars'].update(recipient_merge_data) + else: + data['Vars'] = recipient_merge_data + + if email.addr_spec in self.merge_metadata: + recipient_metadata = self.merge_metadata[email.addr_spec] + if self.metadata: + metadata = self.metadata.copy() # clone toplevel metadata + metadata.update(recipient_metadata) + else: + metadata = recipient_metadata + data["Mj-EventPayLoad"] = self.serialize_json(metadata) + + return data def _populate_sender_from_template(self): # If no From address was given, use the address from the template. @@ -137,42 +154,21 @@ def _populate_sender_from_template(self): email_message=self.message, response=response, backend=self.backend) self.set_from_email(parsed) - def _finish_recipients_with_vars(self): - """Send bulk mail with different variables for each mail.""" - assert "Cc" not in self.data and "Bcc" not in self.data - recipients = [] - merge_data = self.merge_data or {} - for email in self.recipients["to"]: - recipient = { - "Email": email.addr_spec, - "Name": email.display_name, - "Vars": merge_data.get(email.addr_spec) - } - # Strip out empty Name and Vars - recipient = {k: v for k, v in recipient.items() if v} - recipients.append(recipient) - self.data["Recipients"] = recipients - - def _finish_recipients_single(self): - """Send a single mail with some To, Cc and Bcc headers.""" - assert "Recipients" not in self.data - if self.merge_data: - # When Cc and Bcc headers are given, then merge data cannot be set. - raise NotImplementedError("Cannot set merge data with bcc/cc") - for recipient_type, emails in self.recipients.items(): - # Workaround Mailjet 3.0 bug parsing display-name with commas - # (see test_comma_in_display_name in test_mailjet_backend for details) - formatted_emails = [ - email.address if "," not in email.display_name - # else name has a comma, so force it into MIME encoded-word utf-8 syntax: - else EmailAddress(email.display_name.encode('utf-8'), email.addr_spec).formataddr('utf-8') - for email in emails - ] - self.data[recipient_type.capitalize()] = ", ".join(formatted_emails) + def _format_email_for_mailjet(self, email): + """Return EmailAddress email converted to a string that Mailjet can parse properly""" + # Workaround Mailjet 3.0 bug parsing display-name with commas + # (see test_comma_in_display_name in test_mailjet_backend for details) + if "," in email.display_name: + return EmailAddress(email.display_name.encode('utf-8'), email.addr_spec).formataddr('utf-8') + else: + return email.address + + # + # Payload construction + # def init_payload(self): - self.data = { - } + self.data = {} def set_from_email(self, email): self.data["FromEmail"] = email.addr_spec @@ -181,9 +177,10 @@ def set_from_email(self, email): def set_recipients(self, recipient_type, emails): assert recipient_type in ["to", "cc", "bcc"] - # Will be handled later in serialize_data if emails: - self.recipients[recipient_type] = emails + self.recipients[recipient_type] = emails # save for recipient_status processing + self.data[recipient_type.capitalize()] = ", ".join( + [self._format_email_for_mailjet(email) for email in emails]) def set_subject(self, subject): self.data["Subject"] = subject @@ -225,8 +222,8 @@ def set_envelope_sender(self, email): self.data["Sender"] = email.addr_spec # ??? v3 docs unclear def set_metadata(self, metadata): - # Mailjet expects a single string payload self.data["Mj-EventPayLoad"] = self.serialize_json(metadata) + self.metadata = metadata # keep original in case we need to merge with merge_metadata def set_tags(self, tags): # The choices here are CustomID or Campaign, and Campaign seems closer @@ -257,5 +254,9 @@ def set_merge_data(self, merge_data): def set_merge_global_data(self, merge_global_data): self.data["Vars"] = merge_global_data + def set_merge_metadata(self, merge_metadata): + # Will be handled later in serialize_data + self.merge_metadata = merge_metadata + def set_esp_extra(self, extra): self.data.update(extra) diff --git a/anymail/backends/mandrill.py b/anymail/backends/mandrill.py index 15f39842..24a1d31a 100644 --- a/anymail/backends/mandrill.py +++ b/anymail/backends/mandrill.py @@ -79,6 +79,9 @@ def get_api_endpoint(self): def serialize_data(self): self.process_esp_extra() + if self.is_batch(): + # hide recipients from each other + self.data['message']['preserve_recipients'] = False return self.serialize_json(self.data) # @@ -163,7 +166,6 @@ def set_template_id(self, template_id): self.data.setdefault("template_content", []) # Mandrill requires something here def set_merge_data(self, merge_data): - self.data['message']['preserve_recipients'] = False # if merge, hide recipients from each other self.data['message']['merge_vars'] = [ {'rcpt': rcpt, 'vars': [{'name': key, 'content': rcpt_data[key]} for key in sorted(rcpt_data.keys())]} # sort for testing reproducibility @@ -176,6 +178,13 @@ def set_merge_global_data(self, merge_global_data): for var, value in merge_global_data.items() ] + def set_merge_metadata(self, merge_metadata): + # recipient_metadata format is similar to, but not quite the same as, merge_vars: + self.data['message']['recipient_metadata'] = [ + {'rcpt': rcpt, 'values': rcpt_data} + for rcpt, rcpt_data in merge_metadata.items() + ] + def set_esp_extra(self, extra): # late bind in serialize_data, so that obsolete Djrill attrs can contribute self.esp_extra = extra diff --git a/anymail/backends/postmark.py b/anymail/backends/postmark.py index 635b89fc..58a9a659 100644 --- a/anymail/backends/postmark.py +++ b/anymail/backends/postmark.py @@ -156,10 +156,11 @@ def __init__(self, message, defaults, backend, *args, **kwargs): self.to_emails = [] self.cc_and_bcc_emails = [] # need to track (separately) for parse_recipient_status self.merge_data = None + self.merge_metadata = None super(PostmarkPayload, self).__init__(message, defaults, backend, headers=headers, *args, **kwargs) def get_api_endpoint(self): - batch_send = self.merge_data is not None and len(self.to_emails) > 1 + batch_send = self.is_batch() and len(self.to_emails) > 1 if 'TemplateAlias' in self.data or 'TemplateId' in self.data or 'TemplateModel' in self.data: if batch_send: return "email/batchWithTemplates" @@ -197,6 +198,14 @@ def data_for_recipient(self, to): data["TemplateModel"].update(recipient_data) else: data["TemplateModel"] = recipient_data + if self.merge_metadata and to.addr_spec in self.merge_metadata: + recipient_metadata = self.merge_metadata[to.addr_spec] + if "Metadata" in data: + # merge recipient_metadata into toplevel metadata + data["Metadata"] = data["Metadata"].copy() + data["Metadata"].update(recipient_metadata) + else: + data["Metadata"] = recipient_metadata return data # @@ -298,6 +307,10 @@ def set_merge_data(self, merge_data): def set_merge_global_data(self, merge_global_data): self.data["TemplateModel"] = merge_global_data + def set_merge_metadata(self, merge_metadata): + # late-bind + self.merge_metadata = merge_metadata + def set_esp_extra(self, extra): self.data.update(extra) # Special handling for 'server_token': diff --git a/anymail/backends/sparkpost.py b/anymail/backends/sparkpost.py index e51b18a0..8e11b5fe 100644 --- a/anymail/backends/sparkpost.py +++ b/anymail/backends/sparkpost.py @@ -95,11 +95,12 @@ def init_payload(self): self.all_recipients = [] self.to_emails = [] self.merge_data = {} + self.merge_metadata = {} def get_api_params(self): # Compose recipients param from to_emails and merge_data (if any) recipients = [] - if len(self.merge_data) > 0: + if self.is_batch(): # Build JSON recipient structures for email in self.to_emails: rcpt = {'address': {'email': email.addr_spec}} @@ -109,6 +110,10 @@ def get_api_params(self): rcpt['substitution_data'] = self.merge_data[email.addr_spec] except KeyError: pass # no merge_data or none for this recipient + try: + rcpt['metadata'] = self.merge_metadata[email.addr_spec] + except KeyError: + pass # no merge_metadata or none for this recipient recipients.append(rcpt) else: # Just use simple recipients list @@ -213,6 +218,9 @@ def set_template_id(self, template_id): def set_merge_data(self, merge_data): self.merge_data = merge_data # merged into params['recipients'] in get_api_params + def set_merge_metadata(self, merge_metadata): + self.merge_metadata = merge_metadata # merged into params['recipients'] in get_api_params + def set_merge_global_data(self, merge_global_data): self.params['substitution_data'] = merge_global_data diff --git a/anymail/backends/test.py b/anymail/backends/test.py index 1a9c9d39..86b92998 100644 --- a/anymail/backends/test.py +++ b/anymail/backends/test.py @@ -22,6 +22,9 @@ class EmailBackend(AnymailBaseBackend): esp_name = "Test" def __init__(self, *args, **kwargs): + # Allow replacing the payload, for testing. + # (Real backends would generally not implement this option.) + self._payload_class = kwargs.pop('payload_class', TestPayload) super(EmailBackend, self).__init__(*args, **kwargs) if not hasattr(mail, 'outbox'): mail.outbox = [] # see django.core.mail.backends.locmem @@ -32,7 +35,7 @@ def get_esp_message_id(self, message): return mail.outbox.index(message) def build_message_payload(self, message, defaults): - return TestPayload(backend=self, message=message, defaults=defaults) + return self._payload_class(backend=self, message=message, defaults=defaults) def post_to_esp(self, payload, message): # Keep track of the sent messages and params (for test cases) @@ -130,6 +133,9 @@ def set_template_id(self, template_id): def set_merge_data(self, merge_data): self.params['merge_data'] = merge_data + def set_merge_metadata(self, merge_metadata): + self.params['merge_metadata'] = merge_metadata + def set_merge_global_data(self, merge_global_data): self.params['merge_global_data'] = merge_global_data diff --git a/anymail/message.py b/anymail/message.py index 16848164..b908526a 100644 --- a/anymail/message.py +++ b/anymail/message.py @@ -29,6 +29,7 @@ def __init__(self, *args, **kwargs): self.template_id = kwargs.pop('template_id', UNSET) self.merge_data = kwargs.pop('merge_data', UNSET) self.merge_global_data = kwargs.pop('merge_global_data', UNSET) + self.merge_metadata = kwargs.pop('merge_metadata', UNSET) self.anymail_status = AnymailStatus() # noinspection PyArgumentList diff --git a/docs/esps/amazon_ses.rst b/docs/esps/amazon_ses.rst index 2b54dc1b..e80d40fd 100644 --- a/docs/esps/amazon_ses.rst +++ b/docs/esps/amazon_ses.rst @@ -68,6 +68,11 @@ Limitations and quirks :attr:`~anymail.message.AnymailMessage.tags` feature. See :ref:`amazon-ses-tags` below for more information and additional options. +**No merge_metadata** + Amazon SES's batch sending API does not support the custom headers Anymail uses + for metadata, so Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata` + feature is not available. (See :ref:`amazon-ses-tags` below for more information.) + **Open and click tracking overrides** Anymail's :attr:`~anymail.message.AnymailMessage.track_opens` and :attr:`~anymail.message.AnymailMessage.track_clicks` are not supported. @@ -164,7 +169,8 @@ For more complex use cases, set the SES `Tags` parameter directly in Anymail's :ref:`esp_extra `. See the example below. (Because custom headers do not work with SES's SendBulkTemplatedEmail call, esp_extra Tags is the only way to attach data to SES messages also using Anymail's :attr:`~anymail.message.AnymailMessage.template_id` -and :attr:`~anymail.message.AnymailMessage.merge_data` features.) +and :attr:`~anymail.message.AnymailMessage.merge_data` features, and the +:attr:`~anymail.message.AnymailMessage.merge_metadata` cannot be supported.) .. _Introducing Sending Metrics: diff --git a/docs/esps/index.rst b/docs/esps/index.rst index 5bdcd026..6a5fdbfd 100644 --- a/docs/esps/index.rst +++ b/docs/esps/index.rst @@ -38,6 +38,7 @@ Email Service Provider |Amazon SES| |Mailgun| |Mailje --------------------------------------------------------------------------------------------------------------------------------------------------- :attr:`~AnymailMessage.envelope_sender` Yes Domain only Yes Domain only No No No Yes :attr:`~AnymailMessage.metadata` Yes Yes Yes Yes Yes Yes Yes Yes +:attr:`~AnymailMessage.merge_metadata` No Yes Yes Yes Yes Yes No Yes :attr:`~AnymailMessage.send_at` No Yes No Yes No Yes No Yes :attr:`~AnymailMessage.tags` Yes Yes Max 1 tag Yes Max 1 tag Yes Max 1 tag Max 1 tag :attr:`~AnymailMessage.track_clicks` No Yes Yes Yes Yes Yes No Yes diff --git a/docs/esps/mailgun.rst b/docs/esps/mailgun.rst index 0122fa2c..850e1043 100644 --- a/docs/esps/mailgun.rst +++ b/docs/esps/mailgun.rst @@ -179,6 +179,29 @@ Limitations and quirks obvious reasons, only the domain portion applies. You can use anything before the @, and it will be ignored. +**Using merge_metadata with merge_data** + If you use both Anymail's :attr:`~anymail.message.AnymailMessage.merge_data` + and :attr:`~anymail.message.AnymailMessage.merge_metadata` features, make sure your + merge_data keys do not start with ``v:``. (It's a good idea anyway to avoid colons + and other special characters in merge_data keys, as this isn't generally portable + to other ESPs.) + + The same underlying Mailgun feature ("recipient-variables") is used to implement + both Anymail features. To avoid conflicts, Anymail prepends ``v:`` to recipient + variables needed for merge_metadata. (This prefix is stripped as Mailgun prepares + the message to send, so it won't be present in your Mailgun API logs or the metadata + that is sent to tracking webhooks.) + +**merge_metadata values default to empty string** + If you use Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata` feature, + and you supply metadata keys for some recipients but not others, Anymail will first + try to resolve the missing keys in :attr:`~anymail.message.AnymailMessage.metadata`, + and if they are not found there will default them to an empty string value. + + Your tracking webhooks will receive metadata values (either that you provided or the + default empty string) for *every* key used with *any* recipient in the send. + + .. _undocumented API requirement: https://mailgun.uservoice.com/forums/156243-feature-requests/suggestions/35668606 diff --git a/docs/esps/mailjet.rst b/docs/esps/mailjet.rst index e3170e1b..280941b4 100644 --- a/docs/esps/mailjet.rst +++ b/docs/esps/mailjet.rst @@ -11,9 +11,11 @@ Anymail integrates with the `Mailjet`_ email service, using their transactional .. note:: - Mailjet is developing an improved `v3.1 Send API`_ (in public beta as of mid-2017). - Once the v3.1 API is released, Anymail will switch to it. This change should be - largely transparent to your code, unless you are using Anymail's + Mailjet has released a newer `v3.1 Send API`_, but due to mismatches between its + documentation and actual behavior, Anymail has been unable to switch to it. + Anymail's maintainers have reported the problems to Mailjet, and if and when they + are resolved, Anymail will switch to the v3.1 API. This change should be largely + transparent to your code, unless you are using Anymail's :ref:`esp_extra ` feature to set API-specific options. @@ -132,26 +134,26 @@ Limitations and quirks special approval from Mailjet support to use custom senders. **Commas in recipient names** - Mailjet's v3 API does not properly handle commas in recipient display-names - *if* your message also uses the ``cc`` or ``bcc`` fields. + Mailjet's v3 API does not properly handle commas in recipient display-names. (Tested July, 2017, and confirmed with Mailjet API support.) If your message would be affected, Anymail attempts to work around the problem by switching to `MIME encoded-word`_ syntax where needed. Most modern email clients should support this syntax, but if you run - into issues either avoid using ``cc`` and ``bcc``, or strip commas from all + into issues, you might want to strip commas from all recipient names (in ``to``, ``cc``, *and* ``bcc``) before sending. + (This should be resolved in a future release when + Anymail :ref:`switches ` to Mailjet's upcoming v3.1 API.) + .. _MIME encoded-word: https://en.wikipedia.org/wiki/MIME#Encoded-Word -**Merge data not compatible with cc/bcc** - Mailjet's v3 API is not capable of representing both ``cc`` or ``bcc`` fields - and :attr:`~anymail.message.AnymailMessage.merge_data` in the same message. - If you attempt to combine them, Anymail will raise an error at send time. +.. versionchanged:: 6.0 -(The latter two limitations should be resolved in a future release when -Anymail :ref:`switches ` to Mailjet's upcoming v3.1 API.) + Earlier versions of Anymail were unable to mix ``cc`` or ``bcc`` fields + and :attr:`~anymail.message.AnymailMessage.merge_data` in the same Mailjet message. + This limitation was removed in Anymail 6.0. .. _mailjet-templates: diff --git a/docs/esps/sendinblue.rst b/docs/esps/sendinblue.rst index 044d1ace..fb6eda6d 100644 --- a/docs/esps/sendinblue.rst +++ b/docs/esps/sendinblue.rst @@ -178,7 +178,8 @@ SendinBlue supports :ref:`ESP stored templates ` populated with global merge data for all recipients, but does not offer :ref:`batch sending ` with per-recipient merge data. Anymail's :attr:`~anymail.message.AnymailMessage.merge_data` -message attribute is not supported with the SendinBlue backend. +and :attr:`~anymail.message.AnymailMessage.merge_metadata` +message attributes are not supported with the SendinBlue backend. To use a SendinBlue template, set the message's :attr:`~anymail.message.AnymailMessage.template_id` to the numeric diff --git a/docs/sending/anymail_additions.rst b/docs/sending/anymail_additions.rst index 05724c26..c01c01a1 100644 --- a/docs/sending/anymail_additions.rst +++ b/docs/sending/anymail_additions.rst @@ -115,6 +115,31 @@ ESP send options (AnymailMessage) as metadata. See :ref:`formatting-merge-data`. + .. attribute:: merge_metadata + + Set this to a `dict` of *per-recipient* metadata values the ESP should store + with the message, for later search and retrieval. Each key in the dict is a + recipient email (address portion only), and its value is a dict of metadata + for that recipient: + + .. code-block:: python + + message.to = ["wile@example.com", "Mr. Runner "] + message.merge_metadata = { + "wile@example.com": {"customer": 123, "order": "acme-zxyw"}, + "rr@example.com": {"customer": 45678, "order": "acme-wblt"}, + } + + When :attr:`!merge_metadata` is set, Anymail will use the ESP's + :ref:`batch sending ` option, so that each `to` recipient gets an + individual message (and doesn't see the other emails on the `to` list). + + All of the notes on :attr:`metadata` keys and value formatting also apply + to :attr:`!merge_metadata`. If there are conflicting keys, the + :attr:`!merge_metadata` values will take precedence over :attr:`!metadata` + for that recipient. + + .. attribute:: tags Set this to a `list` of `str` tags to apply to the message (usually @@ -131,7 +156,8 @@ ESP send options (AnymailMessage) .. caution:: - Some ESPs put :attr:`metadata` and :attr:`tags` in email headers, + Some ESPs put :attr:`metadata` (and a recipient's :attr:`merge_metadata`) + and :attr:`tags` in email headers, which are included with the email when it is delivered. Anything you put in them **could be exposed to the recipients,** so don't include sensitive data. diff --git a/docs/sending/templates.rst b/docs/sending/templates.rst index 389021a8..f9b21e7b 100644 --- a/docs/sending/templates.rst +++ b/docs/sending/templates.rst @@ -125,7 +125,8 @@ To use batch sending with Anymail (for ESPs that support it): .. caution:: - It's critical to set the :attr:`~AnymailMessage.merge_data` attribute: + It's critical to set the :attr:`~AnymailMessage.merge_data` + (or :attr:`~AnymailMessage.merge_metadata`) attribute: this is how Anymail recognizes the message as a batch send. When you provide merge_data, Anymail will tell the ESP to send an individual customized diff --git a/tests/test_general_backend.py b/tests/test_general_backend.py index 7594efee..7649283c 100644 --- a/tests/test_general_backend.py +++ b/tests/test_general_backend.py @@ -11,7 +11,7 @@ from django.utils.timezone import utc from django.utils.translation import ugettext_lazy -from anymail.backends.test import EmailBackend as TestBackend +from anymail.backends.test import EmailBackend as TestBackend, TestPayload from anymail.exceptions import AnymailConfigurationError, AnymailInvalidAddress, AnymailUnsupportedFeature from anymail.message import AnymailMessage from anymail.utils import get_anymail_setting @@ -425,3 +425,45 @@ def test_spoofed_to_header(self): self.message.extra_headers = {"To": "Apparent Recipient "} with self.assertRaisesMessage(AnymailUnsupportedFeature, "spoofing `To` header"): self.message.send() + + +class BatchSendDetectionTestCase(TestBackendTestCase): + """Tests shared code to consistently determine whether to use batch send""" + + def setUp(self): + super(BatchSendDetectionTestCase, self).setUp() + self.backend = TestBackend() + + def test_default_is_not_batch(self): + payload = self.backend.build_message_payload(self.message, {}) + self.assertFalse(payload.is_batch()) + + def test_merge_data_implies_batch(self): + self.message.merge_data = {} # *anything* (even empty dict) implies batch + payload = self.backend.build_message_payload(self.message, {}) + self.assertTrue(payload.is_batch()) + + def test_merge_metadata_implies_batch(self): + self.message.merge_metadata = {} # *anything* (even empty dict) implies batch + payload = self.backend.build_message_payload(self.message, {}) + self.assertTrue(payload.is_batch()) + + def test_merge_global_data_does_not_imply_batch(self): + self.message.merge_global_data = {} + payload = self.backend.build_message_payload(self.message, {}) + self.assertFalse(payload.is_batch()) + + def test_cannot_call_is_batch_during_init(self): + # It's tempting to try to warn about unsupported batch features in setters, + # but because of the way payload attrs are processed, it won't work... + class ImproperlyImplementedPayload(TestPayload): + def set_cc(self, emails): + if self.is_batch(): # this won't work here! + self.unsupported_feature("cc with batch send") + super(ImproperlyImplementedPayload, self).set_cc(emails) + + connection = mail.get_connection('anymail.backends.test.EmailBackend', + payload_class=ImproperlyImplementedPayload) + with self.assertRaisesMessage(AssertionError, + "Cannot call is_batch before all attributes processed"): + connection.send_messages([self.message]) diff --git a/tests/test_mailgun_backend.py b/tests/test_mailgun_backend.py index d2ad1ab9..842f2d7a 100644 --- a/tests/test_mailgun_backend.py +++ b/tests/test_mailgun_backend.py @@ -98,6 +98,7 @@ def test_email_message(self): self.assertEqual(data['h:Reply-To'], "another@example.com") self.assertEqual(data['h:X-MyHeader'], 'my value') self.assertEqual(data['h:Message-ID'], 'mycustommsgid@example.com') + self.assertNotIn('recipient-variables', data) # multiple recipients, but not a batch send def test_html_message(self): text_content = 'This is an important message.' @@ -387,6 +388,7 @@ def test_metadata(self): data = self.get_api_call_data() self.assertEqual(data['v:user_id'], '12345') self.assertEqual(data['v:items'], '["mail","gun"]') + self.assertNotIn('recipient-variables', data) # shouldn't be needed for non-batch def test_send_at(self): utc_plus_6 = get_fixed_timezone(6 * 60) @@ -484,6 +486,56 @@ def test_only_merge_global_data(self): 'bob@example.com': {'test': "value"}, }) + def test_merge_metadata(self): + self.message.to = ['alice@example.com', 'Bob '] + self.message.merge_metadata = { + 'alice@example.com': {'order_id': 123, 'tier': 'premium'}, + 'bob@example.com': {'order_id': 678}, + } + self.message.metadata = {'tier': 'basic', 'notification_batch': 'zx912'} + self.message.send() + + data = self.get_api_call_data() + # custom-data variables for merge_metadata refer to recipient-variables: + self.assertEqual(data['v:order_id'], '%recipient.v:order_id%') + self.assertEqual(data['v:tier'], '%recipient.v:tier%') + self.assertEqual(data['v:notification_batch'], 'zx912') # metadata constant doesn't need var + # recipient-variables populates them: + self.assertJSONEqual(data['recipient-variables'], { + 'alice@example.com': {'v:order_id': 123, 'v:tier': 'premium'}, + 'bob@example.com': {'v:order_id': 678, 'v:tier': 'basic'}, # tier merged from metadata default + }) + + def test_merge_data_with_merge_metadata(self): + # merge_data and merge_metadata both use recipient-variables + self.message.to = ['alice@example.com', 'Bob '] + self.message.body = "Hi %recipient.name%. Welcome to %recipient.group% at %recipient.site%." + self.message.merge_data = { + 'alice@example.com': {'name': "Alice", 'group': "Developers"}, + 'bob@example.com': {'name': "Bob"}, # and leave group undefined + } + self.message.merge_metadata = { + 'alice@example.com': {'order_id': 123, 'tier': 'premium'}, + 'bob@example.com': {'order_id': 678}, # and leave tier undefined + } + self.message.send() + + data = self.get_api_call_data() + self.assertJSONEqual(data['recipient-variables'], { + 'alice@example.com': {'name': "Alice", 'group': "Developers", + 'v:order_id': 123, 'v:tier': 'premium'}, + 'bob@example.com': {'name': "Bob", # undefined merge_data --> omitted + 'v:order_id': 678, 'v:tier': ''}, # undefined metadata --> empty string + }) + + def test_force_batch(self): + # Mailgun uses presence of recipient-variables to indicate batch send + self.message.to = ['alice@example.com', 'Bob '] + self.message.merge_data = {} + self.message.send() + data = self.get_api_call_data() + self.assertJSONEqual(data['recipient-variables'], {}) + def test_sender_domain(self): """Mailgun send domain can come from from_email, envelope_sender, or esp_extra""" # You could also use MAILGUN_SENDER_DOMAIN in your ANYMAIL settings, as in the next test. diff --git a/tests/test_mailjet_backend.py b/tests/test_mailjet_backend.py index 1eb7ad74..749e3928 100644 --- a/tests/test_mailjet_backend.py +++ b/tests/test_mailjet_backend.py @@ -77,7 +77,7 @@ def test_send_mail(self): self.assertEqual(data['Subject'], "Subject here") self.assertEqual(data['Text-part'], "Here is the message.") self.assertEqual(data['FromEmail'], "from@sender.example.com") - self.assertEqual(data['Recipients'], [{"Email": "to@example.com"}]) + self.assertEqual(data['To'], "to@example.com") def test_name_addr(self): """Make sure RFC2822 name-addr format (with display-name) is allowed @@ -99,25 +99,18 @@ def test_name_addr(self): self.assertEqual(data['Bcc'], 'Blind Copy , bcc2@example.com') def test_comma_in_display_name(self): - # note there are two paths: with cc/bcc, and without - msg = mail.EmailMessage( - 'Subject', 'Message', '"Example, Inc." ', - ['"Recipient, Ltd." ']) - msg.send() - data = self.get_api_call_json() - self.assertEqual(data['FromName'], 'Example, Inc.') - self.assertEqual(data['FromEmail'], 'from@example.com') - self.assertEqual(data['Recipients'][0]["Email"], "to@example.com") - self.assertEqual(data['Recipients'][0]["Name"], "Recipient, Ltd.") # separate Name field works fine - # Mailjet 3.0 API doesn't properly parse RFC-2822 quoted display-names from To/Cc/Bcc: # `To: "Recipient, Ltd." ` tries to send messages to `"Recipient` # and to `Ltd.` (neither of which are actual email addresses). # As a workaround, force MIME "encoded-word" utf-8 encoding, which gets past Mailjet's broken parsing. # (This shouldn't be necessary in Mailjet 3.1, where Name becomes a separate json field for Cc/Bcc.) - msg.cc = ['cc@example.com'] + msg = mail.EmailMessage( + 'Subject', 'Message', '"Example, Inc." ', + ['"Recipient, Ltd." ']) msg.send() data = self.get_api_call_json() + self.assertEqual(data['FromName'], 'Example, Inc.') + self.assertEqual(data['FromEmail'], 'from@example.com') # self.assertEqual(data['To'], '"Recipient, Ltd." ') # this doesn't work self.assertEqual(data['To'], '=?utf-8?q?Recipient=2C_Ltd=2E?= ') # workaround @@ -492,19 +485,50 @@ def test_template_invalid_response(self): self.message.send() def test_merge_data(self): - self.message.to = ['alice@example.com'] + self.message.to = ['alice@example.com', 'Bob '] + self.message.cc = ['cc@example.com'] self.message.template_id = '1234567' self.message.merge_data = { 'alice@example.com': {'name': "Alice", 'group': "Developers"}, + 'bob@example.com': {'name': "Bob"}, } + self.message.merge_global_data = {'group': "Users", 'site': "ExampleCo"} self.message.send() + data = self.get_api_call_json() - self.assertEqual(data['Mj-TemplateID'], '1234567') - self.assertNotIn('Vars', data) - self.assertEqual(data['Recipients'], [{ - 'Email': 'alice@example.com', - 'Vars': {'name': "Alice", 'group': "Developers"} - }]) + messages = data['Messages'] + self.assertEqual(len(messages), 2) + self.assertEqual(messages[0]['To'], 'alice@example.com') + self.assertEqual(messages[0]['Cc'], 'cc@example.com') + self.assertEqual(messages[0]['Mj-TemplateID'], '1234567') + self.assertEqual(messages[0]['Vars'], + {'name': "Alice", 'group': "Developers", 'site': "ExampleCo"}) + + self.assertEqual(messages[1]['To'], 'Bob ') + self.assertEqual(messages[1]['Cc'], 'cc@example.com') + self.assertEqual(messages[1]['Mj-TemplateID'], '1234567') + self.assertEqual(messages[1]['Vars'], + {'name': "Bob", 'group': "Users", 'site': "ExampleCo"}) + + def test_merge_metadata(self): + self.message.to = ['alice@example.com', 'Bob '] + self.message.merge_metadata = { + 'alice@example.com': {'order_id': 123, 'tier': 'premium'}, + 'bob@example.com': {'order_id': 678}, + } + self.message.metadata = {'notification_batch': 'zx912'} + self.message.send() + + data = self.get_api_call_json() + messages = data['Messages'] + self.assertEqual(len(messages), 2) + self.assertEqual(messages[0]['To'], 'alice@example.com') + # metadata and merge_metadata[recipient] are combined: + self.assertJSONEqual(messages[0]['Mj-EventPayLoad'], + {'order_id': 123, 'tier': 'premium', 'notification_batch': 'zx912'}) + self.assertEqual(messages[1]['To'], 'Bob ') + self.assertJSONEqual(messages[1]['Mj-EventPayLoad'], + {'order_id': 678, 'notification_batch': 'zx912'}) def test_default_omits_options(self): """Make sure by default we don't send any ESP-specific options. diff --git a/tests/test_mandrill_backend.py b/tests/test_mandrill_backend.py index 1703b94d..d175d8e1 100644 --- a/tests/test_mandrill_backend.py +++ b/tests/test_mandrill_backend.py @@ -379,7 +379,25 @@ def test_merge_data(self): {'name': "group", 'content': "Users"}, {'name': "site", 'content': "ExampleCo"}, ]) - self.assertEqual(data['message']['preserve_recipients'], False) # we force with merge_data + self.assertIs(data['message']['preserve_recipients'], False) # merge_data implies batch + + def test_merge_metadata(self): + self.message.to = ['alice@example.com', 'Bob '] + self.message.merge_metadata = { + 'alice@example.com': {'order_id': 123, 'tier': 'premium'}, + 'bob@example.com': {'order_id': 678}, + } + self.message.metadata = {'notification_batch': 'zx912'} + self.message.send() + data = self.get_api_call_json() + self.assertCountEqual(data['message']['recipient_metadata'], [{ + 'rcpt': 'alice@example.com', + 'values': {'order_id': 123, 'tier': 'premium'}, + }, { + 'rcpt': 'bob@example.com', + 'values': {'order_id': 678}, + }]) + self.assertIs(data['message']['preserve_recipients'], False) # merge_metadata implies batch def test_missing_from(self): """Make sure a missing from_email omits from* from API call. diff --git a/tests/test_postmark_backend.py b/tests/test_postmark_backend.py index 08a275bf..3629171c 100644 --- a/tests/test_postmark_backend.py +++ b/tests/test_postmark_backend.py @@ -398,8 +398,7 @@ def test_template_alias(self): data = self.get_api_call_json() self.assertEqual(data['TemplateAlias'], 'welcome-message') - def test_merge_data(self): - self.set_mock_response(raw=json.dumps([{ + _mock_batch_response = json.dumps([{ "ErrorCode": 0, "Message": "OK", "To": "alice@example.com", @@ -411,8 +410,10 @@ def test_merge_data(self): "To": "bob@example.com", "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00", "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d", - }]).encode('utf-8')) + }]).encode('utf-8') + def test_merge_data(self): + self.set_mock_response(raw=self._mock_batch_response) message = AnymailMessage( from_email='from@example.com', template_id=1234567, # Postmark only supports merge_data content in a template @@ -451,20 +452,7 @@ def test_merge_data(self): def test_merge_data_no_template(self): # merge_data={} can be used to force batch sending without a template - self.set_mock_response(raw=json.dumps([{ - "ErrorCode": 0, - "Message": "OK", - "To": "alice@example.com", - "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00", - "MessageID": "b7bc2f4a-e38e-4336-af7d-e6c392c2f817", - }, { - "ErrorCode": 0, - "Message": "OK", - "To": "bob@example.com", - "SubmittedAt": "2016-03-12T15:27:50.4468803-05:00", - "MessageID": "e2ecbbfc-fe12-463d-b933-9fe22915106d", - }]).encode('utf-8')) - + self.set_mock_response(raw=self._mock_batch_response) message = AnymailMessage( from_email='from@example.com', to=['alice@example.com', 'Bob '], @@ -496,6 +484,45 @@ def test_merge_data_no_template(self): self.assertEqual(recipients['bob@example.com'].status, 'sent') self.assertEqual(recipients['bob@example.com'].message_id, 'e2ecbbfc-fe12-463d-b933-9fe22915106d') + def test_merge_metadata(self): + self.set_mock_response(raw=self._mock_batch_response) + self.message.to = ['alice@example.com', 'Bob '] + self.message.merge_metadata = { + 'alice@example.com': {'order_id': 123, 'tier': 'premium'}, + 'bob@example.com': {'order_id': 678}, + } + self.message.metadata = {'notification_batch': 'zx912'} + self.message.send() + + self.assert_esp_called('/email/batch') + data = self.get_api_call_json() + self.assertEqual(len(data), 2) + self.assertEqual(data[0]["To"], "alice@example.com") + # metadata and merge_metadata[recipient] are combined: + self.assertEqual(data[0]["Metadata"], {'order_id': 123, 'tier': 'premium', 'notification_batch': 'zx912'}) + self.assertEqual(data[1]["To"], "Bob ") + self.assertEqual(data[1]["Metadata"], {'order_id': 678, 'notification_batch': 'zx912'}) + + def test_merge_metadata_with_template(self): + self.set_mock_response(raw=self._mock_batch_response) + self.message.to = ['alice@example.com', 'Bob '] + self.message.template_id = 1234567 + self.message.merge_metadata = { + 'alice@example.com': {'order_id': 123}, + 'bob@example.com': {'order_id': 678, 'tier': 'premium'}, + } + self.message.send() + + self.assert_esp_called('/email/batchWithTemplates') + data = self.get_api_call_json() + messages = data["Messages"] + self.assertEqual(len(messages), 2) + self.assertEqual(messages[0]["To"], "alice@example.com") + # metadata and merge_metadata[recipient] are combined: + self.assertEqual(messages[0]["Metadata"], {'order_id': 123}) + self.assertEqual(messages[1]["To"], "Bob ") + self.assertEqual(messages[1]["Metadata"], {'order_id': 678, 'tier': 'premium'}) + def test_default_omits_options(self): """Make sure by default we don't send any ESP-specific options. diff --git a/tests/test_sparkpost_backend.py b/tests/test_sparkpost_backend.py index d8c48469..2fdb69c4 100644 --- a/tests/test_sparkpost_backend.py +++ b/tests/test_sparkpost_backend.py @@ -443,6 +443,24 @@ def test_merge_data(self): ]) self.assertEqual(params['substitution_data'], {'group': "Users", 'site': "ExampleCo"}) + def test_merge_metadata(self): + self.set_mock_response(accepted=2) + self.message.to = ['alice@example.com', 'Bob '] + self.message.merge_metadata = { + 'alice@example.com': {'order_id': 123}, + 'bob@example.com': {'order_id': 678, 'tier': 'premium'}, + } + self.message.metadata = {'notification_batch': 'zx912'} + self.message.send() + params = self.get_send_params() + self.assertEqual(params['recipients'], [ + {'address': {'email': 'alice@example.com'}, + 'metadata': {'order_id': 123}}, + {'address': {'email': 'bob@example.com', 'name': 'Bob'}, + 'metadata': {'order_id': 678, 'tier': 'premium'}} + ]) + self.assertEqual(params['metadata'], {'notification_batch': 'zx912'}) + def test_default_omits_options(self): """Make sure by default we don't send any ESP-specific options.