diff --git a/frappe/__init__.py b/frappe/__init__.py index c915ce70a02..bb4e409d611 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -527,16 +527,20 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message if not delayed: now = True - from frappe.email import queue - queue.send(recipients=recipients, sender=sender, + from frappe.email.doctype.email_queue.email_queue import QueueBuilder + builder = QueueBuilder(recipients=recipients, sender=sender, subject=subject, message=message, text_content=text_content, reference_doctype = doctype or reference_doctype, reference_name = name or reference_name, add_unsubscribe_link=add_unsubscribe_link, unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message, attachments=attachments, reply_to=reply_to, cc=cc, bcc=bcc, message_id=message_id, in_reply_to=in_reply_to, send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority, queue_separately=queue_separately, - communication=communication, now=now, read_receipt=read_receipt, is_notification=is_notification, + communication=communication, read_receipt=read_receipt, is_notification=is_notification, inline_images=inline_images, header=header, print_letterhead=print_letterhead, with_container=with_container) + # build email queue and send the email if send_now is True. + builder.process(send_now=now) + + whitelisted = [] guest_methods = [] xss_safe_methods = [] @@ -1692,6 +1696,23 @@ def safe_eval(code, eval_globals=None, eval_locals=None): "round": round } + UNSAFE_ATTRIBUTES = { + # Generator Attributes + "gi_frame", "gi_code", + # Coroutine Attributes + "cr_frame", "cr_code", "cr_origin", + # Async Generator Attributes + "ag_code", "ag_frame", + # Traceback Attributes + "tb_frame", "tb_next", + # Format Attributes + "format", "format_map", + } + + for attribute in UNSAFE_ATTRIBUTES: + if attribute in code: + throw('Illegal rule {0}. Cannot use "{1}"'.format(bold(code), attribute)) + if '__' in code: throw('Illegal rule {0}. Cannot use "__"'.format(bold(code))) diff --git a/frappe/core/doctype/module_def/module_def.json b/frappe/core/doctype/module_def/module_def.json index 7a8bfd76a7b..4de046bbb63 100644 --- a/frappe/core/doctype/module_def/module_def.json +++ b/frappe/core/doctype/module_def/module_def.json @@ -55,7 +55,7 @@ "link_fieldname": "module" } ], - "modified": "2020-08-06 12:39:30.740379", + "modified": "2021-06-02 13:04:53.118716", "modified_by": "Administrator", "module": "Core", "name": "Module Def", @@ -69,6 +69,7 @@ "read": 1, "report": 1, "role": "Administrator", + "select": 1, "share": 1, "write": 1 }, @@ -78,7 +79,14 @@ "read": 1, "report": 1, "role": "System Manager", + "select": 1, "write": 1 + }, + { + "read": 1, + "report": 1, + "role": "All", + "select": 1 } ], "show_name_in_global_search": 1, diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 18076786735..b9dde88126f 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -288,16 +288,6 @@ "fieldname": "autoname", "fieldtype": "Data", "label": "Auto Name" - }, - { - "fieldname": "default_email_template", - "fieldtype": "Link", - "label": "Default Email Template", - "options": "Email Template" - }, - { - "fieldname": "column_break_26", - "fieldtype": "Column Break" } ], "hide_toolbar": 1, @@ -306,7 +296,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2021-04-29 21:21:06.476372", + "modified": "2021-06-02 06:49:16.782806", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 394fe50468c..dad473b8aad 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -9,14 +9,18 @@ import smtplib import quopri from email.parser import Parser +from email.policy import SMTPUTF8 +from html2text import html2text +from six.moves import html_parser as HTMLParser import frappe from frappe import _, safe_encode, task from frappe.model.document import Document -from frappe.email.queue import get_unsubcribed_url -from frappe.email.email_body import add_attachment -from frappe.utils import cint -from email.policy import SMTPUTF8 +from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message +from frappe.email.email_body import add_attachment, get_formatted_html, get_email +from frappe.utils import cint, split_emails, add_days, nowdate, cstr +from frappe.email.doctype.email_account.email_account import EmailAccount + MAX_RETRY_COUNT = 3 class EmailQueue(Document): @@ -41,6 +45,19 @@ def get_duplicate(self, recipients): duplicate.set_recipients(recipients) return duplicate + @classmethod + def new(cls, doc_data, ignore_permissions=False): + data = doc_data.copy() + if not data.get('recipients'): + return + + recipients = data.pop('recipients') + doc = frappe.new_doc(cls.DOCTYPE) + doc.update(data) + doc.set_recipients(recipients) + doc.insert(ignore_permissions=ignore_permissions) + return doc + @classmethod def find(cls, name): return frappe.get_doc(cls.DOCTYPE, name) @@ -74,8 +91,6 @@ def attachments_list(self): return json.loads(self.attachments) if self.attachments else [] def get_email_account(self): - from frappe.email.doctype.email_account.email_account import EmailAccount - if self.email_account: return frappe.get_doc('Email Account', self.email_account) @@ -300,3 +315,283 @@ def send_now(name): def on_doctype_update(): """Add index in `tabCommunication` for `(reference_doctype, reference_name)`""" frappe.db.add_index('Email Queue', ('status', 'send_after', 'priority', 'creation'), 'index_bulk_flush') + +class QueueBuilder: + """Builds Email Queue from the given data + """ + def __init__(self, recipients=None, sender=None, subject=None, message=None, + text_content=None, reference_doctype=None, reference_name=None, + unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, + attachments=None, reply_to=None, cc=None, bcc=None, message_id=None, in_reply_to=None, + send_after=None, expose_recipients=None, send_priority=1, communication=None, + read_receipt=None, queue_separately=False, is_notification=False, + add_unsubscribe_link=1, inline_images=None, header=None, + print_letterhead=False, with_container=False): + """Add email to sending queue (Email Queue) + + :param recipients: List of recipients. + :param sender: Email sender. + :param subject: Email subject. + :param message: Email message. + :param text_content: Text version of email message. + :param reference_doctype: Reference DocType of caller document. + :param reference_name: Reference name of caller document. + :param send_priority: Priority for Email Queue, default 1. + :param unsubscribe_method: URL method for unsubscribe. Default is `/api/method/frappe.email.queue.unsubscribe`. + :param unsubscribe_params: additional params for unsubscribed links. default are name, doctype, email + :param attachments: Attachments to be sent. + :param reply_to: Reply to be captured here (default inbox) + :param in_reply_to: Used to send the Message-Id of a received email back as In-Reply-To. + :param send_after: Send this email after the given datetime. If value is in integer, then `send_after` will be the automatically set to no of days from current date. + :param communication: Communication link to be set in Email Queue record + :param queue_separately: Queue each email separately + :param is_notification: Marks email as notification so will not trigger notifications from system + :param add_unsubscribe_link: Send unsubscribe link in the footer of the Email, default 1. + :param inline_images: List of inline images as {"filename", "filecontent"}. All src properties will be replaced with random Content-Id + :param header: Append header in email (boolean) + :param with_container: Wraps email inside styled container + """ + + self._unsubscribe_method = unsubscribe_method + self._recipients = recipients + self._cc = cc + self._bcc = bcc + self._send_after = send_after + self._sender = sender + self._text_content = text_content + self._message = message + self._add_unsubscribe_link = add_unsubscribe_link + self._unsubscribe_message = unsubscribe_message + self._attachments = attachments + + self._unsubscribed_user_emails = None + self._email_account = None + + self.unsubscribe_params = unsubscribe_params + self.subject = subject + self.reference_doctype = reference_doctype + self.reference_name = reference_name + self.expose_recipients = expose_recipients + self.with_container = with_container + self.header = header + self.reply_to = reply_to + self.message_id = message_id + self.in_reply_to = in_reply_to + self.send_priority = send_priority + self.communication = communication + self.read_receipt = read_receipt + self.queue_separately = queue_separately + self.is_notification = is_notification + self.inline_images = inline_images + self.print_letterhead = print_letterhead + + @property + def unsubscribe_method(self): + return self._unsubscribe_method or '/api/method/frappe.email.queue.unsubscribe' + + def _get_emails_list(self, emails=None): + emails = split_emails(emails) if isinstance(emails, str) else (emails or []) + return [each for each in set(emails) if each] + + @property + def recipients(self): + return self._get_emails_list(self._recipients) + + @property + def cc(self): + return self._get_emails_list(self._cc) + + @property + def bcc(self): + return self._get_emails_list(self._bcc) + + @property + def send_after(self): + if isinstance(self._send_after, int): + return add_days(nowdate(), self._send_after) + return self._send_after + + @property + def sender(self): + if not self._sender or self._sender == "Administrator": + email_account = self.get_outgoing_email_account() + return email_account.default_sender + return self._sender + + def email_text_content(self): + unsubscribe_msg = self.unsubscribe_message() + unsubscribe_text_message = (unsubscribe_msg and unsubscribe_msg.text) or '' + + if self._text_content: + return self._text_content + unsubscribe_text_message + + try: + text_content = html2text(self._message) + except HTMLParser.HTMLParseError: + text_content = "See html attachment" + return text_content + unsubscribe_text_message + + def email_html_content(self): + email_account = self.get_outgoing_email_account() + return get_formatted_html(self.subject, self._message, header=self.header, + email_account=email_account, unsubscribe_link=self.unsubscribe_message(), + with_container=self.with_container) + + def should_include_unsubscribe_link(self): + return (self._add_unsubscribe_link == 1 + and self.reference_doctype + and (self._unsubscribe_message or self.reference_doctype=="Newsletter")) + + def unsubscribe_message(self): + if self.should_include_unsubscribe_link(): + return get_unsubscribe_message(self._unsubscribe_message, self.expose_recipients) + + def get_outgoing_email_account(self): + if self._email_account: + return self._email_account + + self._email_account = EmailAccount.find_outgoing( + match_by_doctype=self.reference_doctype, match_by_email=self._sender, _raise_error=True) + return self._email_account + + def get_unsubscribed_user_emails(self): + if self._unsubscribed_user_emails is not None: + return self._unsubscribed_user_emails + + all_ids = tuple(set(self.recipients + self.cc)) + + unsubscribed = frappe.db.sql_list(''' + SELECT + distinct email + from + `tabEmail Unsubscribe` + where + email in %(all_ids)s + and ( + ( + reference_doctype = %(reference_doctype)s + and reference_name = %(reference_name)s + ) + or global_unsubscribe = 1 + ) + ''', { + 'all_ids': all_ids, + 'reference_doctype': self.reference_doctype, + 'reference_name': self.reference_name, + }) + + self._unsubscribed_user_emails = unsubscribed or [] + return self._unsubscribed_user_emails + + def final_recipients(self): + unsubscribed_emails = self.get_unsubscribed_user_emails() + return [mail_id for mail_id in self.recipients if mail_id not in unsubscribed_emails] + + def final_cc(self): + unsubscribed_emails = self.get_unsubscribed_user_emails() + return [mail_id for mail_id in self.cc if mail_id not in unsubscribed_emails] + + def get_attachments(self): + attachments = [] + if self._attachments: + # store attachments with fid or print format details, to be attached on-demand later + for att in self._attachments: + if att.get('fid'): + attachments.append(att) + elif att.get("print_format_attachment") == 1: + if not att.get('lang', None): + att['lang'] = frappe.local.lang + att['print_letterhead'] = self.print_letterhead + attachments.append(att) + return attachments + + def prepare_email_content(self): + mail = get_email(recipients=self.final_recipients(), + sender=self.sender, + subject=self.subject, + formatted=self.email_html_content(), + text_content=self.email_text_content(), + attachments=self._attachments, + reply_to=self.reply_to, + cc=self.final_cc(), + bcc=self.bcc, + email_account=self.get_outgoing_email_account(), + expose_recipients=self.expose_recipients, + inline_images=self.inline_images, + header=self.header) + + mail.set_message_id(self.message_id, self.is_notification) + if self.read_receipt: + mail.msg_root["Disposition-Notification-To"] = self.sender + if self.in_reply_to: + mail.set_in_reply_to(self.in_reply_to) + return mail + + def process(self, send_now=False): + """Build and return the email queues those are created. + + Sends email incase if it is requested to send now. + """ + final_recipients = self.final_recipients() + queue_separately = (final_recipients and self.queue_separately) or len(final_recipients) > 20 + if not (final_recipients + self.final_cc()): + return [] + + email_queues = [] + queue_data = self.as_dict(include_recipients=False) + if not queue_data: + return [] + + if not queue_separately: + recipients = list(set(final_recipients + self.final_cc() + self.bcc)) + q = EmailQueue.new({**queue_data, **{'recipients': recipients}}, ignore_permissions=True) + email_queues.append(q) + else: + for r in final_recipients: + recipients = [r] if email_queues else list(set([r] + self.final_cc() + self.bcc)) + q = EmailQueue.new({**queue_data, **{'recipients': recipients}}, ignore_permissions=True) + email_queues.append(q) + + if send_now: + for doc in email_queues: + doc.send() + return email_queues + + def as_dict(self, include_recipients=True): + email_account = self.get_outgoing_email_account() + email_account_name = email_account and email_account.is_exists_in_db() and email_account.name + + mail = self.prepare_email_content() + try: + mail_to_string = cstr(mail.as_string()) + except frappe.InvalidEmailAddressError: + # bad Email Address - don't add to queue + frappe.log_error('Invalid Email ID Sender: {0}, Recipients: {1}, \nTraceback: {2} ' + .format(self.sender, ', '.join(self.final_recipients()), traceback.format_exc()), + 'Email Not Sent' + ) + return + + d = { + 'priority': self.send_priority, + 'attachments': json.dumps(self.get_attachments()), + 'message_id': mail.msg_root["Message-Id"].strip(" <>"), + 'message': mail_to_string, + 'sender': self.sender, + 'reference_doctype': self.reference_doctype, + 'reference_name': self.reference_name, + 'add_unsubscribe_link': self._add_unsubscribe_link, + 'unsubscribe_method': self.unsubscribe_method, + 'unsubscribe_params': self.unsubscribe_params, + 'expose_recipients': self.expose_recipients, + 'communication': self.communication, + 'send_after': self.send_after, + 'show_as_cc': ",".join(self.final_cc()), + 'show_as_bcc': ','.join(self.bcc), + 'email_account': email_account_name or None + } + + if include_recipients: + d['recipients'] = self.final_recipients() + + return d diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 3cba8f927c6..97d77549b79 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -6,7 +6,6 @@ from frappe import throw, _ from frappe.website.website_generator import WebsiteGenerator from frappe.utils.verified_command import get_signed_params, verify_request -from frappe.email.queue import send from frappe.email.doctype.email_group.email_group import add_subscribers from frappe.utils import parse_addr, now_datetime, markdown, validate_email_address diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 7aef75b669c..ca96981aa8e 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -2,256 +2,9 @@ # MIT License. See license.txt import frappe -import sys -from html.parser import HTMLParser -import smtplib, quopri, json -from frappe import msgprint, _, safe_decode, safe_encode, enqueue -from frappe.email.smtp import SMTPServer -from frappe.email.doctype.email_account.email_account import EmailAccount -from frappe.email.email_body import get_email, get_formatted_html, add_attachment +from frappe import msgprint, _ from frappe.utils.verified_command import get_signed_params, verify_request -from html2text import html2text -from frappe.utils import get_url, nowdate, now_datetime, add_days, split_emails, cstr, cint -from rq.timeouts import JobTimeoutException -from email.parser import Parser - - -class EmailLimitCrossedError(frappe.ValidationError): pass - -def send(recipients=None, sender=None, subject=None, message=None, text_content=None, reference_doctype=None, - reference_name=None, unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, - attachments=None, reply_to=None, cc=None, bcc=None, message_id=None, in_reply_to=None, send_after=None, - expose_recipients=None, send_priority=1, communication=None, now=False, read_receipt=None, - queue_separately=False, is_notification=False, add_unsubscribe_link=1, inline_images=None, - header=None, print_letterhead=False, with_container=False): - """Add email to sending queue (Email Queue) - - :param recipients: List of recipients. - :param sender: Email sender. - :param subject: Email subject. - :param message: Email message. - :param text_content: Text version of email message. - :param reference_doctype: Reference DocType of caller document. - :param reference_name: Reference name of caller document. - :param send_priority: Priority for Email Queue, default 1. - :param unsubscribe_method: URL method for unsubscribe. Default is `/api/method/frappe.email.queue.unsubscribe`. - :param unsubscribe_params: additional params for unsubscribed links. default are name, doctype, email - :param attachments: Attachments to be sent. - :param reply_to: Reply to be captured here (default inbox) - :param in_reply_to: Used to send the Message-Id of a received email back as In-Reply-To. - :param send_after: Send this email after the given datetime. If value is in integer, then `send_after` will be the automatically set to no of days from current date. - :param communication: Communication link to be set in Email Queue record - :param now: Send immediately (don't send in the background) - :param queue_separately: Queue each email separately - :param is_notification: Marks email as notification so will not trigger notifications from system - :param add_unsubscribe_link: Send unsubscribe link in the footer of the Email, default 1. - :param inline_images: List of inline images as {"filename", "filecontent"}. All src properties will be replaced with random Content-Id - :param header: Append header in email (boolean) - :param with_container: Wraps email inside styled container - """ - if not unsubscribe_method: - unsubscribe_method = "/api/method/frappe.email.queue.unsubscribe" - - if not recipients and not cc: - return - - if not cc: - cc = [] - if not bcc: - bcc = [] - - if isinstance(recipients, str): - recipients = split_emails(recipients) - - if isinstance(cc, str): - cc = split_emails(cc) - - if isinstance(bcc, str): - bcc = split_emails(bcc) - - if isinstance(send_after, int): - send_after = add_days(nowdate(), send_after) - - email_account = EmailAccount.find_outgoing( - match_by_doctype=reference_doctype, match_by_email=sender, _raise_error=True) - - if not sender or sender == "Administrator": - sender = email_account.default_sender - - if not text_content: - try: - text_content = html2text(message) - except HTMLParser.HTMLParseError: - text_content = "See html attachment" - - recipients = list(set(recipients)) - cc = list(set(cc)) - - all_ids = tuple(recipients + cc) - - unsubscribed = frappe.db.sql_list(''' - SELECT - distinct email - from - `tabEmail Unsubscribe` - where - email in %(all_ids)s - and ( - ( - reference_doctype = %(reference_doctype)s - and reference_name = %(reference_name)s - ) - or global_unsubscribe = 1 - ) - ''', { - 'all_ids': all_ids, - 'reference_doctype': reference_doctype, - 'reference_name': reference_name, - }) - - recipients = [r for r in recipients if r and r not in unsubscribed] - - if cc: - cc = [r for r in cc if r and r not in unsubscribed] - - if not recipients and not cc: - # Recipients may have been unsubscribed, exit quietly - return - - email_text_context = text_content - - should_append_unsubscribe = (add_unsubscribe_link - and reference_doctype - and (unsubscribe_message or reference_doctype=="Newsletter") - and add_unsubscribe_link==1) - - unsubscribe_link = None - if should_append_unsubscribe: - unsubscribe_link = get_unsubscribe_message(unsubscribe_message, expose_recipients) - email_text_context += unsubscribe_link.text - - email_content = get_formatted_html(subject, message, - email_account=email_account, header=header, - unsubscribe_link=unsubscribe_link, with_container=with_container) - - # add to queue - add(recipients, sender, subject, - formatted=email_content, - text_content=email_text_context, - reference_doctype=reference_doctype, - reference_name=reference_name, - attachments=attachments, - reply_to=reply_to, - cc=cc, - bcc=bcc, - message_id=message_id, - in_reply_to=in_reply_to, - send_after=send_after, - send_priority=send_priority, - email_account=email_account, - communication=communication, - add_unsubscribe_link=add_unsubscribe_link, - unsubscribe_method=unsubscribe_method, - unsubscribe_params=unsubscribe_params, - expose_recipients=expose_recipients, - read_receipt=read_receipt, - queue_separately=queue_separately, - is_notification = is_notification, - inline_images = inline_images, - header=header, - now=now, - print_letterhead=print_letterhead) - - -def add(recipients, sender, subject, **kwargs): - """Add to Email Queue""" - if kwargs.get('queue_separately') or len(recipients) > 20: - email_queue = None - for r in recipients: - if not email_queue: - email_queue = get_email_queue([r], sender, subject, **kwargs) - if kwargs.get('now'): - email_queue.send() - else: - duplicate = email_queue.get_duplicate([r]) - duplicate.insert(ignore_permissions=True) - - if kwargs.get('now'): - duplicate.send() - - frappe.db.commit() - else: - email_queue = get_email_queue(recipients, sender, subject, **kwargs) - if kwargs.get('now'): - email_queue.send() - -def get_email_queue(recipients, sender, subject, **kwargs): - '''Make Email Queue object''' - e = frappe.new_doc('Email Queue') - e.priority = kwargs.get('send_priority') - attachments = kwargs.get('attachments') - if attachments: - # store attachments with fid or print format details, to be attached on-demand later - _attachments = [] - for att in attachments: - if att.get('fid'): - _attachments.append(att) - elif att.get("print_format_attachment") == 1: - if not att.get('lang', None): - att['lang'] = frappe.local.lang - att['print_letterhead'] = kwargs.get('print_letterhead') - _attachments.append(att) - e.attachments = json.dumps(_attachments) - - try: - mail = get_email(recipients, - sender=sender, - subject=subject, - formatted=kwargs.get('formatted'), - text_content=kwargs.get('text_content'), - attachments=kwargs.get('attachments'), - reply_to=kwargs.get('reply_to'), - cc=kwargs.get('cc'), - bcc=kwargs.get('bcc'), - email_account=kwargs.get('email_account'), - expose_recipients=kwargs.get('expose_recipients'), - inline_images=kwargs.get('inline_images'), - header=kwargs.get('header')) - - mail.set_message_id(kwargs.get('message_id'),kwargs.get('is_notification')) - if kwargs.get('read_receipt'): - mail.msg_root["Disposition-Notification-To"] = sender - if kwargs.get('in_reply_to'): - mail.set_in_reply_to(kwargs.get('in_reply_to')) - - e.message_id = mail.msg_root["Message-Id"].strip(" <>") - e.message = cstr(mail.as_string()) - e.sender = mail.sender - - except frappe.InvalidEmailAddressError: - # bad Email Address - don't add to queue - import traceback - frappe.log_error('Invalid Email ID Sender: {0}, Recipients: {1}, \nTraceback: {2} '.format(mail.sender, - ', '.join(mail.recipients), traceback.format_exc()), 'Email Not Sent') - - recipients = list(set(recipients + kwargs.get('cc', []) + kwargs.get('bcc', []))) - email_account = kwargs.get('email_account') - email_account_name = email_account and email_account.is_exists_in_db() and email_account.name - - e.set_recipients(recipients) - e.reference_doctype = kwargs.get('reference_doctype') - e.reference_name = kwargs.get('reference_name') - e.add_unsubscribe_link = kwargs.get("add_unsubscribe_link") - e.unsubscribe_method = kwargs.get('unsubscribe_method') - e.unsubscribe_params = kwargs.get('unsubscribe_params') - e.expose_recipients = kwargs.get('expose_recipients') - e.communication = kwargs.get('communication') - e.send_after = kwargs.get('send_after') - e.show_as_cc = ",".join(kwargs.get('cc', [])) - e.show_as_bcc = ",".join(kwargs.get('bcc', [])) - e.email_account = email_account_name or None - e.insert(ignore_permissions=True) - return e +from frappe.utils import get_url, now_datetime, cint def get_emails_sent_this_month(): return frappe.db.sql(""" diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 9425f4dbd48..74492c09c3e 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -84,18 +84,19 @@ def session(self): SMTP = smtplib.SMTP_SSL if self.use_ssl else smtplib.SMTP try: - self._session = SMTP(self.server, self.port) - if not self._session: + _session = SMTP(self.server, self.port) + if not _session: frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError) - self.secure_session(self._session) + self.secure_session(_session) if self.login and self.password: - res = self._session.login(str(self.login or ""), str(self.password or "")) + res = _session.login(str(self.login or ""), str(self.password or "")) # check if logged correctly if res[0]!=235: frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError) + self._session = _session return self._session except smtplib.SMTPAuthenticationError as e: diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py index 1961d7801e9..8e637273edd 100644 --- a/frappe/email/test_email_body.py +++ b/frappe/email/test_email_body.py @@ -5,8 +5,7 @@ from frappe.email.receive import Email from frappe.email.email_body import (replace_filename_with_cid, get_email, inline_style_in_html, get_header) -from frappe.email.queue import get_email_queue -from frappe.email.doctype.email_queue.email_queue import SendMailContext +from frappe.email.doctype.email_queue.email_queue import SendMailContext, QueueBuilder class TestEmailBody(unittest.TestCase): @@ -43,27 +42,25 @@ def test_prepare_message_returns_already_encoded_string(self): uni_chr1 = chr(40960) uni_chr2 = chr(1972) - email = get_email_queue( + queue_doc = QueueBuilder( recipients=['test@example.com'], sender='me@example.com', subject='Test Subject', - content='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', - formatted='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', - text_content='whatever') - mail_ctx = SendMailContext(queue_doc = email) + message='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', + text_content='whatever').process()[0] + mail_ctx = SendMailContext(queue_doc = queue_doc) result = mail_ctx.build_message(recipient_email = 'test@test.com') self.assertTrue(b"

=EA=80=80abcd=DE=B4

" in result) def test_prepare_message_returns_cr_lf(self): - email = get_email_queue( + queue_doc = QueueBuilder( recipients=['test@example.com'], sender='me@example.com', subject='Test Subject', - content='

\n this is a test of newlines\n' + '

', - formatted='

\n this is a test of newlines\n' + '

', - text_content='whatever') + message='

\n this is a test of newlines\n' + '

', + text_content='whatever').process()[0] - mail_ctx = SendMailContext(queue_doc = email) + mail_ctx = SendMailContext(queue_doc = queue_doc) result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com')) self.assertTrue(result.count('\n') == result.count("\r")) diff --git a/frappe/public/scss/common/awesomeplete.scss b/frappe/public/scss/common/awesomeplete.scss index 096da1e2fde..619d6a9e3e2 100644 --- a/frappe/public/scss/common/awesomeplete.scss +++ b/frappe/public/scss/common/awesomeplete.scss @@ -31,6 +31,7 @@ margin: 0; padding: var(--padding-xs); z-index: 1; + min-width: 250px; &> li { cursor: pointer; diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index 9811b5515fb..a837a2c572c 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -139,7 +139,8 @@ def test_expired(self): self.assertEqual(len(queue_recipients), 2) def test_unsubscribe(self): - from frappe.email.queue import unsubscribe, send + from frappe.email.queue import unsubscribe + from frappe.email.doctype.email_queue.email_queue import QueueBuilder unsubscribe(doctype="User", name="Administrator", email="test@example.com") self.assertTrue(frappe.db.get_value("Email Unsubscribe", @@ -148,11 +149,11 @@ def test_unsubscribe(self): before = frappe.db.sql("""select count(name) from `tabEmail Queue` where status='Not Sent'""")[0][0] - send(recipients=['test@example.com', 'test1@example.com'], - sender="admin@example.com", - reference_doctype='User', reference_name="Administrator", - subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe") - + builder = QueueBuilder(recipients=['test@example.com', 'test1@example.com'], + sender="admin@example.com", + reference_doctype='User', reference_name="Administrator", + subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe") + builder.process() # this is sent async (?) email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Not Sent'""", diff --git a/frappe/tests/test_password.py b/frappe/tests/test_password.py index 6a5b948d935..731841737c5 100644 --- a/frappe/tests/test_password.py +++ b/frappe/tests/test_password.py @@ -2,8 +2,8 @@ # MIT License. See license.txt import frappe import unittest -from frappe.utils.password import update_password, check_password, passlibctx - +from frappe.utils.password import update_password, check_password, passlibctx, encrypt, decrypt +from cryptography.fernet import Fernet class TestPassword(unittest.TestCase): def setUp(self): frappe.delete_doc('Email Account', 'Test Email Account Password') @@ -104,6 +104,16 @@ def test_password_unset(self): doc.save() self.assertEqual(doc.get_password(raise_exception=False), None) + def test_custom_encryption_key(self): + text = 'Frappe Framework' + custom_encryption_key = Fernet.generate_key().decode() + + encrypted_text = encrypt(text, encryption_key=custom_encryption_key) + decrypted_text = decrypt(encrypted_text, encryption_key=custom_encryption_key) + + self.assertEqual(text, decrypted_text) + + pass def get_password_list(doc): return frappe.db.sql("""SELECT `password` diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 61905b49855..07e34674fea 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -213,7 +213,7 @@ def raise_error_on_no_output(error_message, error_type=None, keep_quiet=None): >>> @raise_error_on_no_output("Ingradients missing") ... def get_indradients(_raise_error=1): return ... - >>> get_indradients() + >>> get_ingradients() `Exception Name`: Ingradients missing """ def decorator_raise_error_on_no_output(func): diff --git a/frappe/utils/password.py b/frappe/utils/password.py index d86275c39ca..428f2e95773 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -156,20 +156,29 @@ def create_auth_table(): frappe.db.create_auth_table() -def encrypt(pwd): - cipher_suite = Fernet(encode(get_encryption_key())) - cipher_text = cstr(cipher_suite.encrypt(encode(pwd))) +def encrypt(txt, encryption_key=None): + # Only use Fernet.generate_key().decode() to enter encyption_key value + + try: + cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) + except Exception: + # encryption_key is not in 32 url-safe base64-encoded format + frappe.throw(_('Encryption key is in invalid format!')) + + cipher_text = cstr(cipher_suite.encrypt(encode(txt))) return cipher_text -def decrypt(pwd): +def decrypt(txt, encryption_key=None): + # Only use encryption_key value generated with Fernet.generate_key().decode() + try: - cipher_suite = Fernet(encode(get_encryption_key())) - plain_text = cstr(cipher_suite.decrypt(encode(pwd))) + cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) + plain_text = cstr(cipher_suite.decrypt(encode(txt))) return plain_text except InvalidToken: # encryption_key in site_config is changed and not valid - frappe.throw(_('Encryption key is invalid, Please check site_config.json')) + frappe.throw(_('Encryption key is invalid' + '!' if encryption_key else ', please check site_config.json.')) def get_encryption_key(): diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index d1307a3a900..632fc5df239 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -150,6 +150,7 @@ def get_safe_globals(): # default writer allows write access out._write_ = _write out._getitem_ = _getitem + out._getattr_ = _getattr # allow iterators and list comprehension out._getiter_ = iter @@ -176,6 +177,27 @@ def _getitem(obj, key): raise SyntaxError('Key starts with _') return obj[key] +def _getattr(object, name, default=None): + # guard function for RestrictedPython + # allow any key to be accessed as long as + # 1. it does not start with an underscore (safer_getattr) + # 2. it is not an UNSAFE_ATTRIBUTES + + UNSAFE_ATTRIBUTES = { + # Generator Attributes + "gi_frame", "gi_code", + # Coroutine Attributes + "cr_frame", "cr_code", "cr_origin", + # Async Generator Attributes + "ag_code", "ag_frame", + # Traceback Attributes + "tb_frame", "tb_next", + } + + if isinstance(name, str) and (name in UNSAFE_ATTRIBUTES): + raise SyntaxError("{name} is an unsafe attribute".format(name=name)) + return RestrictedPython.Guards.safer_getattr(object, name, default=default) + def _write(obj): # guard function for RestrictedPython # allow writing to any object