From 9ab1991689000821a4d86b8617e1c1455a327482 Mon Sep 17 00:00:00 2001 From: Mike Edmunds Date: Mon, 23 Dec 2024 15:01:36 -0800 Subject: [PATCH] Fixed #35581 -- Updated django.core.mail to Python's modern email API. - Changed EmailMessage.message() to construct a "modern email API" email.message.EmailMessage and added policy keyword arg. - Added support for modern MIMEPart objects in EmailMessage.attach() (and EmailMessage constructor, EmailMessage.attachments list). - Updated SMTP EmailBackend to use modern email.policy.SMTP. Deprecated: - Attaching MIMEBase objects (replace with MIMEPart) - BadHeaderError (modern email uses ValueError) - SafeMIMEText, SafeMIMEMultipart (unnecessary for modern email) - django.core.mail.forbid_multi_line_headers() (undocumented, but exposed via `__all__` and in wide use) - django.core.mail.message.sanitize_address() (undocumented, but in wide use) Removed without deprecation (all undocumented): - EmailMessage.mixed_subtype - EmailMultiAlternatives.alternative_subtype - Support for setting (undocumented) EmailMessage.encoding property to a legacy email.charset.Charset object Related changes: - Dropped tests for incorrect RFC 2047 encoding of non-ASCII email address localparts. This is specifically prohibited by RFC 2047, and not supported by any known MTA or email client. (Python still mis-applies encoded-word to non-ASCII localparts, but it is a bug that may be fixed in the future.) - Added tests that try to discourage using Python's legacy email APIs in future updates to django.core.mail. --- django/core/mail/__init__.py | 41 +++- django/core/mail/backends/smtp.py | 51 +++- django/core/mail/message.py | 278 ++++++++++++++-------- docs/internals/deprecation.txt | 12 + docs/releases/6.0.txt | 61 ++++- docs/topics/email.txt | 114 ++++++--- tests/auth_tests/test_forms.py | 4 +- tests/mail/test_deprecated.py | 122 ++++++++++ tests/mail/tests.py | 381 +++++++++++++++++++++++++----- 9 files changed, 846 insertions(+), 218 deletions(-) create mode 100644 tests/mail/test_deprecated.py diff --git a/django/core/mail/__init__.py b/django/core/mail/__init__.py index 4f49d85756..4d7f8bf1d9 100644 --- a/django/core/mail/__init__.py +++ b/django/core/mail/__init__.py @@ -13,13 +13,10 @@ from django.core.exceptions import ImproperlyConfigured # backends and the subsequent reorganization (See #10355) from django.core.mail.message import ( DEFAULT_ATTACHMENT_MIME_TYPE, - BadHeaderError, EmailAlternative, EmailAttachment, EmailMessage, EmailMultiAlternatives, - SafeMIMEMultipart, - SafeMIMEText, forbid_multi_line_headers, make_msgid, ) @@ -33,12 +30,8 @@ __all__ = [ "DNS_NAME", "EmailMessage", "EmailMultiAlternatives", - "SafeMIMEText", - "SafeMIMEMultipart", "DEFAULT_ATTACHMENT_MIME_TYPE", "make_msgid", - "BadHeaderError", - "forbid_multi_line_headers", "get_connection", "send_mail", "send_mass_mail", @@ -46,6 +39,12 @@ __all__ = [ "mail_managers", "EmailAlternative", "EmailAttachment", + # RemovedInDjango70Warning: When the deprecation ends, remove the last + # entries. + "BadHeaderError", + "SafeMIMEText", + "SafeMIMEMultipart", + "forbid_multi_line_headers", ] @@ -224,3 +223,31 @@ def mail_managers( fail_silently=fail_silently, connection=connection, ) + + +# RemovedInDjango70Warning. +_deprecate_on_import = { + "BadHeaderError": "BadHeaderError is deprecated. Replace with ValueError.", + "SafeMIMEText": ( + "SafeMIMEText is deprecated. The return value" + " of EmailMessage.message() is an email.message.EmailMessage." + ), + "SafeMIMEMultipart": ( + "SafeMIMEMultipart is deprecated. The return value" + " of EmailMessage.message() is an email.message.EmailMessage." + ), +} + + +# RemovedInDjango70Warning. +def __getattr__(name): + try: + msg = _deprecate_on_import[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + else: + # Issue deprecation warnings at time of import. + from django.core.mail import message + + warnings.warn(msg, category=RemovedInDjango70Warning) + return getattr(message, name) diff --git a/django/core/mail/backends/smtp.py b/django/core/mail/backends/smtp.py index 6820148ac1..4b79d9f3e1 100644 --- a/django/core/mail/backends/smtp.py +++ b/django/core/mail/backends/smtp.py @@ -1,13 +1,15 @@ """SMTP email backend class.""" +import email.policy import smtplib import ssl import threading +from email.headerregistry import Address, AddressHeader from django.conf import settings from django.core.mail.backends.base import BaseEmailBackend -from django.core.mail.message import sanitize_address from django.core.mail.utils import DNS_NAME +from django.utils.encoding import force_str, punycode from django.utils.functional import cached_property @@ -145,18 +147,47 @@ class EmailBackend(BaseEmailBackend): """A helper method that does the actual sending.""" if not email_message.recipients(): return False - encoding = email_message.encoding or settings.DEFAULT_CHARSET - from_email = sanitize_address(email_message.from_email, encoding) - recipients = [ - sanitize_address(addr, encoding) for addr in email_message.recipients() - ] - message = email_message.message() + from_email = self.prep_address(email_message.from_email) + recipients = [self.prep_address(addr) for addr in email_message.recipients()] + message = email_message.message(policy=email.policy.SMTP) try: - self.connection.sendmail( - from_email, recipients, message.as_bytes(linesep="\r\n") - ) + self.connection.sendmail(from_email, recipients, message.as_bytes()) except smtplib.SMTPException: if not self.fail_silently: raise return False return True + + def prep_address(self, address, force_ascii=True): + """ + Return the addr-spec portion of an email address. Raises ValueError for + invalid addresses, including CR/NL injection. + + If force_ascii is True, apply IDNA encoding to non-ASCII domains, and + raise ValueError for non-ASCII local-parts (which can't be encoded). + Otherwise, leave Unicode characters unencoded (e.g., for sending with + SMTPUTF8). + """ + address = force_str(address) + parsed = AddressHeader.value_parser(address) + defects = set(str(defect) for defect in parsed.all_defects) + # Django allows local mailboxes like "From: webmaster" (#15042). + defects.discard("addr-spec local part with no domain") + if not force_ascii: + # Non-ASCII local-part is valid with SMTPUTF8. Remove once + # https://github.com/python/cpython/issues/81074 is fixed. + defects.discard("local-part contains non-ASCII characters)") + if defects: + raise ValueError(f"Invalid address {address!r}: {'; '.join(defects)}") + + mailboxes = parsed.all_mailboxes + if len(mailboxes) != 1: + raise ValueError(f"Invalid address {address!r}: must be a single address") + + mailbox = mailboxes[0] + if force_ascii and mailbox.domain and not mailbox.domain.isascii(): + # Re-compose an addr-spec with the IDNA encoded domain. + domain = punycode(mailbox.domain) + return str(Address(username=mailbox.local_part, domain=domain)) + else: + return mailbox.addr_spec diff --git a/django/core/mail/message.py b/django/core/mail/message.py index 3fd2ef6656..66d5efeb63 100644 --- a/django/core/mail/message.py +++ b/django/core/mail/message.py @@ -1,17 +1,19 @@ +import email.message +import email.policy import mimetypes +import warnings from collections import namedtuple +from datetime import datetime, timezone from email import charset as Charset -from email import encoders as Encoders -from email import generator, message_from_bytes +from email import generator from email.errors import HeaderParseError from email.header import Header -from email.headerregistry import Address, parser -from email.message import Message +from email.headerregistry import Address, AddressHeader, parser from email.mime.base import MIMEBase from email.mime.message import MIMEMessage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.utils import formataddr, formatdate, getaddresses, make_msgid +from email.utils import formataddr, getaddresses, make_msgid from io import BytesIO, StringIO from pathlib import Path @@ -19,7 +21,9 @@ from django.conf import settings from django.core.mail.utils import DNS_NAME from django.utils.deprecation import RemovedInDjango70Warning, deprecate_posargs from django.utils.encoding import force_bytes, force_str, punycode +from django.utils.timezone import get_current_timezone +# RemovedInDjango70Warning. # Don't BASE64-encode UTF-8 messages so that we avoid unwanted attention from # some spam filters. utf8_charset = Charset.Charset("utf-8") @@ -31,13 +35,17 @@ utf8_charset_qp.body_encoding = Charset.QP # and cannot be guessed). DEFAULT_ATTACHMENT_MIME_TYPE = "application/octet-stream" +# RemovedInDjango70Warning. RFC5322_EMAIL_LINE_LENGTH_LIMIT = 998 -class BadHeaderError(ValueError): - pass - +# RemovedInDjango70Warning. +# BadHeaderError must be ValueError (not subclass it), so that existing code +# with `except BadHeaderError` will catch the ValueError that Python's modern +# email API raises for headers containing CR or NL. +BadHeaderError = ValueError +# RemovedInDjango70Warning. # Header names that contain structured address data (RFC 5322). ADDRESS_HEADERS = { "from", @@ -54,8 +62,16 @@ ADDRESS_HEADERS = { } +# RemovedInDjango70Warning. def forbid_multi_line_headers(name, val, encoding): """Forbid multi-line headers to prevent header injection.""" + warnings.warn( + "The internal API forbid_multi_line_headers() is deprecated." + " Python's modern email API (with email.message.EmailMessage or" + " email.policy.default) will reject multi-line headers.", + RemovedInDjango70Warning, + ) + encoding = encoding or settings.DEFAULT_CHARSET val = str(val) # val may be lazy if "\n" in val or "\r" in val: @@ -77,10 +93,20 @@ def forbid_multi_line_headers(name, val, encoding): return name, val +# RemovedInDjango70Warning. def sanitize_address(addr, encoding): """ Format a pair of (name, address) or an email address string. """ + warnings.warn( + "The internal API sanitize_address() is deprecated." + " Python's modern email API (with email.message.EmailMessage or" + " email.policy.default) will handle most required validation and" + " encoding. Use Python's email.headerregistry.Address to construct" + " formatted addresses from component parts.", + RemovedInDjango70Warning, + ) + address = None if not isinstance(addr, tuple): addr = force_str(addr) @@ -123,6 +149,7 @@ def sanitize_address(addr, encoding): return formataddr((nm, parsed_address.addr_spec)) +# RemovedInDjango70Warning. class MIMEMixin: def as_string(self, unixfrom=False, linesep="\n"): """Return the entire formatted message as a string. @@ -151,6 +178,7 @@ class MIMEMixin: return fp.getvalue() +# RemovedInDjango70Warning. class SafeMIMEMessage(MIMEMixin, MIMEMessage): def __setitem__(self, name, val): # Per RFC 2046 Section 5.2.1, message/rfc822 attachment headers must be @@ -159,6 +187,7 @@ class SafeMIMEMessage(MIMEMixin, MIMEMessage): MIMEMessage.__setitem__(self, name, val) +# RemovedInDjango70Warning. class SafeMIMEText(MIMEMixin, MIMEText): def __init__(self, _text, _subtype="plain", _charset=None): self.encoding = _charset @@ -181,6 +210,7 @@ class SafeMIMEText(MIMEMixin, MIMEText): MIMEText.set_payload(self, payload, charset=charset) +# RemovedInDjango70Warning. class SafeMIMEMultipart(MIMEMixin, MIMEMultipart): def __init__( self, _subtype="mixed", boundary=None, _subparts=None, encoding=None, **_params @@ -201,8 +231,10 @@ class EmailMessage: """A container for email information.""" content_subtype = "plain" - mixed_subtype = "mixed" - encoding = None # None => use settings default + + # Undocumented charset to use for text/* message bodies and attachments. + # If None, defaults to settings.DEFAULT_CHARSET. + encoding = None @deprecate_posargs( RemovedInDjango70Warning, @@ -263,7 +295,10 @@ class EmailMessage: self.attachments = [] if attachments: for attachment in attachments: - if isinstance(attachment, MIMEBase): + if isinstance(attachment, email.message.MIMEPart): + self.attach(attachment) + elif isinstance(attachment, MIMEBase): + # RemovedInDjango70Warning. self.attach(attachment) else: self.attach(*attachment) @@ -277,12 +312,13 @@ class EmailMessage: self.connection = get_connection(fail_silently=fail_silently) return self.connection - def message(self): - encoding = self.encoding or settings.DEFAULT_CHARSET - msg = SafeMIMEText(self.body, self.content_subtype, encoding) - msg = self._create_message(msg) - msg["Subject"] = self.subject - msg["From"] = self.extra_headers.get("From", self.from_email) + def message(self, *, policy=email.policy.default): + msg = email.message.EmailMessage(policy=policy) + self._add_bodies(msg) + self._add_attachments(msg) + + msg["Subject"] = str(self.subject) + msg["From"] = str(self.extra_headers.get("From", self.from_email)) self._set_list_header_if_not_empty(msg, "To", self.to) self._set_list_header_if_not_empty(msg, "Cc", self.cc) self._set_list_header_if_not_empty(msg, "Reply-To", self.reply_to) @@ -291,18 +327,19 @@ class EmailMessage: # accommodate that when doing comparisons. header_names = [key.lower() for key in self.extra_headers] if "date" not in header_names: - # formatdate() uses stdlib methods to format the date, which use - # the stdlib/OS concept of a timezone, however, Django sets the - # TZ environment variable based on the TIME_ZONE setting which - # will get picked up by formatdate(). - msg["Date"] = formatdate(localtime=settings.EMAIL_USE_LOCALTIME) + if settings.EMAIL_USE_LOCALTIME: + tz = get_current_timezone() + else: + tz = timezone.utc + msg["Date"] = datetime.now(tz) if "message-id" not in header_names: # Use cached DNS_NAME for performance msg["Message-ID"] = make_msgid(domain=DNS_NAME) for name, value in self.extra_headers.items(): # Avoid headers handled above. if name.lower() not in {"from", "to", "cc", "reply-to"}: - msg[name] = value + msg[name] = force_str(value, strings_only=True) + self._idna_encode_address_header_domains(msg) return msg def recipients(self): @@ -332,7 +369,19 @@ class EmailMessage: specified as content, decode it as UTF-8. If that fails, set the mimetype to DEFAULT_ATTACHMENT_MIME_TYPE and don't decode the content. """ - if isinstance(filename, MIMEBase): + if isinstance(filename, email.message.MIMEPart): + if content is not None or mimetype is not None: + raise ValueError( + "content and mimetype must not be given when a MIMEPart " + "instance is provided." + ) + self.attachments.append(filename) + elif isinstance(filename, MIMEBase): + warnings.warn( + "MIMEBase attachments are deprecated." + " Use an email.message.MIMEPart instead.", + RemovedInDjango70Warning, + ) if content is not None or mimetype is not None: raise ValueError( "content and mimetype must not be given when a MIMEBase " @@ -376,77 +425,75 @@ class EmailMessage: content = file.read() self.attach(path.name, content, mimetype) - def _create_message(self, msg): - return self._create_attachments(msg) - - def _create_attachments(self, msg): - if self.attachments: + def _add_bodies(self, msg): + if self.body or not self.attachments: encoding = self.encoding or settings.DEFAULT_CHARSET - body_msg = msg - msg = SafeMIMEMultipart(_subtype=self.mixed_subtype, encoding=encoding) - if self.body or body_msg.is_multipart(): - msg.attach(body_msg) + body = force_str( + self.body or "", encoding=encoding, errors="surrogateescape" + ) + msg.set_content(body, subtype=self.content_subtype, charset=encoding) + + def _add_attachments(self, msg): + if self.attachments: + if hasattr(self, "mixed_subtype"): + # RemovedInDjango70Warning. + raise AttributeError( + "EmailMessage no longer supports the" + " undocumented `mixed_subtype` attribute" + ) + msg.make_mixed() for attachment in self.attachments: - if isinstance(attachment, MIMEBase): + if isinstance(attachment, email.message.MIMEPart): + msg.attach(attachment) + elif isinstance(attachment, MIMEBase): + # RemovedInDjango70Warning. msg.attach(attachment) else: - msg.attach(self._create_attachment(*attachment)) - return msg + self._add_attachment(msg, *attachment) - def _create_mime_attachment(self, content, mimetype): - """ - Convert the content, mimetype pair into a MIME attachment object. + def _add_attachment(self, msg, filename, content, mimetype): + encoding = self.encoding or settings.DEFAULT_CHARSET + maintype, subtype = mimetype.split("/", 1) - If the mimetype is message/rfc822, content may be an - email.Message or EmailMessage object, as well as a str. - """ - basetype, subtype = mimetype.split("/", 1) - if basetype == "text" and isinstance(content, bytes): + if maintype == "text" and isinstance(content, bytes): # This duplicates logic from EmailMessage.attach() to properly # handle EmailMessage.attachments not created through attach(). try: content = content.decode() except UnicodeDecodeError: mimetype = DEFAULT_ATTACHMENT_MIME_TYPE - basetype, subtype = mimetype.split("/", 1) + maintype, subtype = mimetype.split("/", 1) - if basetype == "text": - encoding = self.encoding or settings.DEFAULT_CHARSET - attachment = SafeMIMEText(content, subtype, encoding) - elif basetype == "message" and subtype == "rfc822": - # Bug #18967: Per RFC 2046 Section 5.2.1, message/rfc822 - # attachments must not be base64 encoded. - if isinstance(content, EmailMessage): - # convert content into an email.Message first - content = content.message() - elif not isinstance(content, Message): - # For compatibility with existing code, parse the message - # into an email.Message object if it is not one already. - content = message_from_bytes(force_bytes(content)) - - attachment = SafeMIMEMessage(content, subtype) - else: - # Encode non-text attachments with base64. - attachment = MIMEBase(basetype, subtype) - attachment.set_payload(content) - Encoders.encode_base64(attachment) - return attachment - - def _create_attachment(self, filename, content, mimetype=None): - """ - Convert the filename, content, mimetype triple into a MIME attachment - object. - """ - attachment = self._create_mime_attachment(content, mimetype) - if filename: - try: - filename.encode("ascii") - except UnicodeEncodeError: - filename = ("utf-8", "", filename) - attachment.add_header( - "Content-Disposition", "attachment", filename=filename + # See email.contentmanager.set_content() docs for the cases here. + if maintype == "text": + # For text/*, content must be str, and maintype cannot be provided. + msg.add_attachment( + content, subtype=subtype, filename=filename, charset=encoding + ) + elif maintype == "message": + # For message/*, content must be email.message.EmailMessage (or + # legacy email.message.Message), and maintype cannot be provided. + if isinstance(content, EmailMessage): + # Django EmailMessage. + content = content.message(policy=msg.policy) + elif not isinstance( + content, (email.message.EmailMessage, email.message.Message) + ): + content = email.message_from_bytes( + force_bytes(content), policy=msg.policy + ) + msg.add_attachment(content, subtype=subtype, filename=filename) + else: + # For all other types, content must be bytes-like, and both + # maintype and subtype must be provided. + if not isinstance(content, (bytes, bytearray, memoryview)): + content = force_bytes(content) + msg.add_attachment( + content, + maintype=maintype, + subtype=subtype, + filename=filename, ) - return attachment def _set_list_header_if_not_empty(self, msg, header, values): """ @@ -459,6 +506,37 @@ class EmailMessage: if values: msg[header] = ", ".join(str(v) for v in values) + def _idna_encode_address_header_domains(self, msg): + """ + If msg.policy does not permit utf8 in headers, IDNA encode all + non-ASCII domains in its address headers. + """ + # Avoids a problem where Python's email incorrectly converts non-ASCII + # domains to RFC 2047 encoded-words: + # https://github.com/python/cpython/issues/83938. + # This applies to the domain only, not to the localpart (username). + # There is no RFC that permits any 7-bit encoding for non-ASCII + # characters before the '@'. + if not getattr(msg.policy, "utf8", False): + # Not using SMTPUTF8, so apply IDNA encoding in all address + # headers. IDNA encoding does not alter domains that are already + # ASCII. + for field, value in msg.items(): + if isinstance(value, AddressHeader) and any( + not addr.domain.isascii() for addr in value.addresses + ): + msg.replace_header( + field, + [ + Address( + display_name=addr.display_name, + username=addr.username, + domain=punycode(addr.domain), + ) + for addr in value.addresses + ], + ) + class EmailMultiAlternatives(EmailMessage): """ @@ -467,8 +545,6 @@ class EmailMultiAlternatives(EmailMessage): made easier. """ - alternative_subtype = "alternative" - @deprecate_posargs( RemovedInDjango70Warning, [ @@ -522,24 +598,28 @@ class EmailMultiAlternatives(EmailMessage): raise ValueError("Both content and mimetype must be provided.") self.alternatives.append(EmailAlternative(content, mimetype)) - def _create_message(self, msg): - return self._create_attachments(self._create_alternatives(msg)) - - def _create_alternatives(self, msg): - encoding = self.encoding or settings.DEFAULT_CHARSET + def _add_bodies(self, msg): + if self.body or not self.alternatives: + super()._add_bodies(msg) if self.alternatives: - body_msg = msg - msg = SafeMIMEMultipart( - _subtype=self.alternative_subtype, encoding=encoding - ) - if self.body: - msg.attach(body_msg) - for alternative in self.alternatives: - msg.attach( - self._create_mime_attachment( - alternative.content, alternative.mimetype - ) + if hasattr(self, "alternative_subtype"): + # RemovedInDjango70Warning. + raise AttributeError( + "EmailMultiAlternatives no longer supports the" + " undocumented `alternative_subtype` attribute" ) + msg.make_alternative() + encoding = self.encoding or settings.DEFAULT_CHARSET + for alternative in self.alternatives: + maintype, subtype = alternative.mimetype.split("/", 1) + content = alternative.content + if maintype == "text": + if isinstance(content, bytes): + content = content.decode() + msg.add_alternative(content, subtype=subtype, charset=encoding) + else: + content = force_bytes(content, encoding=encoding, strings_only=True) + msg.add_alternative(content, maintype=maintype, subtype=subtype) return msg def body_contains(self, text): diff --git a/docs/internals/deprecation.txt b/docs/internals/deprecation.txt index 98b8eade75..81a9d72860 100644 --- a/docs/internals/deprecation.txt +++ b/docs/internals/deprecation.txt @@ -41,6 +41,18 @@ details on these changes. * The :mod:`django.core.mail` APIs will no longer accept certain parameters as positional arguments. These must be passed as keyword arguments instead. +* Support for passing Python's legacy email ``email.mime.base.MIMEBase`` + object to ``EmailMessage.attach()`` (or including one in the message's + ``attachments`` list) will be removed. + +* The ``django.core.mail.BadHeaderError`` exception will be removed. + +* The ``django.core.mail.SafeMIMEText`` and ``SafeMIMEMultipart`` classes will + be removed. + +* The ``django.core.mail.forbid_multi_line_headers()`` and + ``django.core.mail.message.sanitize_address()`` functions will be removed. + .. _deprecation-removed-in-6.1: 6.1 diff --git a/docs/releases/6.0.txt b/docs/releases/6.0.txt index 9d8120f5c2..7997ff3a38 100644 --- a/docs/releases/6.0.txt +++ b/docs/releases/6.0.txt @@ -74,6 +74,23 @@ To get started, follow the :doc:`CSP how-to guide `. For in-depth guidance, see the :ref:`CSP security overview ` and the :doc:`reference docs `. +Adoption of Python's modern email API +------------------------------------- + +Email handling in Django now uses Python's modern email API, introduced in +Python 3.6. This API, centered around the +:py:class:`email.message.EmailMessage` class, offers a cleaner and +Unicode-friendly interface for composing and sending emails. It replaces use of +Python's older legacy (``Compat32``) API, which relied on lower-level MIME +classes (from :py:mod:`email.mime`) and required more manual handling of +message structure and encoding. + +Notably, the return type of the :class:`EmailMessage.message() +` method is now an instance of Python's +:py:class:`email.message.EmailMessage`. This supports the same API as the +previous ``SafeMIMEText`` and ``SafeMIMEMultipart`` return types, but is not an +instance of those now-deprecated classes. + Minor features -------------- @@ -193,7 +210,13 @@ Decorators Email ~~~~~ -* ... +* The new ``policy`` argument for :class:`EmailMessage.message() + ` allows specifying the email policy, the set + of rules for updating and serializing the representation of the message. + Defaults to :py:data:`email.policy.default`. + +* :class:`EmailMessage.attach() ` now accepts a + :class:`~email.message.MIMEPart` object from Python's modern email API. Error Reporting ~~~~~~~~~~~~~~~ @@ -387,6 +410,22 @@ of each library are the first to add or confirm compatibility with Python 3.12: * ``sqlparse`` 0.5.0 * ``tblib`` 3.0.0 +Email +----- + +* The undocumented ``mixed_subtype`` and ``alternative_subtype`` properties + of :class:`~django.core.mail.EmailMessage` and + :class:`~django.core.mail.EmailMultiAlternatives` are no longer supported. + +* The undocumented ``encoding`` property of + :class:`~django.core.mail.EmailMessage` no longer supports Python legacy + :py:class:`email.charset.Charset` objects. + +* As the internal implementations of :class:`~django.core.mail.EmailMessage` + and :class:`~django.core.mail.EmailMultiAlternatives` have changed + significantly, closely examine any custom subclasses that rely on overriding + undocumented, internal underscore methods. + Miscellaneous ------------- @@ -435,7 +474,9 @@ warning and will raise a :exc:`TypeError` when the deprecation period ends. the first four (``subject``, ``body``, ``from_email``, and ``to``), which may still be passed either as positional or keyword arguments. -.. currentmodule:: None +* :mod:`django.core.mail` APIs now require keyword arguments for less commonly + used parameters. Using positional arguments for these now emits a deprecation + warning and will raise a :exc:`TypeError` when the deprecation period ends: Miscellaneous ------------- @@ -467,6 +508,22 @@ Miscellaneous * Using a percent sign in a column alias or annotation is deprecated. +* Support for passing Python's legacy email :class:`~email.mime.base.MIMEBase` + object to :class:`EmailMessage.attach() ` (or + including one in the message's ``attachments`` list) is deprecated. For + complex attachments requiring additional headers or parameters, switch to the + modern email API's :class:`~email.message.MIMEPart`. + +* The ``django.core.mail.BadHeaderError`` exception is deprecated. Python's + modern email raises a :exc:`!ValueError` for email headers containing + prohibited characters. + +* The ``django.core.mail.SafeMIMEText`` and ``SafeMIMEMultipart`` classes are + deprecated. + +* The undocumented ``django.core.mail.forbid_multi_line_headers()`` and + ``django.core.mail.message.sanitize_address()`` functions are deprecated. + Features removed in 6.0 ======================= diff --git a/docs/topics/email.txt b/docs/topics/email.txt index 44f311ae37..6c39560c6a 100644 --- a/docs/topics/email.txt +++ b/docs/topics/email.txt @@ -249,9 +249,8 @@ The Django email functions outlined above all protect against header injection by forbidding newlines in header values. If any ``subject``, ``from_email`` or ``recipient_list`` contains a newline (in either Unix, Windows or Mac style), the email function (e.g. :meth:`~django.core.mail.send_mail()`) will raise -``django.core.mail.BadHeaderError`` (a subclass of ``ValueError``) and, hence, -will not send the email. It's your responsibility to validate all data before -passing it to the email functions. +:exc:`ValueError` and, hence, will not send the email. It's your responsibility +to validate all data before passing it to the email functions. If a ``message`` contains headers at the start of the string, the headers will be printed as the first bit of the email message. @@ -260,7 +259,7 @@ Here's an example view that takes a ``subject``, ``message`` and ``from_email`` from the request's POST data, sends that to admin@example.com and redirects to "/contact/thanks/" when it's done:: - from django.core.mail import BadHeaderError, send_mail + from django.core.mail import send_mail from django.http import HttpResponse, HttpResponseRedirect @@ -271,7 +270,7 @@ from the request's POST data, sends that to admin@example.com and redirects to if subject and message and from_email: try: send_mail(subject, message, from_email, ["admin@example.com"]) - except BadHeaderError: + except ValueError: return HttpResponse("Invalid header found.") return HttpResponseRedirect("/contact/thanks/") else: @@ -279,6 +278,12 @@ from the request's POST data, sends that to admin@example.com and redirects to # to get proper validation errors. return HttpResponse("Make sure all fields are entered and valid.") + +.. versionchanged:: 6.0 + + Older versions raised ``django.core.mail.BadHeaderError`` for some + invalid headers. This has been replaced with :exc:`!ValueError`. + .. _Header injection: http://www.nyphp.org/phundamentals/8_Preventing-Email-Header-Injection.html .. _emailmessage-and-smtpconnection: @@ -346,8 +351,8 @@ The following parameters must be given as keyword arguments if used: * ``reply_to``: A list or tuple of recipient addresses used in the "Reply-To" header when sending the email. -* ``attachments``: A list of attachments to put on the message. These can - be instances of :class:`~email.mime.base.MIMEBase` or +* ``attachments``: A list of attachments to put on the message. Each can + be an instance of :class:`~email.message.MIMEPart` or :class:`~django.core.mail.EmailAttachment`, or a tuple with attributes ``(filename, content, mimetype)``. @@ -356,6 +361,17 @@ The following parameters must be given as keyword arguments if used: Support for :class:`~django.core.mail.EmailAttachment` items of ``attachments`` was added. + .. versionchanged:: 6.0 + + Support for :class:`~email.message.MIMEPart` objects in the ``attachments`` + list was added. + + .. deprecated:: 6.0 + + Support for Python's legacy :class:`~email.mime.base.MIMEBase` objects in + ``attachments`` is deprecated. Use :class:`~email.message.MIMEPart` + instead. + * ``headers``: A dictionary of extra headers to put on the message. The keys are the header name, values are the header values. It's up to the caller to ensure header names and values are in the correct format for @@ -396,12 +412,27 @@ The class has the following methods: recipients will not raise an exception. It will return ``1`` if the message was sent successfully, otherwise ``0``. -* ``message()`` constructs a ``django.core.mail.SafeMIMEText`` object (a - subclass of Python's :class:`~email.mime.text.MIMEText` class) or a - ``django.core.mail.SafeMIMEMultipart`` object holding the message to be - sent. If you ever need to extend the - :class:`~django.core.mail.EmailMessage` class, you'll probably want to - override this method to put the content you want into the MIME object. +* ``message(policy=email.policy.default)`` constructs and returns a Python + :class:`email.message.EmailMessage` object representing the message to be + sent. + + The keyword argument ``policy`` allows specifying the set of rules for + updating and serializing the representation of the message. It must be an + :py:mod:`email.policy.Policy ` object. Defaults to + :py:data:`email.policy.default`. In certain cases you may want to use + :py:data:`~email.policy.SMTP`, :py:data:`~email.policy.SMTPUTF8` or a custom + policy. For example, :class:`django.core.mail.backends.smtp.EmailBackend` + uses the :py:data:`~email.policy.SMTP` policy to ensure ``\r\n`` line endings + as required by the SMTP protocol. + + If you ever need to extend Django's :class:`~django.core.mail.EmailMessage` + class, you'll probably want to override this method to put the content you + want into the Python EmailMessage object. + + .. versionchanged:: 6.0 + + The ``policy`` keyword argument was added and the return type was updated + to an instance of :py:class:`~email.message.EmailMessage`. * ``recipients()`` returns a list of all the recipients of the message, whether they're recorded in the ``to``, ``cc`` or ``bcc`` attributes. This @@ -410,40 +441,57 @@ The class has the following methods: is sent. If you add another way to specify recipients in your class, they need to be returned from this method as well. -* ``attach()`` creates a new file attachment and adds it to the message. +* ``attach()`` creates a new attachment and adds it to the message. There are two ways to call ``attach()``: - * You can pass it a single argument that is a - :class:`~email.mime.base.MIMEBase` instance. This will be inserted directly - into the resulting message. - - * Alternatively, you can pass ``attach()`` three arguments: - ``filename``, ``content`` and ``mimetype``. ``filename`` is the name - of the file attachment as it will appear in the email, ``content`` is - the data that will be contained inside the attachment and - ``mimetype`` is the optional MIME type for the attachment. If you - omit ``mimetype``, the MIME content type will be guessed from the - filename of the attachment. + * You can pass it three arguments: ``filename``, ``content`` and + ``mimetype``. ``filename`` is the name of the file attachment as it will + appear in the email, ``content`` is the data that will be contained inside + the attachment and ``mimetype`` is the optional MIME type for the + attachment. If you omit ``mimetype``, the MIME content type will be guessed + from the filename of the attachment. For example:: message.attach("design.png", img_data, "image/png") - If you specify a ``mimetype`` of :mimetype:`message/rfc822`, it will also - accept :class:`django.core.mail.EmailMessage` and - :py:class:`email.message.Message`. + If you specify a ``mimetype`` of :mimetype:`message/rfc822`, ``content`` + can be a :class:`django.core.mail.EmailMessage` or Python's + :class:`email.message.EmailMessage` or :class:`email.message.Message`. For a ``mimetype`` starting with :mimetype:`text/`, content is expected to be a string. Binary data will be decoded using UTF-8, and if that fails, the MIME type will be changed to :mimetype:`application/octet-stream` and the data will be attached unchanged. - In addition, :mimetype:`message/rfc822` attachments will no longer be - base64-encoded in violation of :rfc:`2046#section-5.2.1`, which can cause - issues with displaying the attachments in `Evolution`__ and `Thunderbird`__. + * Or for attachments requiring additional headers or parameters, you can pass + ``attach()`` a single Python :class:`~email.message.MIMEPart` object. + This will be attached directly to the resulting message. For example, + to attach an inline image with a :mailheader:`Content-ID`:: - __ https://bugzilla.gnome.org/show_bug.cgi?id=651197 - __ https://bugzilla.mozilla.org/show_bug.cgi?id=333880 + cid = email.utils.make_msgid() + inline_image = email.message.MIMEPart() + inline_image.set_content( + image_data_bytes, + maintype="image", + subtype="png", + disposition="inline", + cid=f"<{cid}>", + ) + message.attach(inline_image) + message.attach_alternative(f'… …', "text/html") + + Python's :meth:`email.contentmanager.set_content` documentation describes + the supported arguments for ``MIMEPart.set_content()``. + + .. versionchanged:: 6.0 + + Support for :class:`~email.message.MIMEPart` attachments was added. + + .. deprecated:: 6.0 + + Support for :class:`email.mime.base.MIMEBase` attachments is + deprecated. Use :class:`~email.message.MIMEPart` instead. * ``attach_file()`` creates a new attachment using a file from your filesystem. Call it with the path of the file to attach and, optionally, diff --git a/tests/auth_tests/test_forms.py b/tests/auth_tests/test_forms.py index 735ac1d237..be55c4369b 100644 --- a/tests/auth_tests/test_forms.py +++ b/tests/auth_tests/test_forms.py @@ -1367,13 +1367,13 @@ class PasswordResetFormTest(TestDataMixin, TestCase): self.assertTrue( re.match( r"^http://example.com/reset/[\w/-]+", - message.get_payload(0).get_payload(), + message.get_payload(0).get_content(), ) ) self.assertTrue( re.match( r'^Link$', - message.get_payload(1).get_payload(), + message.get_payload(1).get_content(), ) ) diff --git a/tests/mail/test_deprecated.py b/tests/mail/test_deprecated.py new file mode 100644 index 0000000000..bdf01e0ef6 --- /dev/null +++ b/tests/mail/test_deprecated.py @@ -0,0 +1,122 @@ +# RemovedInDjango70Warning: This entire file. +from email.mime.text import MIMEText + +from django.core.mail import ( + EmailAlternative, + EmailAttachment, + EmailMessage, + EmailMultiAlternatives, +) +from django.core.mail.message import forbid_multi_line_headers, sanitize_address +from django.test import SimpleTestCase, ignore_warnings +from django.utils.deprecation import RemovedInDjango70Warning + +from .tests import MailTestsMixin + + +class DeprecationWarningTests(MailTestsMixin, SimpleTestCase): + def test_deprecated_on_import(self): + """ + These items are not typically called from user code, + so generate deprecation warnings immediately at the time + they are imported from django.core.mail. + """ + cases = [ + # name, msg + ( + "BadHeaderError", + "BadHeaderError is deprecated. Replace with ValueError.", + ), + ( + "SafeMIMEText", + "SafeMIMEText is deprecated. The return value of" + " EmailMessage.message() is an email.message.EmailMessage.", + ), + ( + "SafeMIMEMultipart", + "SafeMIMEMultipart is deprecated. The return value of" + " EmailMessage.message() is an email.message.EmailMessage.", + ), + ] + for name, msg in cases: + with self.subTest(name=name): + with self.assertWarnsMessage(RemovedInDjango70Warning, msg): + __import__("django.core.mail", fromlist=[name]) + + def test_sanitize_address_deprecated(self): + msg = ( + "The internal API sanitize_address() is deprecated." + " Python's modern email API (with email.message.EmailMessage or" + " email.policy.default) will handle most required validation and" + " encoding. Use Python's email.headerregistry.Address to construct" + " formatted addresses from component parts." + ) + with self.assertWarnsMessage(RemovedInDjango70Warning, msg): + sanitize_address("to@example.com", "ascii") + + def test_forbid_multi_line_headers_deprecated(self): + msg = ( + "The internal API forbid_multi_line_headers() is deprecated." + " Python's modern email API (with email.message.EmailMessage or" + " email.policy.default) will reject multi-line headers." + ) + with self.assertWarnsMessage(RemovedInDjango70Warning, msg): + forbid_multi_line_headers("To", "to@example.com", "ascii") + + +class UndocumentedFeatureErrorTests(SimpleTestCase): + """ + These undocumented features were removed without going through deprecation. + In case they were being used, they now raise errors. + """ + + def test_undocumented_mixed_subtype(self): + """ + Trying to use the previously undocumented, now unsupported + EmailMessage.mixed_subtype causes an error. + """ + msg = ( + "EmailMessage no longer supports" + " the undocumented `mixed_subtype` attribute" + ) + email = EmailMessage( + attachments=[EmailAttachment(None, b"GIF89a...", "image/gif")] + ) + email.mixed_subtype = "related" + with self.assertRaisesMessage(AttributeError, msg): + email.message() + + def test_undocumented_alternative_subtype(self): + """ + Trying to use the previously undocumented, now unsupported + EmailMultiAlternatives.alternative_subtype causes an error. + """ + msg = ( + "EmailMultiAlternatives no longer supports" + " the undocumented `alternative_subtype` attribute" + ) + email = EmailMultiAlternatives( + alternatives=[EmailAlternative("", "text/plain")] + ) + email.alternative_subtype = "multilingual" + with self.assertRaisesMessage(AttributeError, msg): + email.message() + + +@ignore_warnings(category=RemovedInDjango70Warning) +class DeprecatedCompatibilityTests(SimpleTestCase): + def test_bad_header_error(self): + """ + Existing code that catches deprecated BadHeaderError should be + compatible with modern email (which raises ValueError instead). + """ + from django.core.mail import BadHeaderError + + with self.assertRaises(BadHeaderError): + EmailMessage(subject="Bad\r\nHeader").message() + + def test_attachments_mimebase_in_constructor(self): + txt = MIMEText("content1") + msg = EmailMessage(attachments=[txt]) + payload = msg.message().get_payload() + self.assertEqual(payload[0], txt) diff --git a/tests/mail/tests.py b/tests/mail/tests.py index 397fb8c747..f1d7fcf43e 100644 --- a/tests/mail/tests.py +++ b/tests/mail/tests.py @@ -1,16 +1,20 @@ +import ast import mimetypes import os import pickle +import re import shutil import socket import sys import tempfile -from email import charset, message_from_binary_file +from datetime import datetime, timezone +from email import message_from_binary_file from email import message_from_bytes as _message_from_bytes from email import policy from email.headerregistry import Address from email.message import EmailMessage as PyEmailMessage from email.message import Message as PyMessage +from email.message import MIMEPart from email.mime.image import MIMEImage from email.mime.text import MIMEText from io import StringIO @@ -24,7 +28,6 @@ from django.core import mail from django.core.exceptions import ImproperlyConfigured from django.core.mail import ( DNS_NAME, - BadHeaderError, EmailAlternative, EmailAttachment, EmailMessage, @@ -35,9 +38,8 @@ from django.core.mail import ( send_mass_mail, ) from django.core.mail.backends import console, dummy, filebased, locmem, smtp -from django.core.mail.message import sanitize_address from django.test import SimpleTestCase, override_settings -from django.test.utils import requires_tz_support +from django.test.utils import ignore_warnings, requires_tz_support from django.utils.deprecation import RemovedInDjango70Warning from django.utils.translation import gettext_lazy @@ -248,6 +250,8 @@ class MailTests(MailTestsMixin, SimpleTestCase): self.assertEqual(message["From"], "from@example.com") self.assertEqual(message["To"], "to@example.com") + # RemovedInDjango70Warning. + @ignore_warnings(category=RemovedInDjango70Warning) @mock.patch("django.core.mail.message.MIMEText.set_payload") def test_nonascii_as_string_with_ascii_charset(self, mock_set_payload): """Line length check should encode the payload supporting @@ -269,6 +273,7 @@ class MailTests(MailTestsMixin, SimpleTestCase): # This test is specific to Python's legacy MIMEText. This can be safely # removed when EmailMessage.message() uses Python's modern email API. # (Using surrogateescape for non-utf8 is covered in test_encoding().) + from django.core.mail import SafeMIMEText def simplified_set_payload(instance, payload, charset): instance._payload = payload @@ -279,8 +284,7 @@ class MailTests(MailTestsMixin, SimpleTestCase): "Text heavily based in Python's text for non-ascii messages: Föö bär" ).encode("iso-8859-1") body = text.decode("ascii", errors="surrogateescape") - email = EmailMessage("Subject", body, "from@example.com", ["to@example.com"]) - message = email.message() + message = SafeMIMEText(body, "plain", "ascii") mock_set_payload.assert_called_once() self.assertEqual(message.get_payload(decode=True), text) @@ -475,7 +479,7 @@ class MailTests(MailTestsMixin, SimpleTestCase): EmailMessage(reply_to="reply_to@example.com") def test_header_injection(self): - msg = "Header values can't contain newlines " + msg = "Header values may not contain linefeed or carriage return characters" cases = [ {"subject": "Subject\nInjection Test"}, {"subject": gettext_lazy("Lazy Subject\nInjection Test")}, @@ -484,7 +488,7 @@ class MailTests(MailTestsMixin, SimpleTestCase): for kwargs in cases: with self.subTest(case=kwargs): email = EmailMessage(**kwargs) - with self.assertRaisesMessage(BadHeaderError, msg): + with self.assertRaisesMessage(ValueError, msg): email.message() def test_folding_white_space(self): @@ -497,12 +501,9 @@ class MailTests(MailTestsMixin, SimpleTestCase): ) message = email.message() msg_bytes = message.as_bytes() - # Python's legacy email wraps this more than strictly necessary - # (but uses FWS properly at each wrap). Modern email wraps it better. self.assertIn( b"Subject: Long subject lines that get wrapped should contain a space\n" - b" continuation\n" - b" character to comply with RFC 822", + b" continuation character to comply with RFC 822", msg_bytes, ) @@ -522,6 +523,19 @@ class MailTests(MailTestsMixin, SimpleTestCase): }, ) + def test_datetime_in_date_header(self): + """ + A datetime in headers should be passed through to Python email intact, + so that it uses the email header date format. + """ + email = EmailMessage( + headers={"Date": datetime(2001, 11, 9, 1, 8, 47, tzinfo=timezone.utc)}, + ) + message = email.message() + self.assertEqual(message["Date"], "Fri, 09 Nov 2001 01:08:47 +0000") + # Not the default ISO format from force_str(strings_only=False). + self.assertNotEqual(message["Date"], "2001-11-09 01:08:47+00:00") + def test_from_header(self): """ Make sure we can manually set the From header (#9214) @@ -754,6 +768,28 @@ class MailTests(MailTestsMixin, SimpleTestCase): self.assertIn(html_content, msg.message().as_string()) + def test_alternative_alternatives(self): + """ + Alternatives can be attached as either string or bytes + and need not use a text/* mimetype. + """ + cases = [ + # (mimetype, content, expected decoded payload) + ("application/x-ccmail-rtf", b"non-text\x07bytes", b"non-text\x07bytes"), + ("application/x-ccmail-rtf", "non-text\x07string", b"non-text\x07string"), + ("text/x-amp-html", b"text bytes\n", b"text bytes\n"), + ("text/x-amp-html", "text string\n", b"text string\n"), + ] + for mimetype, content, expected in cases: + with self.subTest(case=(mimetype, content)): + email = EmailMultiAlternatives() + email.attach_alternative(content, mimetype) + msg = email.message() + self.assertEqual(msg.get_content_type(), "multipart/alternative") + alternative = msg.get_payload()[0] + self.assertEqual(alternative.get_content_type(), mimetype) + self.assertEqual(alternative.get_payload(decode=True), expected) + def test_alternatives_and_attachment_serializable(self): html_content = "

This is html

" mime_type = "text/html" @@ -774,7 +810,9 @@ class MailTests(MailTestsMixin, SimpleTestCase): def test_none_body(self): msg = EmailMessage("subject", None, "from@example.com", ["to@example.com"]) self.assertEqual(msg.body, "") - self.assertEqual(msg.message().get_payload(), "") + # The modern email API forces trailing newlines on all text/* parts, + # even an empty body. + self.assertEqual(msg.message().get_payload(), "\n") @mock.patch("socket.getfqdn", return_value="漢字") def test_non_ascii_dns_non_unicode_email(self, mocked_getfqdn): @@ -795,13 +833,8 @@ class MailTests(MailTestsMixin, SimpleTestCase): # Check that body is actually encoded with iso-8859-1. msg_bytes = message.as_bytes() - if message["Content-Transfer-Encoding"] == "quoted-printable": - self.assertIn(b"Firstname S=FCrname is a great guy.", msg_bytes) - elif message["Content-Transfer-Encoding"] == "8bit": - self.assertIn(b"Firstname S\xfc", msg_bytes) - else: - self.fail("Unexpected Content-Transfer-Encoding") - + self.assertEqual(message["Content-Transfer-Encoding"], "8bit") + self.assertIn(b"Firstname S\xfc", msg_bytes) parsed = message_from_bytes(msg_bytes) self.assertEqual(parsed.get_content(), "Firstname Sürname is a great guy.\n") @@ -917,8 +950,11 @@ class MailTests(MailTestsMixin, SimpleTestCase): self.assertEqual(payload[0].get_content_type(), "multipart/alternative") self.assertEqual(payload[1].get_content_type(), "application/pdf") - def test_decoded_attachments_MIMEText(self): - txt = MIMEText("content1") + def test_decoded_attachment_text_MIMEPart(self): + # See also test_attach_mime_part() and + # test_attach_mime_part_in_constructor(). + txt = MIMEPart() + txt.set_content("content1") msg = EmailMessage(attachments=[txt]) payload = msg.message().get_payload() self.assertEqual(payload[0], txt) @@ -1070,16 +1106,48 @@ class MailTests(MailTestsMixin, SimpleTestCase): self.assertEqual(attached_message["Content-Transfer-Encoding"], "8bit") self.assertEqual(attached_message.get_content_type(), "text/plain") + # RemovedInDjango70Warning. def test_attach_mime_image(self): """ EmailMessage.attach() docs: "You can pass it a single argument that is a MIMEBase instance." """ + msg = ( + "MIMEBase attachments are deprecated." + " Use an email.message.MIMEPart instead." + ) # This also verifies complex attachments with extra header fields. email = EmailMessage() image = MIMEImage(b"GIF89a...", "gif") image["Content-Disposition"] = "inline" image["Content-ID"] = "" + with self.assertWarnsMessage(RemovedInDjango70Warning, msg): + email.attach(image) + + attachments = self.get_raw_attachments(email) + self.assertEqual(len(attachments), 1) + image_att = attachments[0] + self.assertEqual(image_att.get_content_type(), "image/gif") + self.assertEqual(image_att.get_content_disposition(), "inline") + self.assertEqual(image_att["Content-ID"], "") + self.assertEqual(image_att.get_content(), b"GIF89a...") + self.assertIsNone(image_att.get_filename()) + + def test_attach_mime_part(self): + """ + EmailMessage.attach() docs: "You can pass it + a single argument that is a MIMEPart object." + """ + # This also verifies complex attachments with extra header fields. + email = EmailMessage() + image = MIMEPart() + image.set_content( + b"GIF89a...", + maintype="image", + subtype="gif", + disposition="inline", + cid="", + ) email.attach(image) attachments = self.get_raw_attachments(email) @@ -1091,9 +1159,29 @@ class MailTests(MailTestsMixin, SimpleTestCase): self.assertEqual(image_att.get_content(), b"GIF89a...") self.assertIsNone(image_att.get_filename()) + # RemovedInDjango70Warning. def test_attach_mime_image_in_constructor(self): + msg = ( + "MIMEBase attachments are deprecated." + " Use an email.message.MIMEPart instead." + ) image = MIMEImage(b"\x89PNG...", "png") image["Content-Disposition"] = "attachment; filename=test.png" + with self.assertWarnsMessage(RemovedInDjango70Warning, msg): + email = EmailMessage(attachments=[image]) + + attachments = self.get_raw_attachments(email) + self.assertEqual(len(attachments), 1) + image_att = attachments[0] + self.assertEqual(image_att.get_content_type(), "image/png") + self.assertEqual(image_att.get_content(), b"\x89PNG...") + self.assertEqual(image_att.get_filename(), "test.png") + + def test_attach_mime_part_in_constructor(self): + image = MIMEPart() + image.set_content( + b"\x89PNG...", maintype="image", subtype="png", filename="test.png" + ) email = EmailMessage(attachments=[image]) attachments = self.get_raw_attachments(email) @@ -1163,11 +1251,12 @@ class MailTests(MailTestsMixin, SimpleTestCase): self.assertEqual(child_cte, "7bit") self.assertEqual(attached_message.get_content_type(), "text/plain") - def test_attach_mimebase_prohibits_other_params(self): + def test_attach_mimepart_prohibits_other_params(self): email_msg = EmailMessage() - txt = MIMEText("content") + txt = MIMEPart() + txt.set_content("content") msg = ( - "content and mimetype must not be given when a MIMEBase instance " + "content and mimetype must not be given when a MIMEPart instance " "is provided." ) with self.assertRaisesMessage(ValueError, msg): @@ -1318,9 +1407,7 @@ class MailTests(MailTestsMixin, SimpleTestCase): self.assertIn(b"Content-Transfer-Encoding: 8bit", s) # Long body lines that require folding should use quoted-printable or - # base64, whichever is shorter. However, Python's legacy email API - # avoids re-folding non-ASCII text and just uses CTE 8bit. (The modern - # API would correctly choose base64 here. Any of these is deliverable.) + # base64, whichever is shorter. msg = EmailMessage( body=( "Body with non latin characters: А Б В Г Д Е Ж Ѕ З И І К Л М Н О П.\n" @@ -1329,36 +1416,17 @@ class MailTests(MailTestsMixin, SimpleTestCase): ), ) s = msg.message().as_bytes() - self.assertIn(b"Content-Transfer-Encoding: 8bit", s) - - def test_custom_utf8_encoding(self): - """A UTF-8 charset with a custom body encoding is respected.""" - # Tests that the undocumented EmailMessage.encoding property allows a - # custom, legacy email.charset.Charset object. Modern email doesn't - # support that, and this test will be removed. - # (A str charset name like `msg.encoding = "iso-8859-1"` will still - # work, and is tested elsewhere.) - body = "Body with latin characters: àáä.\n" - msg = EmailMessage("Subject", body, "bounce@example.com", ["to@example.com"]) - encoding = charset.Charset("utf-8") - encoding.body_encoding = charset.QP - msg.encoding = encoding - message = msg.message() - self.assertMessageHasHeaders( - message, - { - ("MIME-Version", "1.0"), - ("Content-Type", 'text/plain; charset="utf-8"'), - ("Content-Transfer-Encoding", "quoted-printable"), - }, - ) - self.assertEqual(message.get_payload(), encoding.body_encode(body)) + self.assertIn(b"Content-Transfer-Encoding: quoted-printable", s) + # RemovedInDjango70Warning. + @ignore_warnings(category=RemovedInDjango70Warning) def test_sanitize_address(self): """Email addresses are properly sanitized.""" # Tests the internal sanitize_address() function. Many of these cases # are duplicated in test_address_header_handling(), which verifies # headers in the generated message. + from django.core.mail.message import sanitize_address + for email_address, encoding, expected_result in ( # ASCII addresses. ("to@example.com", "ascii", "to@example.com"), @@ -1389,6 +1457,7 @@ class MailTests(MailTestsMixin, SimpleTestCase): ( "Tó Example ", "utf-8", + # (Not RFC-2047 compliant.) "=?utf-8?q?T=C3=B3_Example?= <=?utf-8?b?dMOz?=@example.com>", ), # IDNA addresses with display names. @@ -1445,6 +1514,8 @@ class MailTests(MailTestsMixin, SimpleTestCase): sanitize_address(email_address, encoding), expected_result ) + # RemovedInDjango70Warning. + @ignore_warnings(category=RemovedInDjango70Warning) def test_sanitize_address_invalid(self): # Tests the internal sanitize_address() function. Note that Django's # EmailMessage.message() will not catch these cases, as it only calls @@ -1452,6 +1523,8 @@ class MailTests(MailTestsMixin, SimpleTestCase): # Django detects these cases in the SMTP EmailBackend during sending. # See SMTPBackendTests.test_avoids_sending_to_invalid_addresses() # below. + from django.core.mail.message import sanitize_address + for email_address in ( # Invalid address with two @ signs. "to@other.com@example.com", @@ -1467,10 +1540,14 @@ class MailTests(MailTestsMixin, SimpleTestCase): with self.assertRaisesMessage(ValueError, "Invalid address"): sanitize_address(email_address, encoding="utf-8") + # RemovedInDjango70Warning. + @ignore_warnings(category=RemovedInDjango70Warning) def test_sanitize_address_header_injection(self): # Tests the internal sanitize_address() function. These cases are # duplicated in test_address_header_handling(), which verifies headers # in the generated message. + from django.core.mail.message import sanitize_address + msg = "Invalid address; address parts cannot contain newlines." tests = [ "Name\nInjection ", @@ -1549,7 +1626,7 @@ class MailTests(MailTestsMixin, SimpleTestCase): self.assertEqual(actual, expected) def test_address_header_injection(self): - msg = "Header values can't contain newlines" + msg = "Header values may not contain linefeed or carriage return characters" cases = [ "Name\nInjection ", '"Name\nInjection" ', @@ -1823,10 +1900,12 @@ class MailTests(MailTestsMixin, SimpleTestCase): email.to = ["new-to@example.com"] email.bcc = ["new-bcc@example.com"] email.connection = new_connection + image = MIMEPart() + image.set_content(b"GIF89a...", "image", "gif") email.attachments = [ ("new1.txt", "new attachment 1\n", "text/plain"), # plain tuple EmailAttachment("new2.txt", "new attachment 2\n", "text/csv"), - MIMEImage(b"GIF89a...", "gif"), + image, ] email.extra_headers = {"X-Header": "new header"} email.cc = ["new-cc@example.com"] @@ -1855,6 +1934,73 @@ class MailTests(MailTestsMixin, SimpleTestCase): self.assertIs(email.get_connection(), new_connection) self.assertNotIn("original", message.as_string()) + def test_message_is_python_email_message(self): + """ + EmailMessage.message() docs: "returns a Python + email.message.EmailMessage object." + """ + email = EmailMessage() + message = email.message() + self.assertIsInstance(message, PyMessage) + self.assertEqual(message.policy, policy.default) + + def test_message_policy_smtputf8(self): + # With SMTPUTF8, the message uses utf-8 directly in headers (not + # RFC 2047 encoded-words). Note this is the only spec-compliant way + # to send to a non-ASCII localpart. + email = EmailMessage( + subject="Detta ämne innehåller icke-ASCII-tecken", + to=["nøn-åscîi@example.com"], + ) + message = email.message(policy=policy.SMTPUTF8) + self.assertEqual(message.policy, policy.SMTPUTF8) + msg_bytes = message.as_bytes() + self.assertIn( + "Subject: Detta ämne innehåller icke-ASCII-tecken".encode(), msg_bytes + ) + self.assertIn("To: nøn-åscîi@example.com".encode(), msg_bytes) + self.assertNotIn(b"=?utf-8?", msg_bytes) # encoded-word prefix + + def test_message_policy_cte_7bit(self): + """ + Allows a policy that requires 7bit encodings. + """ + email = EmailMessage(body="Detta innehåller icke-ASCII-tecken") + email.attach("file.txt", "يحتوي هذا المرفق على أحرف غير ASCII") + + # Uses 8bit by default. (Test pre-condition.) + self.assertIn(b"Content-Transfer-Encoding: 8bit", email.message().as_bytes()) + + # Uses something 7bit compatible when policy requires it. Should pick + # the shorter of quoted-printable (for this body) or base64 (for this + # attachment), but must not use 8bit. (Decoding to "ascii" verifies + # that.) + policy_7bit = policy.default.clone(cte_type="7bit") + msg_bytes = email.message(policy=policy_7bit).as_bytes() + msg_ascii = msg_bytes.decode("ascii") + self.assertIn("Content-Transfer-Encoding: quoted-printable", msg_ascii) + self.assertIn("Content-Transfer-Encoding: base64", msg_ascii) + self.assertNotIn("Content-Transfer-Encoding: 8bit", msg_ascii) + + def test_message_policy_compat32(self): + """ + Although EmailMessage.message() doesn't support policy=compat32 + (because compat32 doesn't support modern APIs), compat32 _can_ be + used with as_bytes() or as_string() on the resulting message. + """ + # This subject results in different (but equivalent) RFC 2047 encoding + # with compat32 vs. email.policy.default. + email = EmailMessage(subject="Detta ämne innehåller icke-ASCII-tecken") + message = email.message() + self.assertIn( + b"Subject: =?utf-8?q?Detta_=C3=A4mne_inneh=C3=A5ller_icke-ASCII-tecken?=\n", + message.as_bytes(policy=policy.compat32), + ) + self.assertIn( + "Subject: =?utf-8?q?Detta_=C3=A4mne_inneh=C3=A5ller_icke-ASCII-tecken?=\n", + message.as_string(policy=policy.compat32), + ) + # RemovedInDjango70Warning. class MailDeprecatedPositionalArgsTests(SimpleTestCase): @@ -1983,7 +2129,9 @@ class MailTimeZoneTests(MailTestsMixin, SimpleTestCase): EMAIL_USE_LOCALTIME=False creates a datetime in UTC. """ email = EmailMessage() - self.assertEndsWith(email.message()["Date"], "-0000") + # Per RFC 2822/5322 section 3.3, "The form '+0000' SHOULD be used + # to indicate a time zone at Universal Time." + self.assertEndsWith(email.message()["Date"], "+0000") @override_settings( EMAIL_USE_LOCALTIME=True, USE_TZ=True, TIME_ZONE="Africa/Algiers" @@ -1997,6 +2145,7 @@ class MailTimeZoneTests(MailTestsMixin, SimpleTestCase): self.assertEndsWith(email.message()["Date"], "+0100") +# RemovedInDjango70Warning. class PythonGlobalState(SimpleTestCase): """ Tests for #12422 -- Django smarts (#2472/#11212) with charset of utf-8 text @@ -2062,7 +2211,7 @@ class BaseEmailBackendTests(MailTestsMixin): self.assertEqual(num_sent, 1) message = self.get_the_message() self.assertEqual(message["subject"], "Subject") - self.assertEqual(message.get_payload(), "Content\n") + self.assertEqual(message.get_content(), "Content\n") self.assertEqual(message["from"], "from@example.com") self.assertEqual(message.get_all("to"), ["to@example.com"]) @@ -2086,12 +2235,9 @@ class BaseEmailBackendTests(MailTestsMixin): quoted-printable or base64 (whichever is shorter), to avoid having to insert newlines in a way that alters the intended text. """ - # Django with Python's legacy email API uses quoted-printable for both - # cases below. Python's modern API would prefer shorter base64 for the - # first case. cases = [ # (body, expected_cte) - ("В южных морях " * 60, "quoted-printable"), + ("В южных морях " * 60, "base64"), ("I de sørlige hav " * 58, "quoted-printable"), ] for body, expected_cte in cases: @@ -2428,10 +2574,7 @@ class LocmemBackendTests(BaseEmailBackendTests, SimpleTestCase): email_backend = "django.core.mail.backends.locmem.EmailBackend" def get_mailbox_content(self): - # Reparse as modern messages to work with shared BaseEmailBackendTests. - # (Once EmailMessage.message() uses Python's modern email API, this - # can be changed back to `[m.message() for m in mail.outbox]`.) - return [message_from_bytes(m.message().as_bytes()) for m in mail.outbox] + return [m.message() for m in mail.outbox] def flush_mailbox(self): mail.outbox = [] @@ -2453,7 +2596,7 @@ class LocmemBackendTests(BaseEmailBackendTests, SimpleTestCase): def test_validate_multiline_headers(self): # Ticket #18861 - Validate emails when using the locmem backend - with self.assertRaises(BadHeaderError): + with self.assertRaises(ValueError): send_mail( "Subject\nMultiline", "Content", "from@example.com", ["to@example.com"] ) @@ -2908,6 +3051,8 @@ class SMTPBackendTests(BaseEmailBackendTests, SMTPBackendTestsBase): "to@other.com@example.com", # Invalid address without the quotes. "to@other.com ", + # Multiple mailboxes in a single address. + "to@example.com, other@example.com", # Other invalid addresses. "@", "to@", @@ -2976,6 +3121,28 @@ class SMTPBackendTests(BaseEmailBackendTests, SMTPBackendTestsBase): ], ) + def test_rejects_non_ascii_local_part(self): + """ + The SMTP EmailBackend does not currently support non-ASCII local-parts. + (That would require using the RFC 6532 SMTPUTF8 extension.) #35713. + """ + backend = smtp.EmailBackend() + backend.connection = mock.Mock(spec=object()) + email = EmailMessage(to=["nø@example.dk"]) + with self.assertRaisesMessage( + ValueError, + "Invalid address 'nø@example.dk': local-part contains non-ASCII characters", + ): + backend.send_messages([email]) + + def test_prep_address_without_force_ascii(self): + # A subclass implementing SMTPUTF8 could use + # prep_address(force_ascii=False). + backend = smtp.EmailBackend() + for case in ["åh@example.dk", "oh@åh.example.dk", "åh@åh.example.dk"]: + with self.subTest(case=case): + self.assertEqual(backend.prep_address(case, force_ascii=False), case) + @skipUnless(HAS_AIOSMTPD, "No aiosmtpd library detected.") class SMTPBackendStoppedServerTests(SMTPBackendTestsBase): @@ -3005,3 +3172,87 @@ class SMTPBackendStoppedServerTests(SMTPBackendTestsBase): self.backend.open() self.backend.fail_silently = True self.backend.open() + + +class LegacyAPINotUsedTests(SimpleTestCase): + """ + Check django.core.mail does not directly import Python legacy email APIs, + with a few specific exceptions. + """ + + # From "Legacy API:" in https://docs.python.org/3/library/email.html. + legacy_email_apis = { + "email.message.Message", + "email.mime", + "email.header", + "email.charset", + "email.encoders", + "email.utils", + "email.iterators", + } + + allowed_exceptions = { + # Compatibility in EmailMessage.attachments special cases: + "email.message.Message", + # No replacement in modern email API: + "email.utils.make_msgid", + } + + def test_no_legacy_apis_imported(self): + django_core_mail_path = Path(mail.__file__).parent + django_path = django_core_mail_path.parent.parent.parent + for abs_path in django_core_mail_path.glob("**/*.py"): + allowed_exceptions = self.allowed_exceptions.copy() + # RemovedInDjango70Warning. + # The following will be removed after the deprecation period. + if abs_path.name == "message.py": + allowed_exceptions.update( + { + "email.charset", + "email.header.Header", + "email.mime.base.MIMEBase", + "email.mime.message.MIMEMessage", + "email.mime.multipart.MIMEMultipart", + "email.mime.text.MIMEText", + "email.utils.formataddr", + "email.utils.getaddresses", + } + ) + path = abs_path.relative_to(django_path) + with self.subTest(path=str(path)): + collector = self.ImportCollector(abs_path.read_text()) + used_apis = collector.get_matching_imports(self.legacy_email_apis) + used_apis -= allowed_exceptions + self.assertEqual( + "\n".join(sorted(used_apis)), + "", + f"Python legacy email APIs used in {path}", + ) + + class ImportCollector(ast.NodeVisitor): + """ + Collect all imports from an AST as a set of fully-qualified dotted + names. + """ + + def __init__(self, source=None): + self.imports = set() + if source: + tree = ast.parse(source) + self.visit(tree) + + def get_matching_imports(self, base_names): + """ + Return the set of collected imports that start with any + of the fully-qualified dotted names in iterable base_names. + """ + matcher = re.compile( + r"\b(" + r"|".join(re.escape(name) for name in base_names) + r")\b" + ) + return set(name for name in self.imports if matcher.match(name)) + + def visit_Import(self, node): + self.imports.update(alias.name for alias in node.names) + + def visit_ImportFrom(self, node): + self.imports.update(f"{node.module}.{alias.name}" for alias in node.names)