Merge remote-tracking branch 'django/main'

django/contrib/postgres/fields/array.py

@@ -169,7 +169,7 @@ class ArrayField(CheckFieldDefaultMixin, Field):
             else:
                 obj = AttributeSetter(base_field.attname, val)
                 values.append(base_field.value_to_string(obj))
-        return json.dumps(values)
+        return json.dumps(values, ensure_ascii=False)
 
     def get_transform(self, name):
         transform = super().get_transform(name)

django/contrib/postgres/fields/hstore.py

@@ -43,7 +43,7 @@ class HStoreField(CheckFieldDefaultMixin, Field):
         return value
 
     def value_to_string(self, obj):
-        return json.dumps(self.value_from_object(obj))
+        return json.dumps(self.value_from_object(obj), ensure_ascii=False)
 
     def formfield(self, **kwargs):
         return super().formfield(

django/core/management/commands/makemessages.py

@@ -40,7 +40,7 @@ def check_programs(*programs):
 
 
 def is_valid_locale(locale):
-    return re.match(r"^[a-z]+$", locale) or re.match(r"^[a-z]+_[A-Z].*$", locale)
+    return re.match(r"^[a-z]+$", locale) or re.match(r"^[a-z]+_[A-Z0-9].*$", locale)
 
 
 @total_ordering

django/core/management/commands/sqlmigrate.py

@@ -32,10 +32,9 @@ class Command(BaseCommand):
         )
 
     def execute(self, *args, **options):
-        # sqlmigrate doesn't support coloring its output but we need to force
+        # sqlmigrate doesn't support coloring its output, so make the
-        # no_color=True so that the BEGIN/COMMIT statements added by
+        # BEGIN/COMMIT statements added by output_transaction colorless also.
-        # output_transaction don't get colored either.
+        self.style.SQL_KEYWORD = lambda noop: noop
-        options["no_color"] = True
         return super().execute(*args, **options)
 
     def handle(self, *args, **options):

django/db/backends/base/operations.py

@@ -8,7 +8,6 @@ import sqlparse
 
 from django.conf import settings
 from django.db import NotSupportedError, transaction
-from django.db.backends import utils
 from django.db.models.expressions import Col
 from django.utils import timezone
 from django.utils.deprecation import RemovedInDjango60Warning
@@ -586,7 +585,7 @@ class BaseDatabaseOperations:
         Transform a decimal.Decimal value to an object compatible with what is
         expected by the backend driver for decimal (numeric) columns.
         """
-        return utils.format_number(value, max_digits, decimal_places)
+        return value
 
     def adapt_ipaddressfield_value(self, value):
         """

django/db/backends/mysql/operations.py

@@ -166,9 +166,6 @@ class DatabaseOperations(BaseDatabaseOperations):
         """
         return [(None, ("NULL", [], False))]
 
-    def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
-        return value
-
     def last_executed_query(self, cursor, sql, params):
         # With MySQLdb, cursor objects have an (undocumented) "_executed"
         # attribute where the exact query sent to the database is saved.

django/db/backends/oracle/operations.py

@@ -629,9 +629,6 @@ END;
             1900, 1, 1, value.hour, value.minute, value.second, value.microsecond
         )
 
-    def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
-        return value
-
     def combine_expression(self, connector, sub_expressions):
         lhs, rhs = sub_expressions
         if connector == "%%":

django/db/backends/postgresql/operations.py

@@ -346,9 +346,6 @@ class DatabaseOperations(BaseDatabaseOperations):
     def adapt_timefield_value(self, value):
         return value
 
-    def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
-        return value
-
     def adapt_ipaddressfield_value(self, value):
         if value:
             return Inet(value)

django/db/backends/sqlite3/features.py

@@ -50,6 +50,10 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         # The django_format_dtdelta() function doesn't properly handle mixed
         # Date/DateTime fields and timedeltas.
         "expressions.tests.FTimeDeltaTests.test_mixed_comparisons1",
+        # SQLite doesn't parse escaped double quotes in the JSON path notation,
+        # so it cannot match keys that contains double quotes (#35842).
+        "model_fields.test_jsonfield.TestQuerying."
+        "test_lookups_special_chars_double_quotes",
     }
     create_test_table_with_composite_primary_key = """
         CREATE TABLE test_table_composite_pk (

django/db/models/base.py

@@ -726,12 +726,13 @@ class Model(AltersData, metaclass=ModelBase):
         if fields is not None:
             db_instance_qs = db_instance_qs.only(*fields)
         elif deferred_fields:
-            fields = {
+            db_instance_qs = db_instance_qs.only(
+                *{
                     f.attname
                     for f in self._meta.concrete_fields
                     if f.attname not in deferred_fields
                 }
-            db_instance_qs = db_instance_qs.only(*fields)
+            )
 
         db_instance = db_instance_qs.get()
         non_loaded_fields = db_instance.get_deferred_fields()
@@ -748,9 +749,9 @@ class Model(AltersData, metaclass=ModelBase):
                     field.delete_cached_value(self)
 
         # Clear cached relations.
-        for field in self._meta.related_objects:
+        for rel in self._meta.related_objects:
-            if (fields is None or field.name in fields) and field.is_cached(self):
+            if (fields is None or rel.name in fields) and rel.is_cached(self):
-                field.delete_cached_value(self)
+                rel.delete_cached_value(self)
 
         # Clear cached private relations.
         for field in self._meta.private_fields:

django/db/models/fields/__init__.py

@@ -1828,9 +1828,8 @@ class DecimalField(Field):
             )
         return decimal_value
 
-    def get_db_prep_save(self, value, connection):
+    def get_db_prep_value(self, value, connection, prepared=False):
-        if hasattr(value, "as_sql"):
+        value = super().get_db_prep_value(value, connection, prepared)
-            return value
         return connection.ops.adapt_decimalfield_value(
             self.to_python(value), self.max_digits, self.decimal_places
         )

django/db/models/fields/json.py

@@ -193,20 +193,18 @@ class HasKeyLookup(PostgresOperatorLookup):
         # Compile the final key without interpreting ints as array elements.
         return ".%s" % json.dumps(key_transform)
 
-    def as_sql(self, compiler, connection, template=None):
+    def _as_sql_parts(self, compiler, connection):
         # Process JSON path from the left-hand side.
         if isinstance(self.lhs, KeyTransform):
-            lhs, lhs_params, lhs_key_transforms = self.lhs.preprocess_lhs(
+            lhs_sql, lhs_params, lhs_key_transforms = self.lhs.preprocess_lhs(
                 compiler, connection
             )
             lhs_json_path = compile_json_path(lhs_key_transforms)
         else:
-            lhs, lhs_params = self.process_lhs(compiler, connection)
+            lhs_sql, lhs_params = self.process_lhs(compiler, connection)
             lhs_json_path = "$"
-        sql = template % lhs
         # Process JSON path from the right-hand side.
         rhs = self.rhs
-        rhs_params = []
         if not isinstance(rhs, (list, tuple)):
             rhs = [rhs]
         for key in rhs:
@@ -217,24 +215,45 @@ class HasKeyLookup(PostgresOperatorLookup):
             *rhs_key_transforms, final_key = rhs_key_transforms
             rhs_json_path = compile_json_path(rhs_key_transforms, include_root=False)
             rhs_json_path += self.compile_json_path_final_key(final_key)
-            rhs_params.append(lhs_json_path + rhs_json_path)
+            yield lhs_sql, lhs_params, lhs_json_path + rhs_json_path
+
+    def _combine_sql_parts(self, parts):
         # Add condition for each key.
         if self.logical_operator:
-            sql = "(%s)" % self.logical_operator.join([sql] * len(rhs_params))
+            return "(%s)" % self.logical_operator.join(parts)
-        return sql, tuple(lhs_params) + tuple(rhs_params)
+        return "".join(parts)
+
+    def as_sql(self, compiler, connection, template=None):
+        sql_parts = []
+        params = []
+        for lhs_sql, lhs_params, rhs_json_path in self._as_sql_parts(
+            compiler, connection
+        ):
+            sql_parts.append(template % (lhs_sql, "%s"))
+            params.extend(lhs_params + [rhs_json_path])
+        return self._combine_sql_parts(sql_parts), tuple(params)
 
     def as_mysql(self, compiler, connection):
         return self.as_sql(
-            compiler, connection, template="JSON_CONTAINS_PATH(%s, 'one', %%s)"
+            compiler, connection, template="JSON_CONTAINS_PATH(%s, 'one', %s)"
         )
 
     def as_oracle(self, compiler, connection):
-        sql, params = self.as_sql(
+        # Use a custom delimiter to prevent the JSON path from escaping the SQL
-            compiler, connection, template="JSON_EXISTS(%s, '%%s')"
+        # literal. See comment in KeyTransform.
-        )
+        template = "JSON_EXISTS(%s, q'\uffff%s\uffff')"
-        # Add paths directly into SQL because path expressions cannot be passed
+        sql_parts = []
-        # as bind variables on Oracle.
+        params = []
-        return sql % tuple(params), []
+        for lhs_sql, lhs_params, rhs_json_path in self._as_sql_parts(
+            compiler, connection
+        ):
+            # Add right-hand-side directly into SQL because it cannot be passed
+            # as bind variables to JSON_EXISTS. It might result in invalid
+            # queries but it is assumed that it cannot be evaded because the
+            # path is JSON serialized.
+            sql_parts.append(template % (lhs_sql, rhs_json_path))
+            params.extend(lhs_params)
+        return self._combine_sql_parts(sql_parts), tuple(params)
 
     def as_postgresql(self, compiler, connection):
         if isinstance(self.rhs, KeyTransform):
@@ -246,7 +265,7 @@ class HasKeyLookup(PostgresOperatorLookup):
 
     def as_sqlite(self, compiler, connection):
         return self.as_sql(
-            compiler, connection, template="JSON_TYPE(%s, %%s) IS NOT NULL"
+            compiler, connection, template="JSON_TYPE(%s, %s) IS NOT NULL"
         )
 
 
@@ -362,10 +381,24 @@ class KeyTransform(Transform):
         json_path = compile_json_path(key_transforms)
         if connection.features.supports_primitives_in_json_field:
             sql = (
-                "COALESCE(JSON_VALUE(%s, '%s'), JSON_QUERY(%s, '%s' DISALLOW SCALARS))"
+                "COALESCE("
+                "JSON_VALUE(%s, q'\uffff%s\uffff'),"
+                "JSON_QUERY(%s, q'\uffff%s\uffff' DISALLOW SCALARS)"
+                ")"
             )
         else:
-            sql = "COALESCE(JSON_QUERY(%s, '%s'), JSON_VALUE(%s, '%s'))"
+            sql = (
+                "COALESCE("
+                "JSON_QUERY(%s, q'\uffff%s\uffff'),"
+                "JSON_VALUE(%s, q'\uffff%s\uffff')"
+                ")"
+            )
+        # Add paths directly into SQL because path expressions cannot be passed
+        # as bind variables on Oracle. Use a custom delimiter to prevent the
+        # JSON path from escaping the SQL literal. Each key in the JSON path is
+        # passed through json.dumps() with ensure_ascii=True (the default),
+        # which converts the delimiter into the escaped \uffff format. This
+        # ensures that the delimiter is not present in the JSON path.
         return sql % ((lhs, json_path) * 2), tuple(params) * 2
 
     def as_postgresql(self, compiler, connection):
@@ -455,9 +488,9 @@ class KeyTransformIsNull(lookups.IsNull):
         return "(NOT %s OR %s IS NULL)" % (sql, lhs), tuple(params) + tuple(lhs_params)
 
     def as_sqlite(self, compiler, connection):
-        template = "JSON_TYPE(%s, %%s) IS NULL"
+        template = "JSON_TYPE(%s, %s) IS NULL"
         if not self.rhs:
-            template = "JSON_TYPE(%s, %%s) IS NOT NULL"
+            template = "JSON_TYPE(%s, %s) IS NOT NULL"
         return HasKeyOrArrayIndex(self.lhs.lhs, self.lhs.key_name).as_sql(
             compiler,
             connection,

django/forms/forms.py

@@ -298,7 +298,10 @@ class BaseForm(RenderableFormMixin):
                         error_class="nonfield", renderer=self.renderer
                     )
                 else:
-                    self._errors[field] = self.error_class(renderer=self.renderer)
+                    self._errors[field] = self.error_class(
+                        renderer=self.renderer,
+                        field_id=self[field].auto_id,
+                    )
             self._errors[field].extend(error_list)
             if field in self.cleaned_data:
                 del self.cleaned_data[field]

django/forms/jinja2/django/forms/errors/list/ul.html

@@ -1 +1 @@
-{% if errors %}<ul class="{{ error_class }}">{% for error in errors %}<li>{{ error }}</li>{% endfor %}</ul>{% endif %}
+{% if errors %}<ul class="{{ error_class }}"{% if errors.field_id %} id="{{ errors.field_id }}_error"{% endif %}>{% for error in errors %}<li>{{ error }}</li>{% endfor %}</ul>{% endif %}

django/forms/templates/django/forms/errors/list/ul.html

@@ -1 +1 @@
-{% if errors %}<ul class="{{ error_class }}">{% for error in errors %}<li>{{ error }}</li>{% endfor %}</ul>{% endif %}
+{% if errors %}<ul class="{{ error_class }}"{% if errors.field_id %} id="{{ errors.field_id }}_error"{% endif %}>{% for error in errors %}<li>{{ error }}</li>{% endfor %}</ul>{% endif %}

django/forms/utils.py

@@ -147,7 +147,7 @@ class ErrorList(UserList, list, RenderableErrorMixin):
     template_name_text = "django/forms/errors/list/text.txt"
     template_name_ul = "django/forms/errors/list/ul.html"
 
-    def __init__(self, initlist=None, error_class=None, renderer=None):
+    def __init__(self, initlist=None, error_class=None, renderer=None, field_id=None):
         super().__init__(initlist)
 
         if error_class is None:
@@ -155,6 +155,7 @@ class ErrorList(UserList, list, RenderableErrorMixin):
         else:
             self.error_class = "errorlist {}".format(error_class)
         self.renderer = renderer or get_default_renderer()
+        self.field_id = field_id
 
     def as_data(self):
         return ValidationError(self.data).error_list

django/template/loader_tags.py

@@ -242,7 +242,11 @@ def do_block(parser, token):
     return BlockNode(block_name, nodelist)
 
 
-def construct_relative_path(current_template_name, relative_name):
+def construct_relative_path(
+    current_template_name,
+    relative_name,
+    allow_recursion=False,
+):
     """
     Convert a relative path (starting with './' or '../') to the full template
     name based on the current_template_name.
@@ -264,7 +268,7 @@ def construct_relative_path(current_template_name, relative_name):
             "The relative path '%s' points outside the file hierarchy that "
             "template '%s' is in." % (relative_name, current_template_name)
         )
-    if current_template_name.lstrip("/") == new_name:
+    if not allow_recursion and current_template_name.lstrip("/") == new_name:
         raise TemplateSyntaxError(
             "The relative path '%s' was translated to template name '%s', the "
             "same template in which the tag appears."
@@ -346,7 +350,11 @@ def do_include(parser, token):
         options[option] = value
     isolated_context = options.get("only", False)
     namemap = options.get("with", {})
-    bits[1] = construct_relative_path(parser.origin.template_name, bits[1])
+    bits[1] = construct_relative_path(
+        parser.origin.template_name,
+        bits[1],
+        allow_recursion=True,
+    )
     return IncludeNode(
         parser.compile_filter(bits[1]),
         extra_context=namemap,

django/utils/html.py

@@ -8,6 +8,7 @@ from collections.abc import Mapping
 from html.parser import HTMLParser
 from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit, urlunsplit
 
+from django.core.exceptions import SuspiciousOperation
 from django.utils.deprecation import RemovedInDjango60Warning
 from django.utils.encoding import punycode
 from django.utils.functional import Promise, cached_property, keep_lazy, keep_lazy_text
@@ -40,6 +41,7 @@ VOID_ELEMENTS = frozenset(
 )
 
 MAX_URL_LENGTH = 2048
+MAX_STRIP_TAGS_DEPTH = 50
 
 
 @keep_lazy(SafeString)
@@ -211,15 +213,19 @@ def _strip_once(value):
 @keep_lazy_text
 def strip_tags(value):
     """Return the given HTML with all tags stripped."""
-    # Note: in typical case this loop executes _strip_once once. Loop condition
-    # is redundant, but helps to reduce number of executions of _strip_once.
     value = str(value)
+    # Note: in typical case this loop executes _strip_once twice (the second
+    # execution does not remove any more tags).
+    strip_tags_depth = 0
     while "<" in value and ">" in value:
+        if strip_tags_depth >= MAX_STRIP_TAGS_DEPTH:
+            raise SuspiciousOperation
         new_value = _strip_once(value)
         if value.count("<") == new_value.count("<"):
             # _strip_once wasn't able to detect more tags.
             break
         value = new_value
+        strip_tags_depth += 1
     return value
 
 

docs/intro/tutorial03.txt

@@ -234,9 +234,7 @@ Now let's update our ``index`` view in ``polls/views.py`` to use the template:
     def index(request):
         latest_question_list = Question.objects.order_by("-pub_date")[:5]
         template = loader.get_template("polls/index.html")
-        context = {
+        context = {"latest_question_list": latest_question_list}
-            "latest_question_list": latest_question_list,
-        }
         return HttpResponse(template.render(context, request))
 
 That code loads the template called  ``polls/index.html`` and passes it a

docs/ref/contrib/gis/db-api.txt

@@ -407,6 +407,7 @@ Function                              PostGIS  Oracle         MariaDB      MySQL
 :class:`FromWKB`                      X        X              X            X           X
 :class:`FromWKT`                      X        X              X            X           X
 :class:`GeoHash`                      X                       X (≥ 11.7)   X           X (LWGEOM/RTTOPO)
+:class:`GeometryDistance`             X
 :class:`Intersection`                 X        X              X            X           X
 :class:`IsEmpty`                      X
 :class:`IsValid`                      X        X              X (≥ 11.7)   X           X

docs/ref/forms/api.txt

@@ -1025,13 +1025,17 @@ method you're using:
 Customizing the error list format
 ---------------------------------
 
-.. class:: ErrorList(initlist=None, error_class=None, renderer=None)
+.. class:: ErrorList(initlist=None, error_class=None, renderer=None, field_id=None)
 
     By default, forms use ``django.forms.utils.ErrorList`` to format validation
     errors. ``ErrorList`` is a list like object where ``initlist`` is the
     list of errors. In addition this class has the following attributes and
     methods.
 
+    .. versionchanged:: 5.2
+
+        The ``field_id`` argument was added.
+
     .. attribute:: error_class
 
         The CSS classes to be used when rendering the error list. Any provided
@@ -1043,6 +1047,16 @@ Customizing the error list format
         Defaults to ``None`` which means to use the default renderer
         specified by the :setting:`FORM_RENDERER` setting.
 
+    .. attribute:: field_id
+
+        .. versionadded:: 5.2
+
+        An ``id`` for the field for which the errors relate. This allows an
+        HTML ``id`` attribute to be added in the error template and is useful
+        to associate the errors with the field. The default template uses the
+        format ``id="{{ field_id }}_error"`` and a value is provided by
+        :meth:`.Form.add_error` using the field's :attr:`~.BoundField.auto_id`.
+
     .. attribute:: template_name
 
         The name of the template used when calling ``__str__`` or

docs/ref/request-response.txt

@@ -554,12 +554,21 @@ a subclass of dictionary. Exceptions are outlined here:
 
 .. method:: QueryDict.__getitem__(key)
 
-    Returns the value for the given key. If the key has more than one value,
+    Returns the last value for the given key; or an empty list (``[]``) if the
-    it returns the last value. Raises
+    key exists but has no values. Raises
     ``django.utils.datastructures.MultiValueDictKeyError`` if the key does not
     exist. (This is a subclass of Python's standard :exc:`KeyError`, so you can
     stick to catching ``KeyError``.)
 
+    .. code-block:: pycon
+
+        >>> q = QueryDict("a=1&a=2&a=3", mutable=True)
+        >>> q.__getitem__("a")
+        '3'
+        >>> q.__setitem__("b", [])
+        >>> q.__getitem__("b")
+        []
+
 .. method:: QueryDict.__setitem__(key, value)
 
     Sets the given key to ``[value]`` (a list whose single element is

docs/ref/utils.txt

@@ -521,7 +521,7 @@ https://web.archive.org/web/20110718035220/http://diveintomark.org/archives/2004
     The cached value can be treated like an ordinary attribute of the instance::
 
         # clear it, requiring re-computation next time it's called
-        del person.friends  # or delattr(person, "friends")
+        person.__dict__.pop("friends", None)
 
         # set a value manually, that will persist on the instance until cleared
         person.friends = ["Huckleberry Finn", "Tom Sawyer"]

docs/releases/4.2.17.txt

@@ -6,3 +6,28 @@ Django 4.2.17 release notes
 
 Django 4.2.17 fixes one security issue with severity "high" and one security
 issue with severity "moderate" in 4.2.16.
+
+CVE-2024-53907: Denial-of-service possibility in ``strip_tags()``
+=================================================================
+
+:func:`~django.utils.html.strip_tags` would be extremely slow to evaluate
+certain inputs containing large sequences of nested incomplete HTML entities.
+The ``strip_tags()`` method is used to implement the corresponding
+:tfilter:`striptags` template filter, which was thus also vulnerable.
+
+``strip_tags()`` now has an upper limit of recursive calls to ``HTMLParser``
+before raising a :exc:`.SuspiciousOperation` exception.
+
+Remember that absolutely NO guarantee is provided about the results of
+``strip_tags()`` being HTML safe. So NEVER mark safe the result of a
+``strip_tags()`` call without escaping it first, for example with
+:func:`django.utils.html.escape`.
+
+CVE-2024-53908: Potential SQL injection via ``HasKey(lhs, rhs)`` on Oracle
+==========================================================================
+
+Direct usage of the ``django.db.models.fields.json.HasKey`` lookup on Oracle
+was subject to SQL injection if untrusted data was used as a ``lhs`` value.
+
+Applications that use the :lookup:`has_key <jsonfield.has_key>` lookup through
+the ``__`` syntax are unaffected.

docs/releases/5.0.10.txt

@@ -6,3 +6,28 @@ Django 5.0.10 release notes
 
 Django 5.0.10 fixes one security issue with severity "high" and one security
 issue with severity "moderate" in 5.0.9.
+
+CVE-2024-53907: Denial-of-service possibility in ``strip_tags()``
+=================================================================
+
+:func:`~django.utils.html.strip_tags` would be extremely slow to evaluate
+certain inputs containing large sequences of nested incomplete HTML entities.
+The ``strip_tags()`` method is used to implement the corresponding
+:tfilter:`striptags` template filter, which was thus also vulnerable.
+
+``strip_tags()`` now has an upper limit of recursive calls to ``HTMLParser``
+before raising a :exc:`.SuspiciousOperation` exception.
+
+Remember that absolutely NO guarantee is provided about the results of
+``strip_tags()`` being HTML safe. So NEVER mark safe the result of a
+``strip_tags()`` call without escaping it first, for example with
+:func:`django.utils.html.escape`.
+
+CVE-2024-53908: Potential SQL injection via ``HasKey(lhs, rhs)`` on Oracle
+==========================================================================
+
+Direct usage of the ``django.db.models.fields.json.HasKey`` lookup on Oracle
+was subject to SQL injection if untrusted data was used as a ``lhs`` value.
+
+Applications that use the :lookup:`has_key <jsonfield.has_key>` lookup through
+the ``__`` syntax are unaffected.

docs/releases/5.1.4.txt

@@ -7,8 +7,37 @@ Django 5.1.4 release notes
 Django 5.1.4 fixes one security issue with severity "high", one security issue
 with severity "moderate", and several bugs in 5.1.3.
 
+CVE-2024-53907: Denial-of-service possibility in ``strip_tags()``
+=================================================================
+
+:func:`~django.utils.html.strip_tags` would be extremely slow to evaluate
+certain inputs containing large sequences of nested incomplete HTML entities.
+The ``strip_tags()`` method is used to implement the corresponding
+:tfilter:`striptags` template filter, which was thus also vulnerable.
+
+``strip_tags()`` now has an upper limit of recursive calls to ``HTMLParser``
+before raising a :exc:`.SuspiciousOperation` exception.
+
+Remember that absolutely NO guarantee is provided about the results of
+``strip_tags()`` being HTML safe. So NEVER mark safe the result of a
+``strip_tags()`` call without escaping it first, for example with
+:func:`django.utils.html.escape`.
+
+CVE-2024-53908: Potential SQL injection via ``HasKey(lhs, rhs)`` on Oracle
+==========================================================================
+
+Direct usage of the ``django.db.models.fields.json.HasKey`` lookup on Oracle
+was subject to SQL injection if untrusted data was used as a ``lhs`` value.
+
+Applications that use the :lookup:`has_key <jsonfield.has_key>` lookup through
+the ``__`` syntax are unaffected.
+
 Bugfixes
 ========
 
 * Fixed a crash in ``createsuperuser`` on Python 3.13+ caused by an unhandled
   ``OSError`` when the username could not be determined (:ticket:`35942`).
+
+* Fixed a regression in Django 5.1 where relational fields were not updated
+  when calling ``Model.refresh_from_db()`` on instances with deferred fields
+  (:ticket:`35950`).

docs/releases/5.1.5.txt

@@ -0,0 +1,12 @@
+==========================
+Django 5.1.5 release notes
+==========================
+
+*Expected January 7, 2025*
+
+Django 5.1.5 fixes several bugs in 5.1.4.
+
+Bugfixes
+========
+
+* ...

docs/releases/5.2.txt

@@ -249,6 +249,10 @@ Forms
 * The new :class:`~django.forms.TelInput` form widget is for entering telephone
   numbers and renders as ``<input type="tel" ...>``.
 
+* The new ``field_id`` argument for :class:`~django.forms.ErrorList` allows an
+  HTML ``id`` attribute to be added in the error template. See
+  :attr:`.ErrorList.field_id` for details.
+
 Generic Views
 ~~~~~~~~~~~~~
 
@@ -395,6 +399,8 @@ backends.
 * The new :meth:`Model._is_pk_set() <django.db.models.Model._is_pk_set>` method
   allows checking if a Model instance's primary key is defined.
 
+* ``BaseDatabaseOperations.adapt_decimalfield_value()`` is now a no-op, simply
+  returning the given value.
 
 :mod:`django.contrib.gis`
 -------------------------

docs/releases/index.txt

@@ -32,6 +32,7 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   5.1.5
    5.1.4
    5.1.3
    5.1.2

docs/releases/security.txt

@@ -36,6 +36,28 @@ Issues under Django's security process
 All security issues have been handled under versions of Django's security
 process. These are listed below.
 
+December 4, 2024 - :cve:`2024-53907`
+------------------------------------
+
+Potential denial-of-service in ``django.utils.html.strip_tags()``.
+`Full description
+<https://www.djangoproject.com/weblog/2024/dec/04/security-releases/>`__
+
+* Django 5.1 :commit:`(patch) <bbc74a7f7eb7335e913bdb4787f22e83a9be947e>`
+* Django 5.0 :commit:`(patch) <a5a89ea28cc550c1b29b03f9e14ef3c128ec1e84>`
+* Django 4.2 :commit:`(patch) <790eb058b0716c536a2f2e8d1c6d5079d776c22b>`
+
+December 4, 2024 - :cve:`2024-53908`
+------------------------------------
+
+Potential SQL injection in ``HasKey(lhs, rhs)`` on Oracle.
+`Full description
+<https://www.djangoproject.com/weblog/2024/dec/04/security-releases/>`__
+
+* Django 5.1 :commit:`(patch) <6943d61818e63e77b65d8b1ae65941e8f04bd87b>`
+* Django 5.0 :commit:`(patch) <ff08bb6c70aa45f83a5ef3bd0b601c7c9d1a7642>`
+* Django 4.2 :commit:`(patch) <7376bcbf508883282ffcc0f0fac5cf0ed2d6cbc5>`
+
 September 3, 2024 - :cve:`2024-45231`
 -------------------------------------
 

docs/spelling_wordlist

@@ -3,14 +3,12 @@ accessor
 accessors
 Aceh
 admindocs
-affine
 affordances
 Ai
 Alchin
 allowlist
 alphanumerics
 amet
-analytics
 arccosine
 architected
 arcsine
@@ -60,7 +58,6 @@ Bokmål
 Bonham
 bookmarklet
 bookmarklets
-boolean
 booleans
 bpython
 Bronn
@@ -114,23 +111,18 @@ Danga
 Darussalam
 databrowse
 datafile
-dataset
-datasets
 datetimes
 declaratively
-decrementing
 deduplicates
 deduplication
 deepcopy
 deferrable
-DEP
 deprecations
 deserialization
 deserialize
 deserialized
 deserializer
 deserializing
-deterministically
 Deutsch
 dev
 dictConfig
@@ -142,7 +134,6 @@ Disqus
 distro
 django
 djangoproject
-djangotutorial
 dm
 docstring
 docstrings
@@ -165,9 +156,7 @@ esque
 Ess
 ETag
 ETags
-exe
 exfiltration
-extensibility
 fallbacks
 favicon
 fieldset
@@ -177,7 +166,6 @@ filesystems
 flatpage
 flatpages
 focusable
-fooapp
 formatter
 formatters
 formfield
@@ -210,7 +198,6 @@ hashable
 hasher
 hashers
 headerlist
-hoc
 Hoerner
 Holovaty
 Homebrew
@@ -223,7 +210,6 @@ Hypercorn
 ies
 iframe
 Igbo
-incrementing
 indexable
 ing
 ini
@@ -240,7 +226,6 @@ iterable
 iterables
 iteratively
 ize
-Jazzband
 Jinja
 jQuery
 Jupyter
@@ -283,7 +268,6 @@ manouche
 Marino
 memcache
 memcached
-mentorship
 metaclass
 metaclasses
 metre
@@ -338,8 +322,6 @@ orm
 Outdim
 outfile
 paginator
-parallelization
-parallelized
 parameterization
 params
 parens
@@ -361,9 +343,7 @@ pluggable
 pluralizations
 pooler
 postfix
-postgis
 postgres
-postgresql
 pragma
 pre
 precisions
@@ -376,7 +356,6 @@ prefetches
 prefetching
 preload
 preloaded
-prepend
 prepended
 prepending
 prepends
@@ -429,7 +408,6 @@ reflow
 registrable
 reimplement
 reindent
-reindex
 releaser
 releasers
 reloader
@@ -439,7 +417,6 @@ repo
 reportable
 reprojection
 reraising
-resampling
 reST
 reStructuredText
 reusability
@@ -447,7 +424,6 @@ reverter
 roadmap
 Roald
 rss
-runtime
 Sandvik
 savepoint
 savepoints
@@ -458,7 +434,6 @@ screencasts
 semimajor
 semiminor
 serializability
-serializable
 serializer
 serializers
 shapefile
@@ -467,7 +442,6 @@ sharding
 sitewide
 sliceable
 SMTP
-solaris
 Sorani
 sortable
 Spectre
@@ -535,7 +509,6 @@ toolkits
 toolset
 trac
 tracebacks
-transactional
 Transifex
 Tredinnick
 triager

docs/topics/db/fixtures.txt

@@ -4,28 +4,25 @@
 Fixtures
 ========
 
-.. seealso::
-
-    * :doc:`/howto/initial-data`
-
-What is a fixture?
-==================
-
 A *fixture* is a collection of files that contain the serialized contents of
 the database. Each fixture has a unique name, and the files that comprise the
 fixture can be distributed over multiple directories, in multiple applications.
 
-How to produce a fixture?
+.. seealso::
-=========================
+
+    * :doc:`/howto/initial-data`
+
+How to produce a fixture
+========================
 
 Fixtures can be generated by :djadmin:`manage.py dumpdata <dumpdata>`. It's
 also possible to generate custom fixtures by directly using :doc:`serialization
 tools </topics/serialization>` or even by handwriting them.
 
-How to use a fixture?
+How to use a fixture
-=====================
+====================
 
-Fixtures can be used to pre-populate database with data for
+Fixtures can be used to pre-populate the database with data for
 :ref:`tests <topics-testing-fixtures>`:
 
 .. code-block:: python
@@ -40,8 +37,8 @@ or to provide some :ref:`initial data <initial-data-via-fixtures>` using the
 
     django-admin loaddata <fixture label>
 
-Where Django looks for fixtures?
+How fixtures are discovered
-================================
+===========================
 
 Django will search in these locations for fixtures:
 
@@ -116,8 +113,8 @@ example).
 
 .. _MySQL: https://dev.mysql.com/doc/refman/en/constraint-foreign-key.html
 
-How fixtures are saved to the database?
+How fixtures are saved to the database
-=======================================
+======================================
 
 When fixture files are processed, the data is saved to the database as is.
 Model defined :meth:`~django.db.models.Model.save` methods are not called, and

docs/topics/forms/formsets.txt

@@ -571,14 +571,12 @@ happen when the user changes these values:
     ...         {"title": "Article #2", "pub_date": datetime.date(2008, 5, 11)},
     ...     ],
     ... )
-    >>> formset.is_valid()
-    True
     >>> for form in formset.ordered_forms:
     ...     print(form.cleaned_data)
     ...
-    {'pub_date': datetime.date(2008, 5, 1), 'ORDER': 0, 'title': 'Article #3'}
+    {'title': 'Article #3', 'pub_date': datetime.date(2008, 5, 1), 'ORDER': 0}
-    {'pub_date': datetime.date(2008, 5, 11), 'ORDER': 1, 'title': 'Article #2'}
+    {'title': 'Article #2', 'pub_date': datetime.date(2008, 5, 11), 'ORDER': 1}
-    {'pub_date': datetime.date(2008, 5, 10), 'ORDER': 2, 'title': 'Article #1'}
+    {'title': 'Article #1', 'pub_date': datetime.date(2008, 5, 10), 'ORDER': 2}
 
 :class:`~django.forms.formsets.BaseFormSet` also provides an
 :attr:`~django.forms.formsets.BaseFormSet.ordering_widget` attribute and
@@ -690,7 +688,7 @@ delete fields you can access them with ``deleted_forms``:
     ...     ],
     ... )
     >>> [form.cleaned_data for form in formset.deleted_forms]
-    [{'DELETE': True, 'pub_date': datetime.date(2008, 5, 10), 'title': 'Article #1'}]
+    [{'title': 'Article #1', 'pub_date': datetime.date(2008, 5, 10), 'DELETE': True}]
 
 If you are using a :class:`ModelFormSet<django.forms.models.BaseModelFormSet>`,
 model instances for deleted forms will be deleted when you call

tests/composite_pk/test_checks.py

@@ -1,7 +1,7 @@
 from django.core import checks
 from django.db import connection, models
 from django.db.models import F
-from django.test import TestCase
+from django.test import TestCase, skipUnlessAnyDBFeature
 from django.test.utils import isolate_apps
 
 
@@ -217,16 +217,18 @@ class CompositePKChecksTests(TestCase):
             ],
         )
 
+    @skipUnlessAnyDBFeature(
+        "supports_virtual_generated_columns",
+        "supports_stored_generated_columns",
+    )
     def test_composite_pk_cannot_include_generated_field(self):
-        is_oracle = connection.vendor == "oracle"
-
         class Foo(models.Model):
             pk = models.CompositePrimaryKey("id", "foo")
             id = models.IntegerField()
             foo = models.GeneratedField(
                 expression=F("id"),
                 output_field=models.IntegerField(),
-                db_persist=not is_oracle,
+                db_persist=connection.features.supports_stored_generated_columns,
             )
 
         self.assertEqual(

tests/composite_pk/tests.py

@@ -2,7 +2,12 @@ import json
 import unittest
 from uuid import UUID
 
-import yaml
+try:
+    import yaml  # NOQA
+
+    HAS_YAML = True
+except ImportError:
+    HAS_YAML = False
 
 from django import forms
 from django.core import serializers
@@ -35,9 +40,9 @@ class CompositePKTests(TestCase):
         cls.comment = Comment.objects.create(tenant=cls.tenant, id=1, user=cls.user)
 
     @staticmethod
-    def get_constraints(table):
+    def get_primary_key_columns(table):
         with connection.cursor() as cursor:
-            return connection.introspection.get_constraints(cursor, table)
+            return connection.introspection.get_primary_key_columns(cursor, table)
 
     def test_pk_updated_if_field_updated(self):
         user = User.objects.get(pk=self.user.pk)
@@ -125,53 +130,15 @@ class CompositePKTests(TestCase):
         with self.assertRaises(IntegrityError):
             Comment.objects.create(tenant=self.tenant, id=self.comment.id)
 
-    @unittest.skipUnless(connection.vendor == "postgresql", "PostgreSQL specific test")
+    def test_get_primary_key_columns(self):
-    def test_get_constraints_postgresql(self):
+        self.assertEqual(
-        user_constraints = self.get_constraints(User._meta.db_table)
+            self.get_primary_key_columns(User._meta.db_table),
-        user_pk = user_constraints["composite_pk_user_pkey"]
+            ["tenant_id", "id"],
-        self.assertEqual(user_pk["columns"], ["tenant_id", "id"])
+        )
-        self.assertIs(user_pk["primary_key"], True)
+        self.assertEqual(
-
+            self.get_primary_key_columns(Comment._meta.db_table),
-        comment_constraints = self.get_constraints(Comment._meta.db_table)
+            ["tenant_id", "comment_id"],
-        comment_pk = comment_constraints["composite_pk_comment_pkey"]
+        )
-        self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"])
-        self.assertIs(comment_pk["primary_key"], True)
-
-    @unittest.skipUnless(connection.vendor == "sqlite", "SQLite specific test")
-    def test_get_constraints_sqlite(self):
-        user_constraints = self.get_constraints(User._meta.db_table)
-        user_pk = user_constraints["__primary__"]
-        self.assertEqual(user_pk["columns"], ["tenant_id", "id"])
-        self.assertIs(user_pk["primary_key"], True)
-
-        comment_constraints = self.get_constraints(Comment._meta.db_table)
-        comment_pk = comment_constraints["__primary__"]
-        self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"])
-        self.assertIs(comment_pk["primary_key"], True)
-
-    @unittest.skipUnless(connection.vendor == "mysql", "MySQL specific test")
-    def test_get_constraints_mysql(self):
-        user_constraints = self.get_constraints(User._meta.db_table)
-        user_pk = user_constraints["PRIMARY"]
-        self.assertEqual(user_pk["columns"], ["tenant_id", "id"])
-        self.assertIs(user_pk["primary_key"], True)
-
-        comment_constraints = self.get_constraints(Comment._meta.db_table)
-        comment_pk = comment_constraints["PRIMARY"]
-        self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"])
-        self.assertIs(comment_pk["primary_key"], True)
-
-    @unittest.skipUnless(connection.vendor == "oracle", "Oracle specific test")
-    def test_get_constraints_oracle(self):
-        user_constraints = self.get_constraints(User._meta.db_table)
-        user_pk = next(c for c in user_constraints.values() if c["primary_key"])
-        self.assertEqual(user_pk["columns"], ["tenant_id", "id"])
-        self.assertEqual(user_pk["primary_key"], 1)
-
-        comment_constraints = self.get_constraints(Comment._meta.db_table)
-        comment_pk = next(c for c in comment_constraints.values() if c["primary_key"])
-        self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"])
-        self.assertEqual(comment_pk["primary_key"], 1)
 
     def test_in_bulk(self):
         """
@@ -291,6 +258,7 @@ class CompositePKFixturesTests(TestCase):
             },
         )
 
+    @unittest.skipUnless(HAS_YAML, "No yaml library detected")
     def test_serialize_user_yaml(self):
         users = User.objects.filter(pk=(2, 3))
         result = serializers.serialize("yaml", users)

tests/contenttypes_tests/test_fields.py

@@ -57,6 +57,15 @@ class GenericForeignKeyTests(TestCase):
         self.assertIsNot(answer.question, old_question_obj)
         self.assertEqual(answer.question, old_question_obj)
 
+    def test_clear_cached_generic_relation_when_deferred(self):
+        question = Question.objects.create(text="question")
+        Answer.objects.create(text="answer", question=question)
+        answer = Answer.objects.defer("text").get()
+        old_question_obj = answer.question
+        # The reverse relation is refreshed even when the text field is deferred.
+        answer.refresh_from_db()
+        self.assertIsNot(answer.question, old_question_obj)
+
 
 class GenericRelationTests(TestCase):
     def test_value_to_string(self):

tests/defer/tests.py

@@ -290,6 +290,14 @@ class TestDefer2(AssertionMixin, TestCase):
             self.assertEqual(rf2.name, "new foo")
             self.assertEqual(rf2.value, "new bar")
 
+    def test_refresh_when_one_field_deferred(self):
+        s = Secondary.objects.create()
+        PrimaryOneToOne.objects.create(name="foo", value="bar", related=s)
+        s = Secondary.objects.defer("first").get()
+        p_before = s.primary_o2o
+        s.refresh_from_db()
+        self.assertIsNot(s.primary_o2o, p_before)
+
 
 class InvalidDeferTests(SimpleTestCase):
     def test_invalid_defer(self):

tests/forms_tests/tests/test_error_messages.py

@@ -249,7 +249,8 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin):
         form1 = TestForm({"first_name": "John"})
         self.assertHTMLEqual(
             str(form1["last_name"].errors),
-            '<ul class="errorlist"><li>This field is required.</li></ul>',
+            '<ul class="errorlist" id="id_last_name_error"><li>'
+            "This field is required.</li></ul>",
         )
         self.assertHTMLEqual(
             str(form1.errors["__all__"]),
@@ -280,7 +281,7 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin):
         f = SomeForm({"field": "<script>"})
         self.assertHTMLEqual(
             t.render(Context({"form": f})),
-            '<ul class="errorlist"><li>field<ul class="errorlist">'
+            '<ul class="errorlist"><li>field<ul class="errorlist" id="id_field_error">'
             "<li>Select a valid choice. &lt;script&gt; is not one of the "
             "available choices.</li></ul></li></ul>",
         )
@@ -291,7 +292,7 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin):
         f = SomeForm({"field": ["<script>"]})
         self.assertHTMLEqual(
             t.render(Context({"form": f})),
-            '<ul class="errorlist"><li>field<ul class="errorlist">'
+            '<ul class="errorlist"><li>field<ul class="errorlist" id="id_field_error">'
             "<li>Select a valid choice. &lt;script&gt; is not one of the "
             "available choices.</li></ul></li></ul>",
         )
@@ -302,7 +303,7 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin):
         f = SomeForm({"field": ["<script>"]})
         self.assertHTMLEqual(
             t.render(Context({"form": f})),
-            '<ul class="errorlist"><li>field<ul class="errorlist">'
+            '<ul class="errorlist"><li>field<ul class="errorlist" id="id_field_error">'
             "<li>“&lt;script&gt;” is not a valid value.</li>"
             "</ul></li></ul>",
         )

tests/forms_tests/tests/test_forms.py

@@ -181,53 +181,55 @@ class FormsTestCase(SimpleTestCase):
         self.assertHTMLEqual(
             str(p),
             '<div><label for="id_first_name">First name:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_first_name_error"><li>This field is required.'
-            '<input type="text" name="first_name" aria-invalid="true" required '
+            '</li></ul><input type="text" name="first_name" aria-invalid="true" '
-            'id="id_first_name"></div>'
+            'required id="id_first_name"></div>'
             '<div><label for="id_last_name">Last name:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_last_name_error"><li>This field is required.'
-            '<input type="text" name="last_name" aria-invalid="true" required '
+            '</li></ul><input type="text" name="last_name" aria-invalid="true" '
-            'id="id_last_name"></div><div>'
+            'required id="id_last_name"></div><div>'
             '<label for="id_birthday">Birthday:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_birthday_error"><li>This field is required.'
-            '<input type="text" name="birthday" aria-invalid="true" required '
+            '</li></ul><input type="text" name="birthday" aria-invalid="true" required '
             'id="id_birthday"></div>',
         )
         self.assertHTMLEqual(
             p.as_table(),
             """<tr><th><label for="id_first_name">First name:</label></th><td>
-<ul class="errorlist"><li>This field is required.</li></ul>
+<ul class="errorlist" id="id_first_name_error"><li>This field is required.</li></ul>
 <input type="text" name="first_name" id="id_first_name" aria-invalid="true" required>
 </td></tr><tr><th><label for="id_last_name">Last name:</label></th>
-<td><ul class="errorlist"><li>This field is required.</li></ul>
+<td><ul class="errorlist" id="id_last_name_error"><li>This field is required.</li></ul>
 <input type="text" name="last_name" id="id_last_name" aria-invalid="true" required>
 </td></tr><tr><th><label for="id_birthday">Birthday:</label></th>
-<td><ul class="errorlist"><li>This field is required.</li></ul>
+<td><ul class="errorlist" id="id_birthday_error"><li>This field is required.</li></ul>
 <input type="text" name="birthday" id="id_birthday" aria-invalid="true" required>
 </td></tr>""",
         )
         self.assertHTMLEqual(
             p.as_ul(),
-            """<li><ul class="errorlist"><li>This field is required.</li></ul>
+            """<li><ul class="errorlist" id="id_first_name_error">
+<li>This field is required.</li></ul>
 <label for="id_first_name">First name:</label>
 <input type="text" name="first_name" id="id_first_name" aria-invalid="true" required>
-</li><li><ul class="errorlist"><li>This field is required.</li></ul>
+</li><li><ul class="errorlist" id="id_last_name_error"><li>This field is required.</li>
-<label for="id_last_name">Last name:</label>
+</ul><label for="id_last_name">Last name:</label>
 <input type="text" name="last_name" id="id_last_name" aria-invalid="true" required>
-</li><li><ul class="errorlist"><li>This field is required.</li></ul>
+</li><li><ul class="errorlist" id="id_birthday_error"><li>This field is required.</li>
-<label for="id_birthday">Birthday:</label>
+</ul><label for="id_birthday">Birthday:</label>
 <input type="text" name="birthday" id="id_birthday" aria-invalid="true" required>
 </li>""",
         )
         self.assertHTMLEqual(
             p.as_p(),
-            """<ul class="errorlist"><li>This field is required.</li></ul>
+            """<ul class="errorlist" id="id_first_name_error"><li>
+This field is required.</li></ul>
 <p><label for="id_first_name">First name:</label>
 <input type="text" name="first_name" id="id_first_name" aria-invalid="true" required>
-</p><ul class="errorlist"><li>This field is required.</li></ul>
+</p><ul class="errorlist" id="id_last_name_error"><li>This field is required.</li></ul>
 <p><label for="id_last_name">Last name:</label>
 <input type="text" name="last_name" id="id_last_name" aria-invalid="true" required>
-</p><ul class="errorlist"><li>This field is required.</li></ul>
+</p><ul class="errorlist" id="id_birthday_error"><li>This field is required.</li></ul>
 <p><label for="id_birthday">Birthday:</label>
 <input type="text" name="birthday" id="id_birthday" aria-invalid="true" required>
 </p>""",
@@ -235,16 +237,16 @@ class FormsTestCase(SimpleTestCase):
         self.assertHTMLEqual(
             p.as_div(),
             '<div><label for="id_first_name">First name:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_first_name_error"><li>This field is required.'
-            '<input type="text" name="first_name" aria-invalid="true" required '
+            '</li></ul><input type="text" name="first_name" aria-invalid="true" '
-            'id="id_first_name"></div>'
+            'required id="id_first_name"></div>'
             '<div><label for="id_last_name">Last name:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_last_name_error"><li>This field is required.'
-            '<input type="text" name="last_name" aria-invalid="true" required '
+            '</li></ul><input type="text" name="last_name" aria-invalid="true" '
-            'id="id_last_name"></div><div>'
+            'required id="id_last_name"></div><div>'
             '<label for="id_birthday">Birthday:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_birthday_error"><li>This field is required.'
-            '<input type="text" name="birthday" aria-invalid="true" required '
+            '</li></ul><input type="text" name="birthday" aria-invalid="true" required '
             'id="id_birthday"></div>',
         )
 
@@ -387,7 +389,8 @@ class FormsTestCase(SimpleTestCase):
         self.assertEqual(p["first_name"].errors, ["This field is required."])
         self.assertHTMLEqual(
             p["first_name"].errors.as_ul(),
-            '<ul class="errorlist"><li>This field is required.</li></ul>',
+            '<ul class="errorlist" id="id_first_name_error">'
+            "<li>This field is required.</li></ul>",
         )
         self.assertEqual(p["first_name"].errors.as_text(), "* This field is required.")
 
@@ -3706,7 +3709,7 @@ Options: <select multiple name="options" aria-invalid="true" required>
         self.assertHTMLEqual(
             p.as_ul(),
             """
-            <li class="required error"><ul class="errorlist">
+            <li class="required error"><ul class="errorlist" id="id_name_error">
             <li>This field is required.</li></ul>
             <label class="required" for="id_name">Name:</label>
             <input type="text" name="name" id="id_name" aria-invalid="true" required>
@@ -3719,7 +3722,7 @@ Options: <select multiple name="options" aria-invalid="true" required>
             </select></li>
             <li><label for="id_email">Email:</label>
             <input type="email" name="email" id="id_email" maxlength="320"></li>
-            <li class="required error"><ul class="errorlist">
+            <li class="required error"><ul class="errorlist" id="id_age_error">
             <li>This field is required.</li></ul>
             <label class="required" for="id_age">Age:</label>
             <input type="number" name="age" id="id_age" aria-invalid="true" required>
@@ -3729,8 +3732,8 @@ Options: <select multiple name="options" aria-invalid="true" required>
         self.assertHTMLEqual(
             p.as_p(),
             """
-            <ul class="errorlist"><li>This field is required.</li></ul>
+            <ul class="errorlist" id="id_name_error"><li>This field is required.</li>
-            <p class="required error">
+            </ul><p class="required error">
             <label class="required" for="id_name">Name:</label>
             <input type="text" name="name" id="id_name" aria-invalid="true" required>
             </p><p class="required">
@@ -3742,17 +3745,17 @@ Options: <select multiple name="options" aria-invalid="true" required>
             </select></p>
             <p><label for="id_email">Email:</label>
             <input type="email" name="email" id="id_email" maxlength="320"></p>
-            <ul class="errorlist"><li>This field is required.</li></ul>
+            <ul class="errorlist" id="id_age_error"><li>This field is required.</li>
-            <p class="required error"><label class="required" for="id_age">Age:</label>
+            </ul><p class="required error"><label class="required" for="id_age">
-            <input type="number" name="age" id="id_age" aria-invalid="true" required>
+            Age:</label><input type="number" name="age" id="id_age" aria-invalid="true"
-            </p>""",
+            required></p>""",
         )
 
         self.assertHTMLEqual(
             p.as_table(),
             """<tr class="required error">
 <th><label class="required" for="id_name">Name:</label></th>
-<td><ul class="errorlist"><li>This field is required.</li></ul>
+<td><ul class="errorlist" id="id_name_error"><li>This field is required.</li></ul>
 <input type="text" name="name" id="id_name" aria-invalid="true" required></td></tr>
 <tr class="required"><th><label class="required" for="id_is_cool">Is cool:</label></th>
 <td><select name="is_cool" id="id_is_cool">
@@ -3763,14 +3766,14 @@ Options: <select multiple name="options" aria-invalid="true" required>
 <tr><th><label for="id_email">Email:</label></th><td>
 <input type="email" name="email" id="id_email" maxlength="320"></td></tr>
 <tr class="required error"><th><label class="required" for="id_age">Age:</label></th>
-<td><ul class="errorlist"><li>This field is required.</li></ul>
+<td><ul class="errorlist" id="id_age_error"><li>This field is required.</li></ul>
 <input type="number" name="age" id="id_age" aria-invalid="true" required></td></tr>""",
         )
         self.assertHTMLEqual(
             p.as_div(),
             '<div class="required error"><label for="id_name" class="required">Name:'
-            '</label><ul class="errorlist"><li>This field is required.</li></ul>'
+            '</label><ul class="errorlist" id="id_name_error"><li>This field is '
-            '<input type="text" name="name" required id="id_name" '
+            'required.</li></ul><input type="text" name="name" required id="id_name" '
             'aria-invalid="true" /></div>'
             '<div class="required"><label for="id_is_cool" class="required">Is cool:'
             '</label><select name="is_cool" id="id_is_cool">'
@@ -3779,8 +3782,8 @@ Options: <select multiple name="options" aria-invalid="true" required>
             '</select></div><div><label for="id_email">Email:</label>'
             '<input type="email" name="email" id="id_email" maxlength="320"/></div>'
             '<div class="required error"><label for="id_age" class="required">Age:'
-            '</label><ul class="errorlist"><li>This field is required.</li></ul>'
+            '</label><ul class="errorlist" id="id_age_error"><li>This field is '
-            '<input type="number" name="age" required id="id_age" '
+            'required.</li></ul><input type="number" name="age" required id="id_age" '
             'aria-invalid="true" /></div>',
         )
 
@@ -4255,8 +4258,10 @@ Options: <select multiple name="options" aria-invalid="true" required>
 
         errors = form.errors.as_ul()
         control = [
-            '<li>foo<ul class="errorlist"><li>This field is required.</li></ul></li>',
+            '<li>foo<ul class="errorlist" id="id_foo_error"><li>This field is required.'
-            '<li>bar<ul class="errorlist"><li>This field is required.</li></ul></li>',
+            "</li></ul></li>",
+            '<li>bar<ul class="errorlist" id="id_bar_error"><li>This field is required.'
+            "</li></ul></li>",
             '<li>__all__<ul class="errorlist nonfield"><li>Non-field error.</li></ul>'
             "</li>",
         ]
@@ -4461,7 +4466,8 @@ Options: <select multiple name="options" aria-invalid="true" required>
             form.as_ul(),
             '<li><ul class="errorlist nonfield">'
             "<li>(Hidden field hidden) Foo &amp; &quot;bar&quot;!</li></ul></li>"
-            '<li><ul class="errorlist"><li>Foo &amp; &quot;bar&quot;!</li></ul>'
+            '<li><ul class="errorlist" id="id_visible_error"><li>Foo &amp; '
+            "&quot;bar&quot;!</li></ul>"
             '<label for="id_visible">Visible:</label> '
             '<input type="text" name="visible" aria-invalid="true" value="b" '
             'id="id_visible" required>'

tests/forms_tests/tests/test_i18n.py

@@ -97,7 +97,7 @@ class FormsI18nTests(SimpleTestCase):
             f = SomeForm({})
             self.assertHTMLEqual(
                 f.as_p(),
-                '<ul class="errorlist"><li>'
+                '<ul class="errorlist" id="id_somechoice_error"><li>'
                 "\u041e\u0431\u044f\u0437\u0430\u0442\u0435\u043b\u044c"
                 "\u043d\u043e\u0435 \u043f\u043e\u043b\u0435.</li></ul>\n"
                 "<p><label>\xc5\xf8\xdf:</label>"

tests/i18n/test_extraction.py

@@ -179,6 +179,15 @@ class BasicExtractorTests(ExtractorTests):
         self.assertIn("processing locale en_GB", out.getvalue())
         self.assertIs(Path("locale/en_GB/LC_MESSAGES/django.po").exists(), True)
 
+    def test_valid_locale_with_numeric_region_code(self):
+        out = StringIO()
+        management.call_command(
+            "makemessages", locale=["ar_002"], stdout=out, verbosity=1
+        )
+        self.assertNotIn("invalid locale ar_002", out.getvalue())
+        self.assertIn("processing locale ar_002", out.getvalue())
+        self.assertIs(Path("locale/ar_002/LC_MESSAGES/django.po").exists(), True)
+
     def test_valid_locale_tachelhit_latin_morocco(self):
         out = StringIO()
         management.call_command(

tests/mail/tests.py

@@ -4,14 +4,18 @@ import shutil
 import socket
 import sys
 import tempfile
-from email import charset, message_from_binary_file, message_from_bytes
+from email import charset, message_from_binary_file
-from email.header import Header
+from email import message_from_bytes as _message_from_bytes
+from email import policy
+from email.message import EmailMessage as PyEmailMessage
+from email.message import Message as PyMessage
+from email.mime.image import MIMEImage
 from email.mime.text import MIMEText
-from email.utils import parseaddr
 from io import StringIO
 from pathlib import Path
 from smtplib import SMTP, SMTPException
 from ssl import SSLError
+from textwrap import dedent
 from unittest import mock, skipUnless
 
 from django.core import mail
@@ -41,7 +45,15 @@ except ImportError:
     HAS_AIOSMTPD = False
 
 
-class HeadersCheckMixin:
+def message_from_bytes(s):
+    """
+    email.message_from_bytes() using modern email.policy.default.
+    Returns a modern email.message.EmailMessage.
+    """
+    return _message_from_bytes(s, policy=policy.default)
+
+
+class MailTestsMixin:
     def assertMessageHasHeaders(self, message, headers):
         """
         Asserts that the `message` has all `headers`.
@@ -53,36 +65,109 @@ class HeadersCheckMixin:
         if isinstance(message, bytes):
             message = message_from_bytes(message)
         msg_headers = set(message.items())
-        self.assertTrue(
+        if not headers.issubset(msg_headers):
-            headers.issubset(msg_headers),
+            missing = "\n".join(f"  {h}: {v}" for h, v in headers - msg_headers)
-            msg="Message is missing "
+            actual = "\n".join(f"  {h}: {v}" for h, v in msg_headers)
-            "the following headers: %s" % (headers - msg_headers),
+            raise self.failureException(
+                f"Expected headers not found in message.\n"
+                f"Missing headers:\n{missing}\n"
+                f"Actual headers:\n{actual}"
             )
 
+    # In assertStartsWith()/assertEndsWith() failure messages, when truncating
+    # a long first ("haystack") string, include this many characters beyond the
+    # length of the second ("needle") string.
+    START_END_EXTRA_CONTEXT = 15
 
-class MailTests(HeadersCheckMixin, SimpleTestCase):
+    def assertStartsWith(self, first, second):
+        if not first.startswith(second):
+            # Use assertEqual() for failure message with diffs. If first value
+            # is much longer than second, truncate end and add an ellipsis.
+            self.longMessage = True
+            max_len = len(second) + self.START_END_EXTRA_CONTEXT
+            start_of_first = (
+                first
+                if len(first) <= max_len
+                else first[:max_len] + ("…" if isinstance(first, str) else b"...")
+            )
+            self.assertEqual(
+                start_of_first,
+                second,
+                "First string doesn't start with the second.",
+            )
+
+    def assertEndsWith(self, first, second):
+        if not first.endswith(second):
+            # Use assertEqual() for failure message with diffs. If first value
+            # is much longer than second, truncate start and prepend an ellipsis.
+            self.longMessage = True
+            max_len = len(second) + self.START_END_EXTRA_CONTEXT
+            end_of_first = (
+                first
+                if len(first) <= max_len
+                else ("…" if isinstance(first, str) else b"...") + first[-max_len:]
+            )
+            self.assertEqual(
+                end_of_first,
+                second,
+                "First string doesn't end with the second.",
+            )
+
+    def get_raw_attachments(self, django_message):
         """
-    Non-backend specific tests.
+        Return a list of the raw attachment parts in the MIME message generated
+        by serializing django_message and reparsing the result.
+
+        This returns only "top-level" attachments. It will not descend into
+        message/* attached emails to find nested attachments.
         """
+        msg_bytes = django_message.message().as_bytes()
+        message = message_from_bytes(msg_bytes)
+        return list(message.iter_attachments())
 
     def get_decoded_attachments(self, django_message):
         """
-        Encode the specified django.core.mail.message.EmailMessage, then decode
+        Return a list of decoded attachments resulting from serializing
-        it using Python's email.parser module and, for each attachment of the
+        django_message and reparsing the result.
-        message, return a list of tuples with (filename, content, mimetype).
+
+        Each attachment is returned as an EmailAttachment named tuple with
+        fields filename, content, and mimetype. The content will be decoded
+        to str for mimetype text/*; retained as bytes for other mimetypes.
         """
-        msg_bytes = django_message.message().as_bytes()
+        return [
-        email_message = message_from_bytes(msg_bytes)
+            EmailAttachment(
+                attachment.get_filename(),
+                attachment.get_content(),
+                attachment.get_content_type(),
+            )
+            for attachment in self.get_raw_attachments(django_message)
+        ]
 
-        def iter_attachments():
+    def get_message_structure(self, message, level=0):
-            for i in email_message.walk():
+        """
-                if i.get_content_disposition() == "attachment":
+        Return a multiline indented string representation
-                    filename = i.get_filename()
+        of the message's MIME content-type structure, e.g.:
-                    content = i.get_payload(decode=True)
-                    mimetype = i.get_content_type()
-                    yield filename, content, mimetype
 
-        return list(iter_attachments())
+            multipart/mixed
+                multipart/alternative
+                    text/plain
+                    text/html
+                image/jpg
+                text/calendar
+        """
+        # Adapted from email.iterators._structure().
+        indent = " " * (level * 4)
+        structure = [f"{indent}{message.get_content_type()}\n"]
+        if message.is_multipart():
+            for subpart in message.get_payload():
+                structure.append(self.get_message_structure(subpart, level + 1))
+        return "".join(structure)
+
+
+class MailTests(MailTestsMixin, SimpleTestCase):
+    """
+    Non-backend specific tests.
+    """
 
     def test_ascii(self):
         email = EmailMessage(
@@ -238,6 +323,20 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         ).message()
         self.assertEqual(message.get_all("Cc"), ["foo@example.com"])
 
+    def test_bcc_not_in_headers(self):
+        """
+        A bcc address should be in the recipients,
+        but not in the (visible) message headers.
+        """
+        email = EmailMessage(
+            to=["to@example.com"],
+            bcc=["bcc@example.com"],
+        )
+        message = email.message()
+        self.assertNotIn("Bcc", message)
+        self.assertNotIn("bcc@example.com", message.as_string())
+        self.assertEqual(email.recipients(), ["to@example.com", "bcc@example.com"])
+
     def test_reply_to(self):
         email = EmailMessage(
             "Subject",
@@ -303,26 +402,16 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
 
     def test_header_injection(self):
         msg = "Header values can't contain newlines "
-        email = EmailMessage(
+        cases = [
-            "Subject\nInjection Test", "Content", "from@example.com", ["to@example.com"]
+            {"subject": "Subject\nInjection Test"},
-        )
+            {"subject": gettext_lazy("Lazy Subject\nInjection Test")},
+            {"to": ["Name\nInjection test <to@example.com>"]},
+        ]
+        for kwargs in cases:
+            with self.subTest(case=kwargs):
+                email = EmailMessage(**kwargs)
                 with self.assertRaisesMessage(BadHeaderError, msg):
                     email.message()
-        email = EmailMessage(
-            gettext_lazy("Subject\nInjection Test"),
-            "Content",
-            "from@example.com",
-            ["to@example.com"],
-        )
-        with self.assertRaisesMessage(BadHeaderError, msg):
-            email.message()
-        with self.assertRaisesMessage(BadHeaderError, msg):
-            EmailMessage(
-                "Subject",
-                "Content",
-                "from@example.com",
-                ["Name\nInjection test <to@example.com>"],
-            ).message()
 
     def test_folding_white_space(self):
         """
@@ -615,8 +704,8 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
                 ("Content-Transfer-Encoding", "quoted-printable"),
             },
         )
-        self.assertTrue(
+        self.assertEndsWith(
-            payload0.as_bytes().endswith(b"\n\nFirstname S=FCrname is a great guy.")
+            payload0.as_bytes(), b"\n\nFirstname S=FCrname is a great guy."
         )
         # Check the text/html alternative.
         payload1 = message.get_payload(1)
@@ -630,10 +719,9 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
                 ("Content-Transfer-Encoding", "quoted-printable"),
             },
         )
-        self.assertTrue(
+        self.assertEndsWith(
-            payload1.as_bytes().endswith(
+            payload1.as_bytes(),
-                b"\n\n<p>Firstname S=FCrname is a <strong>great</strong> guy.</p>"
+            b"\n\n<p>Firstname S=FCrname is a <strong>great</strong> guy.</p>",
-            )
         )
 
     def test_attachments(self):
@@ -653,7 +741,7 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         self.assertEqual(msg.attachments[0].mimetype, mime_type)
 
         attachments = self.get_decoded_attachments(msg)
-        self.assertEqual(attachments[0], (file_name, file_content.encode(), mime_type))
+        self.assertEqual(attachments[0], (file_name, file_content, mime_type))
 
     def test_attachments_constructor(self):
         file_name = "example.txt"
@@ -675,7 +763,7 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         self.assertEqual(msg.attachments[0].mimetype, mime_type)
 
         attachments = self.get_decoded_attachments(msg)
-        self.assertEqual(attachments[0], (file_name, file_content.encode(), mime_type))
+        self.assertEqual(attachments[0], (file_name, file_content, mime_type))
 
     def test_attachments_constructor_from_tuple(self):
         file_name = "example.txt"
@@ -695,7 +783,7 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         self.assertEqual(msg.attachments[0].mimetype, mime_type)
 
         attachments = self.get_decoded_attachments(msg)
-        self.assertEqual(attachments[0], (file_name, file_content.encode(), mime_type))
+        self.assertEqual(attachments[0], (file_name, file_content, mime_type))
 
     def test_attachments_constructor_omit_mimetype(self):
         """
@@ -736,10 +824,8 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         msg = EmailMessage(body="Content")
         # Unicode in file name
         msg.attach("une pièce jointe.pdf", b"%PDF-1.4.%...", mimetype="application/pdf")
-        msg_bytes = msg.message().as_bytes()
+        attachment = self.get_decoded_attachments(msg)[0]
-        message = message_from_bytes(msg_bytes)
+        self.assertEqual(attachment.filename, "une pièce jointe.pdf")
-        payload = message.get_payload()
-        self.assertEqual(payload[1].get_filename(), "une pièce jointe.pdf")
 
     def test_attach_file(self):
         """
@@ -761,6 +847,9 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
 
         for basename, real_mimetype in files:
             for mimetype in test_mimetypes:
+                with self.subTest(
+                    basename=basename, real_mimetype=real_mimetype, mimetype=mimetype
+                ):
                     self.assertEqual(mimetypes.guess_type(basename)[0], real_mimetype)
                     expected_mimetype = (
                         mimetype or real_mimetype or "application/octet-stream"
@@ -782,20 +871,21 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
                     self.assertEqual(email.attachments[0].mimetype, expected_mimetype)
                     self.assertEqual(email.attachments[0].content, expected_content)
 
-                # Check attachments in generated message.
+                    # Check attachments in the generated message.
                     # (The actual content is not checked as variations in platform
                     # line endings and rfc822 refolding complicate the logic.)
-                actual_attachment = self.get_decoded_attachments(email)[0]
+                    attachments = self.get_decoded_attachments(email)
-                actual_filename, actual_content, actual_mimetype = actual_attachment
+                    self.assertEqual(len(attachments), 1)
-                self.assertEqual(actual_filename, basename)
+                    actual = attachments[0]
-                self.assertEqual(actual_mimetype, expected_mimetype)
+                    self.assertEqual(actual.filename, basename)
+                    self.assertEqual(actual.mimetype, expected_mimetype)
 
     def test_attach_text_as_bytes(self):
         msg = EmailMessage()
         msg.attach("file.txt", b"file content")
         filename, content, mimetype = self.get_decoded_attachments(msg)[0]
         self.assertEqual(filename, "file.txt")
-        self.assertEqual(content, b"file content")
+        self.assertEqual(content, "file content")  # (decoded)
         self.assertEqual(mimetype, "text/plain")
 
     def test_attach_utf8_text_as_bytes(self):
@@ -807,7 +897,7 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         msg.attach("file.txt", b"\xc3\xa4")  # UTF-8 encoded a umlaut.
         filename, content, mimetype = self.get_decoded_attachments(msg)[0]
         self.assertEqual(filename, "file.txt")
-        self.assertEqual(content, b"\xc3\xa4")
+        self.assertEqual(content, "ä")  # (decoded)
         self.assertEqual(mimetype, "text/plain")
 
     def test_attach_non_utf8_text_as_bytes(self):
@@ -823,6 +913,97 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         self.assertEqual(content, b"\xff")
         self.assertEqual(mimetype, "application/octet-stream")
 
+    def test_attach_mime_image(self):
+        """
+        EmailMessage.attach() docs: "You can pass it
+        a single argument that is a MIMEBase instance."
+        """
+        # This also verifies complex attachments with extra header fields.
+        email = EmailMessage()
+        image = MIMEImage(b"GIF89a...", "gif")
+        image["Content-Disposition"] = "inline"
+        image["Content-ID"] = "<content-id@example.org>"
+        email.attach(image)
+
+        attachments = self.get_raw_attachments(email)
+        self.assertEqual(len(attachments), 1)
+        image_att = attachments[0]
+        self.assertEqual(image_att.get_content_type(), "image/gif")
+        self.assertEqual(image_att.get_content_disposition(), "inline")
+        self.assertEqual(image_att["Content-ID"], "<content-id@example.org>")
+        self.assertEqual(image_att.get_content(), b"GIF89a...")
+        self.assertIsNone(image_att.get_filename())
+
+    def test_attach_mime_image_in_constructor(self):
+        image = MIMEImage(b"\x89PNG...", "png")
+        image["Content-Disposition"] = "attachment; filename=test.png"
+        email = EmailMessage(attachments=[image])
+
+        attachments = self.get_raw_attachments(email)
+        self.assertEqual(len(attachments), 1)
+        image_att = attachments[0]
+        self.assertEqual(image_att.get_content_type(), "image/png")
+        self.assertEqual(image_att.get_content(), b"\x89PNG...")
+        self.assertEqual(image_att.get_content_disposition(), "attachment")
+        self.assertEqual(image_att.get_filename(), "test.png")
+
+    def test_attach_rfc822_message(self):
+        """
+        EmailMessage.attach() docs: "If you specify a mimetype of message/rfc822,
+        it will also accept django.core.mail.EmailMessage and email.message.Message."
+        """
+        # django.core.mail.EmailMessage
+        django_email = EmailMessage("child subject", "child body")
+        # email.message.Message
+        py_message = PyMessage()
+        py_message["Subject"] = "child subject"
+        py_message.set_payload("child body")
+        # email.message.EmailMessage
+        py_email_message = PyEmailMessage()
+        py_email_message["Subject"] = "child subject"
+        py_email_message.set_content("child body")
+
+        cases = [
+            django_email,
+            py_message,
+            py_email_message,
+            # Should also allow message serialized as str or bytes.
+            py_message.as_string(),
+            py_message.as_bytes(),
+        ]
+
+        for child_message in cases:
+            with self.subTest(child_type=child_message.__class__):
+                email = EmailMessage("parent message", "parent body")
+                email.attach(content=child_message, mimetype="message/rfc822")
+                self.assertEqual(len(email.attachments), 1)
+                self.assertIsInstance(email.attachments[0], EmailAttachment)
+                self.assertEqual(email.attachments[0].mimetype, "message/rfc822")
+
+                # Make sure it is serialized correctly: a message/rfc822 attachment
+                # whose "body" content (payload) is the "encapsulated" (child) message.
+                attachments = self.get_raw_attachments(email)
+                self.assertEqual(len(attachments), 1)
+                rfc822_attachment = attachments[0]
+                self.assertEqual(rfc822_attachment.get_content_type(), "message/rfc822")
+
+                attached_message = rfc822_attachment.get_content()
+                self.assertEqual(attached_message["Subject"], "child subject")
+                self.assertEqual(attached_message.get_content().rstrip(), "child body")
+
+                # Regression for #18967: Per RFC 2046 5.2.1, "No encoding other
+                # than '7bit', '8bit', or 'binary' is permitted for the body of
+                # a 'message/rfc822' entity." (Default CTE is "7bit".)
+                cte = rfc822_attachment.get("Content-Transfer-Encoding", "7bit")
+                self.assertIn(cte, ("7bit", "8bit", "binary"))
+
+                # Any properly declared CTE is allowed for the attached message itself
+                # (including quoted-printable or base64). For the plain ASCII content
+                # in this test, we'd expect 7bit.
+                child_cte = attached_message.get("Content-Transfer-Encoding", "7bit")
+                self.assertEqual(child_cte, "7bit")
+                self.assertEqual(attached_message.get_content_type(), "text/plain")
+
     def test_attach_mimebase_prohibits_other_params(self):
         email_msg = EmailMessage()
         txt = MIMEText("content")
@@ -987,63 +1168,6 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         s = msg.message().as_string()
         self.assertIn("Content-Transfer-Encoding: 8bit", s)
 
-    def test_dont_base64_encode_message_rfc822(self):
-        # Ticket #18967
-        # Shouldn't use base64 encoding for a child EmailMessage attachment.
-        # Create a child message first
-        child_msg = EmailMessage(
-            "Child Subject",
-            "Some body of child message",
-            "bounce@example.com",
-            ["to@example.com"],
-            headers={"From": "from@example.com"},
-        )
-        child_s = child_msg.message().as_string()
-
-        # Now create a parent
-        parent_msg = EmailMessage(
-            "Parent Subject",
-            "Some parent body",
-            "bounce@example.com",
-            ["to@example.com"],
-            headers={"From": "from@example.com"},
-        )
-
-        # Attach to parent as a string
-        parent_msg.attach(content=child_s, mimetype="message/rfc822")
-        parent_s = parent_msg.message().as_string()
-
-        # The child message header is not base64 encoded
-        self.assertIn("Child Subject", parent_s)
-
-        # Feature test: try attaching email.Message object directly to the mail.
-        parent_msg = EmailMessage(
-            "Parent Subject",
-            "Some parent body",
-            "bounce@example.com",
-            ["to@example.com"],
-            headers={"From": "from@example.com"},
-        )
-        parent_msg.attach(content=child_msg.message(), mimetype="message/rfc822")
-        parent_s = parent_msg.message().as_string()
-
-        # The child message header is not base64 encoded
-        self.assertIn("Child Subject", parent_s)
-
-        # Feature test: try attaching Django's EmailMessage object directly to the mail.
-        parent_msg = EmailMessage(
-            "Parent Subject",
-            "Some parent body",
-            "bounce@example.com",
-            ["to@example.com"],
-            headers={"From": "from@example.com"},
-        )
-        parent_msg.attach(content=child_msg, mimetype="message/rfc822")
-        parent_s = parent_msg.message().as_string()
-
-        # The child message header is not base64 encoded
-        self.assertIn("Child Subject", parent_s)
-
     def test_custom_utf8_encoding(self):
         """A UTF-8 charset with a custom body encoding is respected."""
         body = "Body with latin characters: àáä."
@@ -1187,6 +1311,121 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         with self.assertRaisesMessage(ValueError, msg):
             email_msg.attach_alternative("<p>content</p>", None)
 
+    def test_mime_structure(self):
+        """
+        Check generated messages have the expected MIME parts and nesting.
+        """
+        html_body = EmailAlternative("<p>HTML</p>", "text/html")
+        image = EmailAttachment("image.gif", b"\x89PNG...", "image/png")
+        rfc822_attachment = EmailAttachment(
+            None, EmailMessage(body="text"), "message/rfc822"
+        )
+        cases = [
+            # name, email (EmailMessage or subclass), expected structure
+            (
+                "single body",
+                EmailMessage(body="text"),
+                """
+                text/plain
+                """,
+            ),
+            (
+                "single body with attachment",
+                EmailMessage(body="text", attachments=[image]),
+                """
+                multipart/mixed
+                    text/plain
+                    image/png
+                """,
+            ),
+            (
+                "alternative bodies",
+                EmailMultiAlternatives(body="text", alternatives=[html_body]),
+                """
+                multipart/alternative
+                    text/plain
+                    text/html
+                """,
+            ),
+            (
+                "alternative bodies with attachments",
+                EmailMultiAlternatives(
+                    body="text", alternatives=[html_body], attachments=[image]
+                ),
+                """
+                multipart/mixed
+                    multipart/alternative
+                        text/plain
+                        text/html
+                    image/png
+                """,
+            ),
+            (
+                "alternative bodies with rfc822 attachment",
+                EmailMultiAlternatives(
+                    body="text",
+                    alternatives=[html_body],
+                    attachments=[rfc822_attachment],
+                ),
+                """
+                multipart/mixed
+                    multipart/alternative
+                        text/plain
+                        text/html
+                    message/rfc822
+                        text/plain
+                """,
+            ),
+            (
+                "attachment only",
+                EmailMessage(attachments=[image]),
+                # Avoid empty text/plain body.
+                """
+                multipart/mixed
+                    image/png
+                """,
+            ),
+            (
+                "alternative only",
+                EmailMultiAlternatives(alternatives=[html_body]),
+                # Avoid empty text/plain body.
+                """
+                multipart/alternative
+                    text/html
+                """,
+            ),
+            (
+                "alternative and attachment only",
+                EmailMultiAlternatives(alternatives=[html_body], attachments=[image]),
+                """
+                multipart/mixed
+                    multipart/alternative
+                        text/html
+                    image/png
+                """,
+            ),
+            (
+                "empty EmailMessage",
+                EmailMessage(),
+                """
+                text/plain
+                """,
+            ),
+            (
+                "empty EmailMultiAlternatives",
+                EmailMultiAlternatives(),
+                """
+                text/plain
+                """,
+            ),
+        ]
+        for name, email, expected in cases:
+            expected = dedent(expected).lstrip()
+            with self.subTest(name=name):
+                message = email.message()
+                structure = self.get_message_structure(message)
+                self.assertEqual(structure, expected)
+
     def test_body_contains(self):
         email_msg = EmailMultiAlternatives()
         email_msg.body = "I am content."
@@ -1205,9 +1444,117 @@ class MailTests(HeadersCheckMixin, SimpleTestCase):
         email_msg.attach_alternative(b"I am a song.", "audio/mpeg")
         self.assertIs(email_msg.body_contains("I am content"), True)
 
+    def test_all_params_optional(self):
+        """
+        EmailMessage class docs: "All parameters are optional"
+        """
+        email = EmailMessage()
+        self.assertIsInstance(email.message(), PyMessage)  # force serialization.
+
+        email = EmailMultiAlternatives()
+        self.assertIsInstance(email.message(), PyMessage)  # force serialization.
+
+    def test_positional_arguments_order(self):
+        """
+        EmailMessage class docs: "… is initialized with the following parameters
+        (in the given order, if positional arguments are used)."
+        """
+        connection = mail.get_connection()
+        email = EmailMessage(
+            # (If you need to insert/remove/reorder any params here,
+            # that indicates a breaking change to documented behavior.)
+            "subject",
+            "body",
+            "from@example.com",
+            ["to@example.com"],
+            ["bcc@example.com"],
+            connection,
+            [EmailAttachment("file.txt", "attachment", "text/plain")],
+            {"X-Header": "custom header"},
+            ["cc@example.com"],
+            ["reply-to@example.com"],
+            # (New options can be added below here, ideally as keyword-only args.)
+        )
+
+        message = email.message()
+        self.assertEqual(message.get_all("Subject"), ["subject"])
+        self.assertEqual(message.get_all("From"), ["from@example.com"])
+        self.assertEqual(message.get_all("To"), ["to@example.com"])
+        self.assertEqual(message.get_all("X-Header"), ["custom header"])
+        self.assertEqual(message.get_all("Cc"), ["cc@example.com"])
+        self.assertEqual(message.get_all("Reply-To"), ["reply-to@example.com"])
+        self.assertEqual(message.get_payload(0).get_payload(), "body")
+        self.assertEqual(
+            self.get_decoded_attachments(email),
+            [("file.txt", "attachment", "text/plain")],
+        )
+        self.assertEqual(
+            email.recipients(), ["to@example.com", "cc@example.com", "bcc@example.com"]
+        )
+        self.assertIs(email.get_connection(), connection)
+
+    def test_all_params_can_be_set_before_send(self):
+        """
+        EmailMessage class docs: "All parameters … can be set at any time
+        prior to calling the send() method."
+        """
+        # This is meant to verify EmailMessage.__init__() doesn't apply any
+        # special processing that would be missing for properties set later.
+        original_connection = mail.get_connection(username="original")
+        new_connection = mail.get_connection(username="new")
+        email = EmailMessage(
+            "original subject",
+            "original body",
+            "original-from@example.com",
+            ["original-to@example.com"],
+            ["original-bcc@example.com"],
+            original_connection,
+            [EmailAttachment("original.txt", "original attachment", "text/plain")],
+            {"X-Header": "original header"},
+            ["original-cc@example.com"],
+            ["original-reply-to@example.com"],
+        )
+        email.subject = "new subject"
+        email.body = "new body"
+        email.from_email = "new-from@example.com"
+        email.to = ["new-to@example.com"]
+        email.bcc = ["new-bcc@example.com"]
+        email.connection = new_connection
+        email.attachments = [
+            ("new1.txt", "new attachment 1", "text/plain"),  # plain tuple.
+            EmailAttachment("new2.txt", "new attachment 2", "text/csv"),
+            MIMEImage(b"GIF89a...", "gif"),
+        ]
+        email.extra_headers = {"X-Header": "new header"}
+        email.cc = ["new-cc@example.com"]
+        email.reply_to = ["new-reply-to@example.com"]
+
+        message = email.message()
+        self.assertEqual(message.get_all("Subject"), ["new subject"])
+        self.assertEqual(message.get_all("From"), ["new-from@example.com"])
+        self.assertEqual(message.get_all("To"), ["new-to@example.com"])
+        self.assertEqual(message.get_all("X-Header"), ["new header"])
+        self.assertEqual(message.get_all("Cc"), ["new-cc@example.com"])
+        self.assertEqual(message.get_all("Reply-To"), ["new-reply-to@example.com"])
+        self.assertEqual(message.get_payload(0).get_payload(), "new body")
+        self.assertEqual(
+            self.get_decoded_attachments(email),
+            [
+                ("new1.txt", "new attachment 1", "text/plain"),
+                ("new2.txt", "new attachment 2", "text/csv"),
+                (None, b"GIF89a...", "image/gif"),
+            ],
+        )
+        self.assertEqual(
+            email.recipients(),
+            ["new-to@example.com", "new-cc@example.com", "new-bcc@example.com"],
+        )
+        self.assertIs(email.get_connection(), new_connection)
+        self.assertNotIn("original", message.as_string())
+
 
 @requires_tz_support
-class MailTimeZoneTests(SimpleTestCase):
+class MailTimeZoneTests(MailTestsMixin, SimpleTestCase):
     @override_settings(
         EMAIL_USE_LOCALTIME=False, USE_TZ=True, TIME_ZONE="Africa/Algiers"
     )
@@ -1216,7 +1563,7 @@ class MailTimeZoneTests(SimpleTestCase):
         EMAIL_USE_LOCALTIME=False creates a datetime in UTC.
         """
         email = EmailMessage()
-        self.assertTrue(email.message()["Date"].endswith("-0000"))
+        self.assertEndsWith(email.message()["Date"], "-0000")
 
     @override_settings(
         EMAIL_USE_LOCALTIME=True, USE_TZ=True, TIME_ZONE="Africa/Algiers"
@@ -1226,9 +1573,8 @@ class MailTimeZoneTests(SimpleTestCase):
         EMAIL_USE_LOCALTIME=True creates a datetime in the local time zone.
         """
         email = EmailMessage()
-        self.assertTrue(
+        # Africa/Algiers is UTC+1 year round.
-            email.message()["Date"].endswith("+0100")
+        self.assertEndsWith(email.message()["Date"], "+0100")
-        )  # Africa/Algiers is UTC+1
 
 
 class PythonGlobalState(SimpleTestCase):
@@ -1259,7 +1605,7 @@ class PythonGlobalState(SimpleTestCase):
         self.assertIn("Content-Transfer-Encoding: base64", txt.as_string())
 
 
-class BaseEmailBackendTests(HeadersCheckMixin):
+class BaseEmailBackendTests(MailTestsMixin):
     email_backend = None
 
     @classmethod
@@ -1267,15 +1613,6 @@ class BaseEmailBackendTests(HeadersCheckMixin):
         cls.enterClassContext(override_settings(EMAIL_BACKEND=cls.email_backend))
         super().setUpClass()
 
-    def assertStartsWith(self, first, second):
-        if not first.startswith(second):
-            self.longMessage = True
-            self.assertEqual(
-                first[: len(second)],
-                second,
-                "First string doesn't start with the second.",
-            )
-
     def get_mailbox_content(self):
         raise NotImplementedError(
             "subclasses of BaseEmailBackendTests must provide a get_mailbox_content() "
@@ -1316,10 +1653,9 @@ class BaseEmailBackendTests(HeadersCheckMixin):
         num_sent = mail.get_connection().send_messages([email])
         self.assertEqual(num_sent, 1)
         message = self.get_the_message()
-        self.assertEqual(message["subject"], "=?utf-8?q?Ch=C3=A8re_maman?=")
+        self.assertEqual(message["subject"], "Chère maman")
-        self.assertEqual(
+        self.assertIn(b"Subject: =?utf-8?q?Ch=C3=A8re_maman?=", message.as_bytes())
-            message.get_payload(decode=True).decode(), "Je t'aime très fort"
+        self.assertEqual(message.get_content(), "Je t'aime très fort")
-        )
 
     def test_send_long_lines(self):
         """
@@ -1349,6 +1685,7 @@ class BaseEmailBackendTests(HeadersCheckMixin):
         # send_messages() may take a list or an iterator.
         emails_lists = ([email1, email2], iter((email1, email2)))
         for emails_list in emails_lists:
+            with self.subTest(emails_list=repr(emails_list)):
                 num_sent = mail.get_connection().send_messages(emails_list)
                 self.assertEqual(num_sent, 2)
                 messages = self.get_mailbox_content()
@@ -1364,8 +1701,10 @@ class BaseEmailBackendTests(HeadersCheckMixin):
         )
         email.send()
         message = self.get_the_message()
-        self.assertEqual(
+        self.assertEqual(message["from"], "Firstname Sürname <from@example.com>")
-            message["from"], "=?utf-8?q?Firstname_S=C3=BCrname?= <from@example.com>"
+        self.assertIn(
+            b"From: =?utf-8?q?Firstname_S=C3=BCrname?= <from@example.com>",
+            message.as_bytes(),
         )
 
     def test_plaintext_send_mail(self):
@@ -1597,7 +1936,10 @@ class LocmemBackendTests(BaseEmailBackendTests, SimpleTestCase):
     email_backend = "django.core.mail.backends.locmem.EmailBackend"
 
     def get_mailbox_content(self):
-        return [m.message() for m in mail.outbox]
+        # Reparse as modern messages to work with shared BaseEmailBackendTests.
+        # (Once EmailMessage.message() uses Python's modern email API, this
+        # can be changed back to `[m.message() for m in mail.outbox]`.)
+        return [message_from_bytes(m.message().as_bytes()) for m in mail.outbox]
 
     def flush_mailbox(self):
         mail.outbox = []
@@ -1676,7 +2018,7 @@ class FileBackendTests(BaseEmailBackendTests, SimpleTestCase):
 
         self.assertEqual(len(os.listdir(self.tmp_dir)), 1)
         with open(os.path.join(self.tmp_dir, os.listdir(self.tmp_dir)[0]), "rb") as fp:
-            message = message_from_binary_file(fp)
+            message = message_from_binary_file(fp, policy=policy.default)
         self.assertEqual(message.get_content_type(), "text/plain")
         self.assertEqual(message.get("subject"), "Subject")
         self.assertEqual(message.get("from"), "from@example.com")
@@ -1759,28 +2101,32 @@ class ConsoleBackendTests(BaseEmailBackendTests, SimpleTestCase):
 class SMTPHandler:
     def __init__(self, *args, **kwargs):
         self.mailbox = []
+        self.smtp_envelopes = []
 
     async def handle_DATA(self, server, session, envelope):
         data = envelope.content
         mail_from = envelope.mail_from
 
         message = message_from_bytes(data.rstrip())
-        message_addr = parseaddr(message.get("from"))[1]
+        try:
-        if mail_from != message_addr:
+            header_from = message["from"].addresses[0].addr_spec
-            # According to the spec, mail_from does not necessarily match the
+        except (KeyError, IndexError):
-            # From header - this is the case where the local part isn't
+            header_from = None
-            # encoded, so try to correct that.
-            lp, domain = mail_from.split("@", 1)
-            lp = Header(lp, "utf-8").encode()
-            mail_from = "@".join([lp, domain])
 
-        if mail_from != message_addr:
+        if mail_from != header_from:
-            return f"553 '{mail_from}' != '{message_addr}'"
+            return f"553 '{mail_from}' != '{header_from}'"
         self.mailbox.append(message)
+        self.smtp_envelopes.append(
+            {
+                "mail_from": envelope.mail_from,
+                "rcpt_tos": envelope.rcpt_tos,
+            }
+        )
         return "250 OK"
 
     def flush_mailbox(self):
         self.mailbox[:] = []
+        self.smtp_envelopes[:] = []
 
 
 @skipUnless(HAS_AIOSMTPD, "No aiosmtpd library detected.")
@@ -1827,6 +2173,9 @@ class SMTPBackendTests(BaseEmailBackendTests, SMTPBackendTestsBase):
     def get_mailbox_content(self):
         return self.smtp_handler.mailbox
 
+    def get_smtp_envelopes(self):
+        return self.smtp_handler.smtp_envelopes
+
     @override_settings(
         EMAIL_HOST_USER="not empty username",
         EMAIL_HOST_PASSWORD="not empty password",
@@ -2051,6 +2400,85 @@ class SMTPBackendTests(BaseEmailBackendTests, SMTPBackendTestsBase):
         sent = backend.send_messages([email])
         self.assertEqual(sent, 0)
 
+    def test_avoids_sending_to_invalid_addresses(self):
+        """
+        Verify invalid addresses can't sneak into SMTP commands through
+        EmailMessage.all_recipients() (which is distinct from message header fields).
+        """
+        backend = smtp.EmailBackend()
+        backend.connection = mock.Mock()
+        for email_address in (
+            # Invalid address with two @ signs.
+            "to@other.com@example.com",
+            # Invalid address without the quotes.
+            "to@other.com <to@example.com>",
+            # Other invalid addresses.
+            "@",
+            "to@",
+            "@example.com",
+            # CR/NL in addr-spec. (SMTP strips display-name.)
+            '"evil@example.com\r\nto"@example.com',
+            "to\nevil@example.com",
+        ):
+            with self.subTest(email_address=email_address):
+                # Use bcc (which is only processed by SMTP backend) to ensure
+                # error is coming from SMTP backend, not EmailMessage.message().
+                email = EmailMessage(bcc=[email_address])
+                with self.assertRaisesMessage(ValueError, "Invalid address"):
+                    backend.send_messages([email])
+
+    def test_encodes_idna_in_smtp_commands(self):
+        """
+        SMTP backend must encode non-ASCII domains for the SMTP envelope
+        (which can be distinct from the email headers).
+        """
+        email = EmailMessage(
+            from_email="lists@discussão.example.org",
+            to=["To Example <to@漢字.example.com>"],
+            bcc=["monitor@discussão.example.org"],
+            headers={
+                "From": "Gestor de listas <lists@discussão.example.org>",
+                "To": "Discussão Django <django@discussão.example.org>",
+            },
+        )
+        backend = smtp.EmailBackend()
+        backend.send_messages([email])
+        envelope = self.get_smtp_envelopes()[0]
+        self.assertEqual(envelope["mail_from"], "lists@xn--discusso-xza.example.org")
+        self.assertEqual(
+            envelope["rcpt_tos"],
+            ["to@xn--p8s937b.example.com", "monitor@xn--discusso-xza.example.org"],
+        )
+
+    def test_does_not_reencode_idna(self):
+        """
+        SMTP backend should not downgrade IDNA 2008 to IDNA 2003.
+
+        Django does not currently handle IDNA 2008 encoding, but should retain
+        it for addresses that have been pre-encoded.
+        """
+        # Test all four EmailMessage attrs accessed by the SMTP email backend.
+        # These are IDNA 2008 encoded domains that would be different
+        # in IDNA 2003, from https://www.unicode.org/reports/tr46/#Deviations.
+        email = EmailMessage(
+            from_email='"βόλος" <from@xn--fa-hia.example.com>',
+            to=['"faß" <to@xn--10cl1a0b660p.example.com>'],
+            cc=['"ශ්‍රී" <cc@xn--nxasmm1c.example.com>'],
+            bcc=['"نامه‌ای." <bcc@xn--mgba3gch31f060k.example.com>'],
+        )
+        backend = smtp.EmailBackend()
+        backend.send_messages([email])
+        envelope = self.get_smtp_envelopes()[0]
+        self.assertEqual(envelope["mail_from"], "from@xn--fa-hia.example.com")
+        self.assertEqual(
+            envelope["rcpt_tos"],
+            [
+                "to@xn--10cl1a0b660p.example.com",
+                "cc@xn--nxasmm1c.example.com",
+                "bcc@xn--mgba3gch31f060k.example.com",
+            ],
+        )
+
 
 @skipUnless(HAS_AIOSMTPD, "No aiosmtpd library detected.")
 class SMTPBackendStoppedServerTests(SMTPBackendTestsBase):

tests/migrations/test_commands.py

@@ -9,6 +9,7 @@ from unittest import mock
 
 from django.apps import apps
 from django.core.management import CommandError, call_command
+from django.core.management.base import SystemCheckError
 from django.core.management.commands.makemigrations import (
     Command as MakeMigrationsCommand,
 )
@@ -859,7 +860,7 @@ class MigrateTests(MigrationTestBase):
         sqlmigrate outputs forward looking SQL.
         """
         out = io.StringIO()
-        call_command("sqlmigrate", "migrations", "0001", stdout=out)
+        call_command("sqlmigrate", "migrations", "0001", stdout=out, no_color=True)
 
         lines = out.getvalue().splitlines()
 
@@ -921,7 +922,14 @@ class MigrateTests(MigrationTestBase):
         call_command("migrate", "migrations", verbosity=0)
 
         out = io.StringIO()
-        call_command("sqlmigrate", "migrations", "0001", stdout=out, backwards=True)
+        call_command(
+            "sqlmigrate",
+            "migrations",
+            "0001",
+            stdout=out,
+            backwards=True,
+            no_color=True,
+        )
 
         lines = out.getvalue().splitlines()
         try:
@@ -1098,6 +1106,30 @@ class MigrateTests(MigrationTestBase):
             ],
         )
 
+    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
+    def test_sqlmigrate_transaction_keywords_not_colorized(self):
+        out = io.StringIO()
+        with mock.patch(
+            "django.core.management.color.supports_color", lambda *args: True
+        ):
+            call_command("sqlmigrate", "migrations", "0001", stdout=out, no_color=False)
+        self.assertNotIn("\x1b", out.getvalue())
+
+    @override_settings(
+        MIGRATION_MODULES={"migrations": "migrations.test_migrations_no_operations"},
+        INSTALLED_APPS=["django.contrib.auth"],
+    )
+    def test_sqlmigrate_system_checks_colorized(self):
+        with (
+            mock.patch(
+                "django.core.management.color.supports_color", lambda *args: True
+            ),
+            self.assertRaisesMessage(SystemCheckError, "\x1b"),
+        ):
+            call_command(
+                "sqlmigrate", "migrations", "0001", skip_checks=False, no_color=False
+            )
+
     @override_settings(
         INSTALLED_APPS=[
             "migrations.migrations_test_apps.migrated_app",

tests/model_fields/test_decimalfield.py

@@ -1,9 +1,10 @@
 import math
 from decimal import Decimal
+from unittest import mock
 
 from django.core import validators
 from django.core.exceptions import ValidationError
-from django.db import models
+from django.db import connection, models
 from django.test import TestCase
 
 from .models import BigD, Foo
@@ -48,6 +49,20 @@ class DecimalFieldTests(TestCase):
         self.assertIsNone(f.get_prep_value(None))
         self.assertEqual(f.get_prep_value("2.4"), Decimal("2.4"))
 
+    def test_get_db_prep_value(self):
+        """
+        DecimalField.get_db_prep_value() must call
+        DatabaseOperations.adapt_decimalfield_value().
+        """
+        f = models.DecimalField(max_digits=5, decimal_places=1)
+        # None of the built-in database backends implement
+        # adapt_decimalfield_value(), so this must be confirmed with mocking.
+        with mock.patch.object(
+            connection.ops.__class__, "adapt_decimalfield_value"
+        ) as adapt_decimalfield_value:
+            f.get_db_prep_value("2.4", connection)
+        adapt_decimalfield_value.assert_called_with(Decimal("2.4"), 5, 1)
+
     def test_filter_with_strings(self):
         """
         Should be able to filter decimal fields using strings (#8023).

tests/model_fields/test_jsonfield.py

@@ -29,6 +29,7 @@ from django.db.models import (
 from django.db.models.expressions import RawSQL
 from django.db.models.fields.json import (
     KT,
+    HasKey,
     KeyTextTransform,
     KeyTransform,
     KeyTransformFactory,
@@ -582,6 +583,14 @@ class TestQuerying(TestCase):
                     [expected],
                 )
 
+    def test_has_key_literal_lookup(self):
+        self.assertSequenceEqual(
+            NullableJSONModel.objects.filter(
+                HasKey(Value({"foo": "bar"}, JSONField()), "foo")
+            ).order_by("id"),
+            self.objs,
+        )
+
     def test_has_key_list(self):
         obj = NullableJSONModel.objects.create(value=[{"a": 1}, {"b": "x"}])
         tests = [
@@ -808,6 +817,59 @@ class TestQuerying(TestCase):
         )
         self.assertIs(NullableJSONModel.objects.filter(value__c__lt=5).exists(), False)
 
+    def test_lookups_special_chars(self):
+        test_keys = [
+            "CONTROL",
+            "single'",
+            "dollar$",
+            "dot.dot",
+            "with space",
+            "back\\slash",
+            "question?mark",
+            "user@name",
+            "emo🤡'ji",
+            "com,ma",
+            "curly{{{brace}}}s",
+            "escape\uffff'seq'\uffffue\uffff'nce",
+        ]
+        json_value = {key: "some value" for key in test_keys}
+        obj = NullableJSONModel.objects.create(value=json_value)
+        obj.refresh_from_db()
+        self.assertEqual(obj.value, json_value)
+
+        for key in test_keys:
+            lookups = {
+                "has_key": Q(value__has_key=key),
+                "has_keys": Q(value__has_keys=[key, "CONTROL"]),
+                "has_any_keys": Q(value__has_any_keys=[key, "does_not_exist"]),
+                "exact": Q(**{f"value__{key}": "some value"}),
+            }
+            for lookup, condition in lookups.items():
+                results = NullableJSONModel.objects.filter(condition)
+                with self.subTest(key=key, lookup=lookup):
+                    self.assertSequenceEqual(results, [obj])
+
+    def test_lookups_special_chars_double_quotes(self):
+        test_keys = [
+            'double"',
+            "m\\i@x. m🤡'a,t{{{ch}}}e?d$\"'es\uffff'ca\uffff'pe",
+        ]
+        json_value = {key: "some value" for key in test_keys}
+        obj = NullableJSONModel.objects.create(value=json_value)
+        obj.refresh_from_db()
+        self.assertEqual(obj.value, json_value)
+        self.assertSequenceEqual(
+            NullableJSONModel.objects.filter(value__has_keys=test_keys), [obj]
+        )
+        for key in test_keys:
+            with self.subTest(key=key):
+                results = NullableJSONModel.objects.filter(
+                    Q(value__has_key=key),
+                    Q(value__has_any_keys=[key, "does_not_exist"]),
+                    Q(**{f"value__{key}": "some value"}),
+                )
+                self.assertSequenceEqual(results, [obj])
+
     def test_lookup_exclude(self):
         tests = [
             (Q(value__a="b"), [self.objs[0]]),

tests/model_forms/tests.py

@@ -3207,11 +3207,13 @@ class ModelFormCustomErrorTests(SimpleTestCase):
         errors = CustomErrorMessageForm(data).errors
         self.assertHTMLEqual(
             str(errors["name1"]),
-            '<ul class="errorlist"><li>Form custom error message.</li></ul>',
+            '<ul class="errorlist" id="id_name1_error">'
+            "<li>Form custom error message.</li></ul>",
         )
         self.assertHTMLEqual(
             str(errors["name2"]),
-            '<ul class="errorlist"><li>Model custom error message.</li></ul>',
+            '<ul class="errorlist" id="id_name2_error">'
+            "<li>Model custom error message.</li></ul>",
         )
 
     def test_model_clean_error_messages(self):
@@ -3220,14 +3222,15 @@ class ModelFormCustomErrorTests(SimpleTestCase):
         self.assertFalse(form.is_valid())
         self.assertHTMLEqual(
             str(form.errors["name1"]),
-            '<ul class="errorlist"><li>Model.clean() error messages.</li></ul>',
+            '<ul class="errorlist" id="id_name1_error">'
+            "<li>Model.clean() error messages.</li></ul>",
         )
         data = {"name1": "FORBIDDEN_VALUE2", "name2": "ABC"}
         form = CustomErrorMessageForm(data)
         self.assertFalse(form.is_valid())
         self.assertHTMLEqual(
             str(form.errors["name1"]),
-            '<ul class="errorlist">'
+            '<ul class="errorlist" id="id_name1_error">'
             "<li>Model.clean() error messages (simpler syntax).</li></ul>",
         )
         data = {"name1": "GLOBAL_ERROR", "name2": "ABC"}

tests/postgres_tests/test_array.py

@@ -1008,6 +1008,32 @@ class TestSerialization(PostgreSQLSimpleTestCase):
         self.assertEqual(instance.field, [1, 2, None])
 
 
+class TestStringSerialization(PostgreSQLSimpleTestCase):
+    field_values = [["Django", "Python", None], ["Джанго", "פייתון", None, "król"]]
+
+    @staticmethod
+    def create_json_data(array_field_value):
+        fields = {"field": json.dumps(array_field_value, ensure_ascii=False)}
+        return json.dumps(
+            [{"model": "postgres_tests.chararraymodel", "pk": None, "fields": fields}]
+        )
+
+    def test_encode(self):
+        for field_value in self.field_values:
+            with self.subTest(field_value=field_value):
+                instance = CharArrayModel(field=field_value)
+                data = serializers.serialize("json", [instance])
+                json_data = self.create_json_data(field_value)
+                self.assertEqual(json.loads(data), json.loads(json_data))
+
+    def test_decode(self):
+        for field_value in self.field_values:
+            with self.subTest(field_value=field_value):
+                json_data = self.create_json_data(field_value)
+                instance = list(serializers.deserialize("json", json_data))[0].object
+                self.assertEqual(instance.field, field_value)
+
+
 class TestValidation(PostgreSQLSimpleTestCase):
     def test_unbounded(self):
         field = ArrayField(models.IntegerField())

tests/postgres_tests/test_hstore.py

@@ -297,36 +297,50 @@ class TestChecks(PostgreSQLSimpleTestCase):
 
 
 class TestSerialization(PostgreSQLSimpleTestCase):
-    test_data = json.dumps(
+    field_values = [
-        [
+        ({"a": "b"}, [{"a": "b"}, {"b": "a"}]),
-            {
+        (
-                "model": "postgres_tests.hstoremodel",
+            {"все": "Трурль и Клапауций"},
-                "pk": None,
+            [{"Трурль": "Клапауций"}, {"Клапауций": "Трурль"}],
-                "fields": {
-                    "field": json.dumps({"a": "b"}),
-                    "array_field": json.dumps(
-                        [
-                            json.dumps({"a": "b"}),
-                            json.dumps({"b": "a"}),
-                        ]
         ),
-                },
-            }
     ]
+
+    @staticmethod
+    def create_json_data(field_value, array_field_value):
+        fields = {
+            "field": json.dumps(field_value, ensure_ascii=False),
+            "array_field": json.dumps(
+                [json.dumps(item, ensure_ascii=False) for item in array_field_value],
+                ensure_ascii=False,
+            ),
+        }
+        return json.dumps(
+            [{"model": "postgres_tests.hstoremodel", "pk": None, "fields": fields}]
         )
 
     def test_dumping(self):
-        instance = HStoreModel(field={"a": "b"}, array_field=[{"a": "b"}, {"b": "a"}])
+        for field_value, array_field_value in self.field_values:
+            with self.subTest(field_value=field_value, array_value=array_field_value):
+                instance = HStoreModel(field=field_value, array_field=array_field_value)
                 data = serializers.serialize("json", [instance])
-        self.assertEqual(json.loads(data), json.loads(self.test_data))
+                json_data = self.create_json_data(field_value, array_field_value)
+                self.assertEqual(json.loads(data), json.loads(json_data))
 
     def test_loading(self):
-        instance = list(serializers.deserialize("json", self.test_data))[0].object
+        for field_value, array_field_value in self.field_values:
-        self.assertEqual(instance.field, {"a": "b"})
+            with self.subTest(field_value=field_value, array_value=array_field_value):
-        self.assertEqual(instance.array_field, [{"a": "b"}, {"b": "a"}])
+                json_data = self.create_json_data(field_value, array_field_value)
+                instance = list(serializers.deserialize("json", json_data))[0].object
+                self.assertEqual(instance.field, field_value)
+                self.assertEqual(instance.array_field, array_field_value)
 
     def test_roundtrip_with_null(self):
-        instance = HStoreModel(field={"a": "b", "c": None})
+        for field_value in [
+            {"a": "b", "c": None},
+            {"Енеїда": "Ти знаєш, він який суціга", "Зефір": None},
+        ]:
+            with self.subTest(field_value=field_value):
+                instance = HStoreModel(field=field_value)
                 data = serializers.serialize("json", [instance])
                 new_instance = list(serializers.deserialize("json", data))[0].object
                 self.assertEqual(instance.field, new_instance.field)

tests/runtests.py

@@ -32,9 +32,11 @@ else:
         RemovedInDjango60Warning,
         RemovedInDjango61Warning,
     )
+    from django.utils.functional import classproperty
     from django.utils.log import DEFAULT_LOGGING
     from django.utils.version import PY312, PYPY
 
+
 try:
     import MySQLdb
 except ImportError:
@@ -307,12 +309,12 @@ def setup_run_tests(verbosity, start_at, start_after, test_labels=None):
     apps.set_installed_apps(settings.INSTALLED_APPS)
 
     # Force declaring available_apps in TransactionTestCase for faster tests.
-    def no_available_apps(self):
+    def no_available_apps(cls):
         raise Exception(
             "Please define available_apps in TransactionTestCase and its subclasses."
         )
 
-    TransactionTestCase.available_apps = property(no_available_apps)
+    TransactionTestCase.available_apps = classproperty(no_available_apps)
     TestCase.available_apps = None
 
     # Set an environment variable that other code may consult to see if

tests/serializers/models/data.py

@@ -13,6 +13,15 @@ from django.db import models
 
 from .base import BaseModel
 
+try:
+    from PIL import Image  # NOQA
+except ImportError:
+    ImageData = None
+else:
+
+    class ImageData(models.Model):
+        data = models.ImageField(null=True)
+
 
 class BinaryData(models.Model):
     data = models.BinaryField(null=True)
@@ -62,10 +71,6 @@ class BigIntegerData(models.Model):
     data = models.BigIntegerField(null=True)
 
 
-# class ImageData(models.Model):
-#    data = models.ImageField(null=True)
-
-
 class GenericIPAddressData(models.Model):
     data = models.GenericIPAddressField(null=True)
 

tests/serializers/test_data.py

@@ -10,10 +10,11 @@ forward, backwards and self references.
 import datetime
 import decimal
 import uuid
+from collections import namedtuple
 
 from django.core import serializers
 from django.db import connection, models
-from django.test import TestCase
+from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature
 
 from .models import (
     Anchor,
@@ -46,6 +47,7 @@ from .models import (
     GenericData,
     GenericIPAddressData,
     GenericIPAddressPKData,
+    ImageData,
     InheritAbstractModel,
     InheritBaseModel,
     IntegerData,
@@ -239,24 +241,23 @@ def inherited_compare(testcase, pk, klass, data):
         testcase.assertEqual(value, getattr(instance, key))
 
 
-# Define some data types. Each data type is
+# Define some test helpers. Each has a pair of functions: one to create objects and one
-# actually a pair of functions; one to create
+# to make assertions against objects of a particular type.
-# and one to compare objects of that type
+TestHelper = namedtuple("TestHelper", ["create_object", "compare_object"])
-data_obj = (data_create, data_compare)
+data_obj = TestHelper(data_create, data_compare)
-generic_obj = (generic_create, generic_compare)
+generic_obj = TestHelper(generic_create, generic_compare)
-fk_obj = (fk_create, fk_compare)
+fk_obj = TestHelper(fk_create, fk_compare)
-m2m_obj = (m2m_create, m2m_compare)
+m2m_obj = TestHelper(m2m_create, m2m_compare)
-im2m_obj = (im2m_create, im2m_compare)
+im2m_obj = TestHelper(im2m_create, im2m_compare)
-im_obj = (im_create, im_compare)
+im_obj = TestHelper(im_create, im_compare)
-o2o_obj = (o2o_create, o2o_compare)
+o2o_obj = TestHelper(o2o_create, o2o_compare)
-pk_obj = (pk_create, pk_compare)
+pk_obj = TestHelper(pk_create, pk_compare)
-inherited_obj = (inherited_create, inherited_compare)
+inherited_obj = TestHelper(inherited_create, inherited_compare)
 uuid_obj = uuid.uuid4()
 
 test_data = [
-    # Format: (data type, PK value, Model Class, data)
+    # Format: (test helper, PK value, Model Class, data)
     (data_obj, 1, BinaryData, memoryview(b"\x05\xFD\x00")),
-    (data_obj, 2, BinaryData, None),
     (data_obj, 5, BooleanData, True),
     (data_obj, 6, BooleanData, False),
     (data_obj, 7, BooleanData, None),
@@ -265,7 +266,6 @@ test_data = [
     (data_obj, 12, CharData, "None"),
     (data_obj, 13, CharData, "null"),
     (data_obj, 14, CharData, "NULL"),
-    (data_obj, 15, CharData, None),
     # (We use something that will fit into a latin1 database encoding here,
     # because that is still the default used on many system setups.)
     (data_obj, 16, CharData, "\xa5"),
@@ -274,13 +274,11 @@ test_data = [
     (data_obj, 30, DateTimeData, datetime.datetime(2006, 6, 16, 10, 42, 37)),
     (data_obj, 31, DateTimeData, None),
     (data_obj, 40, EmailData, "hovercraft@example.com"),
-    (data_obj, 41, EmailData, None),
     (data_obj, 42, EmailData, ""),
     (data_obj, 50, FileData, "file:///foo/bar/whiz.txt"),
     # (data_obj, 51, FileData, None),
     (data_obj, 52, FileData, ""),
     (data_obj, 60, FilePathData, "/foo/bar/whiz.txt"),
-    (data_obj, 61, FilePathData, None),
     (data_obj, 62, FilePathData, ""),
     (data_obj, 70, DecimalData, decimal.Decimal("12.345")),
     (data_obj, 71, DecimalData, decimal.Decimal("-12.345")),
@@ -294,7 +292,6 @@ test_data = [
     (data_obj, 81, IntegerData, -123456789),
     (data_obj, 82, IntegerData, 0),
     (data_obj, 83, IntegerData, None),
-    # (XX, ImageData
     (data_obj, 95, GenericIPAddressData, "fe80:1424:2223:6cff:fe8a:2e8a:2151:abcd"),
     (data_obj, 96, GenericIPAddressData, None),
     (data_obj, 110, PositiveBigIntegerData, 9223372036854775807),
@@ -304,7 +301,6 @@ test_data = [
     (data_obj, 130, PositiveSmallIntegerData, 12),
     (data_obj, 131, PositiveSmallIntegerData, None),
     (data_obj, 140, SlugData, "this-is-a-slug"),
-    (data_obj, 141, SlugData, None),
     (data_obj, 142, SlugData, ""),
     (data_obj, 150, SmallData, 12),
     (data_obj, 151, SmallData, -12),
@@ -320,7 +316,6 @@ Several of them.
 The end.""",
     ),
     (data_obj, 161, TextData, ""),
-    (data_obj, 162, TextData, None),
     (data_obj, 170, TimeData, datetime.time(10, 42, 37)),
     (data_obj, 171, TimeData, None),
     (generic_obj, 200, GenericData, ["Generic Object 1", "tag1", "tag2"]),
@@ -388,15 +383,6 @@ The end.""",
     (pk_obj, 750, SmallPKData, 12),
     (pk_obj, 751, SmallPKData, -12),
     (pk_obj, 752, SmallPKData, 0),
-    (
-        pk_obj,
-        760,
-        TextPKData,
-        """This is a long piece of text.
-    It contains line breaks.
-    Several of them.
-    The end.""",
-    ),
     (pk_obj, 770, TimePKData, datetime.time(10, 42, 37)),
     (pk_obj, 791, UUIDData, uuid_obj),
     (fk_obj, 792, FKToUUID, uuid_obj),
@@ -419,71 +405,107 @@ The end.""",
     (data_obj, 1005, LengthModel, 1),
 ]
 
-
+if ImageData is not None:
-# Because Oracle treats the empty string as NULL, Oracle is expected to fail
+    test_data.extend(
-# when field.empty_strings_allowed is True and the value is None; skip these
+        [
-# tests.
+            (data_obj, 86, ImageData, "file:///foo/bar/whiz.png"),
-if connection.features.interprets_empty_strings_as_nulls:
+            # (data_obj, 87, ImageData, None),
-    test_data = [
+            (data_obj, 88, ImageData, ""),
-        data
-        for data in test_data
-        if not (
-            data[0] == data_obj
-            and data[2]._meta.get_field("data").empty_strings_allowed
-            and data[3] is None
-        )
         ]
-
+    )
-
-if not connection.features.supports_index_on_text_field:
-    test_data = [data for data in test_data if data[2] != TextPKData]
 
 
 class SerializerDataTests(TestCase):
     pass
 
 
-def serializerTest(self, format):
+def assert_serializer(self, format, data):
-    # FK to an object with PK of 0. This won't work on MySQL without the
+    # Create all the objects defined in the test data.
-    # NO_AUTO_VALUE_ON_ZERO SQL mode since it won't let you create an object
-    # with an autoincrement primary key of 0.
-    if connection.features.allows_auto_pk_0:
-        test_data.extend(
-            [
-                (data_obj, 0, Anchor, "Anchor 0"),
-                (fk_obj, 465, FKData, 0),
-            ]
-        )
-
-    # Create all the objects defined in the test data
     objects = []
-    instance_count = {}
+    for test_helper, pk, model, data_value in data:
-    for func, pk, klass, datum in test_data:
         with connection.constraint_checks_disabled():
-            objects.extend(func[0](pk, klass, datum))
+            objects.extend(test_helper.create_object(pk, model, data_value))
 
-    # Get a count of the number of objects created for each class
+    # Get a count of the number of objects created for each model class.
-    for klass in instance_count:
+    instance_counts = {}
-        instance_count[klass] = klass.objects.count()
+    for _, _, model, _ in data:
+        if model not in instance_counts:
+            instance_counts[model] = model.objects.count()
 
-    # Add the generic tagged objects to the object list
+    # Add the generic tagged objects to the object list.
     objects.extend(Tag.objects.all())
 
-    # Serialize the test database
+    # Serialize the test database.
     serialized_data = serializers.serialize(format, objects, indent=2)
 
     for obj in serializers.deserialize(format, serialized_data):
         obj.save()
 
-    # Assert that the deserialized data is the same
+    # Assert that the deserialized data is the same as the original source.
-    # as the original source
+    for test_helper, pk, model, data_value in data:
-    for func, pk, klass, datum in test_data:
+        with self.subTest(model=model, data_value=data_value):
-        func[1](self, pk, klass, datum)
+            test_helper.compare_object(self, pk, model, data_value)
 
-    # Assert that the number of objects deserialized is the
+    # Assert no new objects were created.
-    # same as the number that was serialized.
+    for model, count in instance_counts.items():
-    for klass, count in instance_count.items():
+        with self.subTest(model=model, count=count):
-        self.assertEqual(count, klass.objects.count())
+            self.assertEqual(count, model.objects.count())
+
+
+def serializerTest(self, format):
+    assert_serializer(self, format, test_data)
+
+
+@skipUnlessDBFeature("allows_auto_pk_0")
+def serializerTestPK0(self, format):
+    # FK to an object with PK of 0. This won't work on MySQL without the
+    # NO_AUTO_VALUE_ON_ZERO SQL mode since it won't let you create an object
+    # with an autoincrement primary key of 0.
+    data = [
+        (data_obj, 0, Anchor, "Anchor 0"),
+        (fk_obj, 1, FKData, 0),
+    ]
+    assert_serializer(self, format, data)
+
+
+@skipIfDBFeature("interprets_empty_strings_as_nulls")
+def serializerTestNullValueStingField(self, format):
+    data = [
+        (data_obj, 1, BinaryData, None),
+        (data_obj, 2, CharData, None),
+        (data_obj, 3, EmailData, None),
+        (data_obj, 4, FilePathData, None),
+        (data_obj, 5, SlugData, None),
+        (data_obj, 6, TextData, None),
+    ]
+    assert_serializer(self, format, data)
+
+
+@skipUnlessDBFeature("supports_index_on_text_field")
+def serializerTestTextFieldPK(self, format):
+    data = [
+        (
+            pk_obj,
+            1,
+            TextPKData,
+            """This is a long piece of text.
+            It contains line breaks.
+            Several of them.
+            The end.""",
+        ),
+    ]
+    assert_serializer(self, format, data)
 
 
 register_tests(SerializerDataTests, "test_%s_serializer", serializerTest)
+register_tests(SerializerDataTests, "test_%s_serializer_pk_0", serializerTestPK0)
+register_tests(
+    SerializerDataTests,
+    "test_%s_serializer_null_value_string_field",
+    serializerTestNullValueStingField,
+)
+register_tests(
+    SerializerDataTests,
+    "test_%s_serializer_text_field_pk",
+    serializerTestTextFieldPK,
+)

tests/template_tests/syntax_tests/test_include.py

@@ -330,6 +330,7 @@ class IncludeTests(SimpleTestCase):
                 ],
             }
         ]
+        with self.subTest(template="recursive_include.html"):
             engine = Engine(app_dirs=True)
             t = engine.get_template("recursive_include.html")
             self.assertEqual(
@@ -339,6 +340,33 @@ class IncludeTests(SimpleTestCase):
                 .replace("\n", " ")
                 .strip(),
             )
+        with self.subTest(template="recursive_relative_include.html"):
+            engine = Engine(app_dirs=True)
+            t = engine.get_template("recursive_relative_include.html")
+            self.assertEqual(
+                "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
+                t.render(Context({"comments": comments}))
+                .replace(" ", "")
+                .replace("\n", " ")
+                .strip(),
+            )
+        with self.subTest(template="tmpl"):
+            engine = Engine()
+            template = """
+            Recursion!
+            {% for c in comments %}
+              {{ c.comment }}
+              {% if c.children %}{% include tmpl with comments=c.children %}{% endif %}
+            {% endfor %}
+            """
+            outer_tmpl = engine.from_string("{% include tmpl %}")
+            output = outer_tmpl.render(
+                Context({"tmpl": engine.from_string(template), "comments": comments})
+            )
+            self.assertEqual(
+                "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
+                output.replace(" ", "").replace("\n", " ").strip(),
+            )
 
     def test_include_cache(self):
         """

tests/template_tests/templates/recursive_relative_include.html

@@ -0,0 +1,7 @@
+Recursion!
+{% for comment in comments %}
+    {{ comment.comment }}
+    {% if comment.children %}
+        {% include "./recursive_relative_include.html" with comments=comment.children %}
+    {% endif %}
+{% endfor %}

tests/utils_tests/test_html.py

@@ -1,6 +1,7 @@
 import os
 from datetime import datetime
 
+from django.core.exceptions import SuspiciousOperation
 from django.core.serializers.json import DjangoJSONEncoder
 from django.test import SimpleTestCase
 from django.utils.deprecation import RemovedInDjango60Warning
@@ -145,12 +146,18 @@ class TestUtilsHtml(SimpleTestCase):
             ("<script>alert()</script>&h", "alert()h"),
             ("><!" + ("&" * 16000) + "D", "><!" + ("&" * 16000) + "D"),
             ("X<<<<br>br>br>br>X", "XX"),
+            ("<" * 50 + "a>" * 50, ""),
         )
         for value, output in items:
             with self.subTest(value=value, output=output):
                 self.check_output(strip_tags, value, output)
                 self.check_output(strip_tags, lazystr(value), output)
 
+    def test_strip_tags_suspicious_operation(self):
+        value = "<" * 51 + "a>" * 51, "<a>"
+        with self.assertRaises(SuspiciousOperation):
+            strip_tags(value)
+
     def test_strip_tags_files(self):
         # Test with more lengthy content (also catching performance regressions)
         for filename in ("strip_tags1.html", "strip_tags2.txt"):