Merge remote-tracking branch 'django/main'

django/contrib/postgres/fields/array.py

@@ -169,7 +169,7 @@ class ArrayField(CheckFieldDefaultMixin, Field):
             else:
                 obj = AttributeSetter(base_field.attname, val)
                 values.append(base_field.value_to_string(obj))
-        return json.dumps(values)
+        return json.dumps(values, ensure_ascii=False)
 
     def get_transform(self, name):
         transform = super().get_transform(name)

django/contrib/postgres/fields/hstore.py

@@ -43,7 +43,7 @@ class HStoreField(CheckFieldDefaultMixin, Field):
         return value
 
     def value_to_string(self, obj):
-        return json.dumps(self.value_from_object(obj))
+        return json.dumps(self.value_from_object(obj), ensure_ascii=False)
 
     def formfield(self, **kwargs):
         return super().formfield(

django/core/management/commands/makemessages.py

@@ -40,7 +40,7 @@ def check_programs(*programs):
 
 
 def is_valid_locale(locale):
-    return re.match(r"^[a-z]+$", locale) or re.match(r"^[a-z]+_[A-Z].*$", locale)
+    return re.match(r"^[a-z]+$", locale) or re.match(r"^[a-z]+_[A-Z0-9].*$", locale)
 
 
 @total_ordering

django/core/management/commands/sqlmigrate.py

@@ -32,10 +32,9 @@ class Command(BaseCommand):
         )
 
     def execute(self, *args, **options):
-        # sqlmigrate doesn't support coloring its output but we need to force
+        # sqlmigrate doesn't support coloring its output, so make the
-        # no_color=True so that the BEGIN/COMMIT statements added by
+        # BEGIN/COMMIT statements added by output_transaction colorless also.
-        # output_transaction don't get colored either.
+        self.style.SQL_KEYWORD = lambda noop: noop
-        options["no_color"] = True
         return super().execute(*args, **options)
 
     def handle(self, *args, **options):

django/db/backends/base/operations.py

@@ -8,7 +8,6 @@ import sqlparse
 
 from django.conf import settings
 from django.db import NotSupportedError, transaction
-from django.db.backends import utils
 from django.db.models.expressions import Col
 from django.utils import timezone
 from django.utils.deprecation import RemovedInDjango60Warning
@@ -586,7 +585,7 @@ class BaseDatabaseOperations:
         Transform a decimal.Decimal value to an object compatible with what is
         expected by the backend driver for decimal (numeric) columns.
         """
-        return utils.format_number(value, max_digits, decimal_places)
+        return value
 
     def adapt_ipaddressfield_value(self, value):
         """

django/db/backends/mysql/operations.py

@@ -166,9 +166,6 @@ class DatabaseOperations(BaseDatabaseOperations):
         """
         return [(None, ("NULL", [], False))]
 
-    def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
-        return value
-
     def last_executed_query(self, cursor, sql, params):
         # With MySQLdb, cursor objects have an (undocumented) "_executed"
         # attribute where the exact query sent to the database is saved.

django/db/backends/oracle/operations.py

@@ -629,9 +629,6 @@ END;
             1900, 1, 1, value.hour, value.minute, value.second, value.microsecond
         )
 
-    def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
-        return value
-
     def combine_expression(self, connector, sub_expressions):
         lhs, rhs = sub_expressions
         if connector == "%%":

django/db/backends/postgresql/operations.py

@@ -346,9 +346,6 @@ class DatabaseOperations(BaseDatabaseOperations):
     def adapt_timefield_value(self, value):
         return value
 
-    def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
-        return value
-
     def adapt_ipaddressfield_value(self, value):
         if value:
             return Inet(value)

django/db/backends/sqlite3/features.py

@@ -50,6 +50,10 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         # The django_format_dtdelta() function doesn't properly handle mixed
         # Date/DateTime fields and timedeltas.
         "expressions.tests.FTimeDeltaTests.test_mixed_comparisons1",
+        # SQLite doesn't parse escaped double quotes in the JSON path notation,
+        # so it cannot match keys that contains double quotes (#35842).
+        "model_fields.test_jsonfield.TestQuerying."
+        "test_lookups_special_chars_double_quotes",
     }
     create_test_table_with_composite_primary_key = """
         CREATE TABLE test_table_composite_pk (

django/db/models/base.py

@@ -726,12 +726,13 @@ class Model(AltersData, metaclass=ModelBase):
         if fields is not None:
             db_instance_qs = db_instance_qs.only(*fields)
         elif deferred_fields:
-            fields = {
+            db_instance_qs = db_instance_qs.only(
-                f.attname
+                *{
-                for f in self._meta.concrete_fields
+                    f.attname
-                if f.attname not in deferred_fields
+                    for f in self._meta.concrete_fields
-            }
+                    if f.attname not in deferred_fields
-            db_instance_qs = db_instance_qs.only(*fields)
+                }
+            )
 
         db_instance = db_instance_qs.get()
         non_loaded_fields = db_instance.get_deferred_fields()
@@ -748,9 +749,9 @@ class Model(AltersData, metaclass=ModelBase):
                     field.delete_cached_value(self)
 
         # Clear cached relations.
-        for field in self._meta.related_objects:
+        for rel in self._meta.related_objects:
-            if (fields is None or field.name in fields) and field.is_cached(self):
+            if (fields is None or rel.name in fields) and rel.is_cached(self):
-                field.delete_cached_value(self)
+                rel.delete_cached_value(self)
 
         # Clear cached private relations.
         for field in self._meta.private_fields:

django/db/models/fields/__init__.py

@@ -1828,9 +1828,8 @@ class DecimalField(Field):
             )
         return decimal_value
 
-    def get_db_prep_save(self, value, connection):
+    def get_db_prep_value(self, value, connection, prepared=False):
-        if hasattr(value, "as_sql"):
+        value = super().get_db_prep_value(value, connection, prepared)
-            return value
         return connection.ops.adapt_decimalfield_value(
             self.to_python(value), self.max_digits, self.decimal_places
         )

django/db/models/fields/json.py

@@ -193,20 +193,18 @@ class HasKeyLookup(PostgresOperatorLookup):
         # Compile the final key without interpreting ints as array elements.
         return ".%s" % json.dumps(key_transform)
 
-    def as_sql(self, compiler, connection, template=None):
+    def _as_sql_parts(self, compiler, connection):
         # Process JSON path from the left-hand side.
         if isinstance(self.lhs, KeyTransform):
-            lhs, lhs_params, lhs_key_transforms = self.lhs.preprocess_lhs(
+            lhs_sql, lhs_params, lhs_key_transforms = self.lhs.preprocess_lhs(
                 compiler, connection
             )
             lhs_json_path = compile_json_path(lhs_key_transforms)
         else:
-            lhs, lhs_params = self.process_lhs(compiler, connection)
+            lhs_sql, lhs_params = self.process_lhs(compiler, connection)
             lhs_json_path = "$"
-        sql = template % lhs
         # Process JSON path from the right-hand side.
         rhs = self.rhs
-        rhs_params = []
         if not isinstance(rhs, (list, tuple)):
             rhs = [rhs]
         for key in rhs:
@@ -217,24 +215,45 @@ class HasKeyLookup(PostgresOperatorLookup):
             *rhs_key_transforms, final_key = rhs_key_transforms
             rhs_json_path = compile_json_path(rhs_key_transforms, include_root=False)
             rhs_json_path += self.compile_json_path_final_key(final_key)
-            rhs_params.append(lhs_json_path + rhs_json_path)
+            yield lhs_sql, lhs_params, lhs_json_path + rhs_json_path
+
+    def _combine_sql_parts(self, parts):
         # Add condition for each key.
         if self.logical_operator:
-            sql = "(%s)" % self.logical_operator.join([sql] * len(rhs_params))
+            return "(%s)" % self.logical_operator.join(parts)
-        return sql, tuple(lhs_params) + tuple(rhs_params)
+        return "".join(parts)
+
+    def as_sql(self, compiler, connection, template=None):
+        sql_parts = []
+        params = []
+        for lhs_sql, lhs_params, rhs_json_path in self._as_sql_parts(
+            compiler, connection
+        ):
+            sql_parts.append(template % (lhs_sql, "%s"))
+            params.extend(lhs_params + [rhs_json_path])
+        return self._combine_sql_parts(sql_parts), tuple(params)
 
     def as_mysql(self, compiler, connection):
         return self.as_sql(
-            compiler, connection, template="JSON_CONTAINS_PATH(%s, 'one', %%s)"
+            compiler, connection, template="JSON_CONTAINS_PATH(%s, 'one', %s)"
         )
 
     def as_oracle(self, compiler, connection):
-        sql, params = self.as_sql(
+        # Use a custom delimiter to prevent the JSON path from escaping the SQL
-            compiler, connection, template="JSON_EXISTS(%s, '%%s')"
+        # literal. See comment in KeyTransform.
-        )
+        template = "JSON_EXISTS(%s, q'\uffff%s\uffff')"
-        # Add paths directly into SQL because path expressions cannot be passed
+        sql_parts = []
-        # as bind variables on Oracle.
+        params = []
-        return sql % tuple(params), []
+        for lhs_sql, lhs_params, rhs_json_path in self._as_sql_parts(
+            compiler, connection
+        ):
+            # Add right-hand-side directly into SQL because it cannot be passed
+            # as bind variables to JSON_EXISTS. It might result in invalid
+            # queries but it is assumed that it cannot be evaded because the
+            # path is JSON serialized.
+            sql_parts.append(template % (lhs_sql, rhs_json_path))
+            params.extend(lhs_params)
+        return self._combine_sql_parts(sql_parts), tuple(params)
 
     def as_postgresql(self, compiler, connection):
         if isinstance(self.rhs, KeyTransform):
@@ -246,7 +265,7 @@ class HasKeyLookup(PostgresOperatorLookup):
 
     def as_sqlite(self, compiler, connection):
         return self.as_sql(
-            compiler, connection, template="JSON_TYPE(%s, %%s) IS NOT NULL"
+            compiler, connection, template="JSON_TYPE(%s, %s) IS NOT NULL"
         )
 
 
@@ -362,10 +381,24 @@ class KeyTransform(Transform):
         json_path = compile_json_path(key_transforms)
         if connection.features.supports_primitives_in_json_field:
             sql = (
-                "COALESCE(JSON_VALUE(%s, '%s'), JSON_QUERY(%s, '%s' DISALLOW SCALARS))"
+                "COALESCE("
+                "JSON_VALUE(%s, q'\uffff%s\uffff'),"
+                "JSON_QUERY(%s, q'\uffff%s\uffff' DISALLOW SCALARS)"
+                ")"
             )
         else:
-            sql = "COALESCE(JSON_QUERY(%s, '%s'), JSON_VALUE(%s, '%s'))"
+            sql = (
+                "COALESCE("
+                "JSON_QUERY(%s, q'\uffff%s\uffff'),"
+                "JSON_VALUE(%s, q'\uffff%s\uffff')"
+                ")"
+            )
+        # Add paths directly into SQL because path expressions cannot be passed
+        # as bind variables on Oracle. Use a custom delimiter to prevent the
+        # JSON path from escaping the SQL literal. Each key in the JSON path is
+        # passed through json.dumps() with ensure_ascii=True (the default),
+        # which converts the delimiter into the escaped \uffff format. This
+        # ensures that the delimiter is not present in the JSON path.
         return sql % ((lhs, json_path) * 2), tuple(params) * 2
 
     def as_postgresql(self, compiler, connection):
@@ -455,9 +488,9 @@ class KeyTransformIsNull(lookups.IsNull):
         return "(NOT %s OR %s IS NULL)" % (sql, lhs), tuple(params) + tuple(lhs_params)
 
     def as_sqlite(self, compiler, connection):
-        template = "JSON_TYPE(%s, %%s) IS NULL"
+        template = "JSON_TYPE(%s, %s) IS NULL"
         if not self.rhs:
-            template = "JSON_TYPE(%s, %%s) IS NOT NULL"
+            template = "JSON_TYPE(%s, %s) IS NOT NULL"
         return HasKeyOrArrayIndex(self.lhs.lhs, self.lhs.key_name).as_sql(
             compiler,
             connection,

django/forms/forms.py

@@ -298,7 +298,10 @@ class BaseForm(RenderableFormMixin):
                         error_class="nonfield", renderer=self.renderer
                     )
                 else:
-                    self._errors[field] = self.error_class(renderer=self.renderer)
+                    self._errors[field] = self.error_class(
+                        renderer=self.renderer,
+                        field_id=self[field].auto_id,
+                    )
             self._errors[field].extend(error_list)
             if field in self.cleaned_data:
                 del self.cleaned_data[field]

django/forms/jinja2/django/forms/errors/list/ul.html

@@ -1 +1 @@
-{% if errors %}<ul class="{{ error_class }}">{% for error in errors %}<li>{{ error }}</li>{% endfor %}</ul>{% endif %}
+{% if errors %}<ul class="{{ error_class }}"{% if errors.field_id %} id="{{ errors.field_id }}_error"{% endif %}>{% for error in errors %}<li>{{ error }}</li>{% endfor %}</ul>{% endif %}

django/forms/templates/django/forms/errors/list/ul.html

@@ -1 +1 @@
-{% if errors %}<ul class="{{ error_class }}">{% for error in errors %}<li>{{ error }}</li>{% endfor %}</ul>{% endif %}
+{% if errors %}<ul class="{{ error_class }}"{% if errors.field_id %} id="{{ errors.field_id }}_error"{% endif %}>{% for error in errors %}<li>{{ error }}</li>{% endfor %}</ul>{% endif %}

django/forms/utils.py

@@ -147,7 +147,7 @@ class ErrorList(UserList, list, RenderableErrorMixin):
     template_name_text = "django/forms/errors/list/text.txt"
     template_name_ul = "django/forms/errors/list/ul.html"
 
-    def __init__(self, initlist=None, error_class=None, renderer=None):
+    def __init__(self, initlist=None, error_class=None, renderer=None, field_id=None):
         super().__init__(initlist)
 
         if error_class is None:
@@ -155,6 +155,7 @@ class ErrorList(UserList, list, RenderableErrorMixin):
         else:
             self.error_class = "errorlist {}".format(error_class)
         self.renderer = renderer or get_default_renderer()
+        self.field_id = field_id
 
     def as_data(self):
         return ValidationError(self.data).error_list

django/template/loader_tags.py

@@ -242,7 +242,11 @@ def do_block(parser, token):
     return BlockNode(block_name, nodelist)
 
 
-def construct_relative_path(current_template_name, relative_name):
+def construct_relative_path(
+    current_template_name,
+    relative_name,
+    allow_recursion=False,
+):
     """
     Convert a relative path (starting with './' or '../') to the full template
     name based on the current_template_name.
@@ -264,7 +268,7 @@ def construct_relative_path(current_template_name, relative_name):
             "The relative path '%s' points outside the file hierarchy that "
             "template '%s' is in." % (relative_name, current_template_name)
         )
-    if current_template_name.lstrip("/") == new_name:
+    if not allow_recursion and current_template_name.lstrip("/") == new_name:
         raise TemplateSyntaxError(
             "The relative path '%s' was translated to template name '%s', the "
             "same template in which the tag appears."
@@ -346,7 +350,11 @@ def do_include(parser, token):
         options[option] = value
     isolated_context = options.get("only", False)
     namemap = options.get("with", {})
-    bits[1] = construct_relative_path(parser.origin.template_name, bits[1])
+    bits[1] = construct_relative_path(
+        parser.origin.template_name,
+        bits[1],
+        allow_recursion=True,
+    )
     return IncludeNode(
         parser.compile_filter(bits[1]),
         extra_context=namemap,

django/utils/html.py

@@ -8,6 +8,7 @@ from collections.abc import Mapping
 from html.parser import HTMLParser
 from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit, urlunsplit
 
+from django.core.exceptions import SuspiciousOperation
 from django.utils.deprecation import RemovedInDjango60Warning
 from django.utils.encoding import punycode
 from django.utils.functional import Promise, cached_property, keep_lazy, keep_lazy_text
@@ -40,6 +41,7 @@ VOID_ELEMENTS = frozenset(
 )
 
 MAX_URL_LENGTH = 2048
+MAX_STRIP_TAGS_DEPTH = 50
 
 
 @keep_lazy(SafeString)
@@ -211,15 +213,19 @@ def _strip_once(value):
 @keep_lazy_text
 def strip_tags(value):
     """Return the given HTML with all tags stripped."""
-    # Note: in typical case this loop executes _strip_once once. Loop condition
-    # is redundant, but helps to reduce number of executions of _strip_once.
     value = str(value)
+    # Note: in typical case this loop executes _strip_once twice (the second
+    # execution does not remove any more tags).
+    strip_tags_depth = 0
     while "<" in value and ">" in value:
+        if strip_tags_depth >= MAX_STRIP_TAGS_DEPTH:
+            raise SuspiciousOperation
         new_value = _strip_once(value)
         if value.count("<") == new_value.count("<"):
             # _strip_once wasn't able to detect more tags.
             break
         value = new_value
+        strip_tags_depth += 1
     return value
 
 

docs/intro/tutorial03.txt

@@ -234,9 +234,7 @@ Now let's update our ``index`` view in ``polls/views.py`` to use the template:
     def index(request):
         latest_question_list = Question.objects.order_by("-pub_date")[:5]
         template = loader.get_template("polls/index.html")
-        context = {
+        context = {"latest_question_list": latest_question_list}
-            "latest_question_list": latest_question_list,
-        }
         return HttpResponse(template.render(context, request))
 
 That code loads the template called  ``polls/index.html`` and passes it a

docs/ref/contrib/gis/db-api.txt

@@ -407,6 +407,7 @@ Function                              PostGIS  Oracle         MariaDB      MySQL
 :class:`FromWKB`                      X        X              X            X           X
 :class:`FromWKT`                      X        X              X            X           X
 :class:`GeoHash`                      X                       X (≥ 11.7)   X           X (LWGEOM/RTTOPO)
+:class:`GeometryDistance`             X
 :class:`Intersection`                 X        X              X            X           X
 :class:`IsEmpty`                      X
 :class:`IsValid`                      X        X              X (≥ 11.7)   X           X

docs/ref/forms/api.txt

@@ -1025,13 +1025,17 @@ method you're using:
 Customizing the error list format
 ---------------------------------
 
-.. class:: ErrorList(initlist=None, error_class=None, renderer=None)
+.. class:: ErrorList(initlist=None, error_class=None, renderer=None, field_id=None)
 
     By default, forms use ``django.forms.utils.ErrorList`` to format validation
     errors. ``ErrorList`` is a list like object where ``initlist`` is the
     list of errors. In addition this class has the following attributes and
     methods.
 
+    .. versionchanged:: 5.2
+
+        The ``field_id`` argument was added.
+
     .. attribute:: error_class
 
         The CSS classes to be used when rendering the error list. Any provided
@@ -1043,6 +1047,16 @@ Customizing the error list format
         Defaults to ``None`` which means to use the default renderer
         specified by the :setting:`FORM_RENDERER` setting.
 
+    .. attribute:: field_id
+
+        .. versionadded:: 5.2
+
+        An ``id`` for the field for which the errors relate. This allows an
+        HTML ``id`` attribute to be added in the error template and is useful
+        to associate the errors with the field. The default template uses the
+        format ``id="{{ field_id }}_error"`` and a value is provided by
+        :meth:`.Form.add_error` using the field's :attr:`~.BoundField.auto_id`.
+
     .. attribute:: template_name
 
         The name of the template used when calling ``__str__`` or

docs/ref/request-response.txt

@@ -554,12 +554,21 @@ a subclass of dictionary. Exceptions are outlined here:
 
 .. method:: QueryDict.__getitem__(key)
 
-    Returns the value for the given key. If the key has more than one value,
+    Returns the last value for the given key; or an empty list (``[]``) if the
-    it returns the last value. Raises
+    key exists but has no values. Raises
     ``django.utils.datastructures.MultiValueDictKeyError`` if the key does not
     exist. (This is a subclass of Python's standard :exc:`KeyError`, so you can
     stick to catching ``KeyError``.)
 
+    .. code-block:: pycon
+
+        >>> q = QueryDict("a=1&a=2&a=3", mutable=True)
+        >>> q.__getitem__("a")
+        '3'
+        >>> q.__setitem__("b", [])
+        >>> q.__getitem__("b")
+        []
+
 .. method:: QueryDict.__setitem__(key, value)
 
     Sets the given key to ``[value]`` (a list whose single element is

docs/ref/utils.txt

@@ -521,7 +521,7 @@ https://web.archive.org/web/20110718035220/http://diveintomark.org/archives/2004
     The cached value can be treated like an ordinary attribute of the instance::
 
         # clear it, requiring re-computation next time it's called
-        del person.friends  # or delattr(person, "friends")
+        person.__dict__.pop("friends", None)
 
         # set a value manually, that will persist on the instance until cleared
         person.friends = ["Huckleberry Finn", "Tom Sawyer"]

docs/releases/4.2.17.txt

@@ -6,3 +6,28 @@ Django 4.2.17 release notes
 
 Django 4.2.17 fixes one security issue with severity "high" and one security
 issue with severity "moderate" in 4.2.16.
+
+CVE-2024-53907: Denial-of-service possibility in ``strip_tags()``
+=================================================================
+
+:func:`~django.utils.html.strip_tags` would be extremely slow to evaluate
+certain inputs containing large sequences of nested incomplete HTML entities.
+The ``strip_tags()`` method is used to implement the corresponding
+:tfilter:`striptags` template filter, which was thus also vulnerable.
+
+``strip_tags()`` now has an upper limit of recursive calls to ``HTMLParser``
+before raising a :exc:`.SuspiciousOperation` exception.
+
+Remember that absolutely NO guarantee is provided about the results of
+``strip_tags()`` being HTML safe. So NEVER mark safe the result of a
+``strip_tags()`` call without escaping it first, for example with
+:func:`django.utils.html.escape`.
+
+CVE-2024-53908: Potential SQL injection via ``HasKey(lhs, rhs)`` on Oracle
+==========================================================================
+
+Direct usage of the ``django.db.models.fields.json.HasKey`` lookup on Oracle
+was subject to SQL injection if untrusted data was used as a ``lhs`` value.
+
+Applications that use the :lookup:`has_key <jsonfield.has_key>` lookup through
+the ``__`` syntax are unaffected.

docs/releases/5.0.10.txt

@@ -6,3 +6,28 @@ Django 5.0.10 release notes
 
 Django 5.0.10 fixes one security issue with severity "high" and one security
 issue with severity "moderate" in 5.0.9.
+
+CVE-2024-53907: Denial-of-service possibility in ``strip_tags()``
+=================================================================
+
+:func:`~django.utils.html.strip_tags` would be extremely slow to evaluate
+certain inputs containing large sequences of nested incomplete HTML entities.
+The ``strip_tags()`` method is used to implement the corresponding
+:tfilter:`striptags` template filter, which was thus also vulnerable.
+
+``strip_tags()`` now has an upper limit of recursive calls to ``HTMLParser``
+before raising a :exc:`.SuspiciousOperation` exception.
+
+Remember that absolutely NO guarantee is provided about the results of
+``strip_tags()`` being HTML safe. So NEVER mark safe the result of a
+``strip_tags()`` call without escaping it first, for example with
+:func:`django.utils.html.escape`.
+
+CVE-2024-53908: Potential SQL injection via ``HasKey(lhs, rhs)`` on Oracle
+==========================================================================
+
+Direct usage of the ``django.db.models.fields.json.HasKey`` lookup on Oracle
+was subject to SQL injection if untrusted data was used as a ``lhs`` value.
+
+Applications that use the :lookup:`has_key <jsonfield.has_key>` lookup through
+the ``__`` syntax are unaffected.

docs/releases/5.1.4.txt

@@ -7,8 +7,37 @@ Django 5.1.4 release notes
 Django 5.1.4 fixes one security issue with severity "high", one security issue
 with severity "moderate", and several bugs in 5.1.3.
 
+CVE-2024-53907: Denial-of-service possibility in ``strip_tags()``
+=================================================================
+
+:func:`~django.utils.html.strip_tags` would be extremely slow to evaluate
+certain inputs containing large sequences of nested incomplete HTML entities.
+The ``strip_tags()`` method is used to implement the corresponding
+:tfilter:`striptags` template filter, which was thus also vulnerable.
+
+``strip_tags()`` now has an upper limit of recursive calls to ``HTMLParser``
+before raising a :exc:`.SuspiciousOperation` exception.
+
+Remember that absolutely NO guarantee is provided about the results of
+``strip_tags()`` being HTML safe. So NEVER mark safe the result of a
+``strip_tags()`` call without escaping it first, for example with
+:func:`django.utils.html.escape`.
+
+CVE-2024-53908: Potential SQL injection via ``HasKey(lhs, rhs)`` on Oracle
+==========================================================================
+
+Direct usage of the ``django.db.models.fields.json.HasKey`` lookup on Oracle
+was subject to SQL injection if untrusted data was used as a ``lhs`` value.
+
+Applications that use the :lookup:`has_key <jsonfield.has_key>` lookup through
+the ``__`` syntax are unaffected.
+
 Bugfixes
 ========
 
 * Fixed a crash in ``createsuperuser`` on Python 3.13+ caused by an unhandled
   ``OSError`` when the username could not be determined (:ticket:`35942`).
+
+* Fixed a regression in Django 5.1 where relational fields were not updated
+  when calling ``Model.refresh_from_db()`` on instances with deferred fields
+  (:ticket:`35950`).

docs/releases/5.1.5.txt

@@ -0,0 +1,12 @@
+==========================
+Django 5.1.5 release notes
+==========================
+
+*Expected January 7, 2025*
+
+Django 5.1.5 fixes several bugs in 5.1.4.
+
+Bugfixes
+========
+
+* ...

docs/releases/5.2.txt

@@ -249,6 +249,10 @@ Forms
 * The new :class:`~django.forms.TelInput` form widget is for entering telephone
   numbers and renders as ``<input type="tel" ...>``.
 
+* The new ``field_id`` argument for :class:`~django.forms.ErrorList` allows an
+  HTML ``id`` attribute to be added in the error template. See
+  :attr:`.ErrorList.field_id` for details.
+
 Generic Views
 ~~~~~~~~~~~~~
 
@@ -395,6 +399,8 @@ backends.
 * The new :meth:`Model._is_pk_set() <django.db.models.Model._is_pk_set>` method
   allows checking if a Model instance's primary key is defined.
 
+* ``BaseDatabaseOperations.adapt_decimalfield_value()`` is now a no-op, simply
+  returning the given value.
 
 :mod:`django.contrib.gis`
 -------------------------

docs/releases/index.txt

@@ -32,6 +32,7 @@ versions of the documentation contain the release notes for any later releases.
 .. toctree::
    :maxdepth: 1
 
+   5.1.5
    5.1.4
    5.1.3
    5.1.2

docs/releases/security.txt

@@ -36,6 +36,28 @@ Issues under Django's security process
 All security issues have been handled under versions of Django's security
 process. These are listed below.
 
+December 4, 2024 - :cve:`2024-53907`
+------------------------------------
+
+Potential denial-of-service in ``django.utils.html.strip_tags()``.
+`Full description
+<https://www.djangoproject.com/weblog/2024/dec/04/security-releases/>`__
+
+* Django 5.1 :commit:`(patch) <bbc74a7f7eb7335e913bdb4787f22e83a9be947e>`
+* Django 5.0 :commit:`(patch) <a5a89ea28cc550c1b29b03f9e14ef3c128ec1e84>`
+* Django 4.2 :commit:`(patch) <790eb058b0716c536a2f2e8d1c6d5079d776c22b>`
+
+December 4, 2024 - :cve:`2024-53908`
+------------------------------------
+
+Potential SQL injection in ``HasKey(lhs, rhs)`` on Oracle.
+`Full description
+<https://www.djangoproject.com/weblog/2024/dec/04/security-releases/>`__
+
+* Django 5.1 :commit:`(patch) <6943d61818e63e77b65d8b1ae65941e8f04bd87b>`
+* Django 5.0 :commit:`(patch) <ff08bb6c70aa45f83a5ef3bd0b601c7c9d1a7642>`
+* Django 4.2 :commit:`(patch) <7376bcbf508883282ffcc0f0fac5cf0ed2d6cbc5>`
+
 September 3, 2024 - :cve:`2024-45231`
 -------------------------------------
 

docs/spelling_wordlist

@@ -3,14 +3,12 @@ accessor
 accessors
 Aceh
 admindocs
-affine
 affordances
 Ai
 Alchin
 allowlist
 alphanumerics
 amet
-analytics
 arccosine
 architected
 arcsine
@@ -60,7 +58,6 @@ Bokmål
 Bonham
 bookmarklet
 bookmarklets
-boolean
 booleans
 bpython
 Bronn
@@ -114,23 +111,18 @@ Danga
 Darussalam
 databrowse
 datafile
-dataset
-datasets
 datetimes
 declaratively
-decrementing
 deduplicates
 deduplication
 deepcopy
 deferrable
-DEP
 deprecations
 deserialization
 deserialize
 deserialized
 deserializer
 deserializing
-deterministically
 Deutsch
 dev
 dictConfig
@@ -142,7 +134,6 @@ Disqus
 distro
 django
 djangoproject
-djangotutorial
 dm
 docstring
 docstrings
@@ -165,9 +156,7 @@ esque
 Ess
 ETag
 ETags
-exe
 exfiltration
-extensibility
 fallbacks
 favicon
 fieldset
@@ -177,7 +166,6 @@ filesystems
 flatpage
 flatpages
 focusable
-fooapp
 formatter
 formatters
 formfield
@@ -210,7 +198,6 @@ hashable
 hasher
 hashers
 headerlist
-hoc
 Hoerner
 Holovaty
 Homebrew
@@ -223,7 +210,6 @@ Hypercorn
 ies
 iframe
 Igbo
-incrementing
 indexable
 ing
 ini
@@ -240,7 +226,6 @@ iterable
 iterables
 iteratively
 ize
-Jazzband
 Jinja
 jQuery
 Jupyter
@@ -283,7 +268,6 @@ manouche
 Marino
 memcache
 memcached
-mentorship
 metaclass
 metaclasses
 metre
@@ -338,8 +322,6 @@ orm
 Outdim
 outfile
 paginator
-parallelization
-parallelized
 parameterization
 params
 parens
@@ -361,9 +343,7 @@ pluggable
 pluralizations
 pooler
 postfix
-postgis
 postgres
-postgresql
 pragma
 pre
 precisions
@@ -376,7 +356,6 @@ prefetches
 prefetching
 preload
 preloaded
-prepend
 prepended
 prepending
 prepends
@@ -429,7 +408,6 @@ reflow
 registrable
 reimplement
 reindent
-reindex
 releaser
 releasers
 reloader
@@ -439,7 +417,6 @@ repo
 reportable
 reprojection
 reraising
-resampling
 reST
 reStructuredText
 reusability
@@ -447,7 +424,6 @@ reverter
 roadmap
 Roald
 rss
-runtime
 Sandvik
 savepoint
 savepoints
@@ -458,7 +434,6 @@ screencasts
 semimajor
 semiminor
 serializability
-serializable
 serializer
 serializers
 shapefile
@@ -467,7 +442,6 @@ sharding
 sitewide
 sliceable
 SMTP
-solaris
 Sorani
 sortable
 Spectre
@@ -535,7 +509,6 @@ toolkits
 toolset
 trac
 tracebacks
-transactional
 Transifex
 Tredinnick
 triager

docs/topics/db/fixtures.txt

@@ -4,28 +4,25 @@
 Fixtures
 ========
 
-.. seealso::
-
-    * :doc:`/howto/initial-data`
-
-What is a fixture?
-==================
-
 A *fixture* is a collection of files that contain the serialized contents of
 the database. Each fixture has a unique name, and the files that comprise the
 fixture can be distributed over multiple directories, in multiple applications.
 
-How to produce a fixture?
+.. seealso::
-=========================
+
+    * :doc:`/howto/initial-data`
+
+How to produce a fixture
+========================
 
 Fixtures can be generated by :djadmin:`manage.py dumpdata <dumpdata>`. It's
 also possible to generate custom fixtures by directly using :doc:`serialization
 tools </topics/serialization>` or even by handwriting them.
 
-How to use a fixture?
+How to use a fixture
-=====================
+====================
 
-Fixtures can be used to pre-populate database with data for
+Fixtures can be used to pre-populate the database with data for
 :ref:`tests <topics-testing-fixtures>`:
 
 .. code-block:: python
@@ -40,8 +37,8 @@ or to provide some :ref:`initial data <initial-data-via-fixtures>` using the
 
     django-admin loaddata <fixture label>
 
-Where Django looks for fixtures?
+How fixtures are discovered
-================================
+===========================
 
 Django will search in these locations for fixtures:
 
@@ -116,8 +113,8 @@ example).
 
 .. _MySQL: https://dev.mysql.com/doc/refman/en/constraint-foreign-key.html
 
-How fixtures are saved to the database?
+How fixtures are saved to the database
-=======================================
+======================================
 
 When fixture files are processed, the data is saved to the database as is.
 Model defined :meth:`~django.db.models.Model.save` methods are not called, and

docs/topics/forms/formsets.txt

@@ -571,14 +571,12 @@ happen when the user changes these values:
     ...         {"title": "Article #2", "pub_date": datetime.date(2008, 5, 11)},
     ...     ],
     ... )
-    >>> formset.is_valid()
-    True
     >>> for form in formset.ordered_forms:
     ...     print(form.cleaned_data)
     ...
-    {'pub_date': datetime.date(2008, 5, 1), 'ORDER': 0, 'title': 'Article #3'}
+    {'title': 'Article #3', 'pub_date': datetime.date(2008, 5, 1), 'ORDER': 0}
-    {'pub_date': datetime.date(2008, 5, 11), 'ORDER': 1, 'title': 'Article #2'}
+    {'title': 'Article #2', 'pub_date': datetime.date(2008, 5, 11), 'ORDER': 1}
-    {'pub_date': datetime.date(2008, 5, 10), 'ORDER': 2, 'title': 'Article #1'}
+    {'title': 'Article #1', 'pub_date': datetime.date(2008, 5, 10), 'ORDER': 2}
 
 :class:`~django.forms.formsets.BaseFormSet` also provides an
 :attr:`~django.forms.formsets.BaseFormSet.ordering_widget` attribute and
@@ -690,7 +688,7 @@ delete fields you can access them with ``deleted_forms``:
     ...     ],
     ... )
     >>> [form.cleaned_data for form in formset.deleted_forms]
-    [{'DELETE': True, 'pub_date': datetime.date(2008, 5, 10), 'title': 'Article #1'}]
+    [{'title': 'Article #1', 'pub_date': datetime.date(2008, 5, 10), 'DELETE': True}]
 
 If you are using a :class:`ModelFormSet<django.forms.models.BaseModelFormSet>`,
 model instances for deleted forms will be deleted when you call

tests/composite_pk/test_checks.py

@@ -1,7 +1,7 @@
 from django.core import checks
 from django.db import connection, models
 from django.db.models import F
-from django.test import TestCase
+from django.test import TestCase, skipUnlessAnyDBFeature
 from django.test.utils import isolate_apps
 
 
@@ -217,16 +217,18 @@ class CompositePKChecksTests(TestCase):
             ],
         )
 
+    @skipUnlessAnyDBFeature(
+        "supports_virtual_generated_columns",
+        "supports_stored_generated_columns",
+    )
     def test_composite_pk_cannot_include_generated_field(self):
-        is_oracle = connection.vendor == "oracle"
-
         class Foo(models.Model):
             pk = models.CompositePrimaryKey("id", "foo")
             id = models.IntegerField()
             foo = models.GeneratedField(
                 expression=F("id"),
                 output_field=models.IntegerField(),
-                db_persist=not is_oracle,
+                db_persist=connection.features.supports_stored_generated_columns,
             )
 
         self.assertEqual(

tests/composite_pk/tests.py

@@ -2,7 +2,12 @@ import json
 import unittest
 from uuid import UUID
 
-import yaml
+try:
+    import yaml  # NOQA
+
+    HAS_YAML = True
+except ImportError:
+    HAS_YAML = False
 
 from django import forms
 from django.core import serializers
@@ -35,9 +40,9 @@ class CompositePKTests(TestCase):
         cls.comment = Comment.objects.create(tenant=cls.tenant, id=1, user=cls.user)
 
     @staticmethod
-    def get_constraints(table):
+    def get_primary_key_columns(table):
         with connection.cursor() as cursor:
-            return connection.introspection.get_constraints(cursor, table)
+            return connection.introspection.get_primary_key_columns(cursor, table)
 
     def test_pk_updated_if_field_updated(self):
         user = User.objects.get(pk=self.user.pk)
@@ -125,53 +130,15 @@ class CompositePKTests(TestCase):
         with self.assertRaises(IntegrityError):
             Comment.objects.create(tenant=self.tenant, id=self.comment.id)
 
-    @unittest.skipUnless(connection.vendor == "postgresql", "PostgreSQL specific test")
+    def test_get_primary_key_columns(self):
-    def test_get_constraints_postgresql(self):
+        self.assertEqual(
-        user_constraints = self.get_constraints(User._meta.db_table)
+            self.get_primary_key_columns(User._meta.db_table),
-        user_pk = user_constraints["composite_pk_user_pkey"]
+            ["tenant_id", "id"],
-        self.assertEqual(user_pk["columns"], ["tenant_id", "id"])
+        )
-        self.assertIs(user_pk["primary_key"], True)
+        self.assertEqual(
-
+            self.get_primary_key_columns(Comment._meta.db_table),
-        comment_constraints = self.get_constraints(Comment._meta.db_table)
+            ["tenant_id", "comment_id"],
-        comment_pk = comment_constraints["composite_pk_comment_pkey"]
+        )
-        self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"])
-        self.assertIs(comment_pk["primary_key"], True)
-
-    @unittest.skipUnless(connection.vendor == "sqlite", "SQLite specific test")
-    def test_get_constraints_sqlite(self):
-        user_constraints = self.get_constraints(User._meta.db_table)
-        user_pk = user_constraints["__primary__"]
-        self.assertEqual(user_pk["columns"], ["tenant_id", "id"])
-        self.assertIs(user_pk["primary_key"], True)
-
-        comment_constraints = self.get_constraints(Comment._meta.db_table)
-        comment_pk = comment_constraints["__primary__"]
-        self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"])
-        self.assertIs(comment_pk["primary_key"], True)
-
-    @unittest.skipUnless(connection.vendor == "mysql", "MySQL specific test")
-    def test_get_constraints_mysql(self):
-        user_constraints = self.get_constraints(User._meta.db_table)
-        user_pk = user_constraints["PRIMARY"]
-        self.assertEqual(user_pk["columns"], ["tenant_id", "id"])
-        self.assertIs(user_pk["primary_key"], True)
-
-        comment_constraints = self.get_constraints(Comment._meta.db_table)
-        comment_pk = comment_constraints["PRIMARY"]
-        self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"])
-        self.assertIs(comment_pk["primary_key"], True)
-
-    @unittest.skipUnless(connection.vendor == "oracle", "Oracle specific test")
-    def test_get_constraints_oracle(self):
-        user_constraints = self.get_constraints(User._meta.db_table)
-        user_pk = next(c for c in user_constraints.values() if c["primary_key"])
-        self.assertEqual(user_pk["columns"], ["tenant_id", "id"])
-        self.assertEqual(user_pk["primary_key"], 1)
-
-        comment_constraints = self.get_constraints(Comment._meta.db_table)
-        comment_pk = next(c for c in comment_constraints.values() if c["primary_key"])
-        self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"])
-        self.assertEqual(comment_pk["primary_key"], 1)
 
     def test_in_bulk(self):
         """
@@ -291,6 +258,7 @@ class CompositePKFixturesTests(TestCase):
             },
         )
 
+    @unittest.skipUnless(HAS_YAML, "No yaml library detected")
     def test_serialize_user_yaml(self):
         users = User.objects.filter(pk=(2, 3))
         result = serializers.serialize("yaml", users)

tests/contenttypes_tests/test_fields.py

@@ -57,6 +57,15 @@ class GenericForeignKeyTests(TestCase):
         self.assertIsNot(answer.question, old_question_obj)
         self.assertEqual(answer.question, old_question_obj)
 
+    def test_clear_cached_generic_relation_when_deferred(self):
+        question = Question.objects.create(text="question")
+        Answer.objects.create(text="answer", question=question)
+        answer = Answer.objects.defer("text").get()
+        old_question_obj = answer.question
+        # The reverse relation is refreshed even when the text field is deferred.
+        answer.refresh_from_db()
+        self.assertIsNot(answer.question, old_question_obj)
+
 
 class GenericRelationTests(TestCase):
     def test_value_to_string(self):

tests/defer/tests.py

@@ -290,6 +290,14 @@ class TestDefer2(AssertionMixin, TestCase):
             self.assertEqual(rf2.name, "new foo")
             self.assertEqual(rf2.value, "new bar")
 
+    def test_refresh_when_one_field_deferred(self):
+        s = Secondary.objects.create()
+        PrimaryOneToOne.objects.create(name="foo", value="bar", related=s)
+        s = Secondary.objects.defer("first").get()
+        p_before = s.primary_o2o
+        s.refresh_from_db()
+        self.assertIsNot(s.primary_o2o, p_before)
+
 
 class InvalidDeferTests(SimpleTestCase):
     def test_invalid_defer(self):

tests/forms_tests/tests/test_error_messages.py

@@ -249,7 +249,8 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin):
         form1 = TestForm({"first_name": "John"})
         self.assertHTMLEqual(
             str(form1["last_name"].errors),
-            '<ul class="errorlist"><li>This field is required.</li></ul>',
+            '<ul class="errorlist" id="id_last_name_error"><li>'
+            "This field is required.</li></ul>",
         )
         self.assertHTMLEqual(
             str(form1.errors["__all__"]),
@@ -280,7 +281,7 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin):
         f = SomeForm({"field": "<script>"})
         self.assertHTMLEqual(
             t.render(Context({"form": f})),
-            '<ul class="errorlist"><li>field<ul class="errorlist">'
+            '<ul class="errorlist"><li>field<ul class="errorlist" id="id_field_error">'
             "<li>Select a valid choice. &lt;script&gt; is not one of the "
             "available choices.</li></ul></li></ul>",
         )
@@ -291,7 +292,7 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin):
         f = SomeForm({"field": ["<script>"]})
         self.assertHTMLEqual(
             t.render(Context({"form": f})),
-            '<ul class="errorlist"><li>field<ul class="errorlist">'
+            '<ul class="errorlist"><li>field<ul class="errorlist" id="id_field_error">'
             "<li>Select a valid choice. &lt;script&gt; is not one of the "
             "available choices.</li></ul></li></ul>",
         )
@@ -302,7 +303,7 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin):
         f = SomeForm({"field": ["<script>"]})
         self.assertHTMLEqual(
             t.render(Context({"form": f})),
-            '<ul class="errorlist"><li>field<ul class="errorlist">'
+            '<ul class="errorlist"><li>field<ul class="errorlist" id="id_field_error">'
             "<li>“&lt;script&gt;” is not a valid value.</li>"
             "</ul></li></ul>",
         )

tests/forms_tests/tests/test_forms.py

@@ -181,53 +181,55 @@ class FormsTestCase(SimpleTestCase):
         self.assertHTMLEqual(
             str(p),
             '<div><label for="id_first_name">First name:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_first_name_error"><li>This field is required.'
-            '<input type="text" name="first_name" aria-invalid="true" required '
+            '</li></ul><input type="text" name="first_name" aria-invalid="true" '
-            'id="id_first_name"></div>'
+            'required id="id_first_name"></div>'
             '<div><label for="id_last_name">Last name:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_last_name_error"><li>This field is required.'
-            '<input type="text" name="last_name" aria-invalid="true" required '
+            '</li></ul><input type="text" name="last_name" aria-invalid="true" '
-            'id="id_last_name"></div><div>'
+            'required id="id_last_name"></div><div>'
             '<label for="id_birthday">Birthday:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_birthday_error"><li>This field is required.'
-            '<input type="text" name="birthday" aria-invalid="true" required '
+            '</li></ul><input type="text" name="birthday" aria-invalid="true" required '
             'id="id_birthday"></div>',
         )
         self.assertHTMLEqual(
             p.as_table(),
             """<tr><th><label for="id_first_name">First name:</label></th><td>
-<ul class="errorlist"><li>This field is required.</li></ul>
+<ul class="errorlist" id="id_first_name_error"><li>This field is required.</li></ul>
 <input type="text" name="first_name" id="id_first_name" aria-invalid="true" required>
 </td></tr><tr><th><label for="id_last_name">Last name:</label></th>
-<td><ul class="errorlist"><li>This field is required.</li></ul>
+<td><ul class="errorlist" id="id_last_name_error"><li>This field is required.</li></ul>
 <input type="text" name="last_name" id="id_last_name" aria-invalid="true" required>
 </td></tr><tr><th><label for="id_birthday">Birthday:</label></th>
-<td><ul class="errorlist"><li>This field is required.</li></ul>
+<td><ul class="errorlist" id="id_birthday_error"><li>This field is required.</li></ul>
 <input type="text" name="birthday" id="id_birthday" aria-invalid="true" required>
 </td></tr>""",
         )
         self.assertHTMLEqual(
             p.as_ul(),
-            """<li><ul class="errorlist"><li>This field is required.</li></ul>
+            """<li><ul class="errorlist" id="id_first_name_error">
+<li>This field is required.</li></ul>
 <label for="id_first_name">First name:</label>
 <input type="text" name="first_name" id="id_first_name" aria-invalid="true" required>
-</li><li><ul class="errorlist"><li>This field is required.</li></ul>
+</li><li><ul class="errorlist" id="id_last_name_error"><li>This field is required.</li>
-<label for="id_last_name">Last name:</label>
+</ul><label for="id_last_name">Last name:</label>
 <input type="text" name="last_name" id="id_last_name" aria-invalid="true" required>
-</li><li><ul class="errorlist"><li>This field is required.</li></ul>
+</li><li><ul class="errorlist" id="id_birthday_error"><li>This field is required.</li>
-<label for="id_birthday">Birthday:</label>
+</ul><label for="id_birthday">Birthday:</label>
 <input type="text" name="birthday" id="id_birthday" aria-invalid="true" required>
 </li>""",
         )
         self.assertHTMLEqual(
             p.as_p(),
-            """<ul class="errorlist"><li>This field is required.</li></ul>
+            """<ul class="errorlist" id="id_first_name_error"><li>
+This field is required.</li></ul>
 <p><label for="id_first_name">First name:</label>
 <input type="text" name="first_name" id="id_first_name" aria-invalid="true" required>
-</p><ul class="errorlist"><li>This field is required.</li></ul>
+</p><ul class="errorlist" id="id_last_name_error"><li>This field is required.</li></ul>
 <p><label for="id_last_name">Last name:</label>
 <input type="text" name="last_name" id="id_last_name" aria-invalid="true" required>
-</p><ul class="errorlist"><li>This field is required.</li></ul>
+</p><ul class="errorlist" id="id_birthday_error"><li>This field is required.</li></ul>
 <p><label for="id_birthday">Birthday:</label>
 <input type="text" name="birthday" id="id_birthday" aria-invalid="true" required>
 </p>""",
@@ -235,16 +237,16 @@ class FormsTestCase(SimpleTestCase):
         self.assertHTMLEqual(
             p.as_div(),
             '<div><label for="id_first_name">First name:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_first_name_error"><li>This field is required.'
-            '<input type="text" name="first_name" aria-invalid="true" required '
+            '</li></ul><input type="text" name="first_name" aria-invalid="true" '
-            'id="id_first_name"></div>'
+            'required id="id_first_name"></div>'
             '<div><label for="id_last_name">Last name:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_last_name_error"><li>This field is required.'
-            '<input type="text" name="last_name" aria-invalid="true" required '
+            '</li></ul><input type="text" name="last_name" aria-invalid="true" '
-            'id="id_last_name"></div><div>'
+            'required id="id_last_name"></div><div>'
             '<label for="id_birthday">Birthday:</label>'
-            '<ul class="errorlist"><li>This field is required.</li></ul>'
+            '<ul class="errorlist" id="id_birthday_error"><li>This field is required.'
-            '<input type="text" name="birthday" aria-invalid="true" required '
+            '</li></ul><input type="text" name="birthday" aria-invalid="true" required '
             'id="id_birthday"></div>',
         )
 
@@ -387,7 +389,8 @@ class FormsTestCase(SimpleTestCase):
         self.assertEqual(p["first_name"].errors, ["This field is required."])
         self.assertHTMLEqual(
             p["first_name"].errors.as_ul(),
-            '<ul class="errorlist"><li>This field is required.</li></ul>',
+            '<ul class="errorlist" id="id_first_name_error">'
+            "<li>This field is required.</li></ul>",
         )
         self.assertEqual(p["first_name"].errors.as_text(), "* This field is required.")
 
@@ -3706,7 +3709,7 @@ Options: <select multiple name="options" aria-invalid="true" required>
         self.assertHTMLEqual(
             p.as_ul(),
             """
-            <li class="required error"><ul class="errorlist">
+            <li class="required error"><ul class="errorlist" id="id_name_error">
             <li>This field is required.</li></ul>
             <label class="required" for="id_name">Name:</label>
             <input type="text" name="name" id="id_name" aria-invalid="true" required>
@@ -3719,7 +3722,7 @@ Options: <select multiple name="options" aria-invalid="true" required>
             </select></li>
             <li><label for="id_email">Email:</label>
             <input type="email" name="email" id="id_email" maxlength="320"></li>
-            <li class="required error"><ul class="errorlist">
+            <li class="required error"><ul class="errorlist" id="id_age_error">
             <li>This field is required.</li></ul>
             <label class="required" for="id_age">Age:</label>
             <input type="number" name="age" id="id_age" aria-invalid="true" required>
@@ -3729,8 +3732,8 @@ Options: <select multiple name="options" aria-invalid="true" required>
         self.assertHTMLEqual(
             p.as_p(),
             """
-            <ul class="errorlist"><li>This field is required.</li></ul>
+            <ul class="errorlist" id="id_name_error"><li>This field is required.</li>
-            <p class="required error">
+            </ul><p class="required error">
             <label class="required" for="id_name">Name:</label>
             <input type="text" name="name" id="id_name" aria-invalid="true" required>
             </p><p class="required">
@@ -3742,17 +3745,17 @@ Options: <select multiple name="options" aria-invalid="true" required>
             </select></p>
             <p><label for="id_email">Email:</label>
             <input type="email" name="email" id="id_email" maxlength="320"></p>
-            <ul class="errorlist"><li>This field is required.</li></ul>
+            <ul class="errorlist" id="id_age_error"><li>This field is required.</li>
-            <p class="required error"><label class="required" for="id_age">Age:</label>
+            </ul><p class="required error"><label class="required" for="id_age">
-            <input type="number" name="age" id="id_age" aria-invalid="true" required>
+            Age:</label><input type="number" name="age" id="id_age" aria-invalid="true"
-            </p>""",
+            required></p>""",
         )
 
         self.assertHTMLEqual(
             p.as_table(),
             """<tr class="required error">
 <th><label class="required" for="id_name">Name:</label></th>
-<td><ul class="errorlist"><li>This field is required.</li></ul>
+<td><ul class="errorlist" id="id_name_error"><li>This field is required.</li></ul>
 <input type="text" name="name" id="id_name" aria-invalid="true" required></td></tr>
 <tr class="required"><th><label class="required" for="id_is_cool">Is cool:</label></th>
 <td><select name="is_cool" id="id_is_cool">
@@ -3763,14 +3766,14 @@ Options: <select multiple name="options" aria-invalid="true" required>
 <tr><th><label for="id_email">Email:</label></th><td>
 <input type="email" name="email" id="id_email" maxlength="320"></td></tr>
 <tr class="required error"><th><label class="required" for="id_age">Age:</label></th>
-<td><ul class="errorlist"><li>This field is required.</li></ul>
+<td><ul class="errorlist" id="id_age_error"><li>This field is required.</li></ul>
 <input type="number" name="age" id="id_age" aria-invalid="true" required></td></tr>""",
         )
         self.assertHTMLEqual(
             p.as_div(),
             '<div class="required error"><label for="id_name" class="required">Name:'
-            '</label><ul class="errorlist"><li>This field is required.</li></ul>'
+            '</label><ul class="errorlist" id="id_name_error"><li>This field is '
-            '<input type="text" name="name" required id="id_name" '
+            'required.</li></ul><input type="text" name="name" required id="id_name" '
             'aria-invalid="true" /></div>'
             '<div class="required"><label for="id_is_cool" class="required">Is cool:'
             '</label><select name="is_cool" id="id_is_cool">'
@@ -3779,8 +3782,8 @@ Options: <select multiple name="options" aria-invalid="true" required>
             '</select></div><div><label for="id_email">Email:</label>'
             '<input type="email" name="email" id="id_email" maxlength="320"/></div>'
             '<div class="required error"><label for="id_age" class="required">Age:'
-            '</label><ul class="errorlist"><li>This field is required.</li></ul>'
+            '</label><ul class="errorlist" id="id_age_error"><li>This field is '
-            '<input type="number" name="age" required id="id_age" '
+            'required.</li></ul><input type="number" name="age" required id="id_age" '
             'aria-invalid="true" /></div>',
         )
 
@@ -4255,8 +4258,10 @@ Options: <select multiple name="options" aria-invalid="true" required>
 
         errors = form.errors.as_ul()
         control = [
-            '<li>foo<ul class="errorlist"><li>This field is required.</li></ul></li>',
+            '<li>foo<ul class="errorlist" id="id_foo_error"><li>This field is required.'
-            '<li>bar<ul class="errorlist"><li>This field is required.</li></ul></li>',
+            "</li></ul></li>",
+            '<li>bar<ul class="errorlist" id="id_bar_error"><li>This field is required.'
+            "</li></ul></li>",
             '<li>__all__<ul class="errorlist nonfield"><li>Non-field error.</li></ul>'
             "</li>",
         ]
@@ -4461,7 +4466,8 @@ Options: <select multiple name="options" aria-invalid="true" required>
             form.as_ul(),
             '<li><ul class="errorlist nonfield">'
             "<li>(Hidden field hidden) Foo &amp; &quot;bar&quot;!</li></ul></li>"
-            '<li><ul class="errorlist"><li>Foo &amp; &quot;bar&quot;!</li></ul>'
+            '<li><ul class="errorlist" id="id_visible_error"><li>Foo &amp; '
+            "&quot;bar&quot;!</li></ul>"
             '<label for="id_visible">Visible:</label> '
             '<input type="text" name="visible" aria-invalid="true" value="b" '
             'id="id_visible" required>'

tests/forms_tests/tests/test_i18n.py

@@ -97,7 +97,7 @@ class FormsI18nTests(SimpleTestCase):
             f = SomeForm({})
             self.assertHTMLEqual(
                 f.as_p(),
-                '<ul class="errorlist"><li>'
+                '<ul class="errorlist" id="id_somechoice_error"><li>'
                 "\u041e\u0431\u044f\u0437\u0430\u0442\u0435\u043b\u044c"
                 "\u043d\u043e\u0435 \u043f\u043e\u043b\u0435.</li></ul>\n"
                 "<p><label>\xc5\xf8\xdf:</label>"

tests/i18n/test_extraction.py

@@ -179,6 +179,15 @@ class BasicExtractorTests(ExtractorTests):
         self.assertIn("processing locale en_GB", out.getvalue())
         self.assertIs(Path("locale/en_GB/LC_MESSAGES/django.po").exists(), True)
 
+    def test_valid_locale_with_numeric_region_code(self):
+        out = StringIO()
+        management.call_command(
+            "makemessages", locale=["ar_002"], stdout=out, verbosity=1
+        )
+        self.assertNotIn("invalid locale ar_002", out.getvalue())
+        self.assertIn("processing locale ar_002", out.getvalue())
+        self.assertIs(Path("locale/ar_002/LC_MESSAGES/django.po").exists(), True)
+
     def test_valid_locale_tachelhit_latin_morocco(self):
         out = StringIO()
         management.call_command(

tests/migrations/test_commands.py

@@ -9,6 +9,7 @@ from unittest import mock
 
 from django.apps import apps
 from django.core.management import CommandError, call_command
+from django.core.management.base import SystemCheckError
 from django.core.management.commands.makemigrations import (
     Command as MakeMigrationsCommand,
 )
@@ -859,7 +860,7 @@ class MigrateTests(MigrationTestBase):
         sqlmigrate outputs forward looking SQL.
         """
         out = io.StringIO()
-        call_command("sqlmigrate", "migrations", "0001", stdout=out)
+        call_command("sqlmigrate", "migrations", "0001", stdout=out, no_color=True)
 
         lines = out.getvalue().splitlines()
 
@@ -921,7 +922,14 @@ class MigrateTests(MigrationTestBase):
         call_command("migrate", "migrations", verbosity=0)
 
         out = io.StringIO()
-        call_command("sqlmigrate", "migrations", "0001", stdout=out, backwards=True)
+        call_command(
+            "sqlmigrate",
+            "migrations",
+            "0001",
+            stdout=out,
+            backwards=True,
+            no_color=True,
+        )
 
         lines = out.getvalue().splitlines()
         try:
@@ -1098,6 +1106,30 @@ class MigrateTests(MigrationTestBase):
             ],
         )
 
+    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})
+    def test_sqlmigrate_transaction_keywords_not_colorized(self):
+        out = io.StringIO()
+        with mock.patch(
+            "django.core.management.color.supports_color", lambda *args: True
+        ):
+            call_command("sqlmigrate", "migrations", "0001", stdout=out, no_color=False)
+        self.assertNotIn("\x1b", out.getvalue())
+
+    @override_settings(
+        MIGRATION_MODULES={"migrations": "migrations.test_migrations_no_operations"},
+        INSTALLED_APPS=["django.contrib.auth"],
+    )
+    def test_sqlmigrate_system_checks_colorized(self):
+        with (
+            mock.patch(
+                "django.core.management.color.supports_color", lambda *args: True
+            ),
+            self.assertRaisesMessage(SystemCheckError, "\x1b"),
+        ):
+            call_command(
+                "sqlmigrate", "migrations", "0001", skip_checks=False, no_color=False
+            )
+
     @override_settings(
         INSTALLED_APPS=[
             "migrations.migrations_test_apps.migrated_app",

tests/model_fields/test_decimalfield.py

@@ -1,9 +1,10 @@
 import math
 from decimal import Decimal
+from unittest import mock
 
 from django.core import validators
 from django.core.exceptions import ValidationError
-from django.db import models
+from django.db import connection, models
 from django.test import TestCase
 
 from .models import BigD, Foo
@@ -48,6 +49,20 @@ class DecimalFieldTests(TestCase):
         self.assertIsNone(f.get_prep_value(None))
         self.assertEqual(f.get_prep_value("2.4"), Decimal("2.4"))
 
+    def test_get_db_prep_value(self):
+        """
+        DecimalField.get_db_prep_value() must call
+        DatabaseOperations.adapt_decimalfield_value().
+        """
+        f = models.DecimalField(max_digits=5, decimal_places=1)
+        # None of the built-in database backends implement
+        # adapt_decimalfield_value(), so this must be confirmed with mocking.
+        with mock.patch.object(
+            connection.ops.__class__, "adapt_decimalfield_value"
+        ) as adapt_decimalfield_value:
+            f.get_db_prep_value("2.4", connection)
+        adapt_decimalfield_value.assert_called_with(Decimal("2.4"), 5, 1)
+
     def test_filter_with_strings(self):
         """
         Should be able to filter decimal fields using strings (#8023).

tests/model_fields/test_jsonfield.py

@@ -29,6 +29,7 @@ from django.db.models import (
 from django.db.models.expressions import RawSQL
 from django.db.models.fields.json import (
     KT,
+    HasKey,
     KeyTextTransform,
     KeyTransform,
     KeyTransformFactory,
@@ -582,6 +583,14 @@ class TestQuerying(TestCase):
                     [expected],
                 )
 
+    def test_has_key_literal_lookup(self):
+        self.assertSequenceEqual(
+            NullableJSONModel.objects.filter(
+                HasKey(Value({"foo": "bar"}, JSONField()), "foo")
+            ).order_by("id"),
+            self.objs,
+        )
+
     def test_has_key_list(self):
         obj = NullableJSONModel.objects.create(value=[{"a": 1}, {"b": "x"}])
         tests = [
@@ -808,6 +817,59 @@ class TestQuerying(TestCase):
         )
         self.assertIs(NullableJSONModel.objects.filter(value__c__lt=5).exists(), False)
 
+    def test_lookups_special_chars(self):
+        test_keys = [
+            "CONTROL",
+            "single'",
+            "dollar$",
+            "dot.dot",
+            "with space",
+            "back\\slash",
+            "question?mark",
+            "user@name",
+            "emo🤡'ji",
+            "com,ma",
+            "curly{{{brace}}}s",
+            "escape\uffff'seq'\uffffue\uffff'nce",
+        ]
+        json_value = {key: "some value" for key in test_keys}
+        obj = NullableJSONModel.objects.create(value=json_value)
+        obj.refresh_from_db()
+        self.assertEqual(obj.value, json_value)
+
+        for key in test_keys:
+            lookups = {
+                "has_key": Q(value__has_key=key),
+                "has_keys": Q(value__has_keys=[key, "CONTROL"]),
+                "has_any_keys": Q(value__has_any_keys=[key, "does_not_exist"]),
+                "exact": Q(**{f"value__{key}": "some value"}),
+            }
+            for lookup, condition in lookups.items():
+                results = NullableJSONModel.objects.filter(condition)
+                with self.subTest(key=key, lookup=lookup):
+                    self.assertSequenceEqual(results, [obj])
+
+    def test_lookups_special_chars_double_quotes(self):
+        test_keys = [
+            'double"',
+            "m\\i@x. m🤡'a,t{{{ch}}}e?d$\"'es\uffff'ca\uffff'pe",
+        ]
+        json_value = {key: "some value" for key in test_keys}
+        obj = NullableJSONModel.objects.create(value=json_value)
+        obj.refresh_from_db()
+        self.assertEqual(obj.value, json_value)
+        self.assertSequenceEqual(
+            NullableJSONModel.objects.filter(value__has_keys=test_keys), [obj]
+        )
+        for key in test_keys:
+            with self.subTest(key=key):
+                results = NullableJSONModel.objects.filter(
+                    Q(value__has_key=key),
+                    Q(value__has_any_keys=[key, "does_not_exist"]),
+                    Q(**{f"value__{key}": "some value"}),
+                )
+                self.assertSequenceEqual(results, [obj])
+
     def test_lookup_exclude(self):
         tests = [
             (Q(value__a="b"), [self.objs[0]]),

tests/model_forms/tests.py

@@ -3207,11 +3207,13 @@ class ModelFormCustomErrorTests(SimpleTestCase):
         errors = CustomErrorMessageForm(data).errors
         self.assertHTMLEqual(
             str(errors["name1"]),
-            '<ul class="errorlist"><li>Form custom error message.</li></ul>',
+            '<ul class="errorlist" id="id_name1_error">'
+            "<li>Form custom error message.</li></ul>",
         )
         self.assertHTMLEqual(
             str(errors["name2"]),
-            '<ul class="errorlist"><li>Model custom error message.</li></ul>',
+            '<ul class="errorlist" id="id_name2_error">'
+            "<li>Model custom error message.</li></ul>",
         )
 
     def test_model_clean_error_messages(self):
@@ -3220,14 +3222,15 @@ class ModelFormCustomErrorTests(SimpleTestCase):
         self.assertFalse(form.is_valid())
         self.assertHTMLEqual(
             str(form.errors["name1"]),
-            '<ul class="errorlist"><li>Model.clean() error messages.</li></ul>',
+            '<ul class="errorlist" id="id_name1_error">'
+            "<li>Model.clean() error messages.</li></ul>",
         )
         data = {"name1": "FORBIDDEN_VALUE2", "name2": "ABC"}
         form = CustomErrorMessageForm(data)
         self.assertFalse(form.is_valid())
         self.assertHTMLEqual(
             str(form.errors["name1"]),
-            '<ul class="errorlist">'
+            '<ul class="errorlist" id="id_name1_error">'
             "<li>Model.clean() error messages (simpler syntax).</li></ul>",
         )
         data = {"name1": "GLOBAL_ERROR", "name2": "ABC"}

tests/postgres_tests/test_array.py

@@ -1008,6 +1008,32 @@ class TestSerialization(PostgreSQLSimpleTestCase):
         self.assertEqual(instance.field, [1, 2, None])
 
 
+class TestStringSerialization(PostgreSQLSimpleTestCase):
+    field_values = [["Django", "Python", None], ["Джанго", "פייתון", None, "król"]]
+
+    @staticmethod
+    def create_json_data(array_field_value):
+        fields = {"field": json.dumps(array_field_value, ensure_ascii=False)}
+        return json.dumps(
+            [{"model": "postgres_tests.chararraymodel", "pk": None, "fields": fields}]
+        )
+
+    def test_encode(self):
+        for field_value in self.field_values:
+            with self.subTest(field_value=field_value):
+                instance = CharArrayModel(field=field_value)
+                data = serializers.serialize("json", [instance])
+                json_data = self.create_json_data(field_value)
+                self.assertEqual(json.loads(data), json.loads(json_data))
+
+    def test_decode(self):
+        for field_value in self.field_values:
+            with self.subTest(field_value=field_value):
+                json_data = self.create_json_data(field_value)
+                instance = list(serializers.deserialize("json", json_data))[0].object
+                self.assertEqual(instance.field, field_value)
+
+
 class TestValidation(PostgreSQLSimpleTestCase):
     def test_unbounded(self):
         field = ArrayField(models.IntegerField())

tests/postgres_tests/test_hstore.py

@@ -297,39 +297,53 @@ class TestChecks(PostgreSQLSimpleTestCase):
 
 
 class TestSerialization(PostgreSQLSimpleTestCase):
-    test_data = json.dumps(
+    field_values = [
-        [
+        ({"a": "b"}, [{"a": "b"}, {"b": "a"}]),
-            {
+        (
-                "model": "postgres_tests.hstoremodel",
+            {"все": "Трурль и Клапауций"},
-                "pk": None,
+            [{"Трурль": "Клапауций"}, {"Клапауций": "Трурль"}],
-                "fields": {
+        ),
-                    "field": json.dumps({"a": "b"}),
+    ]
-                    "array_field": json.dumps(
+
-                        [
+    @staticmethod
-                            json.dumps({"a": "b"}),
+    def create_json_data(field_value, array_field_value):
-                            json.dumps({"b": "a"}),
+        fields = {
-                        ]
+            "field": json.dumps(field_value, ensure_ascii=False),
-                    ),
+            "array_field": json.dumps(
-                },
+                [json.dumps(item, ensure_ascii=False) for item in array_field_value],
-            }
+                ensure_ascii=False,
-        ]
+            ),
-    )
+        }
+        return json.dumps(
+            [{"model": "postgres_tests.hstoremodel", "pk": None, "fields": fields}]
+        )
 
     def test_dumping(self):
-        instance = HStoreModel(field={"a": "b"}, array_field=[{"a": "b"}, {"b": "a"}])
+        for field_value, array_field_value in self.field_values:
-        data = serializers.serialize("json", [instance])
+            with self.subTest(field_value=field_value, array_value=array_field_value):
-        self.assertEqual(json.loads(data), json.loads(self.test_data))
+                instance = HStoreModel(field=field_value, array_field=array_field_value)
+                data = serializers.serialize("json", [instance])
+                json_data = self.create_json_data(field_value, array_field_value)
+                self.assertEqual(json.loads(data), json.loads(json_data))
 
     def test_loading(self):
-        instance = list(serializers.deserialize("json", self.test_data))[0].object
+        for field_value, array_field_value in self.field_values:
-        self.assertEqual(instance.field, {"a": "b"})
+            with self.subTest(field_value=field_value, array_value=array_field_value):
-        self.assertEqual(instance.array_field, [{"a": "b"}, {"b": "a"}])
+                json_data = self.create_json_data(field_value, array_field_value)
+                instance = list(serializers.deserialize("json", json_data))[0].object
+                self.assertEqual(instance.field, field_value)
+                self.assertEqual(instance.array_field, array_field_value)
 
     def test_roundtrip_with_null(self):
-        instance = HStoreModel(field={"a": "b", "c": None})
+        for field_value in [
-        data = serializers.serialize("json", [instance])
+            {"a": "b", "c": None},
-        new_instance = list(serializers.deserialize("json", data))[0].object
+            {"Енеїда": "Ти знаєш, він який суціга", "Зефір": None},
-        self.assertEqual(instance.field, new_instance.field)
+        ]:
+            with self.subTest(field_value=field_value):
+                instance = HStoreModel(field=field_value)
+                data = serializers.serialize("json", [instance])
+                new_instance = list(serializers.deserialize("json", data))[0].object
+                self.assertEqual(instance.field, new_instance.field)
 
 
 class TestValidation(PostgreSQLSimpleTestCase):

tests/runtests.py

@@ -32,9 +32,11 @@ else:
         RemovedInDjango60Warning,
         RemovedInDjango61Warning,
     )
+    from django.utils.functional import classproperty
     from django.utils.log import DEFAULT_LOGGING
     from django.utils.version import PY312, PYPY
 
+
 try:
     import MySQLdb
 except ImportError:
@@ -307,12 +309,12 @@ def setup_run_tests(verbosity, start_at, start_after, test_labels=None):
     apps.set_installed_apps(settings.INSTALLED_APPS)
 
     # Force declaring available_apps in TransactionTestCase for faster tests.
-    def no_available_apps(self):
+    def no_available_apps(cls):
         raise Exception(
             "Please define available_apps in TransactionTestCase and its subclasses."
         )
 
-    TransactionTestCase.available_apps = property(no_available_apps)
+    TransactionTestCase.available_apps = classproperty(no_available_apps)
     TestCase.available_apps = None
 
     # Set an environment variable that other code may consult to see if

tests/serializers/models/data.py

@@ -13,6 +13,15 @@ from django.db import models
 
 from .base import BaseModel
 
+try:
+    from PIL import Image  # NOQA
+except ImportError:
+    ImageData = None
+else:
+
+    class ImageData(models.Model):
+        data = models.ImageField(null=True)
+
 
 class BinaryData(models.Model):
     data = models.BinaryField(null=True)
@@ -62,10 +71,6 @@ class BigIntegerData(models.Model):
     data = models.BigIntegerField(null=True)
 
 
-# class ImageData(models.Model):
-#    data = models.ImageField(null=True)
-
-
 class GenericIPAddressData(models.Model):
     data = models.GenericIPAddressField(null=True)
 

tests/serializers/test_data.py

@@ -10,10 +10,11 @@ forward, backwards and self references.
 import datetime
 import decimal
 import uuid
+from collections import namedtuple
 
 from django.core import serializers
 from django.db import connection, models
-from django.test import TestCase
+from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature
 
 from .models import (
     Anchor,
@@ -46,6 +47,7 @@ from .models import (
     GenericData,
     GenericIPAddressData,
     GenericIPAddressPKData,
+    ImageData,
     InheritAbstractModel,
     InheritBaseModel,
     IntegerData,
@@ -239,24 +241,23 @@ def inherited_compare(testcase, pk, klass, data):
         testcase.assertEqual(value, getattr(instance, key))
 
 
-# Define some data types. Each data type is
+# Define some test helpers. Each has a pair of functions: one to create objects and one
-# actually a pair of functions; one to create
+# to make assertions against objects of a particular type.
-# and one to compare objects of that type
+TestHelper = namedtuple("TestHelper", ["create_object", "compare_object"])
-data_obj = (data_create, data_compare)
+data_obj = TestHelper(data_create, data_compare)
-generic_obj = (generic_create, generic_compare)
+generic_obj = TestHelper(generic_create, generic_compare)
-fk_obj = (fk_create, fk_compare)
+fk_obj = TestHelper(fk_create, fk_compare)
-m2m_obj = (m2m_create, m2m_compare)
+m2m_obj = TestHelper(m2m_create, m2m_compare)
-im2m_obj = (im2m_create, im2m_compare)
+im2m_obj = TestHelper(im2m_create, im2m_compare)
-im_obj = (im_create, im_compare)
+im_obj = TestHelper(im_create, im_compare)
-o2o_obj = (o2o_create, o2o_compare)
+o2o_obj = TestHelper(o2o_create, o2o_compare)
-pk_obj = (pk_create, pk_compare)
+pk_obj = TestHelper(pk_create, pk_compare)
-inherited_obj = (inherited_create, inherited_compare)
+inherited_obj = TestHelper(inherited_create, inherited_compare)
 uuid_obj = uuid.uuid4()
 
 test_data = [
-    # Format: (data type, PK value, Model Class, data)
+    # Format: (test helper, PK value, Model Class, data)
     (data_obj, 1, BinaryData, memoryview(b"\x05\xFD\x00")),
-    (data_obj, 2, BinaryData, None),
     (data_obj, 5, BooleanData, True),
     (data_obj, 6, BooleanData, False),
     (data_obj, 7, BooleanData, None),
@@ -265,7 +266,6 @@ test_data = [
     (data_obj, 12, CharData, "None"),
     (data_obj, 13, CharData, "null"),
     (data_obj, 14, CharData, "NULL"),
-    (data_obj, 15, CharData, None),
     # (We use something that will fit into a latin1 database encoding here,
     # because that is still the default used on many system setups.)
     (data_obj, 16, CharData, "\xa5"),
@@ -274,13 +274,11 @@ test_data = [
     (data_obj, 30, DateTimeData, datetime.datetime(2006, 6, 16, 10, 42, 37)),
     (data_obj, 31, DateTimeData, None),
     (data_obj, 40, EmailData, "hovercraft@example.com"),
-    (data_obj, 41, EmailData, None),
     (data_obj, 42, EmailData, ""),
     (data_obj, 50, FileData, "file:///foo/bar/whiz.txt"),
     # (data_obj, 51, FileData, None),
     (data_obj, 52, FileData, ""),
     (data_obj, 60, FilePathData, "/foo/bar/whiz.txt"),
-    (data_obj, 61, FilePathData, None),
     (data_obj, 62, FilePathData, ""),
     (data_obj, 70, DecimalData, decimal.Decimal("12.345")),
     (data_obj, 71, DecimalData, decimal.Decimal("-12.345")),
@@ -294,7 +292,6 @@ test_data = [
     (data_obj, 81, IntegerData, -123456789),
     (data_obj, 82, IntegerData, 0),
     (data_obj, 83, IntegerData, None),
-    # (XX, ImageData
     (data_obj, 95, GenericIPAddressData, "fe80:1424:2223:6cff:fe8a:2e8a:2151:abcd"),
     (data_obj, 96, GenericIPAddressData, None),
     (data_obj, 110, PositiveBigIntegerData, 9223372036854775807),
@@ -304,7 +301,6 @@ test_data = [
     (data_obj, 130, PositiveSmallIntegerData, 12),
     (data_obj, 131, PositiveSmallIntegerData, None),
     (data_obj, 140, SlugData, "this-is-a-slug"),
-    (data_obj, 141, SlugData, None),
     (data_obj, 142, SlugData, ""),
     (data_obj, 150, SmallData, 12),
     (data_obj, 151, SmallData, -12),
@@ -320,7 +316,6 @@ Several of them.
 The end.""",
     ),
     (data_obj, 161, TextData, ""),
-    (data_obj, 162, TextData, None),
     (data_obj, 170, TimeData, datetime.time(10, 42, 37)),
     (data_obj, 171, TimeData, None),
     (generic_obj, 200, GenericData, ["Generic Object 1", "tag1", "tag2"]),
@@ -388,15 +383,6 @@ The end.""",
     (pk_obj, 750, SmallPKData, 12),
     (pk_obj, 751, SmallPKData, -12),
     (pk_obj, 752, SmallPKData, 0),
-    (
-        pk_obj,
-        760,
-        TextPKData,
-        """This is a long piece of text.
-    It contains line breaks.
-    Several of them.
-    The end.""",
-    ),
     (pk_obj, 770, TimePKData, datetime.time(10, 42, 37)),
     (pk_obj, 791, UUIDData, uuid_obj),
     (fk_obj, 792, FKToUUID, uuid_obj),
@@ -419,71 +405,107 @@ The end.""",
     (data_obj, 1005, LengthModel, 1),
 ]
 
-
+if ImageData is not None:
-# Because Oracle treats the empty string as NULL, Oracle is expected to fail
+    test_data.extend(
-# when field.empty_strings_allowed is True and the value is None; skip these
+        [
-# tests.
+            (data_obj, 86, ImageData, "file:///foo/bar/whiz.png"),
-if connection.features.interprets_empty_strings_as_nulls:
+            # (data_obj, 87, ImageData, None),
-    test_data = [
+            (data_obj, 88, ImageData, ""),
-        data
+        ]
-        for data in test_data
+    )
-        if not (
-            data[0] == data_obj
-            and data[2]._meta.get_field("data").empty_strings_allowed
-            and data[3] is None
-        )
-    ]
-
-
-if not connection.features.supports_index_on_text_field:
-    test_data = [data for data in test_data if data[2] != TextPKData]
 
 
 class SerializerDataTests(TestCase):
     pass
 
 
-def serializerTest(self, format):
+def assert_serializer(self, format, data):
-    # FK to an object with PK of 0. This won't work on MySQL without the
+    # Create all the objects defined in the test data.
-    # NO_AUTO_VALUE_ON_ZERO SQL mode since it won't let you create an object
-    # with an autoincrement primary key of 0.
-    if connection.features.allows_auto_pk_0:
-        test_data.extend(
-            [
-                (data_obj, 0, Anchor, "Anchor 0"),
-                (fk_obj, 465, FKData, 0),
-            ]
-        )
-
-    # Create all the objects defined in the test data
     objects = []
-    instance_count = {}
+    for test_helper, pk, model, data_value in data:
-    for func, pk, klass, datum in test_data:
         with connection.constraint_checks_disabled():
-            objects.extend(func[0](pk, klass, datum))
+            objects.extend(test_helper.create_object(pk, model, data_value))
 
-    # Get a count of the number of objects created for each class
+    # Get a count of the number of objects created for each model class.
-    for klass in instance_count:
+    instance_counts = {}
-        instance_count[klass] = klass.objects.count()
+    for _, _, model, _ in data:
+        if model not in instance_counts:
+            instance_counts[model] = model.objects.count()
 
-    # Add the generic tagged objects to the object list
+    # Add the generic tagged objects to the object list.
     objects.extend(Tag.objects.all())
 
-    # Serialize the test database
+    # Serialize the test database.
     serialized_data = serializers.serialize(format, objects, indent=2)
 
     for obj in serializers.deserialize(format, serialized_data):
         obj.save()
 
-    # Assert that the deserialized data is the same
+    # Assert that the deserialized data is the same as the original source.
-    # as the original source
+    for test_helper, pk, model, data_value in data:
-    for func, pk, klass, datum in test_data:
+        with self.subTest(model=model, data_value=data_value):
-        func[1](self, pk, klass, datum)
+            test_helper.compare_object(self, pk, model, data_value)
 
-    # Assert that the number of objects deserialized is the
+    # Assert no new objects were created.
-    # same as the number that was serialized.
+    for model, count in instance_counts.items():
-    for klass, count in instance_count.items():
+        with self.subTest(model=model, count=count):
-        self.assertEqual(count, klass.objects.count())
+            self.assertEqual(count, model.objects.count())
+
+
+def serializerTest(self, format):
+    assert_serializer(self, format, test_data)
+
+
+@skipUnlessDBFeature("allows_auto_pk_0")
+def serializerTestPK0(self, format):
+    # FK to an object with PK of 0. This won't work on MySQL without the
+    # NO_AUTO_VALUE_ON_ZERO SQL mode since it won't let you create an object
+    # with an autoincrement primary key of 0.
+    data = [
+        (data_obj, 0, Anchor, "Anchor 0"),
+        (fk_obj, 1, FKData, 0),
+    ]
+    assert_serializer(self, format, data)
+
+
+@skipIfDBFeature("interprets_empty_strings_as_nulls")
+def serializerTestNullValueStingField(self, format):
+    data = [
+        (data_obj, 1, BinaryData, None),
+        (data_obj, 2, CharData, None),
+        (data_obj, 3, EmailData, None),
+        (data_obj, 4, FilePathData, None),
+        (data_obj, 5, SlugData, None),
+        (data_obj, 6, TextData, None),
+    ]
+    assert_serializer(self, format, data)
+
+
+@skipUnlessDBFeature("supports_index_on_text_field")
+def serializerTestTextFieldPK(self, format):
+    data = [
+        (
+            pk_obj,
+            1,
+            TextPKData,
+            """This is a long piece of text.
+            It contains line breaks.
+            Several of them.
+            The end.""",
+        ),
+    ]
+    assert_serializer(self, format, data)
 
 
 register_tests(SerializerDataTests, "test_%s_serializer", serializerTest)
+register_tests(SerializerDataTests, "test_%s_serializer_pk_0", serializerTestPK0)
+register_tests(
+    SerializerDataTests,
+    "test_%s_serializer_null_value_string_field",
+    serializerTestNullValueStingField,
+)
+register_tests(
+    SerializerDataTests,
+    "test_%s_serializer_text_field_pk",
+    serializerTestTextFieldPK,
+)

tests/template_tests/syntax_tests/test_include.py

@@ -330,15 +330,43 @@ class IncludeTests(SimpleTestCase):
                 ],
             }
         ]
-        engine = Engine(app_dirs=True)
+        with self.subTest(template="recursive_include.html"):
-        t = engine.get_template("recursive_include.html")
+            engine = Engine(app_dirs=True)
-        self.assertEqual(
+            t = engine.get_template("recursive_include.html")
-            "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
+            self.assertEqual(
-            t.render(Context({"comments": comments}))
+                "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
-            .replace(" ", "")
+                t.render(Context({"comments": comments}))
-            .replace("\n", " ")
+                .replace(" ", "")
-            .strip(),
+                .replace("\n", " ")
-        )
+                .strip(),
+            )
+        with self.subTest(template="recursive_relative_include.html"):
+            engine = Engine(app_dirs=True)
+            t = engine.get_template("recursive_relative_include.html")
+            self.assertEqual(
+                "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
+                t.render(Context({"comments": comments}))
+                .replace(" ", "")
+                .replace("\n", " ")
+                .strip(),
+            )
+        with self.subTest(template="tmpl"):
+            engine = Engine()
+            template = """
+            Recursion!
+            {% for c in comments %}
+              {{ c.comment }}
+              {% if c.children %}{% include tmpl with comments=c.children %}{% endif %}
+            {% endfor %}
+            """
+            outer_tmpl = engine.from_string("{% include tmpl %}")
+            output = outer_tmpl.render(
+                Context({"tmpl": engine.from_string(template), "comments": comments})
+            )
+            self.assertEqual(
+                "Recursion!  A1  Recursion!  B1   B2   B3  Recursion!  C1",
+                output.replace(" ", "").replace("\n", " ").strip(),
+            )
 
     def test_include_cache(self):
         """

tests/template_tests/templates/recursive_relative_include.html

@@ -0,0 +1,7 @@
+Recursion!
+{% for comment in comments %}
+    {{ comment.comment }}
+    {% if comment.children %}
+        {% include "./recursive_relative_include.html" with comments=comment.children %}
+    {% endif %}
+{% endfor %}

tests/utils_tests/test_html.py

@@ -1,6 +1,7 @@
 import os
 from datetime import datetime
 
+from django.core.exceptions import SuspiciousOperation
 from django.core.serializers.json import DjangoJSONEncoder
 from django.test import SimpleTestCase
 from django.utils.deprecation import RemovedInDjango60Warning
@@ -145,12 +146,18 @@ class TestUtilsHtml(SimpleTestCase):
             ("<script>alert()</script>&h", "alert()h"),
             ("><!" + ("&" * 16000) + "D", "><!" + ("&" * 16000) + "D"),
             ("X<<<<br>br>br>br>X", "XX"),
+            ("<" * 50 + "a>" * 50, ""),
         )
         for value, output in items:
             with self.subTest(value=value, output=output):
                 self.check_output(strip_tags, value, output)
                 self.check_output(strip_tags, lazystr(value), output)
 
+    def test_strip_tags_suspicious_operation(self):
+        value = "<" * 51 + "a>" * 51, "<a>"
+        with self.assertRaises(SuspiciousOperation):
+            strip_tags(value)
+
     def test_strip_tags_files(self):
         # Test with more lengthy content (also catching performance regressions)
         for filename in ("strip_tags1.html", "strip_tags2.txt"):