diff --git a/django/contrib/postgres/fields/array.py b/django/contrib/postgres/fields/array.py index 4171af82f9..a7e40703a3 100644 --- a/django/contrib/postgres/fields/array.py +++ b/django/contrib/postgres/fields/array.py @@ -169,7 +169,7 @@ class ArrayField(CheckFieldDefaultMixin, Field): else: obj = AttributeSetter(base_field.attname, val) values.append(base_field.value_to_string(obj)) - return json.dumps(values) + return json.dumps(values, ensure_ascii=False) def get_transform(self, name): transform = super().get_transform(name) diff --git a/django/contrib/postgres/fields/hstore.py b/django/contrib/postgres/fields/hstore.py index cfc156ab59..300458c0b1 100644 --- a/django/contrib/postgres/fields/hstore.py +++ b/django/contrib/postgres/fields/hstore.py @@ -43,7 +43,7 @@ class HStoreField(CheckFieldDefaultMixin, Field): return value def value_to_string(self, obj): - return json.dumps(self.value_from_object(obj)) + return json.dumps(self.value_from_object(obj), ensure_ascii=False) def formfield(self, **kwargs): return super().formfield( diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py index 076667d41a..23ad424c5c 100644 --- a/django/core/management/commands/makemessages.py +++ b/django/core/management/commands/makemessages.py @@ -40,7 +40,7 @@ def check_programs(*programs): def is_valid_locale(locale): - return re.match(r"^[a-z]+$", locale) or re.match(r"^[a-z]+_[A-Z].*$", locale) + return re.match(r"^[a-z]+$", locale) or re.match(r"^[a-z]+_[A-Z0-9].*$", locale) @total_ordering diff --git a/django/core/management/commands/sqlmigrate.py b/django/core/management/commands/sqlmigrate.py index 3e3151f0cf..076499b3e2 100644 --- a/django/core/management/commands/sqlmigrate.py +++ b/django/core/management/commands/sqlmigrate.py @@ -32,10 +32,9 @@ class Command(BaseCommand): ) def execute(self, *args, **options): - # sqlmigrate doesn't support coloring its output but we need to force - # no_color=True so that the BEGIN/COMMIT statements added by - # output_transaction don't get colored either. - options["no_color"] = True + # sqlmigrate doesn't support coloring its output, so make the + # BEGIN/COMMIT statements added by output_transaction colorless also. + self.style.SQL_KEYWORD = lambda noop: noop return super().execute(*args, **options) def handle(self, *args, **options): diff --git a/django/db/backends/base/operations.py b/django/db/backends/base/operations.py index dba9fcbba8..60de2d6c79 100644 --- a/django/db/backends/base/operations.py +++ b/django/db/backends/base/operations.py @@ -8,7 +8,6 @@ import sqlparse from django.conf import settings from django.db import NotSupportedError, transaction -from django.db.backends import utils from django.db.models.expressions import Col from django.utils import timezone from django.utils.deprecation import RemovedInDjango60Warning @@ -586,7 +585,7 @@ class BaseDatabaseOperations: Transform a decimal.Decimal value to an object compatible with what is expected by the backend driver for decimal (numeric) columns. """ - return utils.format_number(value, max_digits, decimal_places) + return value def adapt_ipaddressfield_value(self, value): """ diff --git a/django/db/backends/mysql/operations.py b/django/db/backends/mysql/operations.py index 9741e6a985..9806303539 100644 --- a/django/db/backends/mysql/operations.py +++ b/django/db/backends/mysql/operations.py @@ -166,9 +166,6 @@ class DatabaseOperations(BaseDatabaseOperations): """ return [(None, ("NULL", [], False))] - def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None): - return value - def last_executed_query(self, cursor, sql, params): # With MySQLdb, cursor objects have an (undocumented) "_executed" # attribute where the exact query sent to the database is saved. diff --git a/django/db/backends/oracle/operations.py b/django/db/backends/oracle/operations.py index 86340bbf4a..79c6da994e 100644 --- a/django/db/backends/oracle/operations.py +++ b/django/db/backends/oracle/operations.py @@ -629,9 +629,6 @@ END; 1900, 1, 1, value.hour, value.minute, value.second, value.microsecond ) - def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None): - return value - def combine_expression(self, connector, sub_expressions): lhs, rhs = sub_expressions if connector == "%%": diff --git a/django/db/backends/postgresql/operations.py b/django/db/backends/postgresql/operations.py index d89f81bf7e..8a0ca36a29 100644 --- a/django/db/backends/postgresql/operations.py +++ b/django/db/backends/postgresql/operations.py @@ -346,9 +346,6 @@ class DatabaseOperations(BaseDatabaseOperations): def adapt_timefield_value(self, value): return value - def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None): - return value - def adapt_ipaddressfield_value(self, value): if value: return Inet(value) diff --git a/django/db/backends/sqlite3/features.py b/django/db/backends/sqlite3/features.py index 2c1aa32506..2a39005f9f 100644 --- a/django/db/backends/sqlite3/features.py +++ b/django/db/backends/sqlite3/features.py @@ -50,6 +50,10 @@ class DatabaseFeatures(BaseDatabaseFeatures): # The django_format_dtdelta() function doesn't properly handle mixed # Date/DateTime fields and timedeltas. "expressions.tests.FTimeDeltaTests.test_mixed_comparisons1", + # SQLite doesn't parse escaped double quotes in the JSON path notation, + # so it cannot match keys that contains double quotes (#35842). + "model_fields.test_jsonfield.TestQuerying." + "test_lookups_special_chars_double_quotes", } create_test_table_with_composite_primary_key = """ CREATE TABLE test_table_composite_pk ( diff --git a/django/db/models/base.py b/django/db/models/base.py index a20e88749f..d948cd2a1c 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -726,12 +726,13 @@ class Model(AltersData, metaclass=ModelBase): if fields is not None: db_instance_qs = db_instance_qs.only(*fields) elif deferred_fields: - fields = { - f.attname - for f in self._meta.concrete_fields - if f.attname not in deferred_fields - } - db_instance_qs = db_instance_qs.only(*fields) + db_instance_qs = db_instance_qs.only( + *{ + f.attname + for f in self._meta.concrete_fields + if f.attname not in deferred_fields + } + ) db_instance = db_instance_qs.get() non_loaded_fields = db_instance.get_deferred_fields() @@ -748,9 +749,9 @@ class Model(AltersData, metaclass=ModelBase): field.delete_cached_value(self) # Clear cached relations. - for field in self._meta.related_objects: - if (fields is None or field.name in fields) and field.is_cached(self): - field.delete_cached_value(self) + for rel in self._meta.related_objects: + if (fields is None or rel.name in fields) and rel.is_cached(self): + rel.delete_cached_value(self) # Clear cached private relations. for field in self._meta.private_fields: diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index 855e8cc28d..c4a730f47b 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -1828,9 +1828,8 @@ class DecimalField(Field): ) return decimal_value - def get_db_prep_save(self, value, connection): - if hasattr(value, "as_sql"): - return value + def get_db_prep_value(self, value, connection, prepared=False): + value = super().get_db_prep_value(value, connection, prepared) return connection.ops.adapt_decimalfield_value( self.to_python(value), self.max_digits, self.decimal_places ) diff --git a/django/db/models/fields/json.py b/django/db/models/fields/json.py index 1b219e620c..8d743c436a 100644 --- a/django/db/models/fields/json.py +++ b/django/db/models/fields/json.py @@ -193,20 +193,18 @@ class HasKeyLookup(PostgresOperatorLookup): # Compile the final key without interpreting ints as array elements. return ".%s" % json.dumps(key_transform) - def as_sql(self, compiler, connection, template=None): + def _as_sql_parts(self, compiler, connection): # Process JSON path from the left-hand side. if isinstance(self.lhs, KeyTransform): - lhs, lhs_params, lhs_key_transforms = self.lhs.preprocess_lhs( + lhs_sql, lhs_params, lhs_key_transforms = self.lhs.preprocess_lhs( compiler, connection ) lhs_json_path = compile_json_path(lhs_key_transforms) else: - lhs, lhs_params = self.process_lhs(compiler, connection) + lhs_sql, lhs_params = self.process_lhs(compiler, connection) lhs_json_path = "$" - sql = template % lhs # Process JSON path from the right-hand side. rhs = self.rhs - rhs_params = [] if not isinstance(rhs, (list, tuple)): rhs = [rhs] for key in rhs: @@ -217,24 +215,45 @@ class HasKeyLookup(PostgresOperatorLookup): *rhs_key_transforms, final_key = rhs_key_transforms rhs_json_path = compile_json_path(rhs_key_transforms, include_root=False) rhs_json_path += self.compile_json_path_final_key(final_key) - rhs_params.append(lhs_json_path + rhs_json_path) + yield lhs_sql, lhs_params, lhs_json_path + rhs_json_path + + def _combine_sql_parts(self, parts): # Add condition for each key. if self.logical_operator: - sql = "(%s)" % self.logical_operator.join([sql] * len(rhs_params)) - return sql, tuple(lhs_params) + tuple(rhs_params) + return "(%s)" % self.logical_operator.join(parts) + return "".join(parts) + + def as_sql(self, compiler, connection, template=None): + sql_parts = [] + params = [] + for lhs_sql, lhs_params, rhs_json_path in self._as_sql_parts( + compiler, connection + ): + sql_parts.append(template % (lhs_sql, "%s")) + params.extend(lhs_params + [rhs_json_path]) + return self._combine_sql_parts(sql_parts), tuple(params) def as_mysql(self, compiler, connection): return self.as_sql( - compiler, connection, template="JSON_CONTAINS_PATH(%s, 'one', %%s)" + compiler, connection, template="JSON_CONTAINS_PATH(%s, 'one', %s)" ) def as_oracle(self, compiler, connection): - sql, params = self.as_sql( - compiler, connection, template="JSON_EXISTS(%s, '%%s')" - ) - # Add paths directly into SQL because path expressions cannot be passed - # as bind variables on Oracle. - return sql % tuple(params), [] + # Use a custom delimiter to prevent the JSON path from escaping the SQL + # literal. See comment in KeyTransform. + template = "JSON_EXISTS(%s, q'\uffff%s\uffff')" + sql_parts = [] + params = [] + for lhs_sql, lhs_params, rhs_json_path in self._as_sql_parts( + compiler, connection + ): + # Add right-hand-side directly into SQL because it cannot be passed + # as bind variables to JSON_EXISTS. It might result in invalid + # queries but it is assumed that it cannot be evaded because the + # path is JSON serialized. + sql_parts.append(template % (lhs_sql, rhs_json_path)) + params.extend(lhs_params) + return self._combine_sql_parts(sql_parts), tuple(params) def as_postgresql(self, compiler, connection): if isinstance(self.rhs, KeyTransform): @@ -246,7 +265,7 @@ class HasKeyLookup(PostgresOperatorLookup): def as_sqlite(self, compiler, connection): return self.as_sql( - compiler, connection, template="JSON_TYPE(%s, %%s) IS NOT NULL" + compiler, connection, template="JSON_TYPE(%s, %s) IS NOT NULL" ) @@ -362,10 +381,24 @@ class KeyTransform(Transform): json_path = compile_json_path(key_transforms) if connection.features.supports_primitives_in_json_field: sql = ( - "COALESCE(JSON_VALUE(%s, '%s'), JSON_QUERY(%s, '%s' DISALLOW SCALARS))" + "COALESCE(" + "JSON_VALUE(%s, q'\uffff%s\uffff')," + "JSON_QUERY(%s, q'\uffff%s\uffff' DISALLOW SCALARS)" + ")" ) else: - sql = "COALESCE(JSON_QUERY(%s, '%s'), JSON_VALUE(%s, '%s'))" + sql = ( + "COALESCE(" + "JSON_QUERY(%s, q'\uffff%s\uffff')," + "JSON_VALUE(%s, q'\uffff%s\uffff')" + ")" + ) + # Add paths directly into SQL because path expressions cannot be passed + # as bind variables on Oracle. Use a custom delimiter to prevent the + # JSON path from escaping the SQL literal. Each key in the JSON path is + # passed through json.dumps() with ensure_ascii=True (the default), + # which converts the delimiter into the escaped \uffff format. This + # ensures that the delimiter is not present in the JSON path. return sql % ((lhs, json_path) * 2), tuple(params) * 2 def as_postgresql(self, compiler, connection): @@ -455,9 +488,9 @@ class KeyTransformIsNull(lookups.IsNull): return "(NOT %s OR %s IS NULL)" % (sql, lhs), tuple(params) + tuple(lhs_params) def as_sqlite(self, compiler, connection): - template = "JSON_TYPE(%s, %%s) IS NULL" + template = "JSON_TYPE(%s, %s) IS NULL" if not self.rhs: - template = "JSON_TYPE(%s, %%s) IS NOT NULL" + template = "JSON_TYPE(%s, %s) IS NOT NULL" return HasKeyOrArrayIndex(self.lhs.lhs, self.lhs.key_name).as_sql( compiler, connection, diff --git a/django/forms/forms.py b/django/forms/forms.py index 452f554e1e..549a3adf6f 100644 --- a/django/forms/forms.py +++ b/django/forms/forms.py @@ -298,7 +298,10 @@ class BaseForm(RenderableFormMixin): error_class="nonfield", renderer=self.renderer ) else: - self._errors[field] = self.error_class(renderer=self.renderer) + self._errors[field] = self.error_class( + renderer=self.renderer, + field_id=self[field].auto_id, + ) self._errors[field].extend(error_list) if field in self.cleaned_data: del self.cleaned_data[field] diff --git a/django/forms/jinja2/django/forms/errors/list/ul.html b/django/forms/jinja2/django/forms/errors/list/ul.html index 752f7c2c8b..59528efccc 100644 --- a/django/forms/jinja2/django/forms/errors/list/ul.html +++ b/django/forms/jinja2/django/forms/errors/list/ul.html @@ -1 +1 @@ -{% if errors %}{% endif %} +{% if errors %}{% endif %} diff --git a/django/forms/templates/django/forms/errors/list/ul.html b/django/forms/templates/django/forms/errors/list/ul.html index 57b34ccb88..c28ce8af67 100644 --- a/django/forms/templates/django/forms/errors/list/ul.html +++ b/django/forms/templates/django/forms/errors/list/ul.html @@ -1 +1 @@ -{% if errors %}{% endif %} \ No newline at end of file +{% if errors %}{% endif %} \ No newline at end of file diff --git a/django/forms/utils.py b/django/forms/utils.py index f4fbf3e241..d24711d1a0 100644 --- a/django/forms/utils.py +++ b/django/forms/utils.py @@ -147,7 +147,7 @@ class ErrorList(UserList, list, RenderableErrorMixin): template_name_text = "django/forms/errors/list/text.txt" template_name_ul = "django/forms/errors/list/ul.html" - def __init__(self, initlist=None, error_class=None, renderer=None): + def __init__(self, initlist=None, error_class=None, renderer=None, field_id=None): super().__init__(initlist) if error_class is None: @@ -155,6 +155,7 @@ class ErrorList(UserList, list, RenderableErrorMixin): else: self.error_class = "errorlist {}".format(error_class) self.renderer = renderer or get_default_renderer() + self.field_id = field_id def as_data(self): return ValidationError(self.data).error_list diff --git a/django/template/loader_tags.py b/django/template/loader_tags.py index 1874d8c528..36703b4782 100644 --- a/django/template/loader_tags.py +++ b/django/template/loader_tags.py @@ -242,7 +242,11 @@ def do_block(parser, token): return BlockNode(block_name, nodelist) -def construct_relative_path(current_template_name, relative_name): +def construct_relative_path( + current_template_name, + relative_name, + allow_recursion=False, +): """ Convert a relative path (starting with './' or '../') to the full template name based on the current_template_name. @@ -264,7 +268,7 @@ def construct_relative_path(current_template_name, relative_name): "The relative path '%s' points outside the file hierarchy that " "template '%s' is in." % (relative_name, current_template_name) ) - if current_template_name.lstrip("/") == new_name: + if not allow_recursion and current_template_name.lstrip("/") == new_name: raise TemplateSyntaxError( "The relative path '%s' was translated to template name '%s', the " "same template in which the tag appears." @@ -346,7 +350,11 @@ def do_include(parser, token): options[option] = value isolated_context = options.get("only", False) namemap = options.get("with", {}) - bits[1] = construct_relative_path(parser.origin.template_name, bits[1]) + bits[1] = construct_relative_path( + parser.origin.template_name, + bits[1], + allow_recursion=True, + ) return IncludeNode( parser.compile_filter(bits[1]), extra_context=namemap, diff --git a/django/utils/html.py b/django/utils/html.py index b34af183d0..bc336d88a6 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -8,6 +8,7 @@ from collections.abc import Mapping from html.parser import HTMLParser from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit, urlunsplit +from django.core.exceptions import SuspiciousOperation from django.utils.deprecation import RemovedInDjango60Warning from django.utils.encoding import punycode from django.utils.functional import Promise, cached_property, keep_lazy, keep_lazy_text @@ -40,6 +41,7 @@ VOID_ELEMENTS = frozenset( ) MAX_URL_LENGTH = 2048 +MAX_STRIP_TAGS_DEPTH = 50 @keep_lazy(SafeString) @@ -211,15 +213,19 @@ def _strip_once(value): @keep_lazy_text def strip_tags(value): """Return the given HTML with all tags stripped.""" - # Note: in typical case this loop executes _strip_once once. Loop condition - # is redundant, but helps to reduce number of executions of _strip_once. value = str(value) + # Note: in typical case this loop executes _strip_once twice (the second + # execution does not remove any more tags). + strip_tags_depth = 0 while "<" in value and ">" in value: + if strip_tags_depth >= MAX_STRIP_TAGS_DEPTH: + raise SuspiciousOperation new_value = _strip_once(value) if value.count("<") == new_value.count("<"): # _strip_once wasn't able to detect more tags. break value = new_value + strip_tags_depth += 1 return value diff --git a/docs/intro/tutorial03.txt b/docs/intro/tutorial03.txt index 3d59809a0c..a19f9c949d 100644 --- a/docs/intro/tutorial03.txt +++ b/docs/intro/tutorial03.txt @@ -234,9 +234,7 @@ Now let's update our ``index`` view in ``polls/views.py`` to use the template: def index(request): latest_question_list = Question.objects.order_by("-pub_date")[:5] template = loader.get_template("polls/index.html") - context = { - "latest_question_list": latest_question_list, - } + context = {"latest_question_list": latest_question_list} return HttpResponse(template.render(context, request)) That code loads the template called ``polls/index.html`` and passes it a diff --git a/docs/ref/contrib/gis/db-api.txt b/docs/ref/contrib/gis/db-api.txt index e33d9a514f..c832f216ad 100644 --- a/docs/ref/contrib/gis/db-api.txt +++ b/docs/ref/contrib/gis/db-api.txt @@ -407,6 +407,7 @@ Function PostGIS Oracle MariaDB MySQL :class:`FromWKB` X X X X X :class:`FromWKT` X X X X X :class:`GeoHash` X X (≥ 11.7) X X (LWGEOM/RTTOPO) +:class:`GeometryDistance` X :class:`Intersection` X X X X X :class:`IsEmpty` X :class:`IsValid` X X X (≥ 11.7) X X diff --git a/docs/ref/forms/api.txt b/docs/ref/forms/api.txt index c6c83dcdfb..4875a1ab72 100644 --- a/docs/ref/forms/api.txt +++ b/docs/ref/forms/api.txt @@ -1025,13 +1025,17 @@ method you're using: Customizing the error list format --------------------------------- -.. class:: ErrorList(initlist=None, error_class=None, renderer=None) +.. class:: ErrorList(initlist=None, error_class=None, renderer=None, field_id=None) By default, forms use ``django.forms.utils.ErrorList`` to format validation errors. ``ErrorList`` is a list like object where ``initlist`` is the list of errors. In addition this class has the following attributes and methods. + .. versionchanged:: 5.2 + + The ``field_id`` argument was added. + .. attribute:: error_class The CSS classes to be used when rendering the error list. Any provided @@ -1043,6 +1047,16 @@ Customizing the error list format Defaults to ``None`` which means to use the default renderer specified by the :setting:`FORM_RENDERER` setting. + .. attribute:: field_id + + .. versionadded:: 5.2 + + An ``id`` for the field for which the errors relate. This allows an + HTML ``id`` attribute to be added in the error template and is useful + to associate the errors with the field. The default template uses the + format ``id="{{ field_id }}_error"`` and a value is provided by + :meth:`.Form.add_error` using the field's :attr:`~.BoundField.auto_id`. + .. attribute:: template_name The name of the template used when calling ``__str__`` or diff --git a/docs/ref/request-response.txt b/docs/ref/request-response.txt index 26fcb5fa08..632e222998 100644 --- a/docs/ref/request-response.txt +++ b/docs/ref/request-response.txt @@ -554,12 +554,21 @@ a subclass of dictionary. Exceptions are outlined here: .. method:: QueryDict.__getitem__(key) - Returns the value for the given key. If the key has more than one value, - it returns the last value. Raises + Returns the last value for the given key; or an empty list (``[]``) if the + key exists but has no values. Raises ``django.utils.datastructures.MultiValueDictKeyError`` if the key does not exist. (This is a subclass of Python's standard :exc:`KeyError`, so you can stick to catching ``KeyError``.) + .. code-block:: pycon + + >>> q = QueryDict("a=1&a=2&a=3", mutable=True) + >>> q.__getitem__("a") + '3' + >>> q.__setitem__("b", []) + >>> q.__getitem__("b") + [] + .. method:: QueryDict.__setitem__(key, value) Sets the given key to ``[value]`` (a list whose single element is diff --git a/docs/ref/utils.txt b/docs/ref/utils.txt index 33e0fceadf..438a38cea0 100644 --- a/docs/ref/utils.txt +++ b/docs/ref/utils.txt @@ -521,7 +521,7 @@ https://web.archive.org/web/20110718035220/http://diveintomark.org/archives/2004 The cached value can be treated like an ordinary attribute of the instance:: # clear it, requiring re-computation next time it's called - del person.friends # or delattr(person, "friends") + person.__dict__.pop("friends", None) # set a value manually, that will persist on the instance until cleared person.friends = ["Huckleberry Finn", "Tom Sawyer"] diff --git a/docs/releases/4.2.17.txt b/docs/releases/4.2.17.txt index 5139d7034d..9a6aee3db6 100644 --- a/docs/releases/4.2.17.txt +++ b/docs/releases/4.2.17.txt @@ -6,3 +6,28 @@ Django 4.2.17 release notes Django 4.2.17 fixes one security issue with severity "high" and one security issue with severity "moderate" in 4.2.16. + +CVE-2024-53907: Denial-of-service possibility in ``strip_tags()`` +================================================================= + +:func:`~django.utils.html.strip_tags` would be extremely slow to evaluate +certain inputs containing large sequences of nested incomplete HTML entities. +The ``strip_tags()`` method is used to implement the corresponding +:tfilter:`striptags` template filter, which was thus also vulnerable. + +``strip_tags()`` now has an upper limit of recursive calls to ``HTMLParser`` +before raising a :exc:`.SuspiciousOperation` exception. + +Remember that absolutely NO guarantee is provided about the results of +``strip_tags()`` being HTML safe. So NEVER mark safe the result of a +``strip_tags()`` call without escaping it first, for example with +:func:`django.utils.html.escape`. + +CVE-2024-53908: Potential SQL injection via ``HasKey(lhs, rhs)`` on Oracle +========================================================================== + +Direct usage of the ``django.db.models.fields.json.HasKey`` lookup on Oracle +was subject to SQL injection if untrusted data was used as a ``lhs`` value. + +Applications that use the :lookup:`has_key ` lookup through +the ``__`` syntax are unaffected. diff --git a/docs/releases/5.0.10.txt b/docs/releases/5.0.10.txt index b06c376038..ae1fbf99e4 100644 --- a/docs/releases/5.0.10.txt +++ b/docs/releases/5.0.10.txt @@ -6,3 +6,28 @@ Django 5.0.10 release notes Django 5.0.10 fixes one security issue with severity "high" and one security issue with severity "moderate" in 5.0.9. + +CVE-2024-53907: Denial-of-service possibility in ``strip_tags()`` +================================================================= + +:func:`~django.utils.html.strip_tags` would be extremely slow to evaluate +certain inputs containing large sequences of nested incomplete HTML entities. +The ``strip_tags()`` method is used to implement the corresponding +:tfilter:`striptags` template filter, which was thus also vulnerable. + +``strip_tags()`` now has an upper limit of recursive calls to ``HTMLParser`` +before raising a :exc:`.SuspiciousOperation` exception. + +Remember that absolutely NO guarantee is provided about the results of +``strip_tags()`` being HTML safe. So NEVER mark safe the result of a +``strip_tags()`` call without escaping it first, for example with +:func:`django.utils.html.escape`. + +CVE-2024-53908: Potential SQL injection via ``HasKey(lhs, rhs)`` on Oracle +========================================================================== + +Direct usage of the ``django.db.models.fields.json.HasKey`` lookup on Oracle +was subject to SQL injection if untrusted data was used as a ``lhs`` value. + +Applications that use the :lookup:`has_key ` lookup through +the ``__`` syntax are unaffected. diff --git a/docs/releases/5.1.4.txt b/docs/releases/5.1.4.txt index 0c21d99566..e768725688 100644 --- a/docs/releases/5.1.4.txt +++ b/docs/releases/5.1.4.txt @@ -7,8 +7,37 @@ Django 5.1.4 release notes Django 5.1.4 fixes one security issue with severity "high", one security issue with severity "moderate", and several bugs in 5.1.3. +CVE-2024-53907: Denial-of-service possibility in ``strip_tags()`` +================================================================= + +:func:`~django.utils.html.strip_tags` would be extremely slow to evaluate +certain inputs containing large sequences of nested incomplete HTML entities. +The ``strip_tags()`` method is used to implement the corresponding +:tfilter:`striptags` template filter, which was thus also vulnerable. + +``strip_tags()`` now has an upper limit of recursive calls to ``HTMLParser`` +before raising a :exc:`.SuspiciousOperation` exception. + +Remember that absolutely NO guarantee is provided about the results of +``strip_tags()`` being HTML safe. So NEVER mark safe the result of a +``strip_tags()`` call without escaping it first, for example with +:func:`django.utils.html.escape`. + +CVE-2024-53908: Potential SQL injection via ``HasKey(lhs, rhs)`` on Oracle +========================================================================== + +Direct usage of the ``django.db.models.fields.json.HasKey`` lookup on Oracle +was subject to SQL injection if untrusted data was used as a ``lhs`` value. + +Applications that use the :lookup:`has_key ` lookup through +the ``__`` syntax are unaffected. + Bugfixes ======== * Fixed a crash in ``createsuperuser`` on Python 3.13+ caused by an unhandled ``OSError`` when the username could not be determined (:ticket:`35942`). + +* Fixed a regression in Django 5.1 where relational fields were not updated + when calling ``Model.refresh_from_db()`` on instances with deferred fields + (:ticket:`35950`). diff --git a/docs/releases/5.1.5.txt b/docs/releases/5.1.5.txt new file mode 100644 index 0000000000..af0dab545c --- /dev/null +++ b/docs/releases/5.1.5.txt @@ -0,0 +1,12 @@ +========================== +Django 5.1.5 release notes +========================== + +*Expected January 7, 2025* + +Django 5.1.5 fixes several bugs in 5.1.4. + +Bugfixes +======== + +* ... diff --git a/docs/releases/5.2.txt b/docs/releases/5.2.txt index 4b05fd3279..907159e36d 100644 --- a/docs/releases/5.2.txt +++ b/docs/releases/5.2.txt @@ -249,6 +249,10 @@ Forms * The new :class:`~django.forms.TelInput` form widget is for entering telephone numbers and renders as ````. +* The new ``field_id`` argument for :class:`~django.forms.ErrorList` allows an + HTML ``id`` attribute to be added in the error template. See + :attr:`.ErrorList.field_id` for details. + Generic Views ~~~~~~~~~~~~~ @@ -395,6 +399,8 @@ backends. * The new :meth:`Model._is_pk_set() ` method allows checking if a Model instance's primary key is defined. +* ``BaseDatabaseOperations.adapt_decimalfield_value()`` is now a no-op, simply + returning the given value. :mod:`django.contrib.gis` ------------------------- diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 536e5917ab..e16ae37f5e 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -32,6 +32,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 5.1.5 5.1.4 5.1.3 5.1.2 diff --git a/docs/releases/security.txt b/docs/releases/security.txt index b83be59dbb..02ea77b54d 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -36,6 +36,28 @@ Issues under Django's security process All security issues have been handled under versions of Django's security process. These are listed below. +December 4, 2024 - :cve:`2024-53907` +------------------------------------ + +Potential denial-of-service in ``django.utils.html.strip_tags()``. +`Full description +`__ + +* Django 5.1 :commit:`(patch) ` +* Django 5.0 :commit:`(patch) ` +* Django 4.2 :commit:`(patch) <790eb058b0716c536a2f2e8d1c6d5079d776c22b>` + +December 4, 2024 - :cve:`2024-53908` +------------------------------------ + +Potential SQL injection in ``HasKey(lhs, rhs)`` on Oracle. +`Full description +`__ + +* Django 5.1 :commit:`(patch) <6943d61818e63e77b65d8b1ae65941e8f04bd87b>` +* Django 5.0 :commit:`(patch) ` +* Django 4.2 :commit:`(patch) <7376bcbf508883282ffcc0f0fac5cf0ed2d6cbc5>` + September 3, 2024 - :cve:`2024-45231` ------------------------------------- diff --git a/docs/spelling_wordlist b/docs/spelling_wordlist index 747a712a62..e9441c62d9 100644 --- a/docs/spelling_wordlist +++ b/docs/spelling_wordlist @@ -3,14 +3,12 @@ accessor accessors Aceh admindocs -affine affordances Ai Alchin allowlist alphanumerics amet -analytics arccosine architected arcsine @@ -60,7 +58,6 @@ Bokmål Bonham bookmarklet bookmarklets -boolean booleans bpython Bronn @@ -114,23 +111,18 @@ Danga Darussalam databrowse datafile -dataset -datasets datetimes declaratively -decrementing deduplicates deduplication deepcopy deferrable -DEP deprecations deserialization deserialize deserialized deserializer deserializing -deterministically Deutsch dev dictConfig @@ -142,7 +134,6 @@ Disqus distro django djangoproject -djangotutorial dm docstring docstrings @@ -165,9 +156,7 @@ esque Ess ETag ETags -exe exfiltration -extensibility fallbacks favicon fieldset @@ -177,7 +166,6 @@ filesystems flatpage flatpages focusable -fooapp formatter formatters formfield @@ -210,7 +198,6 @@ hashable hasher hashers headerlist -hoc Hoerner Holovaty Homebrew @@ -223,7 +210,6 @@ Hypercorn ies iframe Igbo -incrementing indexable ing ini @@ -240,7 +226,6 @@ iterable iterables iteratively ize -Jazzband Jinja jQuery Jupyter @@ -283,7 +268,6 @@ manouche Marino memcache memcached -mentorship metaclass metaclasses metre @@ -338,8 +322,6 @@ orm Outdim outfile paginator -parallelization -parallelized parameterization params parens @@ -361,9 +343,7 @@ pluggable pluralizations pooler postfix -postgis postgres -postgresql pragma pre precisions @@ -376,7 +356,6 @@ prefetches prefetching preload preloaded -prepend prepended prepending prepends @@ -429,7 +408,6 @@ reflow registrable reimplement reindent -reindex releaser releasers reloader @@ -439,7 +417,6 @@ repo reportable reprojection reraising -resampling reST reStructuredText reusability @@ -447,7 +424,6 @@ reverter roadmap Roald rss -runtime Sandvik savepoint savepoints @@ -458,7 +434,6 @@ screencasts semimajor semiminor serializability -serializable serializer serializers shapefile @@ -467,7 +442,6 @@ sharding sitewide sliceable SMTP -solaris Sorani sortable Spectre @@ -535,7 +509,6 @@ toolkits toolset trac tracebacks -transactional Transifex Tredinnick triager diff --git a/docs/topics/db/fixtures.txt b/docs/topics/db/fixtures.txt index ac5b34dae0..6066d34f8e 100644 --- a/docs/topics/db/fixtures.txt +++ b/docs/topics/db/fixtures.txt @@ -4,28 +4,25 @@ Fixtures ======== -.. seealso:: - - * :doc:`/howto/initial-data` - -What is a fixture? -================== - A *fixture* is a collection of files that contain the serialized contents of the database. Each fixture has a unique name, and the files that comprise the fixture can be distributed over multiple directories, in multiple applications. -How to produce a fixture? -========================= +.. seealso:: + + * :doc:`/howto/initial-data` + +How to produce a fixture +======================== Fixtures can be generated by :djadmin:`manage.py dumpdata `. It's also possible to generate custom fixtures by directly using :doc:`serialization tools ` or even by handwriting them. -How to use a fixture? -===================== +How to use a fixture +==================== -Fixtures can be used to pre-populate database with data for +Fixtures can be used to pre-populate the database with data for :ref:`tests `: .. code-block:: python @@ -40,8 +37,8 @@ or to provide some :ref:`initial data ` using the django-admin loaddata -Where Django looks for fixtures? -================================ +How fixtures are discovered +=========================== Django will search in these locations for fixtures: @@ -116,8 +113,8 @@ example). .. _MySQL: https://dev.mysql.com/doc/refman/en/constraint-foreign-key.html -How fixtures are saved to the database? -======================================= +How fixtures are saved to the database +====================================== When fixture files are processed, the data is saved to the database as is. Model defined :meth:`~django.db.models.Model.save` methods are not called, and diff --git a/docs/topics/forms/formsets.txt b/docs/topics/forms/formsets.txt index 14d4962eb6..a452c7640d 100644 --- a/docs/topics/forms/formsets.txt +++ b/docs/topics/forms/formsets.txt @@ -571,14 +571,12 @@ happen when the user changes these values: ... {"title": "Article #2", "pub_date": datetime.date(2008, 5, 11)}, ... ], ... ) - >>> formset.is_valid() - True >>> for form in formset.ordered_forms: ... print(form.cleaned_data) ... - {'pub_date': datetime.date(2008, 5, 1), 'ORDER': 0, 'title': 'Article #3'} - {'pub_date': datetime.date(2008, 5, 11), 'ORDER': 1, 'title': 'Article #2'} - {'pub_date': datetime.date(2008, 5, 10), 'ORDER': 2, 'title': 'Article #1'} + {'title': 'Article #3', 'pub_date': datetime.date(2008, 5, 1), 'ORDER': 0} + {'title': 'Article #2', 'pub_date': datetime.date(2008, 5, 11), 'ORDER': 1} + {'title': 'Article #1', 'pub_date': datetime.date(2008, 5, 10), 'ORDER': 2} :class:`~django.forms.formsets.BaseFormSet` also provides an :attr:`~django.forms.formsets.BaseFormSet.ordering_widget` attribute and @@ -690,7 +688,7 @@ delete fields you can access them with ``deleted_forms``: ... ], ... ) >>> [form.cleaned_data for form in formset.deleted_forms] - [{'DELETE': True, 'pub_date': datetime.date(2008, 5, 10), 'title': 'Article #1'}] + [{'title': 'Article #1', 'pub_date': datetime.date(2008, 5, 10), 'DELETE': True}] If you are using a :class:`ModelFormSet`, model instances for deleted forms will be deleted when you call diff --git a/tests/composite_pk/test_checks.py b/tests/composite_pk/test_checks.py index 02a162c31d..58b580ca85 100644 --- a/tests/composite_pk/test_checks.py +++ b/tests/composite_pk/test_checks.py @@ -1,7 +1,7 @@ from django.core import checks from django.db import connection, models from django.db.models import F -from django.test import TestCase +from django.test import TestCase, skipUnlessAnyDBFeature from django.test.utils import isolate_apps @@ -217,16 +217,18 @@ class CompositePKChecksTests(TestCase): ], ) + @skipUnlessAnyDBFeature( + "supports_virtual_generated_columns", + "supports_stored_generated_columns", + ) def test_composite_pk_cannot_include_generated_field(self): - is_oracle = connection.vendor == "oracle" - class Foo(models.Model): pk = models.CompositePrimaryKey("id", "foo") id = models.IntegerField() foo = models.GeneratedField( expression=F("id"), output_field=models.IntegerField(), - db_persist=not is_oracle, + db_persist=connection.features.supports_stored_generated_columns, ) self.assertEqual( diff --git a/tests/composite_pk/tests.py b/tests/composite_pk/tests.py index 71522cb836..25e5f2fdd5 100644 --- a/tests/composite_pk/tests.py +++ b/tests/composite_pk/tests.py @@ -2,7 +2,12 @@ import json import unittest from uuid import UUID -import yaml +try: + import yaml # NOQA + + HAS_YAML = True +except ImportError: + HAS_YAML = False from django import forms from django.core import serializers @@ -35,9 +40,9 @@ class CompositePKTests(TestCase): cls.comment = Comment.objects.create(tenant=cls.tenant, id=1, user=cls.user) @staticmethod - def get_constraints(table): + def get_primary_key_columns(table): with connection.cursor() as cursor: - return connection.introspection.get_constraints(cursor, table) + return connection.introspection.get_primary_key_columns(cursor, table) def test_pk_updated_if_field_updated(self): user = User.objects.get(pk=self.user.pk) @@ -125,53 +130,15 @@ class CompositePKTests(TestCase): with self.assertRaises(IntegrityError): Comment.objects.create(tenant=self.tenant, id=self.comment.id) - @unittest.skipUnless(connection.vendor == "postgresql", "PostgreSQL specific test") - def test_get_constraints_postgresql(self): - user_constraints = self.get_constraints(User._meta.db_table) - user_pk = user_constraints["composite_pk_user_pkey"] - self.assertEqual(user_pk["columns"], ["tenant_id", "id"]) - self.assertIs(user_pk["primary_key"], True) - - comment_constraints = self.get_constraints(Comment._meta.db_table) - comment_pk = comment_constraints["composite_pk_comment_pkey"] - self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"]) - self.assertIs(comment_pk["primary_key"], True) - - @unittest.skipUnless(connection.vendor == "sqlite", "SQLite specific test") - def test_get_constraints_sqlite(self): - user_constraints = self.get_constraints(User._meta.db_table) - user_pk = user_constraints["__primary__"] - self.assertEqual(user_pk["columns"], ["tenant_id", "id"]) - self.assertIs(user_pk["primary_key"], True) - - comment_constraints = self.get_constraints(Comment._meta.db_table) - comment_pk = comment_constraints["__primary__"] - self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"]) - self.assertIs(comment_pk["primary_key"], True) - - @unittest.skipUnless(connection.vendor == "mysql", "MySQL specific test") - def test_get_constraints_mysql(self): - user_constraints = self.get_constraints(User._meta.db_table) - user_pk = user_constraints["PRIMARY"] - self.assertEqual(user_pk["columns"], ["tenant_id", "id"]) - self.assertIs(user_pk["primary_key"], True) - - comment_constraints = self.get_constraints(Comment._meta.db_table) - comment_pk = comment_constraints["PRIMARY"] - self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"]) - self.assertIs(comment_pk["primary_key"], True) - - @unittest.skipUnless(connection.vendor == "oracle", "Oracle specific test") - def test_get_constraints_oracle(self): - user_constraints = self.get_constraints(User._meta.db_table) - user_pk = next(c for c in user_constraints.values() if c["primary_key"]) - self.assertEqual(user_pk["columns"], ["tenant_id", "id"]) - self.assertEqual(user_pk["primary_key"], 1) - - comment_constraints = self.get_constraints(Comment._meta.db_table) - comment_pk = next(c for c in comment_constraints.values() if c["primary_key"]) - self.assertEqual(comment_pk["columns"], ["tenant_id", "comment_id"]) - self.assertEqual(comment_pk["primary_key"], 1) + def test_get_primary_key_columns(self): + self.assertEqual( + self.get_primary_key_columns(User._meta.db_table), + ["tenant_id", "id"], + ) + self.assertEqual( + self.get_primary_key_columns(Comment._meta.db_table), + ["tenant_id", "comment_id"], + ) def test_in_bulk(self): """ @@ -291,6 +258,7 @@ class CompositePKFixturesTests(TestCase): }, ) + @unittest.skipUnless(HAS_YAML, "No yaml library detected") def test_serialize_user_yaml(self): users = User.objects.filter(pk=(2, 3)) result = serializers.serialize("yaml", users) diff --git a/tests/contenttypes_tests/test_fields.py b/tests/contenttypes_tests/test_fields.py index ab16324fb6..fc49d59b27 100644 --- a/tests/contenttypes_tests/test_fields.py +++ b/tests/contenttypes_tests/test_fields.py @@ -57,6 +57,15 @@ class GenericForeignKeyTests(TestCase): self.assertIsNot(answer.question, old_question_obj) self.assertEqual(answer.question, old_question_obj) + def test_clear_cached_generic_relation_when_deferred(self): + question = Question.objects.create(text="question") + Answer.objects.create(text="answer", question=question) + answer = Answer.objects.defer("text").get() + old_question_obj = answer.question + # The reverse relation is refreshed even when the text field is deferred. + answer.refresh_from_db() + self.assertIsNot(answer.question, old_question_obj) + class GenericRelationTests(TestCase): def test_value_to_string(self): diff --git a/tests/defer/tests.py b/tests/defer/tests.py index 3945b667ba..989b5c63d7 100644 --- a/tests/defer/tests.py +++ b/tests/defer/tests.py @@ -290,6 +290,14 @@ class TestDefer2(AssertionMixin, TestCase): self.assertEqual(rf2.name, "new foo") self.assertEqual(rf2.value, "new bar") + def test_refresh_when_one_field_deferred(self): + s = Secondary.objects.create() + PrimaryOneToOne.objects.create(name="foo", value="bar", related=s) + s = Secondary.objects.defer("first").get() + p_before = s.primary_o2o + s.refresh_from_db() + self.assertIsNot(s.primary_o2o, p_before) + class InvalidDeferTests(SimpleTestCase): def test_invalid_defer(self): diff --git a/tests/forms_tests/tests/test_error_messages.py b/tests/forms_tests/tests/test_error_messages.py index e44c6d6668..f4f5700107 100644 --- a/tests/forms_tests/tests/test_error_messages.py +++ b/tests/forms_tests/tests/test_error_messages.py @@ -249,7 +249,8 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin): form1 = TestForm({"first_name": "John"}) self.assertHTMLEqual( str(form1["last_name"].errors), - '
  • This field is required.
', + '
  • ' + "This field is required.
", ) self.assertHTMLEqual( str(form1.errors["__all__"]), @@ -280,7 +281,7 @@ class FormsErrorMessagesTestCase(SimpleTestCase, AssertFormErrorsMixin): f = SomeForm({"field": "&h", "alert()h"), (">br>br>br>X", "XX"), + ("<" * 50 + "a>" * 50, ""), ) for value, output in items: with self.subTest(value=value, output=output): self.check_output(strip_tags, value, output) self.check_output(strip_tags, lazystr(value), output) + def test_strip_tags_suspicious_operation(self): + value = "<" * 51 + "a>" * 51, "" + with self.assertRaises(SuspiciousOperation): + strip_tags(value) + def test_strip_tags_files(self): # Test with more lengthy content (also catching performance regressions) for filename in ("strip_tags1.html", "strip_tags2.txt"):