1
0
mirror of https://github.com/django/django.git synced 2024-12-23 01:25:58 +00:00

Fixed #25367 -- Allowed boolean expressions in QuerySet.filter() and exclude().

This allows using expressions that have an output_field that is a
BooleanField to be used directly in a queryset filters, or in the
When() clauses of a Case() expression.

Thanks Josh Smeaton, Tim Graham, Simon Charette, Mariusz Felisiak, and
Adam Johnson for reviews.

Co-Authored-By: NyanKiyoshi <hello@vanille.bid>
This commit is contained in:
Matthew Schinckel 2017-02-27 19:31:52 +10:30 committed by Mariusz Felisiak
parent 069bee7c12
commit 4137fc2efc
10 changed files with 184 additions and 23 deletions

View File

@ -581,6 +581,13 @@ class BaseDatabaseOperations:
""" """
pass pass
def conditional_expression_supported_in_where_clause(self, expression):
"""
Return True, if the conditional expression is supported in the WHERE
clause.
"""
return True
def combine_expression(self, connector, sub_expressions): def combine_expression(self, connector, sub_expressions):
""" """
Combine a list of subexpressions into a single expression, using Combine a list of subexpressions into a single expression, using

View File

@ -6,6 +6,8 @@ from functools import lru_cache
from django.conf import settings from django.conf import settings
from django.db.backends.base.operations import BaseDatabaseOperations from django.db.backends.base.operations import BaseDatabaseOperations
from django.db.backends.utils import strip_quotes, truncate_name from django.db.backends.utils import strip_quotes, truncate_name
from django.db.models.expressions import Exists, ExpressionWrapper
from django.db.models.query_utils import Q
from django.db.utils import DatabaseError from django.db.utils import DatabaseError
from django.utils import timezone from django.utils import timezone
from django.utils.encoding import force_bytes, force_str from django.utils.encoding import force_bytes, force_str
@ -607,3 +609,14 @@ END;
if fields: if fields:
return self.connection.features.max_query_params // len(fields) return self.connection.features.max_query_params // len(fields)
return len(objs) return len(objs)
def conditional_expression_supported_in_where_clause(self, expression):
"""
Oracle supports only EXISTS(...) or filters in the WHERE clause, others
must be compared with True.
"""
if isinstance(expression, Exists):
return True
if isinstance(expression, ExpressionWrapper) and isinstance(expression.expression, Q):
return True
return False

View File

@ -90,6 +90,8 @@ class Combinable:
return self._combine(other, self.POW, False) return self._combine(other, self.POW, False)
def __and__(self, other): def __and__(self, other):
if getattr(self, 'conditional', False) and getattr(other, 'conditional', False):
return Q(self) & Q(other)
raise NotImplementedError( raise NotImplementedError(
"Use .bitand() and .bitor() for bitwise logical operations." "Use .bitand() and .bitor() for bitwise logical operations."
) )
@ -104,6 +106,8 @@ class Combinable:
return self._combine(other, self.BITRIGHTSHIFT, False) return self._combine(other, self.BITRIGHTSHIFT, False)
def __or__(self, other): def __or__(self, other):
if getattr(self, 'conditional', False) and getattr(other, 'conditional', False):
return Q(self) | Q(other)
raise NotImplementedError( raise NotImplementedError(
"Use .bitand() and .bitor() for bitwise logical operations." "Use .bitand() and .bitor() for bitwise logical operations."
) )
@ -245,6 +249,10 @@ class BaseExpression:
]) ])
return c return c
@property
def conditional(self):
return isinstance(self.output_field, fields.BooleanField)
@property @property
def field(self): def field(self):
return self.output_field return self.output_field
@ -873,12 +881,17 @@ class ExpressionWrapper(Expression):
class When(Expression): class When(Expression):
template = 'WHEN %(condition)s THEN %(result)s' template = 'WHEN %(condition)s THEN %(result)s'
# This isn't a complete conditional expression, must be used in Case().
conditional = False
def __init__(self, condition=None, then=None, **lookups): def __init__(self, condition=None, then=None, **lookups):
if lookups and condition is None: if lookups and condition is None:
condition, lookups = Q(**lookups), None condition, lookups = Q(**lookups), None
if condition is None or not getattr(condition, 'conditional', False) or lookups: if condition is None or not getattr(condition, 'conditional', False) or lookups:
raise TypeError("__init__() takes either a Q object or lookups as keyword arguments") raise TypeError(
'When() supports a Q object, a boolean expression, or lookups '
'as a condition.'
)
if isinstance(condition, Q) and not condition: if isinstance(condition, Q) and not condition:
raise ValueError("An empty Q() can't be used as a When() condition.") raise ValueError("An empty Q() can't be used as a When() condition.")
super().__init__(output_field=None) super().__init__(output_field=None)
@ -1090,6 +1103,7 @@ class Exists(Subquery):
class OrderBy(BaseExpression): class OrderBy(BaseExpression):
template = '%(expression)s %(ordering)s' template = '%(expression)s %(ordering)s'
conditional = False
def __init__(self, expression, descending=False, nulls_first=False, nulls_last=False): def __init__(self, expression, descending=False, nulls_first=False, nulls_last=False):
if nulls_first and nulls_last: if nulls_first and nulls_last:

View File

@ -1229,6 +1229,16 @@ class Query(BaseExpression):
""" """
if isinstance(filter_expr, dict): if isinstance(filter_expr, dict):
raise FieldError("Cannot parse keyword query as dict") raise FieldError("Cannot parse keyword query as dict")
if hasattr(filter_expr, 'resolve_expression') and getattr(filter_expr, 'conditional', False):
if connections[DEFAULT_DB_ALIAS].ops.conditional_expression_supported_in_where_clause(filter_expr):
condition = filter_expr.resolve_expression(self)
else:
# Expression is not supported in the WHERE clause, add
# comparison with True.
condition = self.build_lookup(['exact'], filter_expr.resolve_expression(self), True)
clause = self.where_class()
clause.add(condition, AND)
return clause, []
arg, value = filter_expr arg, value = filter_expr
if not arg: if not arg:
raise FieldError("Cannot parse keyword query %r" % arg) raise FieldError("Cannot parse keyword query %r" % arg)

View File

@ -42,9 +42,15 @@ We'll be using the following model in the subsequent examples::
A ``When()`` object is used to encapsulate a condition and its result for use A ``When()`` object is used to encapsulate a condition and its result for use
in the conditional expression. Using a ``When()`` object is similar to using in the conditional expression. Using a ``When()`` object is similar to using
the :meth:`~django.db.models.query.QuerySet.filter` method. The condition can the :meth:`~django.db.models.query.QuerySet.filter` method. The condition can
be specified using :ref:`field lookups <field-lookups>` or be specified using :ref:`field lookups <field-lookups>`,
:class:`~django.db.models.Q` objects. The result is provided using the ``then`` :class:`~django.db.models.Q` objects, or :class:`~django.db.models.Expression`
keyword. objects that have an ``output_field`` that is a
:class:`~django.db.models.BooleanField`. The result is provided using the
``then`` keyword.
.. versionchanged:: 3.0
Support for boolean :class:`~django.db.models.Expression` was added.
Some examples:: Some examples::
@ -60,6 +66,12 @@ Some examples::
>>> # Complex conditions can be created using Q objects >>> # Complex conditions can be created using Q objects
>>> When(Q(name__startswith="John") | Q(name__startswith="Paul"), >>> When(Q(name__startswith="John") | Q(name__startswith="Paul"),
... then='name') ... then='name')
>>> # Condition can be created using boolean expressions.
>>> from django.db.models import Exists, OuterRef
>>> non_unique_account_type = Client.objects.filter(
... account_type=OuterRef('account_type'),
... ).exclude(pk=OuterRef('pk')).values('pk')
>>> When(Exists(non_unique_account_type), then=Value('non unique'))
Keep in mind that each of these values can be an expression. Keep in mind that each of these values can be an expression.
@ -158,9 +170,9 @@ registered more than a year ago::
Advanced queries Advanced queries
================ ================
Conditional expressions can be used in annotations, aggregations, lookups, and Conditional expressions can be used in annotations, aggregations, filters,
updates. They can also be combined and nested with other expressions. This lookups, and updates. They can also be combined and nested with other
allows you to make powerful conditional queries. expressions. This allows you to make powerful conditional queries.
Conditional update Conditional update
------------------ ------------------
@ -236,3 +248,29 @@ On other databases, this is emulated using a ``CASE`` statement:
The two SQL statements are functionally equivalent but the more explicit The two SQL statements are functionally equivalent but the more explicit
``FILTER`` may perform better. ``FILTER`` may perform better.
Conditional filter
------------------
.. versionadded:: 3.0
When a conditional expression returns a boolean value, it is possible to use it
directly in filters. This means that it will not be added to the ``SELECT``
columns, but you can still use it to filter results::
>>> non_unique_account_type = Client.objects.filter(
... account_type=OuterRef('account_type'),
... ).exclude(pk=OuterRef('pk')).values('pk')
>>> Client.objects.filter(~Exists(non_unique_account_type))
In SQL terms, that evaluates to:
.. code-block:: sql
SELECT ...
FROM client c0
WHERE NOT EXISTS (
SELECT c1.id
FROM client c1
WHERE c1.account_type = c0.account_type AND NOT c1.id = c0.id
)

View File

@ -5,10 +5,11 @@ Query Expressions
.. currentmodule:: django.db.models .. currentmodule:: django.db.models
Query expressions describe a value or a computation that can be used as part of Query expressions describe a value or a computation that can be used as part of
an update, create, filter, order by, annotation, or aggregate. There are a an update, create, filter, order by, annotation, or aggregate. When an
number of built-in expressions (documented below) that can be used to help you expression outputs a boolean value, it may be used directly in filters. There
write queries. Expressions can be combined, or in some cases nested, to form are a number of built-in expressions (documented below) that can be used to
more complex computations. help you write queries. Expressions can be combined, or in some cases nested,
to form more complex computations.
Supported arithmetic Supported arithmetic
==================== ====================
@ -69,6 +70,12 @@ Some examples
CharField.register_lookup(Length) CharField.register_lookup(Length)
Company.objects.order_by('name__length') Company.objects.order_by('name__length')
# Boolean expression can be used directly in filters.
from django.db.models import Exists
Company.objects.filter(
Exists(Employee.objects.filter(company=OuterRef('pk'), salary__gt=10))
)
Built-in Expressions Built-in Expressions
==================== ====================
@ -626,22 +633,25 @@ degrade performance, it's automatically removed.
You can query using ``NOT EXISTS`` with ``~Exists()``. You can query using ``NOT EXISTS`` with ``~Exists()``.
Filtering on a ``Subquery`` expression Filtering on a ``Subquery()`` or ``Exists()`` expressions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
It's not possible to filter directly using ``Subquery`` and ``Exists``, e.g.:: ``Subquery()`` that returns a boolean value and ``Exists()`` may be used as a
``condition`` in :class:`~django.db.models.expressions.When` expressions, or to
directly filter a queryset::
>>> recent_comments = Comment.objects.filter(...) # From above
>>> Post.objects.filter(Exists(recent_comments)) >>> Post.objects.filter(Exists(recent_comments))
...
TypeError: 'Exists' object is not iterable
This will ensure that the subquery will not be added to the ``SELECT`` columns,
which may result in a better performance.
You must filter on a subquery expression by first annotating the queryset .. versionchanged:: 3.0
and then filtering based on that annotation::
>>> Post.objects.annotate( In previous versions of Django, it was necessary to first annotate and then
... recent_comment=Exists(recent_comments), filter against the annotation. This resulted in the annotated value always
... ).filter(recent_comment=True) being present in the query result, and often resulted in a query that took
more time to execute.
Using aggregates within a ``Subquery`` expression Using aggregates within a ``Subquery`` expression
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

View File

@ -74,6 +74,13 @@ enable adding exclusion constraints on PostgreSQL. Constraints are added to
models using the models using the
:attr:`Meta.constraints <django.db.models.Options.constraints>` option. :attr:`Meta.constraints <django.db.models.Options.constraints>` option.
Filter expressions
------------------
Expressions that outputs :class:`~django.db.models.BooleanField` may now be
used directly in ``QuerySet`` filters, without having to first annotate and
then filter against the annotation.
Minor features Minor features
-------------- --------------

View File

@ -34,6 +34,7 @@ class Company(models.Model):
related_name='company_point_of_contact_set', related_name='company_point_of_contact_set',
null=True, null=True,
) )
based_in_eu = models.BooleanField(default=False)
def __str__(self): def __str__(self):
return self.name return self.name

View File

@ -37,7 +37,7 @@ class BasicExpressionsTests(TestCase):
ceo=Employee.objects.create(firstname="Joe", lastname="Smith", salary=10) ceo=Employee.objects.create(firstname="Joe", lastname="Smith", salary=10)
) )
cls.foobar_ltd = Company.objects.create( cls.foobar_ltd = Company.objects.create(
name="Foobar Ltd.", num_employees=3, num_chairs=4, name="Foobar Ltd.", num_employees=3, num_chairs=4, based_in_eu=True,
ceo=Employee.objects.create(firstname="Frank", lastname="Meyer", salary=20) ceo=Employee.objects.create(firstname="Frank", lastname="Meyer", salary=20)
) )
cls.max = Employee.objects.create(firstname='Max', lastname='Mustermann', salary=30) cls.max = Employee.objects.create(firstname='Max', lastname='Mustermann', salary=30)
@ -83,6 +83,14 @@ class BasicExpressionsTests(TestCase):
2, 2,
) )
def test_filtering_on_q_that_is_boolean(self):
self.assertEqual(
Company.objects.filter(
ExpressionWrapper(Q(num_employees__gt=3), output_field=models.BooleanField())
).count(),
2,
)
def test_filter_inter_attribute(self): def test_filter_inter_attribute(self):
# We can filter on attribute relationships on same model obj, e.g. # We can filter on attribute relationships on same model obj, e.g.
# find companies where the number of employees is greater # find companies where the number of employees is greater
@ -642,6 +650,56 @@ class BasicExpressionsTests(TestCase):
with self.assertRaisesMessage(FieldError, "Cannot resolve keyword 'nope' into field."): with self.assertRaisesMessage(FieldError, "Cannot resolve keyword 'nope' into field."):
list(Company.objects.filter(ceo__pk=F('point_of_contact__nope'))) list(Company.objects.filter(ceo__pk=F('point_of_contact__nope')))
def test_exists_in_filter(self):
inner = Company.objects.filter(ceo=OuterRef('pk')).values('pk')
qs1 = Employee.objects.filter(Exists(inner))
qs2 = Employee.objects.annotate(found=Exists(inner)).filter(found=True)
self.assertCountEqual(qs1, qs2)
self.assertFalse(Employee.objects.exclude(Exists(inner)).exists())
self.assertCountEqual(qs2, Employee.objects.exclude(~Exists(inner)))
def test_subquery_in_filter(self):
inner = Company.objects.filter(ceo=OuterRef('pk')).values('based_in_eu')
self.assertSequenceEqual(
Employee.objects.filter(Subquery(inner)),
[self.foobar_ltd.ceo],
)
def test_case_in_filter_if_boolean_output_field(self):
is_ceo = Company.objects.filter(ceo=OuterRef('pk'))
is_poc = Company.objects.filter(point_of_contact=OuterRef('pk'))
qs = Employee.objects.filter(
Case(
When(Exists(is_ceo), then=True),
When(Exists(is_poc), then=True),
default=False,
output_field=models.BooleanField(),
),
)
self.assertSequenceEqual(qs, [self.example_inc.ceo, self.foobar_ltd.ceo, self.max])
def test_boolean_expression_combined(self):
is_ceo = Company.objects.filter(ceo=OuterRef('pk'))
is_poc = Company.objects.filter(point_of_contact=OuterRef('pk'))
self.gmbh.point_of_contact = self.max
self.gmbh.save()
self.assertSequenceEqual(
Employee.objects.filter(Exists(is_ceo) | Exists(is_poc)),
[self.example_inc.ceo, self.foobar_ltd.ceo, self.max],
)
self.assertSequenceEqual(
Employee.objects.filter(Exists(is_ceo) & Exists(is_poc)),
[self.max],
)
self.assertSequenceEqual(
Employee.objects.filter(Exists(is_ceo) & Q(salary__gte=30)),
[self.max],
)
self.assertSequenceEqual(
Employee.objects.filter(Exists(is_poc) | Q(salary__lt=15)),
[self.example_inc.ceo, self.max],
)
class IterableLookupInnerExpressionsTests(TestCase): class IterableLookupInnerExpressionsTests(TestCase):
@classmethod @classmethod

View File

@ -1327,7 +1327,10 @@ class CaseWhenTests(SimpleTestCase):
Case(When(Q(pk__in=[])), object()) Case(When(Q(pk__in=[])), object())
def test_invalid_when_constructor_args(self): def test_invalid_when_constructor_args(self):
msg = '__init__() takes either a Q object or lookups as keyword arguments' msg = (
'When() supports a Q object, a boolean expression, or lookups as '
'a condition.'
)
with self.assertRaisesMessage(TypeError, msg): with self.assertRaisesMessage(TypeError, msg):
When(condition=object()) When(condition=object())
with self.assertRaisesMessage(TypeError, msg): with self.assertRaisesMessage(TypeError, msg):