From b5e0eede406633b88c6222031171f80876d8f5e1 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 22 Mar 2015 13:30:57 -0700 Subject: [PATCH] Fixed #22394 -- Refactored built-in datetime lookups to transforms. --- AUTHORS | 1 + django/db/models/fields/__init__.py | 166 ++++++++++++++++++++++++---- django/db/models/lookups.py | 58 ---------- docs/ref/models/querysets.txt | 63 +++++++++-- docs/releases/1.9.txt | 4 + tests/custom_lookups/tests.py | 40 +++---- tests/lookup/tests.py | 28 +++++ 7 files changed, 251 insertions(+), 109 deletions(-) diff --git a/AUTHORS b/AUTHORS index 79f5ae1b3b..ebf5c491b1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -347,6 +347,7 @@ answer newbie questions, and generally made Django that much better: John Paulett John Shaffer Jökull Sólberg Auðunsson + Jon Dufresne Jonathan Buchanan Jonathan Daugherty (cygnus) Jonathan Feignberg diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index b305330f3b..c64535ff99 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -12,7 +12,7 @@ from base64 import b64decode, b64encode from django.apps import apps from django.db import connection -from django.db.models.lookups import default_lookups, RegisterLookupMixin +from django.db.models.lookups import default_lookups, RegisterLookupMixin, Transform, Lookup from django.db.models.query_utils import QueryWrapper from django.conf import settings from django import forms @@ -724,7 +724,6 @@ class Field(RegisterLookupMixin): if lookup_type in { 'iexact', 'contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', - 'month', 'day', 'week_day', 'hour', 'minute', 'second', 'isnull', 'search', 'regex', 'iregex', }: return value @@ -732,12 +731,6 @@ class Field(RegisterLookupMixin): return self.get_prep_value(value) elif lookup_type in ('range', 'in'): return [self.get_prep_value(v) for v in value] - elif lookup_type == 'year': - try: - return int(value) - except ValueError: - raise ValueError("The __year lookup type requires an integer " - "argument") return self.get_prep_value(value) def get_db_prep_lookup(self, lookup_type, value, connection, @@ -761,8 +754,7 @@ class Field(RegisterLookupMixin): sql, params = value._as_sql(connection=connection) return QueryWrapper(('(%s)' % sql), params) - if lookup_type in ('month', 'day', 'week_day', 'hour', 'minute', - 'second', 'search', 'regex', 'iregex', 'contains', + if lookup_type in ('search', 'regex', 'iregex', 'contains', 'icontains', 'iexact', 'startswith', 'endswith', 'istartswith', 'iendswith'): return [value] @@ -774,13 +766,6 @@ class Field(RegisterLookupMixin): prepared=prepared) for v in value] elif lookup_type == 'isnull': return [] - elif lookup_type == 'year': - if isinstance(self, DateTimeField): - return connection.ops.year_lookup_bounds_for_datetime_field(value) - elif isinstance(self, DateField): - return connection.ops.year_lookup_bounds_for_date_field(value) - else: - return [value] # this isn't supposed to happen else: return [value] @@ -1302,13 +1287,6 @@ class DateField(DateTimeCheckMixin, Field): curry(cls._get_next_or_previous_by_FIELD, field=self, is_next=False)) - def get_prep_lookup(self, lookup_type, value): - # For dates lookups, convert the value to an int - # so the database backend always sees a consistent type. - if lookup_type in ('month', 'day', 'week_day', 'hour', 'minute', 'second'): - return int(value) - return super(DateField, self).get_prep_lookup(lookup_type, value) - def get_prep_value(self, value): value = super(DateField, self).get_prep_value(value) return self.to_python(value) @@ -2408,3 +2386,143 @@ class UUIDField(Field): } defaults.update(kwargs) return super(UUIDField, self).formfield(**defaults) + + +class DateTransform(Transform): + def as_sql(self, compiler, connection): + sql, params = compiler.compile(self.lhs) + lhs_output_field = self.lhs.output_field + if isinstance(lhs_output_field, DateTimeField): + tzname = timezone.get_current_timezone_name() if settings.USE_TZ else None + sql, tz_params = connection.ops.datetime_extract_sql(self.lookup_name, sql, tzname) + params.extend(tz_params) + else: + # DateField and TimeField. + sql = connection.ops.date_extract_sql(self.lookup_name, sql) + return sql, params + + @cached_property + def output_field(self): + return IntegerField() + + +class YearTransform(DateTransform): + lookup_name = 'year' + + +class YearLookup(Lookup): + def year_lookup_bounds(self, connection, year): + output_field = self.lhs.lhs.output_field + if isinstance(output_field, DateTimeField): + bounds = connection.ops.year_lookup_bounds_for_datetime_field(year) + else: + bounds = connection.ops.year_lookup_bounds_for_date_field(year) + return bounds + + +@YearTransform.register_lookup +class YearExact(YearLookup): + lookup_name = 'exact' + + def as_sql(self, compiler, connection): + # We will need to skip the extract part and instead go + # directly with the originating field, that is self.lhs.lhs. + lhs_sql, params = self.process_lhs(compiler, connection, self.lhs.lhs) + rhs_sql, rhs_params = self.process_rhs(compiler, connection) + bounds = self.year_lookup_bounds(connection, rhs_params[0]) + params.extend(bounds) + return '%s BETWEEN %%s AND %%s' % lhs_sql, params + + +class YearComparisonLookup(YearLookup): + def as_sql(self, compiler, connection): + # We will need to skip the extract part and instead go + # directly with the originating field, that is self.lhs.lhs. + lhs_sql, params = self.process_lhs(compiler, connection, self.lhs.lhs) + rhs_sql, rhs_params = self.process_rhs(compiler, connection) + rhs_sql = self.get_rhs_op(connection, rhs_sql) + start, finish = self.year_lookup_bounds(connection, rhs_params[0]) + params.append(self.get_bound(start, finish)) + return '%s %s' % (lhs_sql, rhs_sql), params + + def get_rhs_op(self, connection, rhs): + return connection.operators[self.lookup_name] % rhs + + def get_bound(self): + raise NotImplementedError( + 'subclasses of YearComparisonLookup must provide a get_bound() method' + ) + + +@YearTransform.register_lookup +class YearGt(YearComparisonLookup): + lookup_name = 'gt' + + def get_bound(self, start, finish): + return finish + + +@YearTransform.register_lookup +class YearGte(YearComparisonLookup): + lookup_name = 'gte' + + def get_bound(self, start, finish): + return start + + +@YearTransform.register_lookup +class YearLt(YearComparisonLookup): + lookup_name = 'lt' + + def get_bound(self, start, finish): + return start + + +@YearTransform.register_lookup +class YearLte(YearComparisonLookup): + lookup_name = 'lte' + + def get_bound(self, start, finish): + return finish + + +class MonthTransform(DateTransform): + lookup_name = 'month' + + +class DayTransform(DateTransform): + lookup_name = 'day' + + +class WeekDayTransform(DateTransform): + lookup_name = 'week_day' + + +class HourTransform(DateTransform): + lookup_name = 'hour' + + +class MinuteTransform(DateTransform): + lookup_name = 'minute' + + +class SecondTransform(DateTransform): + lookup_name = 'second' + + +DateField.register_lookup(YearTransform) +DateField.register_lookup(MonthTransform) +DateField.register_lookup(DayTransform) +DateField.register_lookup(WeekDayTransform) + +TimeField.register_lookup(HourTransform) +TimeField.register_lookup(MinuteTransform) +TimeField.register_lookup(SecondTransform) + +DateTimeField.register_lookup(YearTransform) +DateTimeField.register_lookup(MonthTransform) +DateTimeField.register_lookup(DayTransform) +DateTimeField.register_lookup(WeekDayTransform) +DateTimeField.register_lookup(HourTransform) +DateTimeField.register_lookup(MinuteTransform) +DateTimeField.register_lookup(SecondTransform) diff --git a/django/db/models/lookups.py b/django/db/models/lookups.py index de64129e19..45974093b4 100644 --- a/django/db/models/lookups.py +++ b/django/db/models/lookups.py @@ -1,8 +1,6 @@ import inspect from copy import copy -from django.conf import settings -from django.utils import timezone from django.utils.functional import cached_property from django.utils.six.moves import range @@ -408,11 +406,6 @@ class Between(BuiltinLookup): return "BETWEEN %s AND %s" % (rhs, rhs) -class Year(Between): - lookup_name = 'year' -default_lookups['year'] = Year - - class Range(BuiltinLookup): lookup_name = 'range' @@ -430,57 +423,6 @@ class Range(BuiltinLookup): default_lookups['range'] = Range -class DateLookup(BuiltinLookup): - def process_lhs(self, compiler, connection, lhs=None): - from django.db.models import DateTimeField - lhs, params = super(DateLookup, self).process_lhs(compiler, connection, lhs) - if isinstance(self.lhs.output_field, DateTimeField): - tzname = timezone.get_current_timezone_name() if settings.USE_TZ else None - sql, tz_params = connection.ops.datetime_extract_sql(self.extract_type, lhs, tzname) - return connection.ops.lookup_cast(self.lookup_name) % sql, tz_params - else: - return connection.ops.date_extract_sql(self.lookup_name, lhs), [] - - def get_rhs_op(self, connection, rhs): - return '= %s' % rhs - - -class Month(DateLookup): - lookup_name = 'month' - extract_type = 'month' -default_lookups['month'] = Month - - -class Day(DateLookup): - lookup_name = 'day' - extract_type = 'day' -default_lookups['day'] = Day - - -class WeekDay(DateLookup): - lookup_name = 'week_day' - extract_type = 'week_day' -default_lookups['week_day'] = WeekDay - - -class Hour(DateLookup): - lookup_name = 'hour' - extract_type = 'hour' -default_lookups['hour'] = Hour - - -class Minute(DateLookup): - lookup_name = 'minute' - extract_type = 'minute' -default_lookups['minute'] = Minute - - -class Second(DateLookup): - lookup_name = 'second' - extract_type = 'second' -default_lookups['second'] = Second - - class IsNull(BuiltinLookup): lookup_name = 'isnull' diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index 733025a5f1..6fb3dc6aae 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -2431,36 +2431,45 @@ numbers and even characters. year ~~~~ -For date and datetime fields, an exact year match. Takes an integer year. +For date and datetime fields, an exact year match. Allows chaining additional +field lookups. Takes an integer year. Example:: Entry.objects.filter(pub_date__year=2005) + Entry.objects.filter(pub_date__year__gte=2005) SQL equivalent:: SELECT ... WHERE pub_date BETWEEN '2005-01-01' AND '2005-12-31'; + SELECT ... WHERE pub_date >= '2005-01-01'; (The exact SQL syntax varies for each database engine.) When :setting:`USE_TZ` is ``True``, datetime fields are converted to the current time zone before filtering. +.. versionchanged:: 1.9 + + Allowed chaining additional field lookups. + .. fieldlookup:: month month ~~~~~ -For date and datetime fields, an exact month match. Takes an integer 1 -(January) through 12 (December). +For date and datetime fields, an exact month match. Allows chaining additional +field lookups. Takes an integer 1 (January) through 12 (December). Example:: Entry.objects.filter(pub_date__month=12) + Entry.objects.filter(pub_date__month__gte=6) SQL equivalent:: SELECT ... WHERE EXTRACT('month' FROM pub_date) = '12'; + SELECT ... WHERE EXTRACT('month' FROM pub_date) >= '6'; (The exact SQL syntax varies for each database engine.) @@ -2468,20 +2477,27 @@ When :setting:`USE_TZ` is ``True``, datetime fields are converted to the current time zone before filtering. This requires :ref:`time zone definitions in the database `. +.. versionchanged:: 1.9 + + Allowed chaining additional field lookups. + .. fieldlookup:: day day ~~~ -For date and datetime fields, an exact day match. Takes an integer day. +For date and datetime fields, an exact day match. Allows chaining additional +field lookups. Takes an integer day. Example:: Entry.objects.filter(pub_date__day=3) + Entry.objects.filter(pub_date__day__gte=3) SQL equivalent:: SELECT ... WHERE EXTRACT('day' FROM pub_date) = '3'; + SELECT ... WHERE EXTRACT('day' FROM pub_date) >= '3'; (The exact SQL syntax varies for each database engine.) @@ -2492,12 +2508,17 @@ When :setting:`USE_TZ` is ``True``, datetime fields are converted to the current time zone before filtering. This requires :ref:`time zone definitions in the database `. +.. versionchanged:: 1.9 + + Allowed chaining additional field lookups. + .. fieldlookup:: week_day week_day ~~~~~~~~ -For date and datetime fields, a 'day of the week' match. +For date and datetime fields, a 'day of the week' match. Allows chaining +additional field lookups. Takes an integer value representing the day of week from 1 (Sunday) to 7 (Saturday). @@ -2505,6 +2526,7 @@ Takes an integer value representing the day of week from 1 (Sunday) to 7 Example:: Entry.objects.filter(pub_date__week_day=2) + Entry.objects.filter(pub_date__week_day__gte=2) (No equivalent SQL code fragment is included for this lookup because implementation of the relevant query varies among different database engines.) @@ -2517,66 +2539,91 @@ When :setting:`USE_TZ` is ``True``, datetime fields are converted to the current time zone before filtering. This requires :ref:`time zone definitions in the database `. +.. versionchanged:: 1.9 + + Allowed chaining additional field lookups. + .. fieldlookup:: hour hour ~~~~ -For datetime fields, an exact hour match. Takes an integer between 0 and 23. +For datetime fields, an exact hour match. Allows chaining additional field +lookups. Takes an integer between 0 and 23. Example:: Event.objects.filter(timestamp__hour=23) + Event.objects.filter(timestamp__hour__gte=12) SQL equivalent:: SELECT ... WHERE EXTRACT('hour' FROM timestamp) = '23'; + SELECT ... WHERE EXTRACT('hour' FROM timestamp) >= '12'; (The exact SQL syntax varies for each database engine.) When :setting:`USE_TZ` is ``True``, values are converted to the current time zone before filtering. +.. versionchanged:: 1.9 + + Allowed chaining additional field lookups. + .. fieldlookup:: minute minute ~~~~~~ -For datetime fields, an exact minute match. Takes an integer between 0 and 59. +For datetime fields, an exact minute match. Allows chaining additional field +lookups. Takes an integer between 0 and 59. Example:: Event.objects.filter(timestamp__minute=29) + Event.objects.filter(timestamp__minute__gte=29) SQL equivalent:: SELECT ... WHERE EXTRACT('minute' FROM timestamp) = '29'; + SELECT ... WHERE EXTRACT('minute' FROM timestamp) >= '29'; (The exact SQL syntax varies for each database engine.) When :setting:`USE_TZ` is ``True``, values are converted to the current time zone before filtering. +.. versionchanged:: 1.9 + + Allowed chaining additional field lookups. + .. fieldlookup:: second second ~~~~~~ -For datetime fields, an exact second match. Takes an integer between 0 and 59. +For datetime fields, an exact second match. Allows chaining additional field +lookups. Takes an integer between 0 and 59. Example:: Event.objects.filter(timestamp__second=31) + Event.objects.filter(timestamp__second__gte=31) SQL equivalent:: SELECT ... WHERE EXTRACT('second' FROM timestamp) = '31'; + SELECT ... WHERE EXTRACT('second' FROM timestamp) >= '31'; (The exact SQL syntax varies for each database engine.) When :setting:`USE_TZ` is ``True``, values are converted to the current time zone before filtering. +.. versionchanged:: 1.9 + + Allowed chaining additional field lookups. + .. fieldlookup:: isnull isnull diff --git a/docs/releases/1.9.txt b/docs/releases/1.9.txt index 9fe8e9b195..0d0f3d5e01 100644 --- a/docs/releases/1.9.txt +++ b/docs/releases/1.9.txt @@ -184,6 +184,10 @@ Models * Added a system check to prevent defining both ``Meta.ordering`` and ``order_with_respect_to`` on the same model. +* :lookup:`Date and time ` lookups can be chained with other lookups + (such as :lookup:`exact`, :lookup:`gt`, :lookup:`lt`, etc.). For example: + ``Entry.objects.filter(pub_date__month__gt=6)``. + CSRF ^^^^ diff --git a/tests/custom_lookups/tests.py b/tests/custom_lookups/tests.py index 1b45be8451..1b069047e4 100644 --- a/tests/custom_lookups/tests.py +++ b/tests/custom_lookups/tests.py @@ -63,7 +63,8 @@ class UpperBilateralTransform(models.Transform): class YearTransform(models.Transform): - lookup_name = 'year' + # Use a name that avoids collision with the built-in year lookup. + lookup_name = 'testyear' def as_sql(self, compiler, connection): lhs_sql, params = compiler.compile(self.lhs) @@ -400,19 +401,19 @@ class YearLteTests(TestCase): def test_year_lte(self): baseqs = Author.objects.order_by('name') self.assertQuerysetEqual( - baseqs.filter(birthdate__year__lte=2012), + baseqs.filter(birthdate__testyear__lte=2012), [self.a1, self.a2, self.a3, self.a4], lambda x: x) self.assertQuerysetEqual( - baseqs.filter(birthdate__year=2012), + baseqs.filter(birthdate__testyear=2012), [self.a2, self.a3, self.a4], lambda x: x) - self.assertNotIn('BETWEEN', str(baseqs.filter(birthdate__year=2012).query)) + self.assertNotIn('BETWEEN', str(baseqs.filter(birthdate__testyear=2012).query)) self.assertQuerysetEqual( - baseqs.filter(birthdate__year__lte=2011), + baseqs.filter(birthdate__testyear__lte=2011), [self.a1], lambda x: x) # The non-optimized version works, too. self.assertQuerysetEqual( - baseqs.filter(birthdate__year__lt=2012), + baseqs.filter(birthdate__testyear__lt=2012), [self.a1], lambda x: x) @unittest.skipUnless(connection.vendor == 'postgresql', "PostgreSQL specific SQL used") @@ -425,10 +426,10 @@ class YearLteTests(TestCase): self.a4.save() baseqs = Author.objects.order_by('name') self.assertQuerysetEqual( - baseqs.filter(birthdate__year__lte=models.F('age')), + baseqs.filter(birthdate__testyear__lte=models.F('age')), [self.a3, self.a4], lambda x: x) self.assertQuerysetEqual( - baseqs.filter(birthdate__year__lt=models.F('age')), + baseqs.filter(birthdate__testyear__lt=models.F('age')), [self.a4], lambda x: x) def test_year_lte_sql(self): @@ -437,16 +438,16 @@ class YearLteTests(TestCase): # error - not running YearLte SQL at all. baseqs = Author.objects.order_by('name') self.assertIn( - '<= (2011 || ', str(baseqs.filter(birthdate__year__lte=2011).query)) + '<= (2011 || ', str(baseqs.filter(birthdate__testyear__lte=2011).query)) self.assertIn( - '-12-31', str(baseqs.filter(birthdate__year__lte=2011).query)) + '-12-31', str(baseqs.filter(birthdate__testyear__lte=2011).query)) def test_postgres_year_exact(self): baseqs = Author.objects.order_by('name') self.assertIn( - '= (2011 || ', str(baseqs.filter(birthdate__year=2011).query)) + '= (2011 || ', str(baseqs.filter(birthdate__testyear=2011).query)) self.assertIn( - '-12-31', str(baseqs.filter(birthdate__year=2011).query)) + '-12-31', str(baseqs.filter(birthdate__testyear=2011).query)) def test_custom_implementation_year_exact(self): try: @@ -462,7 +463,7 @@ class YearLteTests(TestCase): setattr(YearExact, 'as_' + connection.vendor, as_custom_sql) self.assertIn( 'concat(', - str(Author.objects.filter(birthdate__year=2012).query)) + str(Author.objects.filter(birthdate__testyear=2012).query)) finally: delattr(YearExact, 'as_' + connection.vendor) try: @@ -483,14 +484,15 @@ class YearLteTests(TestCase): YearTransform.register_lookup(CustomYearExact) self.assertIn( 'CONCAT(', - str(Author.objects.filter(birthdate__year=2012).query)) + str(Author.objects.filter(birthdate__testyear=2012).query)) finally: YearTransform._unregister_lookup(CustomYearExact) YearTransform.register_lookup(YearExact) class TrackCallsYearTransform(YearTransform): - lookup_name = 'year' + # Use a name that avoids collision with the built-in year lookup. + lookup_name = 'testyear' call_order = [] def as_sql(self, compiler, connection): @@ -516,23 +518,23 @@ class LookupTransformCallOrderTests(TestCase): try: # junk lookup - tries lookup, then transform, then fails with self.assertRaises(FieldError): - Author.objects.filter(birthdate__year__junk=2012) + Author.objects.filter(birthdate__testyear__junk=2012) self.assertEqual(TrackCallsYearTransform.call_order, ['lookup', 'transform']) TrackCallsYearTransform.call_order = [] # junk transform - tries transform only, then fails with self.assertRaises(FieldError): - Author.objects.filter(birthdate__year__junk__more_junk=2012) + Author.objects.filter(birthdate__testyear__junk__more_junk=2012) self.assertEqual(TrackCallsYearTransform.call_order, ['transform']) TrackCallsYearTransform.call_order = [] # Just getting the year (implied __exact) - lookup only - Author.objects.filter(birthdate__year=2012) + Author.objects.filter(birthdate__testyear=2012) self.assertEqual(TrackCallsYearTransform.call_order, ['lookup']) TrackCallsYearTransform.call_order = [] # Just getting the year (explicit __exact) - lookup only - Author.objects.filter(birthdate__year__exact=2012) + Author.objects.filter(birthdate__testyear__exact=2012) self.assertEqual(TrackCallsYearTransform.call_order, ['lookup']) diff --git a/tests/lookup/tests.py b/tests/lookup/tests.py index 36b60a033d..0468811a54 100644 --- a/tests/lookup/tests.py +++ b/tests/lookup/tests.py @@ -713,6 +713,34 @@ class LookupTests(TestCase): self.assertEqual(Player.objects.filter(games__season__year__gt=2010).distinct().count(), 2) self.assertEqual(Player.objects.filter(games__season__gt__gt=222).distinct().count(), 2) + def test_chain_date_time_lookups(self): + self.assertQuerysetEqual( + Article.objects.filter(pub_date__month__gt=7), + ['', ''], + ordered=False + ) + self.assertQuerysetEqual( + Article.objects.filter(pub_date__day__gte=27), + ['', '', + '', ''], + ordered=False + ) + self.assertQuerysetEqual( + Article.objects.filter(pub_date__hour__lt=8), + ['', '', + '', '', + ''], + ordered=False + ) + self.assertQuerysetEqual( + Article.objects.filter(pub_date__minute__lte=0), + ['', '', + '', '', + '', '', + ''], + ordered=False + ) + class LookupTransactionTests(TransactionTestCase): available_apps = ['lookup']