From 57554442fe3e209c135e15dda4ea45123e579e58 Mon Sep 17 00:00:00 2001 From: Marc Tamlyn Date: Thu, 24 Jul 2014 13:57:24 +0100 Subject: [PATCH] Fixed #2443 -- Added DurationField. A field for storing periods of time - modeled in Python by timedelta. It is stored in the native interval data type on PostgreSQL and as a bigint of microseconds on other backends. Also includes significant changes to the internals of time related maths in expressions, including the removal of DateModifierNode. Thanks to Tim and Josh in particular for reviews. --- django/db/backends/__init__.py | 16 ++++ django/db/backends/mysql/base.py | 9 +- django/db/backends/mysql/creation.py | 1 + django/db/backends/oracle/base.py | 13 +-- django/db/backends/oracle/creation.py | 1 + .../db/backends/postgresql_psycopg2/base.py | 1 + .../backends/postgresql_psycopg2/creation.py | 1 + .../postgresql_psycopg2/operations.py | 17 ---- django/db/backends/sqlite3/base.py | 54 ++++++++---- django/db/backends/sqlite3/creation.py | 1 + django/db/models/expressions.py | 87 ++++++++++--------- django/db/models/fields/__init__.py | 51 ++++++++++- django/forms/fields.py | 23 ++++- django/utils/dateparse.py | 41 +++++++++ django/utils/duration.py | 21 +++++ docs/ref/forms/fields.txt | 17 ++++ docs/ref/models/fields.txt | 17 ++++ docs/ref/utils.txt | 9 ++ docs/releases/1.8.txt | 13 ++- tests/expressions/models.py | 1 + tests/expressions/tests.py | 73 +++++++--------- tests/forms_tests/tests/test_fields.py | 37 ++++++-- tests/model_fields/models.py | 4 + tests/model_fields/test_durationfield.py | 67 ++++++++++++++ tests/utils_tests/test_dateparse.py | 44 +++++++++- tests/utils_tests/test_duration.py | 43 +++++++++ 26 files changed, 524 insertions(+), 138 deletions(-) create mode 100644 django/utils/duration.py create mode 100644 tests/model_fields/test_durationfield.py create mode 100644 tests/utils_tests/test_duration.py diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index bdf841cbd9..104744b220 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -1,5 +1,6 @@ from collections import deque import datetime +import decimal import time import warnings @@ -18,6 +19,7 @@ from django.db.backends.signals import connection_created from django.db.backends import utils from django.db.transaction import TransactionManagementError from django.db.utils import DatabaseError, DatabaseErrorWrapper, ProgrammingError +from django.utils.dateparse import parse_duration from django.utils.deprecation import RemovedInDjango19Warning from django.utils.functional import cached_property from django.utils import six @@ -575,6 +577,9 @@ class BaseDatabaseFeatures(object): supports_binary_field = True + # Is there a true datatype for timedeltas? + has_native_duration_field = False + # Do time/datetime fields have microsecond precision? supports_microsecond_precision = True @@ -1251,8 +1256,16 @@ class BaseDatabaseOperations(object): Some field types on some backends do not provide data in the correct format, this is the hook for coverter functions. """ + if not self.connection.features.has_native_duration_field and internal_type == 'DurationField': + return [self.convert_durationfield_value] return [] + def convert_durationfield_value(self, value, field): + if value is not None: + value = str(decimal.Decimal(value) / decimal.Decimal(1000000)) + value = parse_duration(value) + return value + def check_aggregate_support(self, aggregate_func): """Check that the backend supports the provided aggregate @@ -1272,6 +1285,9 @@ class BaseDatabaseOperations(object): conn = ' %s ' % connector return conn.join(sub_expressions) + def combine_duration_expression(self, connector, sub_expressions): + return self.combine_expression(connector, sub_expressions) + def modify_insert_params(self, placeholders, params): """Allow modification of insert parameters. Needed for Oracle Spatial backend due to #10888. diff --git a/django/db/backends/mysql/base.py b/django/db/backends/mysql/base.py index 566ce6c0a7..8fcc9e0db1 100644 --- a/django/db/backends/mysql/base.py +++ b/django/db/backends/mysql/base.py @@ -289,9 +289,12 @@ class DatabaseOperations(BaseDatabaseOperations): sql = "CAST(DATE_FORMAT(%s, '%s') AS DATETIME)" % (field_name, format_str) return sql, params - def date_interval_sql(self, sql, connector, timedelta): - return "(%s %s INTERVAL '%d 0:0:%d:%d' DAY_MICROSECOND)" % (sql, connector, - timedelta.days, timedelta.seconds, timedelta.microseconds) + def date_interval_sql(self, timedelta): + return "INTERVAL '%d 0:0:%d:%d' DAY_MICROSECOND" % ( + timedelta.days, timedelta.seconds, timedelta.microseconds), [] + + def format_for_duration_arithmetic(self, sql): + return 'INTERVAL %s MICROSECOND' % sql def drop_foreignkey_sql(self): return "DROP FOREIGN KEY" diff --git a/django/db/backends/mysql/creation.py b/django/db/backends/mysql/creation.py index 40d9251235..efa0ea06e7 100644 --- a/django/db/backends/mysql/creation.py +++ b/django/db/backends/mysql/creation.py @@ -16,6 +16,7 @@ class DatabaseCreation(BaseDatabaseCreation): 'DateField': 'date', 'DateTimeField': 'datetime', 'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)', + 'DurationField': 'bigint', 'FileField': 'varchar(%(max_length)s)', 'FilePathField': 'varchar(%(max_length)s)', 'FloatField': 'double precision', diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index 51abca9137..db64803c0a 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -198,19 +198,22 @@ WHEN (new.%(col_name)s IS NULL) # http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions050.htm return "EXTRACT(%s FROM %s)" % (lookup_type.upper(), field_name) - def date_interval_sql(self, sql, connector, timedelta): + def date_interval_sql(self, timedelta): """ Implements the interval functionality for expressions format for Oracle: - (datefield + INTERVAL '3 00:03:20.000000' DAY(1) TO SECOND(6)) + INTERVAL '3 00:03:20.000000' DAY(1) TO SECOND(6) """ minutes, seconds = divmod(timedelta.seconds, 60) hours, minutes = divmod(minutes, 60) days = str(timedelta.days) day_precision = len(days) - fmt = "(%s %s INTERVAL '%s %02d:%02d:%02d.%06d' DAY(%d) TO SECOND(6))" - return fmt % (sql, connector, days, hours, minutes, seconds, - timedelta.microseconds, day_precision) + fmt = "INTERVAL '%s %02d:%02d:%02d.%06d' DAY(%d) TO SECOND(6)" + return fmt % (days, hours, minutes, seconds, timedelta.microseconds, + day_precision), [] + + def format_for_duration_arithmetic(self, sql): + return "NUMTODSINTERVAL(%s / 100000, 'SECOND')" % sql def date_trunc_sql(self, lookup_type, field_name): # http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions230.htm#i1002084 diff --git a/django/db/backends/oracle/creation.py b/django/db/backends/oracle/creation.py index d470d5ac7c..ae4f5f42ec 100644 --- a/django/db/backends/oracle/creation.py +++ b/django/db/backends/oracle/creation.py @@ -29,6 +29,7 @@ class DatabaseCreation(BaseDatabaseCreation): 'DateField': 'DATE', 'DateTimeField': 'TIMESTAMP', 'DecimalField': 'NUMBER(%(max_digits)s, %(decimal_places)s)', + 'DurationField': 'NUMBER(19)', 'FileField': 'NVARCHAR2(%(max_length)s)', 'FilePathField': 'NVARCHAR2(%(max_length)s)', 'FloatField': 'DOUBLE PRECISION', diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py index 66200c0064..5fb5d68b5b 100644 --- a/django/db/backends/postgresql_psycopg2/base.py +++ b/django/db/backends/postgresql_psycopg2/base.py @@ -47,6 +47,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): needs_datetime_string_cast = False can_return_id_from_insert = True has_real_datatype = True + has_native_duration_field = True can_defer_constraint_checks = True has_select_for_update = True has_select_for_update_nowait = True diff --git a/django/db/backends/postgresql_psycopg2/creation.py b/django/db/backends/postgresql_psycopg2/creation.py index 9573107271..45ce939165 100644 --- a/django/db/backends/postgresql_psycopg2/creation.py +++ b/django/db/backends/postgresql_psycopg2/creation.py @@ -16,6 +16,7 @@ class DatabaseCreation(BaseDatabaseCreation): 'DateField': 'date', 'DateTimeField': 'timestamp with time zone', 'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)', + 'DurationField': 'interval', 'FileField': 'varchar(%(max_length)s)', 'FilePathField': 'varchar(%(max_length)s)', 'FloatField': 'double precision', diff --git a/django/db/backends/postgresql_psycopg2/operations.py b/django/db/backends/postgresql_psycopg2/operations.py index 0e512b91fa..3447bfe682 100644 --- a/django/db/backends/postgresql_psycopg2/operations.py +++ b/django/db/backends/postgresql_psycopg2/operations.py @@ -16,23 +16,6 @@ class DatabaseOperations(BaseDatabaseOperations): else: return "EXTRACT('%s' FROM %s)" % (lookup_type, field_name) - def date_interval_sql(self, sql, connector, timedelta): - """ - implements the interval functionality for expressions - format for Postgres: - (datefield + interval '3 days 200 seconds 5 microseconds') - """ - modifiers = [] - if timedelta.days: - modifiers.append('%s days' % timedelta.days) - if timedelta.seconds: - modifiers.append('%s seconds' % timedelta.seconds) - if timedelta.microseconds: - modifiers.append('%s microseconds' % timedelta.microseconds) - mods = ' '.join(modifiers) - conn = ' %s ' % connector - return '(%s)' % conn.join([sql, 'interval \'%s\'' % mods]) - def date_trunc_sql(self, lookup_type, field_name): # http://www.postgresql.org/docs/current/static/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC return "DATE_TRUNC('%s', %s)" % (lookup_type, field_name) diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index db2ac07ce9..5e23474db1 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -21,7 +21,8 @@ from django.db.backends.sqlite3.creation import DatabaseCreation from django.db.backends.sqlite3.introspection import DatabaseIntrospection from django.db.backends.sqlite3.schema import DatabaseSchemaEditor from django.db.models import fields, aggregates -from django.utils.dateparse import parse_date, parse_datetime, parse_time +from django.utils.dateparse import parse_date, parse_datetime, parse_time, parse_duration +from django.utils.duration import duration_string from django.utils.encoding import force_text from django.utils.functional import cached_property from django.utils.safestring import SafeBytes @@ -175,15 +176,12 @@ class DatabaseOperations(BaseDatabaseOperations): # cause a collision with a field name). return "django_date_extract('%s', %s)" % (lookup_type.lower(), field_name) - def date_interval_sql(self, sql, connector, timedelta): - # It would be more straightforward if we could use the sqlite strftime - # function, but it does not allow for keeping six digits of fractional - # second information, nor does it allow for formatting date and datetime - # values differently. So instead we register our own function that - # formats the datetime combined with the delta in a manner suitable - # for comparisons. - return 'django_format_dtdelta(%s, "%s", "%d", "%d", "%d")' % (sql, - connector, timedelta.days, timedelta.seconds, timedelta.microseconds) + def date_interval_sql(self, timedelta): + return "'%s'" % duration_string(timedelta), [] + + def format_for_duration_arithmetic(self, sql): + """Do nothing here, we will handle it in the custom function.""" + return sql def date_trunc_sql(self, lookup_type, field_name): # sqlite doesn't support DATE_TRUNC, so we fake it with a user-defined @@ -314,6 +312,14 @@ class DatabaseOperations(BaseDatabaseOperations): return 'django_power(%s)' % ','.join(sub_expressions) return super(DatabaseOperations, self).combine_expression(connector, sub_expressions) + def combine_duration_expression(self, connector, sub_expressions): + if connector not in ['+', '-']: + raise utils.DatabaseError('Invalid connector for timedelta: %s.' % connector) + fn_params = ["'%s'" % connector] + sub_expressions + if len(fn_params) > 3: + raise ValueError('Too many params for timedelta operations.') + return "django_format_dtdelta(%s)" % ', '.join(fn_params) + def integer_field_range(self, internal_type): # SQLite doesn't enforce any integer constraints return (None, None) @@ -408,7 +414,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): conn.create_function("django_datetime_extract", 3, _sqlite_datetime_extract) conn.create_function("django_datetime_trunc", 3, _sqlite_datetime_trunc) conn.create_function("regexp", 2, _sqlite_regexp) - conn.create_function("django_format_dtdelta", 5, _sqlite_format_dtdelta) + conn.create_function("django_format_dtdelta", 3, _sqlite_format_dtdelta) conn.create_function("django_power", 2, _sqlite_power) return conn @@ -585,19 +591,33 @@ def _sqlite_datetime_trunc(lookup_type, dt, tzname): return "%i-%02i-%02i %02i:%02i:%02i" % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) -def _sqlite_format_dtdelta(dt, conn, days, secs, usecs): +def _sqlite_format_dtdelta(conn, lhs, rhs): + """ + LHS and RHS can be either: + - An integer number of microseconds + - A string representing a timedelta object + - A string representing a datetime + """ try: - dt = backend_utils.typecast_timestamp(dt) - delta = datetime.timedelta(int(days), int(secs), int(usecs)) + if isinstance(lhs, int): + lhs = str(decimal.Decimal(lhs) / decimal.Decimal(1000000)) + real_lhs = parse_duration(lhs) + if real_lhs is None: + real_lhs = backend_utils.typecast_timestamp(lhs) + if isinstance(rhs, int): + rhs = str(decimal.Decimal(rhs) / decimal.Decimal(1000000)) + real_rhs = parse_duration(rhs) + if real_rhs is None: + real_rhs = backend_utils.typecast_timestamp(rhs) if conn.strip() == '+': - dt = dt + delta + out = real_lhs + real_rhs else: - dt = dt - delta + out = real_lhs - real_rhs except (ValueError, TypeError): return None # typecast_timestamp returns a date or a datetime without timezone. # It will be formatted as "%Y-%m-%d" or "%Y-%m-%d %H:%M:%S[.%f]" - return str(dt) + return str(out) def _sqlite_regexp(re_pattern, re_string): diff --git a/django/db/backends/sqlite3/creation.py b/django/db/backends/sqlite3/creation.py index ea91c4a2b7..78a1cb5463 100644 --- a/django/db/backends/sqlite3/creation.py +++ b/django/db/backends/sqlite3/creation.py @@ -18,6 +18,7 @@ class DatabaseCreation(BaseDatabaseCreation): 'DateField': 'date', 'DateTimeField': 'datetime', 'DecimalField': 'decimal', + 'DurationField': 'bigint', 'FileField': 'varchar(%(max_length)s)', 'FilePathField': 'varchar(%(max_length)s)', 'FloatField': 'real', diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py index 81b81673d7..de0d339af7 100644 --- a/django/db/models/expressions.py +++ b/django/db/models/expressions.py @@ -34,12 +34,12 @@ class CombinableMixin(object): BITOR = '|' def _combine(self, other, connector, reversed, node=None): - if isinstance(other, datetime.timedelta): - return DateModifierNode(self, connector, other) - if not hasattr(other, 'resolve_expression'): # everything must be resolvable to an expression - other = Value(other) + if isinstance(other, datetime.timedelta): + other = DurationValue(other, output_field=fields.DurationField()) + else: + other = Value(other) if reversed: return Expression(other, connector, self) @@ -333,6 +333,18 @@ class Expression(ExpressionNode): self.lhs, self.rhs = exprs def as_sql(self, compiler, connection): + try: + lhs_output = self.lhs.output_field + except FieldError: + lhs_output = None + try: + rhs_output = self.rhs.output_field + except FieldError: + rhs_output = None + if (not connection.features.has_native_duration_field and + ((lhs_output and lhs_output.get_internal_type() == 'DurationField') + or (rhs_output and rhs_output.get_internal_type() == 'DurationField'))): + return DurationExpression(self.lhs, self.connector, self.rhs).as_sql(compiler, connection) expressions = [] expression_params = [] sql, params = compiler.compile(self.lhs) @@ -354,45 +366,31 @@ class Expression(ExpressionNode): return c -class DateModifierNode(Expression): - """ - Node that implements the following syntax: - filter(end_date__gt=F('start_date') + datetime.timedelta(days=3, seconds=200)) - - which translates into: - POSTGRES: - WHERE end_date > (start_date + INTERVAL '3 days 200 seconds') - - MYSQL: - WHERE end_date > (start_date + INTERVAL '3 0:0:200:0' DAY_MICROSECOND) - - ORACLE: - WHERE end_date > (start_date + INTERVAL '3 00:03:20.000000' DAY(1) TO SECOND(6)) - - SQLITE: - WHERE end_date > django_format_dtdelta(start_date, "+" "3", "200", "0") - (A custom function is used in order to preserve six digits of fractional - second information on sqlite, and to format both date and datetime values.) - - Note that microsecond comparisons are not well supported with MySQL, since - MySQL does not store microsecond information. - - Only adding and subtracting timedeltas is supported, attempts to use other - operations raise a TypeError. - """ - def __init__(self, lhs, connector, rhs): - if not isinstance(rhs, datetime.timedelta): - raise TypeError('rhs must be a timedelta.') - if connector not in (self.ADD, self.SUB): - raise TypeError('Connector must be + or -, not %s' % connector) - super(DateModifierNode, self).__init__(lhs, connector, Value(rhs)) +class DurationExpression(Expression): + def compile(self, side, compiler, connection): + if not isinstance(side, DurationValue): + try: + output = side.output_field + except FieldError: + pass + if output.get_internal_type() == 'DurationField': + sql, params = compiler.compile(side) + return connection.ops.format_for_duration_arithmetic(sql), params + return compiler.compile(side) def as_sql(self, compiler, connection): - timedelta = self.rhs.value - sql, params = compiler.compile(self.lhs) - if (timedelta.days == timedelta.seconds == timedelta.microseconds == 0): - return sql, params - return connection.ops.date_interval_sql(sql, self.connector, timedelta), params + expressions = [] + expression_params = [] + sql, params = self.compile(self.lhs, compiler, connection) + expressions.append(sql) + expression_params.extend(params) + sql, params = self.compile(self.rhs, compiler, connection) + expressions.append(sql) + expression_params.extend(params) + # order of precedence + expression_wrapper = '(%s)' + sql = connection.ops.combine_duration_expression(self.connector, expressions) + return expression_wrapper % sql, expression_params class F(CombinableMixin): @@ -488,6 +486,13 @@ class Value(ExpressionNode): return '%s', [self.value] +class DurationValue(Value): + def as_sql(self, compiler, connection): + if connection.features.has_native_duration_field: + return super(DurationValue, self).as_sql(compiler, connection) + return connection.ops.date_interval_sql(self.value) + + class Col(ExpressionNode): def __init__(self, alias, target, source=None): if source is None: diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index de754c83e5..92ad769e4c 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -19,8 +19,9 @@ from django.conf import settings from django import forms from django.core import exceptions, validators, checks from django.utils.datastructures import DictWrapper -from django.utils.dateparse import parse_date, parse_datetime, parse_time +from django.utils.dateparse import parse_date, parse_datetime, parse_time, parse_duration from django.utils.deprecation import RemovedInDjango19Warning +from django.utils.duration import duration_string from django.utils.functional import cached_property, curry, total_ordering, Promise from django.utils.text import capfirst from django.utils import timezone @@ -36,8 +37,8 @@ from django.utils.itercompat import is_iterable __all__ = [str(x) for x in ( 'AutoField', 'BLANK_CHOICE_DASH', 'BigIntegerField', 'BinaryField', 'BooleanField', 'CharField', 'CommaSeparatedIntegerField', 'DateField', - 'DateTimeField', 'DecimalField', 'EmailField', 'Empty', 'Field', - 'FieldDoesNotExist', 'FilePathField', 'FloatField', + 'DateTimeField', 'DecimalField', 'DurationField', 'EmailField', 'Empty', + 'Field', 'FieldDoesNotExist', 'FilePathField', 'FloatField', 'GenericIPAddressField', 'IPAddressField', 'IntegerField', 'NOT_PROVIDED', 'NullBooleanField', 'PositiveIntegerField', 'PositiveSmallIntegerField', 'SlugField', 'SmallIntegerField', 'TextField', 'TimeField', 'URLField', @@ -1573,6 +1574,50 @@ class DecimalField(Field): return super(DecimalField, self).formfield(**defaults) +class DurationField(Field): + """Stores timedelta objects. + + Uses interval on postgres, bigint of microseconds on other databases. + """ + empty_strings_allowed = False + default_error_messages = { + 'invalid': _("'%(value)s' value has an invalid format. It must be in " + "[DD] [HH:[MM:]]ss[.uuuuuu] format.") + } + description = _("Duration") + + def get_internal_type(self): + return "DurationField" + + def to_python(self, value): + if value is None: + return value + if isinstance(value, datetime.timedelta): + return value + try: + parsed = parse_duration(value) + except ValueError: + pass + else: + if parsed is not None: + return parsed + + raise exceptions.ValidationError( + self.error_messages['invalid'], + code='invalid', + params={'value': value}, + ) + + def get_db_prep_value(self, value, connection, prepared=False): + if connection.features.has_native_duration_field: + return value + return value.total_seconds() * 1000000 + + def value_to_string(self, obj): + val = self._get_val_from_obj(obj) + return '' if val is None else duration_string(val) + + class EmailField(CharField): default_validators = [validators.validate_email] description = _("Email address") diff --git a/django/forms/fields.py b/django/forms/fields.py index 6f0dc16c35..891d8ccd6e 100644 --- a/django/forms/fields.py +++ b/django/forms/fields.py @@ -26,7 +26,9 @@ from django.forms.widgets import ( from django.utils import formats from django.utils.encoding import smart_text, force_str, force_text from django.utils.ipv6 import clean_ipv6_address +from django.utils.dateparse import parse_duration from django.utils.deprecation import RemovedInDjango19Warning, RemovedInDjango20Warning, RenameMethodsBase +from django.utils.duration import duration_string from django.utils import six from django.utils.six.moves.urllib.parse import urlsplit, urlunsplit from django.utils.translation import ugettext_lazy as _, ungettext_lazy @@ -37,7 +39,7 @@ from django.core.validators import EMPTY_VALUES # NOQA __all__ = ( 'Field', 'CharField', 'IntegerField', - 'DateField', 'TimeField', 'DateTimeField', + 'DateField', 'TimeField', 'DateTimeField', 'DurationField', 'RegexField', 'EmailField', 'FileField', 'ImageField', 'URLField', 'BooleanField', 'NullBooleanField', 'ChoiceField', 'MultipleChoiceField', 'ComboField', 'MultiValueField', 'FloatField', 'DecimalField', @@ -518,6 +520,25 @@ class DateTimeField(BaseTemporalField): return datetime.datetime.strptime(force_str(value), format) +class DurationField(Field): + default_error_messages = { + 'invalid': _('Enter a valid duration.'), + } + + def prepare_value(self, value): + return duration_string(value) + + def to_python(self, value): + if value in self.empty_values: + return None + if isinstance(value, datetime.timedelta): + return value + value = parse_duration(value) + if value is None: + raise ValidationError(self.error_messages['invalid'], code='invalid') + return value + + class RegexField(CharField): def __init__(self, regex, max_length=None, min_length=None, error_message=None, *args, **kwargs): """ diff --git a/django/utils/dateparse.py b/django/utils/dateparse.py index 422f55b9c5..2c749f015c 100644 --- a/django/utils/dateparse.py +++ b/django/utils/dateparse.py @@ -27,6 +27,29 @@ datetime_re = re.compile( r'(?PZ|[+-]\d{2}(?::?\d{2})?)?$' ) +standard_duration_re = re.compile( + r'^' + r'(?:(?P-?\d+) )?' + r'((?:(?P\d+):)(?=\d+:\d+))?' + r'(?:(?P\d+):)?' + r'(?P\d+)' + r'(?:\.(?P\d{1,6})\d{0,6})?' + r'$' +) + +# Support the sections of ISO 8601 date representation that are accepted by +# timedelta +iso8601_duration_re = re.compile( + r'^P' + r'(?:(?P\d+(.\d+)?)D)?' + r'(?:T' + r'(?:(?P\d+(.\d+)?)H)?' + r'(?:(?P\d+(.\d+)?)M)?' + r'(?:(?P\d+(.\d+)?)S)?' + r')?' + r'$' +) + def parse_date(value): """Parses a string and return a datetime.date. @@ -84,3 +107,21 @@ def parse_datetime(value): kw = {k: int(v) for k, v in six.iteritems(kw) if v is not None} kw['tzinfo'] = tzinfo return datetime.datetime(**kw) + + +def parse_duration(value): + """Parses a duration string and returns a datetime.timedelta. + + The preferred format for durations in Django is '%d %H:%M:%S.%f'. + + Also supports ISO 8601 representation. + """ + match = standard_duration_re.match(value) + if not match: + match = iso8601_duration_re.match(value) + if match: + kw = match.groupdict() + if kw.get('microseconds'): + kw['microseconds'] = kw['microseconds'].ljust(6, '0') + kw = {k: float(v) for k, v in six.iteritems(kw) if v is not None} + return datetime.timedelta(**kw) diff --git a/django/utils/duration.py b/django/utils/duration.py new file mode 100644 index 0000000000..c37c885b91 --- /dev/null +++ b/django/utils/duration.py @@ -0,0 +1,21 @@ +"""Version of str(timedelta) which is not English specific.""" + + +def duration_string(duration): + days = duration.days + seconds = duration.seconds + microseconds = duration.microseconds + + minutes = seconds // 60 + seconds = seconds % 60 + + hours = minutes // 60 + minutes = minutes % 60 + + string = '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds) + if days: + string = '{} '.format(days) + string + if microseconds: + string += '.{:06d}'.format(microseconds) + + return string diff --git a/docs/ref/forms/fields.txt b/docs/ref/forms/fields.txt index 0f13adc249..b63826d9b1 100644 --- a/docs/ref/forms/fields.txt +++ b/docs/ref/forms/fields.txt @@ -546,6 +546,23 @@ For each field, we describe the default widget used if you don't specify The maximum number of decimal places permitted. +``DurationField`` +~~~~~~~~~~~~~~~~~ + +.. versionadded:: 1.8 + +.. class:: DurationField(**kwargs) + + * Default widget: :class:`TextInput` + * Empty value: ``None`` + * Normalizes to: A Python :class:`~python:datetime.timedelta`. + * Validates that the given value is a string which can be converted into a + ``timedelta``. + * Error message keys: ``required``, ``invalid``. + + Accepts any format understood by + :func:`~django.utils.dateparse.parse_duration`. + ``EmailField`` ~~~~~~~~~~~~~~ diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index 2429406d68..203241ae5f 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -542,6 +542,23 @@ The default form widget for this field is a :class:`~django.forms.TextInput`. :class:`FloatField` and :class:`DecimalField` classes, please see :ref:`FloatField vs. DecimalField `. +``DurationField`` +----------------- + +.. versionadded:: 1.8 + +.. class:: DurationField([**options]) + +A field for storing periods of time - modeled in Python by +:class:`~python:datetime.timedelta`. When used on PostgreSQL, the data type +used is an ``interval``, otherwise a ``bigint`` of microseconds is used. + +.. note:: + + Arithmetic with ``DurationField`` works in most cases. However on all + databases other than PostgreSQL, comparing the value of a ``DurationField`` + to arithmetic on ``DateTimeField`` instances will not work as expected. + ``EmailField`` -------------- diff --git a/docs/ref/utils.txt b/docs/ref/utils.txt index 5939ff6fb1..327f44bbfc 100644 --- a/docs/ref/utils.txt +++ b/docs/ref/utils.txt @@ -158,6 +158,15 @@ The functions defined in this module share the following properties: ``tzinfo`` attribute is a :class:`~django.utils.tzinfo.FixedOffset` instance. +.. function:: parse_duration(value) + + .. versionadded:: 1.8 + + Parses a string and returns a :class:`datetime.timedelta`. + + Expects data in the format ``"DD HH:MM:SS.uuuuuu"`` or as specified by ISO + 8601 (e.g. ``P4DT1H15M20S`` which is equivalent to ``4 1:15:20``). + ``django.utils.decorators`` =========================== diff --git a/docs/releases/1.8.txt b/docs/releases/1.8.txt index fccf50bc70..3a7cfea8e7 100644 --- a/docs/releases/1.8.txt +++ b/docs/releases/1.8.txt @@ -54,9 +54,16 @@ New data types ~~~~~~~~~~~~~~ * Django now has a :class:`~django.db.models.UUIDField` for storing - universally unique identifiers. There is a corresponding :class:`form field - `. It is stored as the native ``uuid`` data type on - PostgreSQL and as a fixed length character field on other backends. + universally unique identifiers. It is stored as the native ``uuid`` data type + on PostgreSQL and as a fixed length character field on other backends. There + is a corresponding :class:`form field `. + +* Django now has a :class:`~django.db.models.DurationField` for storing periods + of time - modeled in Python by :class:`~python:datetime.timedelta`. It is + stored in the native ``interval`` data type on PostgreSQL and as a ``bigint`` + of microseconds on other backends. Date and time related arithmetic has also + been improved on all backends. There is a corresponding :class:`form field + `. Query Expressions ~~~~~~~~~~~~~~~~~ diff --git a/tests/expressions/models.py b/tests/expressions/models.py index 3a25e0862e..53eb54ec48 100644 --- a/tests/expressions/models.py +++ b/tests/expressions/models.py @@ -47,6 +47,7 @@ class Experiment(models.Model): name = models.CharField(max_length=24) assigned = models.DateField() completed = models.DateField() + estimated_time = models.DurationField() start = models.DateTimeField() end = models.DateTimeField() diff --git a/tests/expressions/tests.py b/tests/expressions/tests.py index 6949dbf43a..0e9bd57e91 100644 --- a/tests/expressions/tests.py +++ b/tests/expressions/tests.py @@ -4,7 +4,7 @@ from copy import deepcopy import datetime from django.core.exceptions import FieldError -from django.db import connection, transaction +from django.db import connection, transaction, DatabaseError from django.db.models import F from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature from django.test.utils import Approximate @@ -602,7 +602,7 @@ class FTimeDeltaTests(TestCase): # e0: started same day as assigned, zero duration end = stime + delta0 e0 = Experiment.objects.create(name='e0', assigned=sday, start=stime, - end=end, completed=end.date()) + end=end, completed=end.date(), estimated_time=delta0) self.deltas.append(delta0) self.delays.append(e0.start - datetime.datetime.combine(e0.assigned, midnight)) @@ -617,7 +617,7 @@ class FTimeDeltaTests(TestCase): delay = datetime.timedelta(1) end = stime + delay + delta1 e1 = Experiment.objects.create(name='e1', assigned=sday, - start=stime + delay, end=end, completed=end.date()) + start=stime + delay, end=end, completed=end.date(), estimated_time=delta1) self.deltas.append(delta1) self.delays.append(e1.start - datetime.datetime.combine(e1.assigned, midnight)) @@ -627,7 +627,7 @@ class FTimeDeltaTests(TestCase): end = stime + delta2 e2 = Experiment.objects.create(name='e2', assigned=sday - datetime.timedelta(3), start=stime, end=end, - completed=end.date()) + completed=end.date(), estimated_time=datetime.timedelta(hours=1)) self.deltas.append(delta2) self.delays.append(e2.start - datetime.datetime.combine(e2.assigned, midnight)) @@ -637,7 +637,7 @@ class FTimeDeltaTests(TestCase): delay = datetime.timedelta(4) end = stime + delay + delta3 e3 = Experiment.objects.create(name='e3', - assigned=sday, start=stime + delay, end=end, completed=end.date()) + assigned=sday, start=stime + delay, end=end, completed=end.date(), estimated_time=delta3) self.deltas.append(delta3) self.delays.append(e3.start - datetime.datetime.combine(e3.assigned, midnight)) @@ -647,7 +647,7 @@ class FTimeDeltaTests(TestCase): end = stime + delta4 e4 = Experiment.objects.create(name='e4', assigned=sday - datetime.timedelta(10), start=stime, end=end, - completed=end.date()) + completed=end.date(), estimated_time=delta4 - datetime.timedelta(1)) self.deltas.append(delta4) self.delays.append(e4.start - datetime.datetime.combine(e4.assigned, midnight)) @@ -675,6 +675,10 @@ class FTimeDeltaTests(TestCase): Experiment.objects.filter(end__lt=F('start') + delta)] self.assertEqual(test_set, self.expnames[:i]) + test_set = [e.name for e in + Experiment.objects.filter(end__lt=delta + F('start'))] + self.assertEqual(test_set, self.expnames[:i]) + test_set = [e.name for e in Experiment.objects.filter(end__lte=F('start') + delta)] self.assertEqual(test_set, self.expnames[:i + 1]) @@ -756,42 +760,29 @@ class FTimeDeltaTests(TestCase): self.assertEqual(expected_ends, new_ends) self.assertEqual(expected_durations, new_durations) - def test_delta_invalid_op_mult(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start') * self.deltas[0])) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to multiply datetime by timedelta.") + def test_invalid_operator(self): + with self.assertRaises(DatabaseError): + list(Experiment.objects.filter(start=F('start') * datetime.timedelta(0))) - def test_delta_invalid_op_div(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start') / self.deltas[0])) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to divide datetime by timedelta.") + def test_durationfield_add(self): + zeros = [e.name for e in + Experiment.objects.filter(start=F('start') + F('estimated_time'))] + self.assertEqual(zeros, ['e0']) - def test_delta_invalid_op_mod(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start') % self.deltas[0])) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to modulo divide datetime by timedelta.") + end_less = [e.name for e in + Experiment.objects.filter(end__lt=F('start') + F('estimated_time'))] + self.assertEqual(end_less, ['e2']) - def test_delta_invalid_op_and(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start').bitand(self.deltas[0]))) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to binary and a datetime with a timedelta.") + delta_math = [e.name for e in + Experiment.objects.filter(end__gte=F('start') + F('estimated_time') + datetime.timedelta(hours=1))] + self.assertEqual(delta_math, ['e4']) - def test_delta_invalid_op_or(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start').bitor(self.deltas[0]))) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to binary or a datetime with a timedelta.") + @skipUnlessDBFeature("has_native_duration_field") + def test_date_subtraction(self): + under_estimate = [e.name for e in + Experiment.objects.filter(estimated_time__gt=F('end') - F('start'))] + self.assertEqual(under_estimate, ['e2']) + + over_estimate = [e.name for e in + Experiment.objects.filter(estimated_time__lt=F('end') - F('start'))] + self.assertEqual(over_estimate, ['e4']) diff --git a/tests/forms_tests/tests/test_fields.py b/tests/forms_tests/tests/test_fields.py index 3caf902349..b2f0869c47 100644 --- a/tests/forms_tests/tests/test_fields.py +++ b/tests/forms_tests/tests/test_fields.py @@ -43,11 +43,12 @@ except ImportError: from django.core.files.uploadedfile import SimpleUploadedFile from django.forms import ( BooleanField, CharField, ChoiceField, ComboField, DateField, DateTimeField, - DecimalField, EmailField, Field, FileField, FilePathField, FloatField, - Form, forms, HiddenInput, ImageField, IntegerField, MultipleChoiceField, - NullBooleanField, NumberInput, PasswordInput, RadioSelect, RegexField, - SplitDateTimeField, TextInput, Textarea, TimeField, TypedChoiceField, - TypedMultipleChoiceField, URLField, UUIDField, ValidationError, Widget, + DecimalField, DurationField, EmailField, Field, FileField, FilePathField, + FloatField, Form, forms, HiddenInput, ImageField, IntegerField, + MultipleChoiceField, NullBooleanField, NumberInput, PasswordInput, + RadioSelect, RegexField, SplitDateTimeField, TextInput, Textarea, + TimeField, TypedChoiceField, TypedMultipleChoiceField, URLField, UUIDField, + ValidationError, Widget, ) from django.test import SimpleTestCase from django.utils import formats @@ -611,6 +612,32 @@ class FieldsTests(SimpleTestCase): # RegexField ################################################################## + def test_durationfield_1(self): + f = DurationField() + self.assertEqual(datetime.timedelta(seconds=30), f.clean('30')) + self.assertEqual( + datetime.timedelta(minutes=15, seconds=30), + f.clean('15:30') + ) + self.assertEqual( + datetime.timedelta(hours=1, minutes=15, seconds=30), + f.clean('1:15:30') + ) + self.assertEqual( + datetime.timedelta( + days=1, hours=1, minutes=15, seconds=30, milliseconds=300), + f.clean('1 1:15:30.3') + ) + + def test_durationfield_2(self): + class DurationForm(Form): + duration = DurationField(initial=datetime.timedelta(hours=1)) + f = DurationForm() + self.assertHTMLEqual( + '', + str(f['duration']) + ) + def test_regexfield_1(self): f = RegexField('^[0-9][A-F][0-9]$') self.assertEqual('2A2', f.clean('2A2')) diff --git a/tests/model_fields/models.py b/tests/model_fields/models.py index bb75208905..e9e287f04f 100644 --- a/tests/model_fields/models.py +++ b/tests/model_fields/models.py @@ -121,6 +121,10 @@ class DateTimeModel(models.Model): t = models.TimeField() +class DurationModel(models.Model): + field = models.DurationField() + + class PrimaryKeyCharModel(models.Model): string = models.CharField(max_length=10, primary_key=True) diff --git a/tests/model_fields/test_durationfield.py b/tests/model_fields/test_durationfield.py new file mode 100644 index 0000000000..fc2c22af61 --- /dev/null +++ b/tests/model_fields/test_durationfield.py @@ -0,0 +1,67 @@ +import datetime +import json + +from django.core import exceptions, serializers +from django.db import models +from django.test import TestCase + +from .models import DurationModel + + +class TestSaveLoad(TestCase): + + def test_simple_roundtrip(self): + duration = datetime.timedelta(days=123, seconds=123, microseconds=123) + DurationModel.objects.create(field=duration) + loaded = DurationModel.objects.get() + self.assertEqual(loaded.field, duration) + + +class TestQuerying(TestCase): + + @classmethod + def setUpTestData(cls): + cls.objs = [ + DurationModel.objects.create(field=datetime.timedelta(days=1)), + DurationModel.objects.create(field=datetime.timedelta(seconds=1)), + DurationModel.objects.create(field=datetime.timedelta(seconds=-1)), + ] + + def test_exact(self): + self.assertSequenceEqual( + DurationModel.objects.filter(field=datetime.timedelta(days=1)), + [self.objs[0]] + ) + + def test_gt(self): + self.assertSequenceEqual( + DurationModel.objects.filter(field__gt=datetime.timedelta(days=0)), + [self.objs[0], self.objs[1]] + ) + + +class TestSerialization(TestCase): + test_data = '[{"fields": {"field": "1 01:00:00"}, "model": "model_fields.durationmodel", "pk": null}]' + + def test_dumping(self): + instance = DurationModel(field=datetime.timedelta(days=1, hours=1)) + data = serializers.serialize('json', [instance]) + self.assertEqual(json.loads(data), json.loads(self.test_data)) + + def test_loading(self): + instance = list(serializers.deserialize('json', self.test_data))[0].object + self.assertEqual(instance.field, datetime.timedelta(days=1, hours=1)) + + +class TestValidation(TestCase): + + def test_invalid_string(self): + field = models.DurationField() + with self.assertRaises(exceptions.ValidationError) as cm: + field.clean('not a datetime', None) + self.assertEqual(cm.exception.code, 'invalid') + self.assertEqual( + cm.exception.message % cm.exception.params, + "'not a datetime' value has an invalid format. " + "It must be in [DD] [HH:[MM:]]ss[.uuuuuu] format." + ) diff --git a/tests/utils_tests/test_dateparse.py b/tests/utils_tests/test_dateparse.py index cdf91c039e..a224e3b174 100644 --- a/tests/utils_tests/test_dateparse.py +++ b/tests/utils_tests/test_dateparse.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals -from datetime import date, time, datetime +from datetime import date, time, datetime, timedelta import unittest -from django.utils.dateparse import parse_date, parse_time, parse_datetime +from django.utils.dateparse import parse_date, parse_time, parse_datetime, parse_duration from django.utils.timezone import get_fixed_timezone @@ -46,3 +46,43 @@ class DateParseTests(unittest.TestCase): # Invalid inputs self.assertEqual(parse_datetime('20120423091500'), None) self.assertRaises(ValueError, parse_datetime, '2012-04-56T09:15:90') + + +class DurationParseTests(unittest.TestCase): + def test_seconds(self): + self.assertEqual(parse_duration('30'), timedelta(seconds=30)) + + def test_minutes_seconds(self): + self.assertEqual(parse_duration('15:30'), timedelta(minutes=15, seconds=30)) + self.assertEqual(parse_duration('5:30'), timedelta(minutes=5, seconds=30)) + + def test_hours_minutes_seconds(self): + self.assertEqual(parse_duration('10:15:30'), timedelta(hours=10, minutes=15, seconds=30)) + self.assertEqual(parse_duration('1:15:30'), timedelta(hours=1, minutes=15, seconds=30)) + self.assertEqual(parse_duration('100:200:300'), timedelta(hours=100, minutes=200, seconds=300)) + + def test_days(self): + self.assertEqual(parse_duration('4 15:30'), timedelta(days=4, minutes=15, seconds=30)) + self.assertEqual(parse_duration('4 10:15:30'), timedelta(days=4, hours=10, minutes=15, seconds=30)) + + def test_fractions_of_seconds(self): + self.assertEqual(parse_duration('15:30.1'), timedelta(minutes=15, seconds=30, milliseconds=100)) + self.assertEqual(parse_duration('15:30.01'), timedelta(minutes=15, seconds=30, milliseconds=10)) + self.assertEqual(parse_duration('15:30.001'), timedelta(minutes=15, seconds=30, milliseconds=1)) + self.assertEqual(parse_duration('15:30.0001'), timedelta(minutes=15, seconds=30, microseconds=100)) + self.assertEqual(parse_duration('15:30.00001'), timedelta(minutes=15, seconds=30, microseconds=10)) + self.assertEqual(parse_duration('15:30.000001'), timedelta(minutes=15, seconds=30, microseconds=1)) + + def test_negative(self): + self.assertEqual(parse_duration('-4 15:30'), timedelta(days=-4, minutes=15, seconds=30)) + + def test_iso_8601(self): + self.assertEqual(parse_duration('P4Y'), None) + self.assertEqual(parse_duration('P4M'), None) + self.assertEqual(parse_duration('P4W'), None) + self.assertEqual(parse_duration('P4D'), timedelta(days=4)) + self.assertEqual(parse_duration('P0.5D'), timedelta(hours=12)) + self.assertEqual(parse_duration('PT5H'), timedelta(hours=5)) + self.assertEqual(parse_duration('PT5M'), timedelta(minutes=5)) + self.assertEqual(parse_duration('PT5S'), timedelta(seconds=5)) + self.assertEqual(parse_duration('PT0.000005S'), timedelta(microseconds=5)) diff --git a/tests/utils_tests/test_duration.py b/tests/utils_tests/test_duration.py new file mode 100644 index 0000000000..559d2ef16f --- /dev/null +++ b/tests/utils_tests/test_duration.py @@ -0,0 +1,43 @@ +import datetime +import unittest + +from django.utils.dateparse import parse_duration +from django.utils.duration import duration_string + + +class TestDurationString(unittest.TestCase): + + def test_simple(self): + duration = datetime.timedelta(hours=1, minutes=3, seconds=5) + self.assertEqual(duration_string(duration), '01:03:05') + + def test_days(self): + duration = datetime.timedelta(days=1, hours=1, minutes=3, seconds=5) + self.assertEqual(duration_string(duration), '1 01:03:05') + + def test_microseconds(self): + duration = datetime.timedelta(hours=1, minutes=3, seconds=5, microseconds=12345) + self.assertEqual(duration_string(duration), '01:03:05.012345') + + def test_negative(self): + duration = datetime.timedelta(days=-1, hours=1, minutes=3, seconds=5) + self.assertEqual(duration_string(duration), '-1 01:03:05') + + +class TestParseDurationRoundtrip(unittest.TestCase): + + def test_simple(self): + duration = datetime.timedelta(hours=1, minutes=3, seconds=5) + self.assertEqual(parse_duration(duration_string(duration)), duration) + + def test_days(self): + duration = datetime.timedelta(days=1, hours=1, minutes=3, seconds=5) + self.assertEqual(parse_duration(duration_string(duration)), duration) + + def test_microseconds(self): + duration = datetime.timedelta(hours=1, minutes=3, seconds=5, microseconds=12345) + self.assertEqual(parse_duration(duration_string(duration)), duration) + + def test_negative(self): + duration = datetime.timedelta(days=-1, hours=1, minutes=3, seconds=5) + self.assertEqual(parse_duration(duration_string(duration)), duration)