mirrors/django
1
0
Fork 0
mirror of https://github.com/django/django.git synced 2025-10-31 09:41:08 +00:00
Code Issues 32 Releases Wiki Activity

Refs #29444 -- Allowed returning multiple fields from INSERT statements on PostgreSQL.

Browse Source 
Thanks Florian Apolloner, Tim Graham, Simon Charette, Nick Pope, and
Mariusz Felisiak for reviews.
This commit is contained in:
Johannes Hoppe
2019-07-24 08:42:41 +02:00
committed by Mariusz Felisiak
parent 736e7d44de
commit 7254f1138d
16 changed files with 209 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
django/db/backends/base/features.py
View File

@@ -23,6 +23,7 @@ class BaseDatabaseFeatures:
can_use_chunked_reads = True
can_return_columns_from_insert = False
can_return_multiple_columns_from_insert = False
can_return_rows_from_bulk_insert = False
has_bulk_insert = True
uses_savepoints = True

16
django/db/backends/base/operations.py
View File

@@ -176,13 +176,12 @@ class BaseDatabaseOperations:
else:
return ['DISTINCT'], []
def fetch_returned_insert_id(self, cursor):
def fetch_returned_insert_columns(self, cursor):
"""
Given a cursor object that has just performed an INSERT...RETURNING
statement into a table that has an auto-incrementing ID, return the
newly created ID.
statement into a table, return the newly created data.
"""
return cursor.fetchone()[0]
return cursor.fetchone()
def field_cast_sql(self, db_type, internal_type):
"""
@@ -314,12 +313,11 @@ class BaseDatabaseOperations:
"""
return value
def return_insert_id(self, field):
def return_insert_columns(self, fields):
"""
For backends that support returning the last insert ID as part of an
insert query, return the SQL and params to append to the INSERT query.
The returned fragment should contain a format string to hold the
appropriate column.
For backends that support returning columns as part of an insert query,
return the SQL and params to append to the INSERT query. The returned
fragment should contain a format string to hold the appropriate column.
"""
pass

14
django/db/backends/oracle/operations.py
View File

@@ -248,7 +248,7 @@ END;
def deferrable_sql(self):
return " DEFERRABLE INITIALLY DEFERRED"
def fetch_returned_insert_id(self, cursor):
def fetch_returned_insert_columns(self, cursor):
value = cursor._insert_id_var.getvalue()
if value is None or value == []:
# cx_Oracle < 6.3 returns None, >= 6.3 returns empty list.
@@ -258,7 +258,7 @@ END;
'Oracle OCI library (see https://code.djangoproject.com/ticket/28859).'
)
# cx_Oracle < 7 returns value, >= 7 returns list with single value.
return value[0] if isinstance(value, list) else value
return value if isinstance(value, list) else [value]
def field_cast_sql(self, db_type, internal_type):
if db_type and db_type.endswith('LOB'):
@@ -341,8 +341,14 @@ END;
match_option = "'i'"
return 'REGEXP_LIKE(%%s, %%s, %s)' % match_option
def return_insert_id(self, field):
return 'RETURNING %s INTO %%s', (InsertVar(field),)
def return_insert_columns(self, fields):
if not fields:
return '', ()
sql = 'RETURNING %s.%s INTO %%s' % (
self.quote_name(fields[0].model._meta.db_table),
self.quote_name(fields[0].column),
)
return sql, (InsertVar(fields[0]),)
def __foreign_key_constraints(self, table_name, recursive):
with self.connection.cursor() as cursor:

1
django/db/backends/postgresql/features.py
View File

@@ -8,6 +8,7 @@ from django.utils.functional import cached_property
class DatabaseFeatures(BaseDatabaseFeatures):
allows_group_by_selected_pks = True
can_return_columns_from_insert = True
can_return_multiple_columns_from_insert = True
can_return_rows_from_bulk_insert = True
has_real_datatype = True
has_native_uuid_field = True

19
django/db/backends/postgresql/operations.py
View File

@@ -76,13 +76,12 @@ class DatabaseOperations(BaseDatabaseOperations):
def deferrable_sql(self):
return " DEFERRABLE INITIALLY DEFERRED"
def fetch_returned_insert_ids(self, cursor):
def fetch_returned_insert_rows(self, cursor):
"""
Given a cursor object that has just performed an INSERT...RETURNING
statement into a table that has an auto-incrementing ID, return the
list of newly created IDs.
statement into a table, return the tuple of returned data.
"""
return [item[0] for item in cursor.fetchall()]
return cursor.fetchall()
def lookup_cast(self, lookup_type, internal_type=None):
lookup = '%s'
@@ -236,8 +235,16 @@ class DatabaseOperations(BaseDatabaseOperations):
return cursor.query.decode()
return None
def return_insert_id(self, field):
return "RETURNING %s", ()
def return_insert_columns(self, fields):
if not fields:
return '', ()
columns = [
'%s.%s' % (
self.quote_name(field.model._meta.db_table),
self.quote_name(field.column),
) for field in fields
]
return 'RETURNING %s' % ', '.join(columns), ()
def bulk_insert_sql(self, fields, placeholder_rows):
placeholder_rows_sql = (", ".join(row) for row in placeholder_rows)

20
django/db/models/base.py
View File

@@ -876,10 +876,10 @@ class Model(metaclass=ModelBase):
if not pk_set:
fields = [f for f in fields if f is not meta.auto_field]
update_pk = meta.auto_field and not pk_set
result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)
if update_pk:
setattr(self, meta.pk.attname, result)
returning_fields = meta.db_returning_fields
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
for result, field in zip(results, returning_fields):
setattr(self, field.attname, result)
return updated
def _do_update(self, base_qs, using, pk_val, values, update_fields, forced_update):
@@ -909,13 +909,15 @@ class Model(metaclass=ModelBase):
)
return filtered._update(values) > 0
def _do_insert(self, manager, using, fields, update_pk, raw):
def _do_insert(self, manager, using, fields, returning_fields, raw):
"""
Do an INSERT. If update_pk is defined then this method should return
the new pk for the model.
Do an INSERT. If returning_fields is defined then this method should
return the newly created data for the model.
"""
return manager._insert([self], fields=fields, return_id=update_pk,
using=using, raw=raw)
return manager._insert(
[self], fields=fields, returning_fields=returning_fields,
using=using, raw=raw,
)
def delete(self, using=None, keep_parents=False):
using = using or router.db_for_write(self.__class__, instance=self)

9
django/db/models/fields/__init__.py
View File

@@ -735,6 +735,14 @@ class Field(RegisterLookupMixin):
def db_tablespace(self):
return self._db_tablespace or settings.DEFAULT_INDEX_TABLESPACE
@property
def db_returning(self):
"""
Private API intended only to be used by Django itself. Currently only
the PostgreSQL backend supports returning multiple fields on a model.
"""
return False
def set_attributes_from_name(self, name):
self.name = self.name or name
self.attname, self.column = self.get_attname_column()
@@ -2311,6 +2319,7 @@ class UUIDField(Field):
class AutoFieldMixin:
db_returning = True
def __init__(self, *args, **kwargs):
kwargs['blank'] = True

11
django/db/models/options.py
View File

@@ -842,3 +842,14 @@ class Options:
if isinstance(attr, property):
names.append(name)
return frozenset(names)
@cached_property
def db_returning_fields(self):
"""
Private API intended only to be used by Django itself.
Fields to be returned after a database insert.
"""
return [
field for field in self._get_fields(forward=True, reverse=False, include_parents=PROXY_PARENTS)
if getattr(field, 'db_returning', False)
]

41
django/db/models/query.py
View File

@@ -470,23 +470,33 @@ class QuerySet:
return objs
self._for_write = True
connection = connections[self.db]
fields = self.model._meta.concrete_fields
opts = self.model._meta
fields = opts.concrete_fields
objs = list(objs)
self._populate_pk_values(objs)
with transaction.atomic(using=self.db, savepoint=False):
objs_with_pk, objs_without_pk = partition(lambda o: o.pk is None, objs)
if objs_with_pk:
self._batched_insert(objs_with_pk, fields, batch_size, ignore_conflicts=ignore_conflicts)
returned_columns = self._batched_insert(
objs_with_pk, fields, batch_size, ignore_conflicts=ignore_conflicts,
)
for obj_with_pk, results in zip(objs_with_pk, returned_columns):
for result, field in zip(results, opts.db_returning_fields):
if field != opts.pk:
setattr(obj_with_pk, field.attname, result)
for obj_with_pk in objs_with_pk:
obj_with_pk._state.adding = False
obj_with_pk._state.db = self.db
if objs_without_pk:
fields = [f for f in fields if not isinstance(f, AutoField)]
ids = self._batched_insert(objs_without_pk, fields, batch_size, ignore_conflicts=ignore_conflicts)
returned_columns = self._batched_insert(
objs_without_pk, fields, batch_size, ignore_conflicts=ignore_conflicts,
)
if connection.features.can_return_rows_from_bulk_insert and not ignore_conflicts:
assert len(ids) == len(objs_without_pk)
for obj_without_pk, pk in zip(objs_without_pk, ids):
obj_without_pk.pk = pk
assert len(returned_columns) == len(objs_without_pk)
for obj_without_pk, results in zip(objs_without_pk, returned_columns):
for result, field in zip(results, opts.db_returning_fields):
setattr(obj_without_pk, field.attname, result)
obj_without_pk._state.adding = False
obj_without_pk._state.db = self.db
@@ -1181,7 +1191,7 @@ class QuerySet:
# PRIVATE METHODS #
###################
def _insert(self, objs, fields, return_id=False, raw=False, using=None, ignore_conflicts=False):
def _insert(self, objs, fields, returning_fields=None, raw=False, using=None, ignore_conflicts=False):
"""
Insert a new record for the given model. This provides an interface to
the InsertQuery class and is how Model.save() is implemented.
@@ -1191,7 +1201,7 @@ class QuerySet:
using = self.db
query = sql.InsertQuery(self.model, ignore_conflicts=ignore_conflicts)
query.insert_values(fields, objs, raw=raw)
return query.get_compiler(using=using).execute_sql(return_id)
return query.get_compiler(using=using).execute_sql(returning_fields)
_insert.alters_data = True
_insert.queryset_only = False
@@ -1203,21 +1213,22 @@ class QuerySet:
raise NotSupportedError('This database backend does not support ignoring conflicts.')
ops = connections[self.db].ops
batch_size = (batch_size or max(ops.bulk_batch_size(fields, objs), 1))
inserted_ids = []
inserted_rows = []
bulk_return = connections[self.db].features.can_return_rows_from_bulk_insert
for item in [objs[i:i + batch_size] for i in range(0, len(objs), batch_size)]:
if bulk_return and not ignore_conflicts:
inserted_id = self._insert(
item, fields=fields, using=self.db, return_id=True,
inserted_columns = self._insert(
item, fields=fields, using=self.db,
returning_fields=self.model._meta.db_returning_fields,
ignore_conflicts=ignore_conflicts,
)
if isinstance(inserted_id, list):
inserted_ids.extend(inserted_id)
if isinstance(inserted_columns, list):
inserted_rows.extend(inserted_columns)
else:
inserted_ids.append(inserted_id)
inserted_rows.append(inserted_columns)
else:
self._insert(item, fields=fields, using=self.db, ignore_conflicts=ignore_conflicts)
return inserted_ids
return inserted_rows
def _chain(self, **kwargs):
"""

41
django/db/models/sql/compiler.py
View File

@@ -1159,7 +1159,7 @@ class SQLCompiler:
class SQLInsertCompiler(SQLCompiler):
return_id = False
returning_fields = None
def field_as_sql(self, field, val):
"""
@@ -1290,14 +1290,14 @@ class SQLInsertCompiler(SQLCompiler):
# queries and generate their own placeholders. Doing that isn't
# necessary and it should be possible to use placeholders and
# expressions in bulk inserts too.
can_bulk = (not self.return_id and self.connection.features.has_bulk_insert)
can_bulk = (not self.returning_fields and self.connection.features.has_bulk_insert)
placeholder_rows, param_rows = self.assemble_as_sql(fields, value_rows)
ignore_conflicts_suffix_sql = self.connection.ops.ignore_conflicts_suffix_sql(
ignore_conflicts=self.query.ignore_conflicts
)
if self.return_id and self.connection.features.can_return_columns_from_insert:
if self.returning_fields and self.connection.features.can_return_columns_from_insert:
if self.connection.features.can_return_rows_from_bulk_insert:
result.append(self.connection.ops.bulk_insert_sql(fields, placeholder_rows))
params = param_rows
@@ -1306,12 +1306,11 @@ class SQLInsertCompiler(SQLCompiler):
params = [param_rows[0]]
if ignore_conflicts_suffix_sql:
result.append(ignore_conflicts_suffix_sql)
col = "%s.%s" % (qn(opts.db_table), qn(opts.pk.column))
r_fmt, r_params = self.connection.ops.return_insert_id(opts.pk)
# Skip empty r_fmt to allow subclasses to customize behavior for
# Skip empty r_sql to allow subclasses to customize behavior for
# 3rd party backends. Refs #19096.
if r_fmt:
result.append(r_fmt % col)
r_sql, r_params = self.connection.ops.return_insert_columns(self.returning_fields)
if r_sql:
result.append(r_sql)
params += [r_params]
return [(" ".join(result), tuple(chain.from_iterable(params)))]
@@ -1328,25 +1327,33 @@ class SQLInsertCompiler(SQLCompiler):
for p, vals in zip(placeholder_rows, param_rows)
]
def execute_sql(self, return_id=False):
def execute_sql(self, returning_fields=None):
assert not (
return_id and len(self.query.objs) != 1 and
returning_fields and len(self.query.objs) != 1 and
not self.connection.features.can_return_rows_from_bulk_insert
)
self.return_id = return_id
self.returning_fields = returning_fields
with self.connection.cursor() as cursor:
for sql, params in self.as_sql():
cursor.execute(sql, params)
if not return_id:
return
if not self.returning_fields:
return []
if self.connection.features.can_return_rows_from_bulk_insert and len(self.query.objs) > 1:
return self.connection.ops.fetch_returned_insert_ids(cursor)
return self.connection.ops.fetch_returned_insert_rows(cursor)
if self.connection.features.can_return_columns_from_insert:
if (
len(self.returning_fields) > 1 and
not self.connection.features.can_return_multiple_columns_from_insert
):
raise NotSupportedError(
'Returning multiple columns from INSERT statements is '
'not supported on this database backend.'
)
assert len(self.query.objs) == 1
return self.connection.ops.fetch_returned_insert_id(cursor)
return self.connection.ops.last_insert_id(
return self.connection.ops.fetch_returned_insert_columns(cursor)
return [self.connection.ops.last_insert_id(
cursor, self.query.get_meta().db_table, self.query.get_meta().pk.column
)
)]
class SQLDeleteCompiler(SQLCompiler):