mirrors/django
1
0
Fork 0
mirror of https://github.com/django/django.git synced 2025-09-15 05:29:11 +00:00
Code Issues 32 Releases Wiki Activity

magic-removal: Modified bulk delete algorithnm to remove objects and relations in chunks. This prevents the WHERE x IN (...) statements causing a stack overflow when there are lots objects, or lots of of relationships between objects.

Browse Source 
git-svn-id: http://code.djangoproject.com/svn/django/branches/magic-removal@2488 bcc190cf-cafb-0310-a4f2-bffc1f526a37
This commit is contained in:
Russell Keith-Magee 2006-03-04 23:40:15 +00:00
parent fdd1e3682f
commit 6e82705741
1 changed files with 42 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
django/db/models/query.py
View File

@ -232,15 +232,23 @@ class QuerySet(object):
del_query._select_related = False del_query._select_related = False
del_query._order_by = [] del_query._order_by = []
# Collect all the objects to be deleted, and all the objects that are related to # Delete objects in chunks to prevent an the list of
# the objects that are to be deleted # related objects from becoming too long
seen_objs = SortedDict() more_objects = True
for object in del_query: while more_objects:
object._collect_sub_objects(seen_objs) # Collect all the objects to be deleted in this chunk, and all the objects
# that are related to the objects that are to be deleted
# Delete the objects seen_objs = SortedDict()
delete_objects(seen_objs) more_objects = False
for object in del_query[0:GET_ITERATOR_CHUNK_SIZE]:
more_objects = True
object._collect_sub_objects(seen_objs)
# If one or more objects were found, delete them.
# Otherwise, stop looping.
if more_objects:
delete_objects(seen_objs)
# Clear the result cache, in case this QuerySet gets reused. # Clear the result cache, in case this QuerySet gets reused.
self._result_cache = None self._result_cache = None
delete.alters_data = True delete.alters_data = True
@ -832,35 +840,39 @@ def delete_objects(seen_objs):
pk_list = [pk for pk,instance in seen_objs[cls]] pk_list = [pk for pk,instance in seen_objs[cls]]
for related in cls._meta.get_all_related_many_to_many_objects(): for related in cls._meta.get_all_related_many_to_many_objects():
cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \ for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
(backend.quote_name(related.field.m2m_db_table()), cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
backend.quote_name(related.field.m2m_reverse_name()), (backend.quote_name(related.field.m2m_db_table()),
','.join(['%s' for pk in pk_list])), backend.quote_name(related.field.m2m_reverse_name()),
pk_list) ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE])
for f in cls._meta.many_to_many: for f in cls._meta.many_to_many:
cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \ for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
(backend.quote_name(f.m2m_db_table()), cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
backend.quote_name(f.m2m_column_name()), (backend.quote_name(f.m2m_db_table()),
','.join(['%s' for pk in pk_list])), backend.quote_name(f.m2m_column_name()),
pk_list) ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE])
for field in cls._meta.fields: for field in cls._meta.fields:
if field.rel and field.null and field.rel.to in seen_objs: if field.rel and field.null and field.rel.to in seen_objs:
cursor.execute("UPDATE %s SET %s=NULL WHERE %s IN (%s)" % \ for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
(backend.quote_name(cls._meta.db_table), cursor.execute("UPDATE %s SET %s=NULL WHERE %s IN (%s)" % \
backend.quote_name(field.column), (backend.quote_name(cls._meta.db_table),
backend.quote_name(cls._meta.pk.column), backend.quote_name(field.column),
','.join(['%s' for pk in pk_list])), backend.quote_name(cls._meta.pk.column),
pk_list) ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE])
# Now delete the actual data # Now delete the actual data
for cls in ordered_classes: for cls in ordered_classes:
seen_objs[cls].reverse() seen_objs[cls].reverse()
pk_list = [pk for pk,instance in seen_objs[cls]] pk_list = [pk for pk,instance in seen_objs[cls]]
cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \ for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
(backend.quote_name(cls._meta.db_table), cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
backend.quote_name(cls._meta.pk.column), (backend.quote_name(cls._meta.db_table),
','.join(['%s' for pk in pk_list])), backend.quote_name(cls._meta.pk.column),
pk_list) ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE])
# Last cleanup; set NULLs where there once was a reference to the object, # Last cleanup; set NULLs where there once was a reference to the object,
# NULL the primary key of the found objects, and perform post-notification. # NULL the primary key of the found objects, and perform post-notification.