mirrors/django
1
0
Fork 0
mirror of https://github.com/django/django.git synced 2025-10-24 06:06:09 +00:00
Code Issues 32 Releases Wiki Activity

Fixed #17788 -- Added batch_size argument to qs.bulk_create()

Browse Source 
The qs.bulk_create() method did not work with large batches together
with SQLite3. This commit adds a way to split the bulk into smaller
batches. The default batch size is unlimited except for SQLite3 where
the batch size is limited to 999 SQL parameters per batch.

Thanks to everybody who participated in the discussions at Trac.
This commit is contained in:
Anssi Kääriäinen
2012-04-29 04:22:05 +03:00
parent fcad6c48f0
commit 29132ebdef
8 changed files with 110 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
tests/regressiontests/bulk_create/tests.py
View File

@@ -2,9 +2,11 @@ from __future__ import absolute_import
from operator import attrgetter
from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature
from django.db import connection
from django.test import TestCase, skipIfDBFeature
from django.test.utils import override_settings
from .models import Country, Restaurant, Pizzeria, State
from .models import Country, Restaurant, Pizzeria, State, TwoFields
class BulkCreateTests(TestCase):
@@ -27,7 +29,6 @@ class BulkCreateTests(TestCase):
self.assertEqual(created, [])
self.assertEqual(Country.objects.count(), 4)
@skipUnlessDBFeature("has_bulk_insert")
def test_efficiency(self):
with self.assertNumQueries(1):
Country.objects.bulk_create(self.data)
@@ -69,3 +70,42 @@ class BulkCreateTests(TestCase):
invalid_country = Country(id=0, name='Poland', iso_two_letter='PL')
with self.assertRaises(ValueError):
Country.objects.bulk_create([valid_country, invalid_country])
def test_large_batch(self):
with override_settings(DEBUG=True):
connection.queries = []
TwoFields.objects.bulk_create([
TwoFields(f1=i, f2=i+1) for i in range(0, 1001)
])
self.assertTrue(len(connection.queries) < 10)
self.assertEqual(TwoFields.objects.count(), 1001)
self.assertEqual(
TwoFields.objects.filter(f1__gte=450, f1__lte=550).count(),
101)
self.assertEqual(TwoFields.objects.filter(f2__gte=901).count(), 101)
def test_large_batch_mixed(self):
"""
Test inserting a large batch with objects having primary key set
mixed together with objects without PK set.
"""
with override_settings(DEBUG=True):
connection.queries = []
TwoFields.objects.bulk_create([
TwoFields(id=i if i % 2 == 0 else None, f1=i, f2=i+1)
for i in range(100000, 101000)])
self.assertTrue(len(connection.queries) < 10)
self.assertEqual(TwoFields.objects.count(), 1000)
# We can't assume much about the ID's created, except that the above
# created IDs must exist.
id_range = range(100000, 101000, 2)
self.assertEqual(TwoFields.objects.filter(id__in=id_range).count(), 500)
self.assertEqual(TwoFields.objects.exclude(id__in=id_range).count(), 500)
def test_explicit_batch_size(self):
objs = [TwoFields(f1=i, f2=i) for i in range(0, 100)]
with self.assertNumQueries(2):
TwoFields.objects.bulk_create(objs, 50)
TwoFields.objects.all().delete()
with self.assertNumQueries(1):
TwoFields.objects.bulk_create(objs, len(objs))