mirrors/django
1
0
Fork 0
mirror of https://github.com/django/django.git synced 2025-07-05 10:19:20 +00:00
Code Issues 32 Releases Wiki Activity

[soc2009/multidb] Modified the fixture loading, fixture dumping and synchronization process to be multi-db aware. Patch from Russell Keith-Magee.

Browse Source 
git-svn-id: http://code.djangoproject.com/svn/django/branches/soc2009/multidb@11763 bcc190cf-cafb-0310-a4f2-bffc1f526a37
This commit is contained in:
Alex Gaynor 2009-11-23 16:42:38 +00:00
parent 1fc8f9a279
commit 0c167ae0ff
14 changed files with 317 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
TODO
View File

@ -13,10 +13,6 @@ Required for v1.2
* Resolve the public facing UI issues around using multi-db
* Should we take the opportunity to modify DB backends to use fully qualified paths?
* Meta.using? Is is still required/desirable?
* syncdb
* Add --exclude/--include argument? (not sure this approach will work due to flush)
* Flush - which models are flushed?
* Fixture loading over multiple DBs
* Testing infrastructure
* Most tests don't need multidb. Some absolutely require it, but only to prove you
can write to a different db. Second DB could be a SQLite temp file. Need to have

6
django/core/management/commands/createcachetable.py
View File

@ -10,9 +10,9 @@ class Command(LabelCommand):
option_list = LabelCommand.option_list + (
make_option('--database', action='store', dest='database',
default=DEFAULT_DB_ALIAS, help='Nominates a specific database to '
'install the cache table to. Defaults to the "default" '
'database.'),
default=DEFAULT_DB_ALIAS, help='Nominates a database onto '
'which the cache table will be installed. '
'Defaults to the "default" database.'),
)
requires_model_validation = False

4
django/core/management/commands/dbshell.py
View File

@ -9,8 +9,8 @@ class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--database', action='store', dest='database',
default=DEFAULT_DB_ALIAS, help='Nominates a database to open a '
'shell for. Defaults to the "default" database.'),
default=DEFAULT_DB_ALIAS, help='Nominates a database onto which to '
'open a shell. Defaults to the "default" database.'),
)
requires_model_validation = False

14
django/core/management/commands/dumpdata.py
View File

@ -1,6 +1,7 @@
from django.core.exceptions import ImproperlyConfigured
from django.core.management.base import BaseCommand, CommandError
from django.core import serializers
from django.db import connections, DEFAULT_DB_ALIAS
from django.utils.datastructures import SortedDict
from optparse import make_option
@ -11,6 +12,9 @@ class Command(BaseCommand):
help='Specifies the output serialization format for fixtures.'),
make_option('--indent', default=None, dest='indent', type='int',
help='Specifies the indent level to use when pretty-printing output'),
make_option('--database', action='store', dest='database',
default=DEFAULT_DB_ALIAS, help='Nominates a specific database to load '
'fixtures into. Defaults to the "default" database.'),
make_option('-e', '--exclude', dest='exclude',action='append', default=[],
help='App to exclude (use multiple --exclude to exclude multiple apps).'),
)
@ -22,6 +26,8 @@ class Command(BaseCommand):
format = options.get('format','json')
indent = options.get('indent',None)
using = options['database']
connection = connections[using]
exclude = options.get('exclude',[])
show_traceback = options.get('traceback', False)
@ -67,14 +73,18 @@ class Command(BaseCommand):
except KeyError:
raise CommandError("Unknown serialization format: %s" % format)
# Get a list of synchronized tables
tables = connection.introspection.table_names()
objects = []
for app, model_list in app_list.items():
if model_list is None:
model_list = get_models(app)
for model in model_list:
if not model._meta.proxy:
objects.extend(model._default_manager.all())
# Don't serialize proxy models, or models that haven't been synchronized
if not model._meta.proxy and model._meta.db_table in tables:
objects.extend(model._default_manager.using(using).all())
try:
return serializers.serialize(format, objects, indent=indent)

6
django/core/management/commands/flush.py
View File

@ -1,7 +1,7 @@
from optparse import make_option
from django.conf import settings
from django.db import connections, transaction, models
from django.db import connections, transaction, models, DEFAULT_DB_ALIAS
from django.core.management import call_command
from django.core.management.base import NoArgsCommand, CommandError
from django.core.management.color import no_style
@ -15,8 +15,8 @@ class Command(NoArgsCommand):
make_option('--noinput', action='store_false', dest='interactive', default=True,
help='Tells Django to NOT prompt the user for input of any kind.'),
make_option('--database', action='store', dest='database',
default=None, help='Nominates a database to flush. Defaults to '
'flushing all databases.'),
default=DEFAULT_DB_ALIAS, help='Nominates a database to flush. '
'Defaults to the "default" database.'),
)
help = "Executes ``sqlflush`` on the current database."

24
django/core/management/commands/loaddata.py
View File

@ -10,7 +10,7 @@ from django.core.management.base import BaseCommand
from django.core.management.color import no_style
from django.db import connections, transaction, DEFAULT_DB_ALIAS
from django.db.models import get_apps
from django.utils.itercompat import product
try:
set
@ -30,11 +30,15 @@ class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--database', action='store', dest='database',
default=DEFAULT_DB_ALIAS, help='Nominates a specific database to load '
'fixtures into. By default uses the "default" database.'),
'fixtures into. Defaults to the "default" database.'),
make_option('-e', '--exclude', dest='exclude',action='append', default=[],
help='App to exclude (use multiple --exclude to exclude multiple apps).'),
)
def handle(self, *fixture_labels, **options):
using = options['database']
excluded_apps = options.get('exclude', [])
connection = connections[using]
self.style = no_style()
@ -125,13 +129,14 @@ class Command(BaseCommand):
print "Checking %s for fixtures..." % humanize(fixture_dir)
label_found = False
for format in formats:
for compression_format in compression_formats:
if compression_format:
file_name = '.'.join([fixture_name, format,
compression_format])
else:
file_name = '.'.join([fixture_name, format])
for combo in product([using, None], formats, compression_formats):
database, format, compression_format = combo
file_name = '.'.join(
p for p in [
fixture_name, database, format, compression_format
]
if p
)
if verbosity > 1:
print "Trying %s for %s fixture '%s'..." % \
@ -156,6 +161,7 @@ class Command(BaseCommand):
try:
objects = serializers.deserialize(format, fixture)
for obj in objects:
if obj.object._meta.app_label not in excluded_apps:
objects_in_fixture += 1
models.add(obj.object.__class__)
obj.save(using=using)

28
django/core/management/commands/reset.py
View File

@ -4,15 +4,15 @@ from django.conf import settings
from django.core.management.base import AppCommand, CommandError
from django.core.management.color import no_style
from django.core.management.sql import sql_reset
from django.db import connections, transaction
from django.db import connections, transaction, DEFAULT_DB_ALIAS
class Command(AppCommand):
option_list = AppCommand.option_list + (
make_option('--noinput', action='store_false', dest='interactive', default=True,
help='Tells Django to NOT prompt the user for input of any kind.'),
make_option('--database', action='store', dest='database',
default='', help='Nominates a database to reset. Defaults to '
'reseting all databases.'),
default=DEFAULT_DB_ALIAS, help='Nominates a database to reset. '
'Defaults to the "default" database.'),
)
help = "Executes ``sqlreset`` for the given app(s) in the current database."
args = '[appname ...]'
@ -20,26 +20,22 @@ class Command(AppCommand):
output_transaction = True
def handle_app(self, app, **options):
if not options['database']:
dbs = connections.all()
else:
dbs = [options['database']]
using = options['database']
connection = connections[using]
app_name = app.__name__.split('.')[-2]
self.style = no_style()
for connection in dbs:
sql_list = sql_reset(app, self.style, connection)
if options.get('interactive'):
confirm = raw_input("""
You have requested a database reset.
This will IRREVERSIBLY DESTROY any data for
the "%s" application in the database "%s".
Are you sure you want to do this?
You have requested a database reset.
This will IRREVERSIBLY DESTROY any data for
the "%s" application in the database "%s".
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % (app_name, connection.settings_dict['DATABASE_NAME']))
Type 'yes' to continue, or 'no' to cancel: """ % (app_name, connection.settings_dict['DATABASE_NAME']))
else:
confirm = 'yes'
@ -54,8 +50,8 @@ class Command(AppCommand):
* The database isn't running or isn't configured correctly.
* At least one of the database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlreset %s'. That's the SQL this command wasn't able to run.
The full error: %s""" % (app_name, app_name, e))
Hint: Look at the output of 'django-admin.py sqlreset %s'. That's the SQL this command wasn't able to run.
The full error: %s""" % (app_name, app_name, e))
transaction.commit_unless_managed()
else:
print "Reset cancelled."

17
django/core/management/commands/syncdb.py
View File

@ -14,8 +14,10 @@ class Command(NoArgsCommand):
make_option('--noinput', action='store_false', dest='interactive', default=True,
help='Tells Django to NOT prompt the user for input of any kind.'),
make_option('--database', action='store', dest='database',
default=DEFAULT_DB_ALIAS, help='Nominates a database to sync. '
default=DEFAULT_DB_ALIAS, help='Nominates a database to synchronize. '
'Defaults to the "default" database.'),
make_option('-e', '--exclude', dest='exclude',action='append', default=[],
help='App to exclude (use multiple --exclude to exclude multiple apps).'),
)
help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created."
@ -24,6 +26,7 @@ class Command(NoArgsCommand):
verbosity = int(options.get('verbosity', 1))
interactive = options.get('interactive')
show_traceback = options.get('traceback', False)
exclude = options.get('exclude', [])
self.style = no_style()
@ -56,8 +59,11 @@ class Command(NoArgsCommand):
created_models = set()
pending_references = {}
excluded_apps = [models.get_app(app_label) for app_label in exclude]
app_list = [app for app in models.get_apps() if app not in excluded_apps]
# Create the tables for each model
for app in models.get_apps():
for app in app_list:
app_name = app.__name__.split('.')[-2]
model_list = models.get_models(app, include_auto_created=True)
for model in model_list:
@ -95,7 +101,7 @@ class Command(NoArgsCommand):
# Install custom SQL for the app (but only if this
# is a model we've just created)
for app in models.get_apps():
for app in app_list:
app_name = app.__name__.split('.')[-2]
for model in models.get_models(app):
if model in created_models:
@ -118,8 +124,9 @@ class Command(NoArgsCommand):
else:
if verbosity >= 2:
print "No custom SQL for %s.%s model" % (app_name, model._meta.object_name)
# Install SQL indicies for all newly created models
for app in models.get_apps():
for app in app_list:
app_name = app.__name__.split('.')[-2]
for model in models.get_models(app):
if model in created_models:
@ -138,4 +145,4 @@ class Command(NoArgsCommand):
transaction.commit_unless_managed(using=db)
from django.core.management import call_command
call_command('loaddata', 'initial_data', verbosity=verbosity, database=db)
call_command('loaddata', 'initial_data', verbosity=verbosity, exclude=exclude, database=db)

15
django/utils/itercompat.py
View File

@ -45,6 +45,19 @@ def groupby(iterable, keyfunc=None):
l.append(item)
yield lastkey, l
def product(*args, **kwds):
"""
Taken from http://docs.python.org/library/itertools.html#itertools.product
"""
# product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
# product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
pools = map(tuple, args) * kwds.get('repeat', 1)
result = [[]]
for pool in pools:
result = [x+[y] for x in result for y in pool]
for prod in result:
yield tuple(prod)
# Not really in itertools, since it's a builtin in Python 2.4 and later, but it
# does operate as an iterator.
def reversed(data):
@ -57,6 +70,8 @@ else:
tee = compat_tee
if hasattr(itertools, 'groupby'):
groupby = itertools.groupby
if hasattr(itertools, 'product'):
product = itertools.product
def is_iterable(x):
"A implementation independent way of checking for iterables"

186
docs/ref/django-admin.txt
View File

@ -121,11 +121,10 @@ createcachetable
Creates a cache table named ``tablename`` for use with the database cache
backend. See :ref:`topics-cache` for more information.
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to install the cachetable to. By default uses the
``'default'`` alias.
The :djadminopt:`--database` option can be used to specify the database
onto which the cachetable will be installed.
createsuperuser
---------------
@ -173,11 +172,10 @@ the program name (``psql``, ``mysql``, ``sqlite3``) will find the program in
the right place. There's no way to specify the location of the program
manually.
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to open the shell for. By default uses the
``'default'`` alias.
The :djadminopt:`--database` option can be used to specify the database
onto which to open a shell.
diffsettings
@ -212,21 +210,6 @@ records to dump. If you're using a :ref:`custom manager <custom-managers>` as
the default manager and it filters some of the available records, not all of the
objects will be dumped.
.. django-admin-option:: --exclude
.. versionadded:: 1.0
Exclude a specific application from the applications whose contents is
output. For example, to specifically exclude the `auth` application from
the output, you would call::
django-admin.py dumpdata --exclude=auth
If you want to exclude multiple applications, use multiple ``--exclude``
directives::
django-admin.py dumpdata --exclude=auth --exclude=contenttypes
.. django-admin-option:: --format <fmt>
By default, ``dumpdata`` will format its output in JSON, but you can use the
@ -239,6 +222,11 @@ By default, ``dumpdata`` will output all data on a single line. This isn't
easy for humans to read, so you can use the ``--indent`` option to
pretty-print the output with a number of indentation spaces.
.. versionadded:: 1.0
The :djadminopt:`--exclude` option may be provided to prevent specific
applications from being dumped.
.. versionadded:: 1.1
In addition to specifying application names, you can provide a list of
@ -247,6 +235,11 @@ name to ``dumpdata``, the dumped output will be restricted to that model,
rather than the entire application. You can also mix application names and
model names.
.. versionadded:: 1.2
The :djadminopt:`--database` option can be used to specify the database
onto which the data will be loaded.
flush
-----
@ -260,10 +253,10 @@ fixture will be re-installed.
The :djadminopt:`--noinput` option may be provided to suppress all user
prompts.
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to flush. By default flushes all databases.
The :djadminopt:`--database` option may be used to specify the database
to flush.
inspectdb
@ -309,11 +302,10 @@ needed.
``inspectdb`` works with PostgreSQL, MySQL and SQLite. Foreign-key detection
only works in PostgreSQL and with certain types of MySQL tables.
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to introspect. By default uses the ``'default'``
alias.
The :djadminopt:`--database` option may be used to specify the
database to introspect.
loaddata <fixture fixture ...>
@ -323,6 +315,11 @@ loaddata <fixture fixture ...>
Searches for and loads the contents of the named fixture into the database.
.. versionadded:: 1.2
The :djadminopt:`--database` option can be used to specify the database
onto which the data will be loaded.
What's a "fixture"?
~~~~~~~~~~~~~~~~~~~
@ -404,6 +401,37 @@ installation will be aborted, and any data installed in the call to
references in your data files - MySQL doesn't provide a mechanism to
defer checking of row constraints until a transaction is committed.
Database-specific fixtures
~~~~~~~~~~~~~~~~~~~~~~~~~~
If you are in a multi-database setup, you may have fixture data that
you want to load onto one database, but not onto another. In this
situation, you can add database identifier into . If your
:setting:`DATABASES` setting has a 'master' database defined, you can
define the fixture ``mydata.master.json`` or
``mydata.master.json.gz``. This fixture will only be loaded if you
have specified that you want to load data onto the ``master``
database.
Excluding applications from loading
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 1.2
The :djadminopt:`--exclude` option may be provided to prevent specific
applications from being loaded.
For example, if you wanted to exclude models from ``django.contrib.auth``
from being loaded into your database, you would call::
django-admin.py loaddata mydata.json --exclude auth
This will look for for a JSON fixture called ``mydata`` in all the
usual locations - including the ``fixtures`` directory of the
``django.contrib.auth`` application. However, any fixture object that
identifies itself as belonging to the ``auth`` application (e.g.,
instance of ``auth.User``) would be ignored by loaddata.
makemessages
------------
@ -465,10 +493,10 @@ Executes the equivalent of ``sqlreset`` for the given app name(s).
The :djadminopt:`--noinput` option may be provided to suppress all user
prompts.
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to reset. By default resets all databases.
The :djadminopt:`--database` option can be used to specify the alias
of the database to reset.
Executes the equivalent of ``sqlreset`` for the given app name(s).
@ -592,12 +620,10 @@ sql <appname appname ...>
Prints the CREATE TABLE SQL statements for the given app name(s).
--database
~~~~~~~~~~
The alias for the database to print the SQL for. By default uses the
``'default'`` alias.
.. versionadded:: 1.2
The :djadminopt:`--database` option can be used to specify the database for
which to print the SQL.
sqlall <appname appname ...>
----------------------------
@ -609,11 +635,10 @@ Prints the CREATE TABLE and initial-data SQL statements for the given app name(s
Refer to the description of ``sqlcustom`` for an explanation of how to
specify initial data.
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to print the SQL for. By default uses the
``'default'`` alias.
The :djadminopt:`--database` option can be used to specify the database for
which to print the SQL.
sqlclear <appname appname ...>
------------------------------
@ -622,11 +647,10 @@ sqlclear <appname appname ...>
Prints the DROP TABLE SQL statements for the given app name(s).
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to print the SQL for. By default uses the
``'default'`` alias.
The :djadminopt:`--database` option can be used to specify the database for
which to print the SQL.
sqlcustom <appname appname ...>
-------------------------------
@ -649,11 +673,10 @@ table modifications, or insert any SQL functions into the database.
Note that the order in which the SQL files are processed is undefined.
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to print the SQL for. By default uses the
``'default'`` alias.
The :djadminopt:`--database` option can be used to specify the database for
which to print the SQL.
sqlflush
--------
@ -663,11 +686,10 @@ sqlflush
Prints the SQL statements that would be executed for the :djadmin:`flush`
command.
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to print the SQL for. By default uses the
``'default'`` alias.
The :djadminopt:`--database` option can be used to specify the database for
which to print the SQL.
sqlindexes <appname appname ...>
--------------------------------
@ -676,11 +698,10 @@ sqlindexes <appname appname ...>
Prints the CREATE INDEX SQL statements for the given app name(s).
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to print the SQL for. By default uses the
``'default'`` alias.
The :djadminopt:`--database` option can be used to specify the database for
which to print the SQL.
sqlreset <appname appname ...>
------------------------------
@ -689,11 +710,10 @@ sqlreset <appname appname ...>
Prints the DROP TABLE SQL, then the CREATE TABLE SQL, for the given app name(s).
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to print the SQL for. By default uses the
``'default'`` alias.
The :djadminopt:`--database` option can be used to specify the database for
which to print the SQL.
sqlsequencereset <appname appname ...>
--------------------------------------
@ -708,11 +728,10 @@ number for automatically incremented fields.
Use this command to generate SQL which will fix cases where a sequence is out
of sync with its automatically incremented field data.
--database
~~~~~~~~~~
.. versionadded:: 1.2
The alias for the database to print the SQL for. By default uses the
``'default'`` alias.
The :djadminopt:`--database` option can be used to specify the database for
which to print the SQL.
startapp <appname>
------------------
@ -770,17 +789,16 @@ with an appropriate extension (e.g. ``json`` or ``xml``). See the
documentation for ``loaddata`` for details on the specification of fixture
data files.
--database
~~~~~~~~~~
The alias for the database install the tables for. By default uses the
``'default'`` alias.
--noinput
~~~~~~~~~
The :djadminopt:`--noinput` option may be provided to suppress all user
prompts.
.. versionadded:: 1.2
The :djadminopt:`--database` option can be used to specify the database to
synchronize.
test <app or test identifier>
-----------------------------
@ -922,6 +940,30 @@ Common options
The following options are not available on every commands, but they are
common to a number of commands.
.. django-admin-option:: --database
.. versionadded:: 1.2
Used to specify the database on which a command will operate. If not
specified, this option will default to an alias of ``default``.
For example, to dump data from the database with the alias ``master``::
django-admin.py dumpdata --database=master
.. django-admin-option:: --exclude
Exclude a specific application from the applications whose contents is
output. For example, to specifically exclude the `auth` application from
the output of dumpdata, you would call::
django-admin.py dumpdata --exclude=auth
If you want to exclude multiple applications, use multiple ``--exclude``
directives::
django-admin.py dumpdata --exclude=auth --exclude=contenttypes
.. django-admin-option:: --locale
Use the ``--locale`` or ``-l`` option to specify the locale to process.

10
tests/modeltests/fixtures/fixtures/fixture6.default.json Normal file
View File

@ -0,0 +1,10 @@
[
{
"pk": "6",
"model": "fixtures.article",
"fields": {
"headline": "Who needs more than one database?",
"pub_date": "2006-06-16 14:00:00"
}
}
]

BIN
tests/modeltests/fixtures/fixtures/fixture7.default.json.gz Normal file
View File

Binary file not shown.

10
tests/modeltests/fixtures/fixtures/fixture8.nosuchdb.json Normal file
View File

@ -0,0 +1,10 @@
[
{
"pk": "8",
"model": "fixtures.article",
"fields": {
"headline": "There is no spoon.",
"pub_date": "2006-06-16 14:00:00"
}
}
]

37
tests/modeltests/fixtures/models.py
View File

@ -159,6 +159,43 @@ Multiple fixtures named 'fixture2' in '...fixtures'. Aborting.
>>> management.call_command('loaddata', 'fixture5', verbosity=0) # doctest: +ELLIPSIS
Multiple fixtures named 'fixture5' in '...fixtures'. Aborting.
>>> management.call_command('flush', verbosity=0, interactive=False)
# Load fixtures 6 and 7. These will load using the 'default' database identifier implicitly
>>> management.call_command('loaddata', 'fixture6', verbosity=0)
>>> management.call_command('loaddata', 'fixture7', verbosity=0)
>>> Article.objects.all()
[<Article: Who needs more than one database?>, <Article: Who needs to use compressed data?>, <Article: Python program becomes self aware>]
>>> management.call_command('flush', verbosity=0, interactive=False)
# Load fixtures 6 and 7. These will load using the 'default' database identifier explicitly
>>> management.call_command('loaddata', 'fixture6', verbosity=0, using='default')
>>> management.call_command('loaddata', 'fixture7', verbosity=0, using='default')
>>> Article.objects.all()
[<Article: Who needs more than one database?>, <Article: Who needs to use compressed data?>, <Article: Python program becomes self aware>]
>>> management.call_command('flush', verbosity=0, interactive=False)
# Try to load fixture 8. This won't load because the database identifier doesn't match
>>> management.call_command('loaddata', 'fixture8', verbosity=0)
>>> Article.objects.all()
[<Article: Python program becomes self aware>]
>>> management.call_command('loaddata', 'fixture8', verbosity=0, using='default')
>>> Article.objects.all()
[<Article: Python program becomes self aware>]
>>> management.call_command('flush', verbosity=0, interactive=False)
# Try to load fixture 1, but this time, exclude the 'fixtures' app.
>>> management.call_command('loaddata', 'fixture1', verbosity=0, exclude='fixtures')
>>> Article.objects.all()
[<Article: Python program becomes self aware>]
>>> Category.objects.all()
[]
"""
from django.test import TestCase