From 315ab41e416c777d4f42932d42df07872e8f8895 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 7 Jun 2013 18:47:17 +0100 Subject: [PATCH] Initial stab at a migrate command, it's probably quite a way off. --- django/core/management/commands/migrate.py | 210 +++++++++++++++++++++ django/core/management/commands/syncdb.py | 152 +-------------- django/db/migrations/executor.py | 17 +- django/db/migrations/loader.py | 4 +- django/db/migrations/migration.py | 3 + django/utils/termcolors.py | 6 + 6 files changed, 241 insertions(+), 151 deletions(-) create mode 100644 django/core/management/commands/migrate.py diff --git a/django/core/management/commands/migrate.py b/django/core/management/commands/migrate.py new file mode 100644 index 0000000000..68897fe19c --- /dev/null +++ b/django/core/management/commands/migrate.py @@ -0,0 +1,210 @@ +from optparse import make_option +import itertools +import traceback + +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import NoArgsCommand +from django.core.management.color import color_style +from django.core.management.sql import custom_sql_for_model, emit_post_sync_signal, emit_pre_sync_signal +from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS +from django.db.migrations.executor import MigrationExecutor +from django.utils.datastructures import SortedDict +from django.utils.importlib import import_module + + +class Command(NoArgsCommand): + option_list = NoArgsCommand.option_list + ( + make_option('--noinput', action='store_false', dest='interactive', default=True, + help='Tells Django to NOT prompt the user for input of any kind.'), + make_option('--no-initial-data', action='store_false', dest='load_initial_data', default=True, + help='Tells Django not to load any initial data after database synchronization.'), + make_option('--database', action='store', dest='database', + default=DEFAULT_DB_ALIAS, help='Nominates a database to synchronize. ' + 'Defaults to the "default" database.'), + ) + + help = "Updates database schema. Manages both apps with migrations and those without." + + def handle_noargs(self, **options): + + self.verbosity = int(options.get('verbosity')) + self.interactive = options.get('interactive') + self.show_traceback = options.get('traceback') + self.load_initial_data = options.get('load_initial_data') + + self.style = color_style() + + # Import the 'management' module within each installed app, to register + # dispatcher events. + for app_name in settings.INSTALLED_APPS: + try: + import_module('.management', app_name) + except ImportError as exc: + # This is slightly hackish. We want to ignore ImportErrors + # if the "management" module itself is missing -- but we don't + # want to ignore the exception if the management module exists + # but raises an ImportError for some reason. The only way we + # can do this is to check the text of the exception. Note that + # we're a bit broad in how we check the text, because different + # Python implementations may not use the same text. + # CPython uses the text "No module named management" + # PyPy uses "No module named myproject.myapp.management" + msg = exc.args[0] + if not msg.startswith('No module named') or 'management' not in msg: + raise + + # Get the database we're operating from + db = options.get('database') + connection = connections[db] + + # Work out which apps have migrations and which do not + if self.verbosity >= 1: + self.stdout.write(self.style.MIGRATE_HEADING("Calculating migration plan:")) + executor = MigrationExecutor(connection, self.migration_progress_callback) + if self.verbosity >= 1: + self.stdout.write(self.style.MIGRATE_LABEL(" Apps without migrations: ") + (", ".join(executor.loader.unmigrated_apps) or "(none)")) + + # Work out what targets they want, and then make a migration plan + # TODO: Let users select targets + targets = executor.loader.graph.leaf_nodes() + plan = executor.migration_plan(targets) + + if self.verbosity >= 1: + self.stdout.write(self.style.MIGRATE_LABEL(" Apps with migrations: ") + (", ".join(executor.loader.disk_migrations) or "(none)")) + + # Run the syncdb phase. + # If you ever manage to get rid of this, I owe you many, many drinks. + self.stdout.write(self.style.MIGRATE_HEADING("Synchronizing apps without migrations:")) + self.sync_apps(connection, executor.loader.unmigrated_apps) + + # Migrate! + if self.verbosity >= 1: + self.stdout.write(self.style.MIGRATE_HEADING("Running migrations:")) + if not plan: + if self.verbosity >= 1: + self.stdout.write(" No migrations needed.") + else: + executor.migrate(targets, plan) + + def migration_progress_callback(self, action, migration): + if self.verbosity >= 1: + if action == "apply_start": + self.stdout.write(" Applying %s... " % migration) + self.stdout.flush() + elif action == "apply_success": + self.stdout.write(" OK\n") + elif action == "unapply_start": + self.stdout.write(" Unapplying %s... " % migration) + self.stdout.flush() + elif action == "unapply_success": + self.stdout.write(" OK\n") + + def sync_apps(self, connection, apps): + "Runs the old syncdb-style operation on a list of apps." + cursor = connection.cursor() + + # Get a list of already installed *models* so that references work right. + tables = connection.introspection.table_names() + seen_models = connection.introspection.installed_models(tables) + created_models = set() + pending_references = {} + + # Build the manifest of apps and models that are to be synchronized + all_models = [ + (app.__name__.split('.')[-2], + [ + m for m in models.get_models(app, include_auto_created=True) + if router.allow_syncdb(connection.alias, m) + ]) + for app in models.get_apps() if app.__name__.split('.')[-2] in apps + ] + + def model_installed(model): + opts = model._meta + converter = connection.introspection.table_name_converter + # Note that if a model is unmanaged we short-circuit and never try to install it + return not ((converter(opts.db_table) in tables) or + (opts.auto_created and converter(opts.auto_created._meta.db_table) in tables)) + + manifest = SortedDict( + (app_name, list(filter(model_installed, model_list))) + for app_name, model_list in all_models + ) + + create_models = set([x for x in itertools.chain(*manifest.values())]) + emit_pre_sync_signal(create_models, self.verbosity, self.interactive, connection.alias) + + # Create the tables for each model + if self.verbosity >= 1: + self.stdout.write(" Creating tables...\n") + with transaction.commit_on_success_unless_managed(using=connection.alias): + for app_name, model_list in manifest.items(): + for model in model_list: + # Create the model's database table, if it doesn't already exist. + if self.verbosity >= 3: + self.stdout.write(" Processing %s.%s model\n" % (app_name, model._meta.object_name)) + sql, references = connection.creation.sql_create_model(model, self.style, seen_models) + seen_models.add(model) + created_models.add(model) + for refto, refs in references.items(): + pending_references.setdefault(refto, []).extend(refs) + if refto in seen_models: + sql.extend(connection.creation.sql_for_pending_references(refto, self.style, pending_references)) + sql.extend(connection.creation.sql_for_pending_references(model, self.style, pending_references)) + if self.verbosity >= 1 and sql: + self.stdout.write(" Creating table %s\n" % model._meta.db_table) + for statement in sql: + cursor.execute(statement) + tables.append(connection.introspection.table_name_converter(model._meta.db_table)) + + # Send the post_syncdb signal, so individual apps can do whatever they need + # to do at this point. + emit_post_sync_signal(created_models, self.verbosity, self.interactive, connection.alias) + + # The connection may have been closed by a syncdb handler. + cursor = connection.cursor() + + # Install custom SQL for the app (but only if this + # is a model we've just created) + if self.verbosity >= 1: + self.stdout.write(" Installing custom SQL...\n") + for app_name, model_list in manifest.items(): + for model in model_list: + if model in created_models: + custom_sql = custom_sql_for_model(model, self.style, connection) + if custom_sql: + if self.verbosity >= 2: + self.stdout.write(" Installing custom SQL for %s.%s model\n" % (app_name, model._meta.object_name)) + try: + with transaction.commit_on_success_unless_managed(using=connection.alias): + for sql in custom_sql: + cursor.execute(sql) + except Exception as e: + self.stderr.write(" Failed to install custom SQL for %s.%s model: %s\n" % (app_name, model._meta.object_name, e)) + if self.show_traceback: + traceback.print_exc() + else: + if self.verbosity >= 3: + self.stdout.write(" No custom SQL for %s.%s model\n" % (app_name, model._meta.object_name)) + + if self.verbosity >= 1: + self.stdout.write(" Installing indexes...\n") + # Install SQL indices for all newly created models + for app_name, model_list in manifest.items(): + for model in model_list: + if model in created_models: + index_sql = connection.creation.sql_indexes_for_model(model, self.style) + if index_sql: + if self.verbosity >= 2: + self.stdout.write(" Installing index for %s.%s model\n" % (app_name, model._meta.object_name)) + try: + with transaction.commit_on_success_unless_managed(using=connection.alias): + for sql in index_sql: + cursor.execute(sql) + except Exception as e: + self.stderr.write(" Failed to install index for %s.%s model: %s\n" % (app_name, model._meta.object_name, e)) + + # Load initial_data fixtures (unless that has been disabled) + if self.load_initial_data: + call_command('loaddata', 'initial_data', verbosity=self.verbosity, database=connection.alias, skip_validation=True) diff --git a/django/core/management/commands/syncdb.py b/django/core/management/commands/syncdb.py index 51470d7bda..17ea51f4d5 100644 --- a/django/core/management/commands/syncdb.py +++ b/django/core/management/commands/syncdb.py @@ -1,15 +1,8 @@ +import warnings from optparse import make_option -import itertools -import traceback - -from django.conf import settings +from django.db import DEFAULT_DB_ALIAS from django.core.management import call_command from django.core.management.base import NoArgsCommand -from django.core.management.color import no_style -from django.core.management.sql import custom_sql_for_model, emit_post_sync_signal, emit_pre_sync_signal -from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS -from django.utils.datastructures import SortedDict -from django.utils.importlib import import_module class Command(NoArgsCommand): @@ -22,143 +15,8 @@ class Command(NoArgsCommand): default=DEFAULT_DB_ALIAS, help='Nominates a database to synchronize. ' 'Defaults to the "default" database.'), ) - help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created." + help = "Deprecated - use 'migrate' instead." def handle_noargs(self, **options): - - verbosity = int(options.get('verbosity')) - interactive = options.get('interactive') - show_traceback = options.get('traceback') - load_initial_data = options.get('load_initial_data') - - self.style = no_style() - - # Import the 'management' module within each installed app, to register - # dispatcher events. - for app_name in settings.INSTALLED_APPS: - try: - import_module('.management', app_name) - except ImportError as exc: - # This is slightly hackish. We want to ignore ImportErrors - # if the "management" module itself is missing -- but we don't - # want to ignore the exception if the management module exists - # but raises an ImportError for some reason. The only way we - # can do this is to check the text of the exception. Note that - # we're a bit broad in how we check the text, because different - # Python implementations may not use the same text. - # CPython uses the text "No module named management" - # PyPy uses "No module named myproject.myapp.management" - msg = exc.args[0] - if not msg.startswith('No module named') or 'management' not in msg: - raise - - db = options.get('database') - connection = connections[db] - cursor = connection.cursor() - - # Get a list of already installed *models* so that references work right. - tables = connection.introspection.table_names() - seen_models = connection.introspection.installed_models(tables) - created_models = set() - pending_references = {} - - # Build the manifest of apps and models that are to be synchronized - all_models = [ - (app.__name__.split('.')[-2], - [m for m in models.get_models(app, include_auto_created=True) - if router.allow_syncdb(db, m)]) - for app in models.get_apps() - ] - - def model_installed(model): - opts = model._meta - converter = connection.introspection.table_name_converter - # Note that if a model is unmanaged we short-circuit and never try to install it - return not ((converter(opts.db_table) in tables) or - (opts.auto_created and converter(opts.auto_created._meta.db_table) in tables)) - - - manifest = SortedDict( - (app_name, list(filter(model_installed, model_list))) - for app_name, model_list in all_models - ) - - create_models = set([x for x in itertools.chain(*manifest.values())]) - emit_pre_sync_signal(create_models, verbosity, interactive, db) - - # Create the tables for each model - if verbosity >= 1: - self.stdout.write("Creating tables ...\n") - with transaction.commit_on_success_unless_managed(using=db): - for app_name, model_list in manifest.items(): - for model in model_list: - # Create the model's database table, if it doesn't already exist. - if verbosity >= 3: - self.stdout.write("Processing %s.%s model\n" % (app_name, model._meta.object_name)) - sql, references = connection.creation.sql_create_model(model, self.style, seen_models) - seen_models.add(model) - created_models.add(model) - for refto, refs in references.items(): - pending_references.setdefault(refto, []).extend(refs) - if refto in seen_models: - sql.extend(connection.creation.sql_for_pending_references(refto, self.style, pending_references)) - sql.extend(connection.creation.sql_for_pending_references(model, self.style, pending_references)) - if verbosity >= 1 and sql: - self.stdout.write("Creating table %s\n" % model._meta.db_table) - for statement in sql: - cursor.execute(statement) - tables.append(connection.introspection.table_name_converter(model._meta.db_table)) - - # Send the post_syncdb signal, so individual apps can do whatever they need - # to do at this point. - emit_post_sync_signal(created_models, verbosity, interactive, db) - - # The connection may have been closed by a syncdb handler. - cursor = connection.cursor() - - # Install custom SQL for the app (but only if this - # is a model we've just created) - if verbosity >= 1: - self.stdout.write("Installing custom SQL ...\n") - for app_name, model_list in manifest.items(): - for model in model_list: - if model in created_models: - custom_sql = custom_sql_for_model(model, self.style, connection) - if custom_sql: - if verbosity >= 2: - self.stdout.write("Installing custom SQL for %s.%s model\n" % (app_name, model._meta.object_name)) - try: - with transaction.commit_on_success_unless_managed(using=db): - for sql in custom_sql: - cursor.execute(sql) - except Exception as e: - self.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \ - (app_name, model._meta.object_name, e)) - if show_traceback: - traceback.print_exc() - else: - if verbosity >= 3: - self.stdout.write("No custom SQL for %s.%s model\n" % (app_name, model._meta.object_name)) - - if verbosity >= 1: - self.stdout.write("Installing indexes ...\n") - # Install SQL indices for all newly created models - for app_name, model_list in manifest.items(): - for model in model_list: - if model in created_models: - index_sql = connection.creation.sql_indexes_for_model(model, self.style) - if index_sql: - if verbosity >= 2: - self.stdout.write("Installing index for %s.%s model\n" % (app_name, model._meta.object_name)) - try: - with transaction.commit_on_success_unless_managed(using=db): - for sql in index_sql: - cursor.execute(sql) - except Exception as e: - self.stderr.write("Failed to install index for %s.%s model: %s\n" % \ - (app_name, model._meta.object_name, e)) - - # Load initial_data fixtures (unless that has been disabled) - if load_initial_data: - call_command('loaddata', 'initial_data', verbosity=verbosity, - database=db, skip_validation=True) + warnings.warn("The syncdb command will be removed in Django 1.9", PendingDeprecationWarning) + call_command("migrate", **options) diff --git a/django/db/migrations/executor.py b/django/db/migrations/executor.py index 0f16001500..82601be7c0 100644 --- a/django/db/migrations/executor.py +++ b/django/db/migrations/executor.py @@ -8,10 +8,12 @@ class MigrationExecutor(object): up or down to a specified set of targets. """ - def __init__(self, connection): + def __init__(self, connection, progress_callback=None): self.connection = connection self.loader = MigrationLoader(self.connection) + self.loader.load_disk() self.recorder = MigrationRecorder(self.connection) + self.progress_callback = progress_callback def migration_plan(self, targets): """ @@ -34,11 +36,12 @@ class MigrationExecutor(object): applied.add(migration) return plan - def migrate(self, targets): + def migrate(self, targets, plan=None): """ Migrates the database up to the given targets. """ - plan = self.migration_plan(targets) + if plan is None: + plan = self.migration_plan(targets) for migration, backwards in plan: if not backwards: self.apply_migration(migration) @@ -49,16 +52,24 @@ class MigrationExecutor(object): """ Runs a migration forwards. """ + if self.progress_callback: + self.progress_callback("apply_start", migration) with self.connection.schema_editor() as schema_editor: project_state = self.loader.graph.project_state((migration.app_label, migration.name), at_end=False) migration.apply(project_state, schema_editor) self.recorder.record_applied(migration.app_label, migration.name) + if self.progress_callback: + self.progress_callback("apply_success", migration) def unapply_migration(self, migration): """ Runs a migration backwards. """ + if self.progress_callback: + self.progress_callback("unapply_start", migration) with self.connection.schema_editor() as schema_editor: project_state = self.loader.graph.project_state((migration.app_label, migration.name), at_end=False) migration.unapply(project_state, schema_editor) self.recorder.record_unapplied(migration.app_label, migration.name) + if self.progress_callback: + self.progress_callback("unapply_success", migration) diff --git a/django/db/migrations/loader.py b/django/db/migrations/loader.py index ce9fb7c8de..894d8c91f2 100644 --- a/django/db/migrations/loader.py +++ b/django/db/migrations/loader.py @@ -41,6 +41,7 @@ class MigrationLoader(object): Loads the migrations from all INSTALLED_APPS from disk. """ self.disk_migrations = {} + self.unmigrated_apps = set() for app in cache.get_apps(): # Get the migrations module directory module_name = ".".join(app.__name__.split(".")[:-1] + ["migrations"]) @@ -50,7 +51,8 @@ class MigrationLoader(object): except ImportError as e: # I hate doing this, but I don't want to squash other import errors. # Might be better to try a directory check directly. - if "No module named migrations" in str(e): + if "No module named" in str(e) and "migrations" in str(e): + self.unmigrated_apps.add(app_label) continue directory = os.path.dirname(module.__file__) # Scan for .py[c|o] files diff --git a/django/db/migrations/migration.py b/django/db/migrations/migration.py index ed7fad747b..277c5faa3f 100644 --- a/django/db/migrations/migration.py +++ b/django/db/migrations/migration.py @@ -47,6 +47,9 @@ class Migration(object): def __repr__(self): return "" % (self.app_label, self.name) + def __str__(self): + return "%s.%s" % (self.app_label, self.name) + def __hash__(self): return hash("%s.%s" % (self.app_label, self.name)) diff --git a/django/utils/termcolors.py b/django/utils/termcolors.py index bb14837716..3562fa4fb5 100644 --- a/django/utils/termcolors.py +++ b/django/utils/termcolors.py @@ -86,6 +86,8 @@ PALETTES = { 'HTTP_BAD_REQUEST': {}, 'HTTP_NOT_FOUND': {}, 'HTTP_SERVER_ERROR': {}, + 'MIGRATE_HEADING': {}, + 'MIGRATE_LABEL': {}, }, DARK_PALETTE: { 'ERROR': { 'fg': 'red', 'opts': ('bold',) }, @@ -101,6 +103,8 @@ PALETTES = { 'HTTP_BAD_REQUEST': { 'fg': 'red', 'opts': ('bold',) }, 'HTTP_NOT_FOUND': { 'fg': 'yellow' }, 'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) }, + 'MIGRATE_HEADING': { 'fg': 'cyan', 'opts': ('bold',) }, + 'MIGRATE_LABEL': { 'opts': ('bold',) }, }, LIGHT_PALETTE: { 'ERROR': { 'fg': 'red', 'opts': ('bold',) }, @@ -116,6 +120,8 @@ PALETTES = { 'HTTP_BAD_REQUEST': { 'fg': 'red', 'opts': ('bold',) }, 'HTTP_NOT_FOUND': { 'fg': 'red' }, 'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) }, + 'MIGRATE_HEADING': { 'fg': 'cyan', 'opts': ('bold',) }, + 'MIGRATE_LABEL': { 'opts': ('bold',) }, } } DEFAULT_PALETTE = DARK_PALETTE