From 246580573d7f1226aca39615dbf39956c520e6d7 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 14 Oct 2011 21:49:43 +0000 Subject: [PATCH] Fixed #12308 -- Added tablespace support to the PostgreSQL backend. git-svn-id: http://code.djangoproject.com/svn/django/trunk@16987 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/db/backends/__init__.py | 12 ++- django/db/backends/creation.py | 16 ++-- django/db/backends/oracle/base.py | 7 +- .../db/backends/postgresql_psycopg2/base.py | 2 +- .../backends/postgresql_psycopg2/creation.py | 8 +- .../postgresql_psycopg2/operations.py | 6 ++ django/db/models/fields/related.py | 5 +- docs/howto/custom-model-fields.txt | 6 +- docs/ref/databases.txt | 43 --------- docs/ref/models/fields.txt | 9 +- docs/ref/models/options.txt | 6 +- docs/ref/settings.txt | 4 +- docs/releases/1.4.txt | 2 + docs/topics/db/index.txt | 1 + docs/topics/db/tablespaces.txt | 73 +++++++++++++++ tests/modeltests/tablespaces/__init__.py | 0 tests/modeltests/tablespaces/models.py | 32 +++++++ tests/modeltests/tablespaces/tests.py | 92 +++++++++++++++++++ 18 files changed, 251 insertions(+), 73 deletions(-) create mode 100644 docs/topics/db/tablespaces.txt create mode 100644 tests/modeltests/tablespaces/__init__.py create mode 100644 tests/modeltests/tablespaces/models.py create mode 100644 tests/modeltests/tablespaces/tests.py diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index 5e96fefd05..d6bd380886 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -365,6 +365,10 @@ class BaseDatabaseFeatures(object): # date_interval_sql can properly handle mixed Date/DateTime fields and timedeltas supports_mixed_date_datetime_comparisons = True + # Does the backend support tablespaces? Default to False because it isn't + # in the SQL standard. + supports_tablespaces = False + # Features that need to be confirmed at runtime # Cache whether the confirmation has been performed. _confirmed = False @@ -696,8 +700,12 @@ class BaseDatabaseOperations(object): def tablespace_sql(self, tablespace, inline=False): """ - Returns the SQL that will be appended to tables or rows to define - a tablespace. Returns '' if the backend doesn't use tablespaces. + Returns the SQL that will be used in a query to define the tablespace. + + Returns '' if the backend doesn't support tablespaces. + + If inline is True, the SQL is appended to a row; otherwise it's appended + to the entire CREATE TABLE or CREATE INDEX statement. """ return '' diff --git a/django/db/backends/creation.py b/django/db/backends/creation.py index fa0feffeac..f03df5d380 100644 --- a/django/db/backends/creation.py +++ b/django/db/backends/creation.py @@ -57,7 +57,9 @@ class BaseDatabaseCreation(object): if tablespace and f.unique: # We must specify the index tablespace inline, because we # won't be generating a CREATE INDEX statement for this field. - field_output.append(self.connection.ops.tablespace_sql(tablespace, inline=True)) + tablespace_sql = self.connection.ops.tablespace_sql(tablespace, inline=True) + if tablespace_sql: + field_output.append(tablespace_sql) if f.rel: ref_output, pending = self.sql_for_inline_foreign_key_references(f, known_models, style) if pending: @@ -74,7 +76,9 @@ class BaseDatabaseCreation(object): full_statement.append(' %s%s' % (line, i < len(table_output)-1 and ',' or '')) full_statement.append(')') if opts.db_tablespace: - full_statement.append(self.connection.ops.tablespace_sql(opts.db_tablespace)) + tablespace_sql = self.connection.ops.tablespace_sql(opts.db_tablespace) + if tablespace_sql: + full_statement.append(tablespace_sql) full_statement.append(';') final_output.append('\n'.join(full_statement)) @@ -149,11 +153,9 @@ class BaseDatabaseCreation(object): qn = self.connection.ops.quote_name tablespace = f.db_tablespace or model._meta.db_tablespace if tablespace: - sql = self.connection.ops.tablespace_sql(tablespace) - if sql: - tablespace_sql = ' ' + sql - else: - tablespace_sql = '' + tablespace_sql = self.connection.ops.tablespace_sql(tablespace) + if tablespace_sql: + tablespace_sql = ' ' + tablespace_sql else: tablespace_sql = '' i_name = '%s_%s' % (model._meta.db_table, self._digest(f.column)) diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index bd45f9bd3d..c9830f5a7c 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -79,6 +79,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): can_defer_constraint_checks = True ignores_nulls_in_unique_constraints = False has_bulk_insert = True + supports_tablespaces = True class DatabaseOperations(BaseDatabaseOperations): compiler_module = "django.db.backends.oracle.compiler" @@ -326,8 +327,10 @@ WHEN (new.%(col_name)s IS NULL) return '' def tablespace_sql(self, tablespace, inline=False): - return "%sTABLESPACE %s" % ((inline and "USING INDEX " or ""), - self.quote_name(tablespace)) + if inline: + return "USING INDEX TABLESPACE %s" % self.quote_name(tablespace) + else: + return "TABLESPACE %s" % self.quote_name(tablespace) def value_to_db_datetime(self, value): # Oracle doesn't support tz-aware datetimes diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py index f0a89e50a8..74342e9ac6 100644 --- a/django/db/backends/postgresql_psycopg2/base.py +++ b/django/db/backends/postgresql_psycopg2/base.py @@ -75,7 +75,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): has_select_for_update = True has_select_for_update_nowait = True has_bulk_insert = True - + supports_tablespaces = True class DatabaseWrapper(BaseDatabaseWrapper): vendor = 'postgresql' diff --git a/django/db/backends/postgresql_psycopg2/creation.py b/django/db/backends/postgresql_psycopg2/creation.py index 1f5609a158..e4e7428ea7 100644 --- a/django/db/backends/postgresql_psycopg2/creation.py +++ b/django/db/backends/postgresql_psycopg2/creation.py @@ -44,11 +44,9 @@ class DatabaseCreation(BaseDatabaseCreation): db_table = model._meta.db_table tablespace = f.db_tablespace or model._meta.db_tablespace if tablespace: - sql = self.connection.ops.tablespace_sql(tablespace) - if sql: - tablespace_sql = ' ' + sql - else: - tablespace_sql = '' + tablespace_sql = self.connection.ops.tablespace_sql(tablespace) + if tablespace_sql: + tablespace_sql = ' ' + tablespace_sql else: tablespace_sql = '' diff --git a/django/db/backends/postgresql_psycopg2/operations.py b/django/db/backends/postgresql_psycopg2/operations.py index c3a23c2073..acfeeaf7ac 100644 --- a/django/db/backends/postgresql_psycopg2/operations.py +++ b/django/db/backends/postgresql_psycopg2/operations.py @@ -99,6 +99,12 @@ class DatabaseOperations(BaseDatabaseOperations): else: return [] + def tablespace_sql(self, tablespace, inline=False): + if inline: + return "USING INDEX TABLESPACE %s" % self.quote_name(tablespace) + else: + return "TABLESPACE %s" % self.quote_name(tablespace) + def sequence_reset_sql(self, style, model_list): from django.db import models output = [] diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 8c054e7303..848fd6ed74 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -1078,6 +1078,7 @@ def create_many_to_many_intermediary_model(field, klass): 'managed': managed, 'auto_created': klass, 'app_label': klass._meta.app_label, + 'db_tablespace': klass._meta.db_tablespace, 'unique_together': (from_, to), 'verbose_name': '%(from)s-%(to)s relationship' % {'from': from_, 'to': to}, 'verbose_name_plural': '%(from)s-%(to)s relationships' % {'from': from_, 'to': to}, @@ -1086,8 +1087,8 @@ def create_many_to_many_intermediary_model(field, klass): return type(name, (models.Model,), { 'Meta': meta, '__module__': klass.__module__, - from_: models.ForeignKey(klass, related_name='%s+' % name), - to: models.ForeignKey(to_model, related_name='%s+' % name) + from_: models.ForeignKey(klass, related_name='%s+' % name, db_tablespace=field.db_tablespace), + to: models.ForeignKey(to_model, related_name='%s+' % name, db_tablespace=field.db_tablespace) }) class ManyToManyField(RelatedField, Field): diff --git a/docs/howto/custom-model-fields.txt b/docs/howto/custom-model-fields.txt index 1cf4ddb729..7f0ea3799d 100644 --- a/docs/howto/custom-model-fields.txt +++ b/docs/howto/custom-model-fields.txt @@ -219,9 +219,9 @@ parameters: * :attr:`~django.db.models.Field.choices` * :attr:`~django.db.models.Field.help_text` * :attr:`~django.db.models.Field.db_column` -* :attr:`~django.db.models.Field.db_tablespace`: Currently only used with - the Oracle backend and only for index creation. You can usually ignore - this option. +* :attr:`~django.db.models.Field.db_tablespace`: Only for index creation, if the + backend supports :doc:`tablespaces `. You can usually + ignore this option. * :attr:`~django.db.models.Field.auto_created`: True if the field was automatically created, as for the `OneToOneField` used by model inheritance. For advanced use only. diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index 9a83e059e5..9ea0fe72cc 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -646,49 +646,6 @@ The ``RETURNING INTO`` clause can be disabled by setting the In this case, the Oracle backend will use a separate ``SELECT`` query to retrieve AutoField values. -Tablespace options ------------------- - -A common paradigm for optimizing performance in Oracle-based systems is the -use of `tablespaces`_ to organize disk layout. The Oracle backend supports -this use case by adding ``db_tablespace`` options to the ``Meta`` and -``Field`` classes. (When you use a backend that lacks support for tablespaces, -Django ignores these options.) - -.. _`tablespaces`: http://en.wikipedia.org/wiki/Tablespace - -A tablespace can be specified for the table(s) generated by a model by -supplying the ``db_tablespace`` option inside the model's ``class Meta``. -Additionally, you can pass the ``db_tablespace`` option to a ``Field`` -constructor to specify an alternate tablespace for the ``Field``'s column -index. If no index would be created for the column, the ``db_tablespace`` -option is ignored:: - - class TablespaceExample(models.Model): - name = models.CharField(max_length=30, db_index=True, db_tablespace="indexes") - data = models.CharField(max_length=255, db_index=True) - edges = models.ManyToManyField(to="self", db_tablespace="indexes") - - class Meta: - db_tablespace = "tables" - -In this example, the tables generated by the ``TablespaceExample`` model -(i.e., the model table and the many-to-many table) would be stored in the -``tables`` tablespace. The index for the name field and the indexes on the -many-to-many table would be stored in the ``indexes`` tablespace. The ``data`` -field would also generate an index, but no tablespace for it is specified, so -it would be stored in the model tablespace ``tables`` by default. - -Use the :setting:`DEFAULT_TABLESPACE` and :setting:`DEFAULT_INDEX_TABLESPACE` -settings to specify default values for the db_tablespace options. -These are useful for setting a tablespace for the built-in Django apps and -other applications whose code you cannot control. - -Django does not create the tablespaces for you. Please refer to `Oracle's -documentation`_ for details on creating and managing tablespaces. - -.. _`Oracle's documentation`: http://download.oracle.com/docs/cd/B19306_01/server.102/b14200/statements_7003.htm#SQLRF01403 - Naming issues ------------- diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index 3ae35f0dc2..f3a4ef4f77 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -178,10 +178,11 @@ If ``True``, djadmin:`django-admin.py sqlindexes ` will output a .. attribute:: Field.db_tablespace -The name of the database tablespace to use for this field's index, if this field -is indexed. The default is the project's :setting:`DEFAULT_INDEX_TABLESPACE` -setting, if set, or the :attr:`~Field.db_tablespace` of the model, if any. If -the backend doesn't support tablespaces, this option is ignored. +The name of the :doc:`database tablespace ` to use for +this field's index, if this field is indexed. The default is the project's +:setting:`DEFAULT_INDEX_TABLESPACE` setting, if set, or the +:attr:`~Options.db_tablespace` of the model, if any. If the backend doesn't +support tablespaces for indexes, this option is ignored. ``default`` ----------- diff --git a/docs/ref/models/options.txt b/docs/ref/models/options.txt index 14446b69b4..4ae105af92 100644 --- a/docs/ref/models/options.txt +++ b/docs/ref/models/options.txt @@ -73,8 +73,10 @@ Django quotes column and table names behind the scenes. .. attribute:: Options.db_tablespace - The name of the database tablespace to use for the model. If the backend - doesn't support tablespaces, this option is ignored. + The name of the :doc:`database tablespace ` to use + for this model. The default is the project's :setting:`DEFAULT_TABLESPACE` + setting, if set. If the backend doesn't support tablespaces, this option is + ignored. ``get_latest_by`` ----------------- diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 0669750565..4ff5d4d068 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -864,7 +864,7 @@ DEFAULT_INDEX_TABLESPACE Default: ``''`` (Empty string) Default tablespace to use for indexes on fields that don't specify -one, if the backend supports it. +one, if the backend supports it (see :doc:`/topics/db/tablespaces`). .. setting:: DEFAULT_TABLESPACE @@ -874,7 +874,7 @@ DEFAULT_TABLESPACE Default: ``''`` (Empty string) Default tablespace to use for models that don't specify one, if the -backend supports it. +backend supports it (see :doc:`/topics/db/tablespaces`). .. setting:: DISALLOWED_USER_AGENTS diff --git a/docs/releases/1.4.txt b/docs/releases/1.4.txt index a035f44567..3e364fbe23 100644 --- a/docs/releases/1.4.txt +++ b/docs/releases/1.4.txt @@ -405,6 +405,8 @@ Django 1.4 also includes several smaller improvements worth noting: code are slightly emphasized. This change makes it easier to scan a stacktrace for issues in user code. +* :doc:`Tablespace support ` in PostgreSQL. + * Customizable names for :meth:`~django.template.Library.simple_tag`. * In the documentation, a helpful :doc:`security overview ` diff --git a/docs/topics/db/index.txt b/docs/topics/db/index.txt index 24a6202564..0e0fc8adb6 100644 --- a/docs/topics/db/index.txt +++ b/docs/topics/db/index.txt @@ -17,4 +17,5 @@ model maps to a single database table. sql transactions multi-db + tablespaces optimization diff --git a/docs/topics/db/tablespaces.txt b/docs/topics/db/tablespaces.txt new file mode 100644 index 0000000000..7fcd5588e7 --- /dev/null +++ b/docs/topics/db/tablespaces.txt @@ -0,0 +1,73 @@ +=========== +Tablespaces +=========== + +A common paradigm for optimizing performance in database systems is the use of +`tablespaces`_ to organize disk layout. + +.. _`tablespaces`: http://en.wikipedia.org/wiki/Tablespace + +.. warning:: + Django does not create the tablespaces for you. Please refer to your + database engine's documentation for details on creating and managing + tablespaces. + + +Declaring tablespaces for tables +-------------------------------- + +A tablespace can be specified for the table generated by a model by supplying +the :attr:`~django.db.models.Options.db_tablespace` option inside the model's +``class Meta``. This option also affects tables automatically created for +:class:`~django.db.models.ManyToManyField`\ s in the model. + +You can use the :setting:`DEFAULT_TABLESPACE` setting to specify a default value +for :attr:`~django.db.models.Options.db_tablespace`. This is useful for setting +a tablespace for the built-in Django apps and other applications whose code you +cannot control. + +Declaring tablespaces for indexes +--------------------------------- + +You can pass the :attr:`~django.db.models.Field.db_tablespace` option to a +``Field`` constructor to specify an alternate tablespace for the ``Field``'s +column index. If no index would be created for the column, the option is +ignored. + +You can use the :setting:`DEFAULT_INDEX_TABLESPACE` setting to specify +a default value for :attr:`~django.db.models.Field.db_tablespace`. + +If :attr:`~django.db.models.Field.db_tablespace` isn't specified and you didn't +set :setting:`DEFAULT_INDEX_TABLESPACE`, the index is created in the same +tablespace as the tables. + +An example +---------- + +.. code-block:: python + + class TablespaceExample(models.Model): + name = models.CharField(max_length=30, db_index=True, db_tablespace="indexes") + data = models.CharField(max_length=255, db_index=True) + edges = models.ManyToManyField(to="self", db_tablespace="indexes") + + class Meta: + db_tablespace = "tables" + +In this example, the tables generated by the ``TablespaceExample`` model (i.e. +the model table and the many-to-many table) would be stored in the ``tables`` +tablespace. The index for the name field and the indexes on the many-to-many +table would be stored in the ``indexes`` tablespace. The ``data`` field would +also generate an index, but no tablespace for it is specified, so it would be +stored in the model tablespace ``tables`` by default. + +Database support +---------------- + +PostgreSQL and Oracle support tablespaces. SQLite and MySQL don't. + +When you use a backend that lacks support for tablespaces, Django ignores all +tablespace-related options. + +.. versionchanged:: 1.4 + Since Django 1.4, the PostgreSQL backend supports tablespaces. diff --git a/tests/modeltests/tablespaces/__init__.py b/tests/modeltests/tablespaces/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/modeltests/tablespaces/models.py b/tests/modeltests/tablespaces/models.py new file mode 100644 index 0000000000..afc376c931 --- /dev/null +++ b/tests/modeltests/tablespaces/models.py @@ -0,0 +1,32 @@ +from django.db import models + +# Since the test database doesn't have tablespaces, it's impossible for Django +# to create the tables for models where db_tablespace is set. To avoid this +# problem, we mark the models as unmanaged, and temporarily revert them to +# managed during each tes. See setUp and tearDown -- it isn't possible to use +# setUpClass and tearDownClass because they're called before Django flushes the +# tables, so Django attempts to flush a non-existing table. + +class ScientistRef(models.Model): + name = models.CharField(max_length=50) + +class ArticleRef(models.Model): + title = models.CharField(max_length=50, unique=True) + code = models.CharField(max_length=50, unique=True) + authors = models.ManyToManyField(ScientistRef, related_name='articles_written_set') + reviewers = models.ManyToManyField(ScientistRef, related_name='articles_reviewed_set') + +class Scientist(models.Model): + name = models.CharField(max_length=50) + class Meta: + db_tablespace = 'tbl_tbsp' + managed = False + +class Article(models.Model): + title = models.CharField(max_length=50, unique=True) + code = models.CharField(max_length=50, unique=True, db_tablespace='idx_tbsp') + authors = models.ManyToManyField(Scientist, related_name='articles_written_set') + reviewers = models.ManyToManyField(Scientist, related_name='articles_reviewed_set', db_tablespace='idx_tbsp') + class Meta: + db_tablespace = 'tbl_tbsp' + managed = False diff --git a/tests/modeltests/tablespaces/tests.py b/tests/modeltests/tablespaces/tests.py new file mode 100644 index 0000000000..763c8a7fdb --- /dev/null +++ b/tests/modeltests/tablespaces/tests.py @@ -0,0 +1,92 @@ +import copy + +from django.db import connection +from django.db import models +from django.db.models.loading import cache +from django.core.management.color import no_style +from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature + +from models import Article, ArticleRef, Scientist, ScientistRef + +# Automatically created models +Authors = Article._meta.get_field('authors').rel.through +Reviewers = Article._meta.get_field('reviewers').rel.through + +# We can't test the DEFAULT_TABLESPACE and DEFAULT_INDEX_TABLESPACE settings +# because they're evaluated when the model class is defined. As a consequence, +# @override_settings doesn't work. + +def sql_for_table(model): + return '\n'.join(connection.creation.sql_create_model(model, no_style())[0]) + +def sql_for_index(model): + return '\n'.join(connection.creation.sql_indexes_for_model(model, no_style())) + + +class TablespacesTests(TestCase): + + def setUp(self): + # The unmanaged models need to be removed after the test in order to + # prevent bad interactions with other tests (proxy_models_inheritance). + self.old_app_models = copy.deepcopy(cache.app_models) + self.old_app_store = copy.deepcopy(cache.app_store) + + for model in Article, Authors, Reviewers, Scientist: + model._meta.managed = True + + def tearDown(self): + for model in Article, Authors, Reviewers, Scientist: + model._meta.managed = False + + cache.app_models = self.old_app_models + cache.app_store = self.old_app_store + cache._get_models_cache = {} + + def assertNumContains(self, haystack, needle, count): + real_count = haystack.count(needle) + self.assertEqual(real_count, count, "Found %d instances of '%s', " + "expected %d" % (real_count, needle, count)) + + @skipUnlessDBFeature('supports_tablespaces') + def test_tablespace_for_model(self): + # 1 for the table + 1 for the index on the primary key + self.assertNumContains(sql_for_table(Scientist).lower(), 'tbl_tbsp', 2) + + @skipIfDBFeature('supports_tablespaces') + def test_tablespace_ignored_for_model(self): + # No tablespace-related SQL + self.assertEqual(sql_for_table(Scientist), + sql_for_table(ScientistRef).replace('ref', '')) + + @skipUnlessDBFeature('supports_tablespaces') + def test_tablespace_for_indexed_field(self): + # 1 for the table + 1 for the primary key + 1 for the index on name + self.assertNumContains(sql_for_table(Article).lower(), 'tbl_tbsp', 3) + # 1 for the index on reference + self.assertNumContains(sql_for_table(Article).lower(), 'idx_tbsp', 1) + + @skipIfDBFeature('supports_tablespaces') + def test_tablespace_ignored_for_indexed_field(self): + # No tablespace-related SQL + self.assertEqual(sql_for_table(Article), + sql_for_table(ArticleRef).replace('ref', '')) + + @skipUnlessDBFeature('supports_tablespaces') + def test_tablespace_for_many_to_many_field(self): + # The join table of the ManyToManyField always goes to the tablespace + # of the model. + self.assertNumContains(sql_for_table(Authors).lower(), 'tbl_tbsp', 2) + self.assertNumContains(sql_for_table(Authors).lower(), 'idx_tbsp', 0) + # The ManyToManyField declares no db_tablespace, indexes for the two + # foreign keys in the join table go to the tablespace of the model. + self.assertNumContains(sql_for_index(Authors).lower(), 'tbl_tbsp', 2) + self.assertNumContains(sql_for_index(Authors).lower(), 'idx_tbsp', 0) + + # The join table of the ManyToManyField always goes to the tablespace + # of the model. + self.assertNumContains(sql_for_table(Reviewers).lower(), 'tbl_tbsp', 2) + self.assertNumContains(sql_for_table(Reviewers).lower(), 'idx_tbsp', 0) + # The ManyToManyField declares db_tablespace, indexes for the two + # foreign keys in the join table go to this tablespace. + self.assertNumContains(sql_for_index(Reviewers).lower(), 'tbl_tbsp', 0) + self.assertNumContains(sql_for_index(Reviewers).lower(), 'idx_tbsp', 2)