boulder-oracle-sprint: Refactored all Oracle references outside django.db into the

backend.


git-svn-id: http://code.djangoproject.com/svn/django/branches/boulder-oracle-sprint@4082 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/core/management.py

@@ -6,7 +6,6 @@ from django.core.exceptions import ImproperlyConfigured
 import os, re, shutil, sys, textwrap
 from optparse import OptionParser
 from django.utils import termcolors
-from django.conf import settings
 
 # For Python 2.3
 if not hasattr(__builtins__, 'set'):
@@ -56,7 +55,7 @@ def _is_valid_dir_name(s):
 
 def _get_installed_models(table_list):
     "Gets a set of all models that are installed, given a list of existing tables"
-    from django.db import models, backend
+    from django.db import backend, models
     all_models = []
     for app in models.get_apps():
         for model in models.get_models(app):
@@ -89,7 +88,8 @@ def get_version():
 
 def get_sql_create(app):
     "Returns a list of the CREATE TABLE SQL statements for the given app."
-    from django.db import models, get_creation_module, backend
+    from django.db import get_creation_module, models
+
     data_types = get_creation_module().DATA_TYPES
 
     if not data_types:
@@ -145,7 +145,6 @@ def _get_sql_model_create(model, known_models=set()):
     Returns list_of_sql, pending_references_dict
     """
     from django.db import backend, get_creation_module, models
-    from django.db.backends.util import truncate_name
     data_types = get_creation_module().DATA_TYPES
 
     opts = model._meta
@@ -165,13 +164,10 @@ def _get_sql_model_create(model, known_models=set()):
             field_output = [style.SQL_FIELD(backend.quote_name(f.column)),
                 style.SQL_COLTYPE(col_type % rel_field.__dict__)]
             field_output.append(style.SQL_KEYWORD('%sNULL' % (not f.null and 'NOT ' or '')))
-            if f.unique:
+            if f.unique and (not f.primary_key or backend.allows_unique_and_pk):
                 field_output.append(style.SQL_KEYWORD('UNIQUE'))
             if f.primary_key:
                 field_output.append(style.SQL_KEYWORD('PRIMARY KEY'))
-            if settings.DATABASE_ENGINE == 'oracle' and f.unique and f.primary_key:
-                # Suppress UNIQUE/PRIMARY KEY for Oracle (ORA-02259)
-                field_output.remove(style.SQL_KEYWORD('UNIQUE'))
             if f.rel:
                 if f.rel.to in known_models:
                     field_output.append(style.SQL_KEYWORD('REFERENCES') + ' ' + \
@@ -197,21 +193,12 @@ def _get_sql_model_create(model, known_models=set()):
     full_statement.append(');')
     final_output.append('\n'.join(full_statement))
 
-    # To simulate auto-incrementing primary keys in Oracle -- creating primary tables
+    if opts.has_auto_field:
-    if settings.DATABASE_ENGINE == 'oracle' and opts.has_auto_field:
+        # Add any extra SQL needed to support auto-incrementing primary keys
-        name_length = backend.get_max_name_length() - 3
+        autoinc_sql = backend.get_autoinc_sql(opts.db_table)
-        sequence_name = '%s_sq' % truncate_name(opts.db_table, name_length)
+        if autoinc_sql:
-        sequence_statement = 'CREATE SEQUENCE %s;' % sequence_name
+            for stmt in autoinc_sql:
-        final_output.append(sequence_statement)
+                final_output.append(stmt)
-        trigger_statement = '' + \
-            'CREATE OR REPLACE TRIGGER %s\n'    % ('%s_tr' % truncate_name(opts.db_table, name_length)) + \
-            '  BEFORE INSERT ON %s\n'           % backend.quote_name(opts.db_table) + \
-            '  FOR EACH ROW\n'  + \
-            '  WHEN (new.id IS NULL)\n' + \
-            '    BEGIN\n' + \
-            '      SELECT %s.nextval INTO :new.id FROM dual;\n' % sequence_name + \
-            '    END;\n'
-        final_output.append(trigger_statement)
 
     return final_output, pending_references
 
@@ -243,7 +230,6 @@ def _get_sql_for_pending_references(model, pending_references):
 def _get_many_to_many_sql_for_model(model):
     from django.db import backend, get_creation_module
     from django.db.models import GenericRel
-    from django.db.backends.util import truncate_name
 
     data_types = get_creation_module().DATA_TYPES
 
@@ -276,22 +262,12 @@ def _get_many_to_many_sql_for_model(model):
             table_output.append(');')
             final_output.append('\n'.join(table_output))
 
-            # To simulate auto-incrementing primary keys in Oracle -- creating m2m tables
+            # Add any extra SQL needed to support auto-incrementing PKs
-            if settings.DATABASE_ENGINE == 'oracle':
+            autoinc_sql = backend.get_autoinc_sql(f.m2m_db_table())
-                name_length = backend.get_max_name_length() - 3
+            if autoinc_sql:
-                m_table = f.m2m_db_table()
+                for stmt in autoinc_sql:
-                sequence_name = '%s_sq' % truncate_name(m_table, name_length)
+                    final_output.append(stmt)
-                sequence_statement = 'CREATE SEQUENCE %s;' % sequence_name
+
-                final_output.append(sequence_statement)
-                trigger_statement = '' + \
-                    'CREATE OR REPLACE TRIGGER %s\n'    % ('%s_tr' % truncate_name(m_table, name_length)) + \
-                    '  BEFORE INSERT ON %s\n'           % backend.quote_name(m_table) + \
-                    '  FOR EACH ROW\n'  + \
-                    '  WHEN (new.id IS NULL)\n' + \
-                    '    BEGIN\n' + \
-                    '      SELECT %s.nextval INTO :new.id FROM dual;\n' % sequence_name + \
-                    '    END;\n'
-                final_output.append(trigger_statement)
     return final_output
 
 def get_sql_delete(app):
@@ -1450,8 +1426,7 @@ def execute_from_command_line(action_mapping=DEFAULT_ACTION_MAPPING, argv=None):
         if not mod_list:
             parser.print_usage_and_exit()
         if action not in NO_SQL_TRANSACTION:
-            if settings.DATABASE_ENGINE != 'oracle':
+            print style.SQL_KEYWORD("BEGIN;")
-                print style.SQL_KEYWORD("BEGIN;")
         for mod in mod_list:
             if action == 'reset':
                 output = action_mapping[action](mod, options.interactive)
@@ -1460,8 +1435,7 @@ def execute_from_command_line(action_mapping=DEFAULT_ACTION_MAPPING, argv=None):
             if output:
                 print '\n'.join(output)
         if action not in NO_SQL_TRANSACTION:
-            if settings.DATABASE_ENGINE != 'oracle':
+            print style.SQL_KEYWORD("COMMIT;")
-                print style.SQL_KEYWORD("COMMIT;")
 
 def setup_environ(settings_mod):
     """

django/db/backends/ado_mssql/base.py

@@ -87,6 +87,9 @@ class DatabaseWrapper(local):
             self.connection.close()
             self.connection = None
 
+allows_group_by_ordinal = True
+allows_unique_and_pk = True
+returns_dates_as_strings = False
 supports_constraints = True
 uses_case_insensitive_names = False
 
@@ -116,6 +119,9 @@ def get_date_trunc_sql(lookup_type, field_name):
     if lookup_type=='day':
         return "Convert(datetime, Convert(varchar(12), %s))" % field_name
 
+def get_datetime_cast_sql():
+    return None
+
 def get_limit_offset_sql(limit, offset=None):
     # TODO: This is a guess. Make sure this is correct.
     sql = "LIMIT %s" % limit
@@ -138,6 +144,9 @@ def get_pk_default_value():
 def get_max_name_length():
     return None
 
+def get_autoinc_sql(table):
+    return None
+
 OPERATOR_MAPPING = {
     'exact': '= %s',
     'iexact': 'LIKE %s',

django/db/backends/mysql/base.py

@@ -130,6 +130,9 @@ class DatabaseWrapper(local):
             self.server_version = tuple([int(x) for x in m.groups()])
         return self.server_version
 
+allows_group_by_ordinal = True
+allows_unique_and_pk = True
+returns_dates_as_strings = True     # MySQLdb requires a typecast for dates
 supports_constraints = True
 uses_case_insensitive_names = False
 
@@ -164,6 +167,9 @@ def get_date_trunc_sql(lookup_type, field_name):
         sql = "CAST(DATE_FORMAT(%s, '%s') AS DATETIME)" % (field_name, format_str)
     return sql
 
+def get_datetime_cast_sql():
+    return None
+
 def get_limit_offset_sql(limit, offset=None):
     sql = "LIMIT "
     if offset and offset != 0:
@@ -185,6 +191,9 @@ def get_pk_default_value():
 def get_max_name_length():
     return 64;
 
+def get_autoinc_sql(table):
+    return None
+
 OPERATOR_MAPPING = {
     'exact': '= %s',
     'iexact': 'LIKE %s',

django/db/backends/oracle/base.py

@@ -59,6 +59,9 @@ class DatabaseWrapper(local):
             self.connection.close()
             self.connection = None
 
+allows_group_by_ordinal = False
+allows_unique_and_pk = False        # Suppress UNIQUE/PK for Oracle (ORA-02259)
+returns_dates_as_strings = False
 supports_constraints = True
 uses_case_insensitive_names = True
 
@@ -118,6 +121,9 @@ def get_date_trunc_sql(lookup_type, field_name):
         sql = "TRUNC(%s, '%s')" % (field_name, lookup_type)
     return sql
 
+def get_datetime_cast_sql():
+    return "TO_TIMESTAMP(%s, 'YYYY-MM-DD HH24:MI:SS')"
+
 def get_limit_offset_sql(limit, offset=None):
     # Limits and offset are too complicated to be handled here.
     # Instead, they are handled in django/db/backends/oracle/query.py.
@@ -138,11 +144,27 @@ def get_pk_default_value():
 def get_max_name_length():
     return 30
 
+def get_autoinc_sql(table):
+    # To simulate auto-incrementing primary keys in Oracle, we have to
+    # create a sequence and a trigger.
+    name_length = get_max_name_length() - 3
+    sq_name = '%s_sq' % util.truncate_name(table, name_length)
+    tr_name = '%s_tr' % util.truncate_name(table, name_length)
+    sequence_sql = 'CREATE SEQUENCE %s;' % sq_name
+    trigger_sql = """CREATE OR REPLACE TRIGGER %s
+  BEFORE INSERT ON %s
+  FOR EACH ROW
+  WHEN (new.id IS NULL)
+    BEGIN
+      SELECT %s.nextval INTO :new.id FROM dual;
+    END;\n""" % (tr_name, quote_name(table), sq_name)
+    return sequence_sql, trigger_sql
+
 OPERATOR_MAPPING = {
     'exact': '= %s',
     'iexact': "LIKE %s ESCAPE '\\'",
     'contains': "LIKE %s ESCAPE '\\'",
-    'icontains': "LIKE %s ESCAPE '\\'",
+    'icontains': "LIKE LOWER(%s) ESCAPE '\\'",
     'gt': '> %s',
     'gte': '>= %s',
     'lt': '< %s',

django/db/backends/postgresql/base.py

@@ -61,6 +61,9 @@ class DatabaseWrapper(local):
             self.connection.close()
             self.connection = None
 
+allows_group_by_ordinal = True
+allows_unique_and_pk = True
+returns_dates_as_strings = False
 supports_constraints = True
 uses_case_insensitive_names = False
 
@@ -95,6 +98,9 @@ def get_date_trunc_sql(lookup_type, field_name):
     # http://www.postgresql.org/docs/8.0/static/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC
     return "DATE_TRUNC('%s', %s)" % (lookup_type, field_name)
 
+def get_datetime_cast_sql():
+    return None
+
 def get_limit_offset_sql(limit, offset=None):
     sql = "LIMIT %s" % limit
     if offset and offset != 0:
@@ -116,6 +122,9 @@ def get_pk_default_value():
 def get_max_name_length():
     return None
 
+def get_autoinc_sql(table):
+    return None
+
 # Register these custom typecasts, because Django expects dates/times to be
 # in Python's native (standard-library) datetime/time format, whereas psycopg
 # use mx.DateTime by default.

django/db/backends/postgresql_psycopg2/base.py

@@ -62,6 +62,9 @@ class DatabaseWrapper(local):
             self.connection.close()
             self.connection = None
 
+allows_group_by_ordinal = True
+allows_unique_and_pk = True
+returns_dates_as_strings = False
 supports_constraints = True
 uses_case_insensitive_names = True
 
@@ -88,6 +91,9 @@ def get_date_trunc_sql(lookup_type, field_name):
     # http://www.postgresql.org/docs/8.0/static/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC
     return "DATE_TRUNC('%s', %s)" % (lookup_type, field_name)
 
+def get_datetime_cast_sql():
+    return None
+
 def get_limit_offset_sql(limit, offset=None):
     sql = "LIMIT %s" % limit
     if offset and offset != 0:
@@ -109,6 +115,9 @@ def get_pk_default_value():
 def get_max_name_length():
     return None
 
+def get_autoinc_sql(table):
+    return None
+
 OPERATOR_MAPPING = {
     'exact': '= %s',
     'iexact': 'ILIKE %s',

django/db/backends/sqlite3/base.py

@@ -98,6 +98,9 @@ class SQLiteCursorWrapper(Database.Cursor):
     def convert_query(self, query, num_params):
         return query % tuple("?" * num_params)
 
+allows_group_by_ordinal = True
+allows_unique_and_pk = True
+returns_dates_as_strings = True
 supports_constraints = False
 uses_case_insensitive_names = False
 
@@ -131,6 +134,9 @@ def get_date_trunc_sql(lookup_type, field_name):
     # sqlite doesn't support DATE_TRUNC, so we fake it as above.
     return 'django_date_trunc("%s", %s)' % (lookup_type.lower(), field_name)
 
+def get_datetime_cast_sql():
+    return None
+
 def get_limit_offset_sql(limit, offset=None):
     sql = "LIMIT %s" % limit
     if offset and offset != 0:
@@ -152,6 +158,9 @@ def get_pk_default_value():
 def get_max_name_length():
     return None
 
+def get_autoinc_sql(table):
+    return None
+
 def _sqlite_date_trunc(lookup_type, dt):
     try:
         dt = util.typecast_timestamp(dt)

django/db/models/query.py

@@ -1,5 +1,4 @@
-from django import db
+from django.db import backend, connection, get_query_module, transaction
-from django.db import backend, connection, transaction
 from django.db.models.fields import DateField, FieldDoesNotExist
 from django.db.models import signals
 from django.dispatch import dispatcher
@@ -509,13 +508,12 @@ class _QuerySet(object):
         else:
             assert self._offset is None, "'offset' is not allowed without 'limit'"
 
-        return select, " ".join(sql), params
+        return select, " ".join(sql), params, None
 
-# Check to see if the DB backend would like to define its own QuerySet class
+# Use the backend's QuerySet class if it defines one, otherwise use _QuerySet.
-# and otherwise use the default.
+backend_query_module = get_query_module()
-backend_query_module = db.get_query_module()
+if hasattr(backend_query_module, 'get_query_set_class'):
-if hasattr(backend_query_module, "get_query_set_class"):
+    QuerySet = backend_query_module.get_query_set_class(_QuerySet)
-    QuerySet = db.get_query_module().get_query_set_class(_QuerySet)
 else:
     QuerySet = _QuerySet
 
@@ -561,21 +559,18 @@ class DateQuerySet(QuerySet):
         field_name = backend.quote_name(self._field.column)
         date_trunc_sql = backend.get_date_trunc_sql(self._kind,
                                                     '%s.%s' % (table_name, field_name))
-        fmt = 'SELECT %s %s GROUP BY %s ORDER BY 1 %s'
+        if backend.allows_group_by_ordinal:
-
+            group_by = '1'
-        if settings.DATABASE_ENGINE == 'oracle':
-            sql = fmt % (date_trunc_sql, sql, date_trunc_sql, self._order)
-            cursor = connection.cursor()
-            cursor.execute(sql, params)
-            return [row[0] for row in cursor.fetchall()]
         else:
-            sql = fmt % (date_trunc_sql, sql, 1, self._order_by)
+            group_by = date_trunc_sql
-            cursor = connection.cursor()
+        fmt = 'SELECT %s %s GROUP BY %s ORDER BY 1 %s'
-            cursor.execute(sql, params)
+        stmt = fmt % (date_trunc_sql, sql, group_by, self._order)
-            # We have to manually run typecast_timestamp(str()) on the results, because
+        cursor = connection.cursor()
-            # MySQL doesn't automatically cast the result of date functions as datetime
+        cursor.execute(stmt, params)
-            # objects -- MySQL returns the values as strings, instead.
+        if backend.returns_dates_as_strings:
             return [typecast_timestamp(str(row[0])) for row in cursor.fetchall()]
+        else:
+            return [row[0] for row in cursor.fetchall()]
 
     def _clone(self, klass=None, **kwargs):
         c = super(DateQuerySet, self)._clone(klass, **kwargs)
@@ -657,15 +652,13 @@ def get_where_clause(lookup_type, table_prefix, field_name, value):
     if table_prefix.endswith('.'):
         table_prefix = backend.quote_name(table_prefix[:-1])+'.'
     field_name = backend.quote_name(field_name)
-    # TODO: move this into django.db.backends.oracle somehow
+    if type(value) == datetime.datetime and backend.get_datetime_cast_sql():
-    if settings.DATABASE_ENGINE == 'oracle':
+        cast_sql = backend.get_datetime_cast_sql()
-        if lookup_type == 'icontains':
+    else:
-            return 'lower(%s%s) %s' % (table_prefix, field_name, (backend.OPERATOR_MAPPING[lookup_type] % '%s'))
+        cast_sql = '%s'
-        elif type(value) == datetime.datetime:
-            return "%s%s %s" % (table_prefix, field_name,
-                 (backend.OPERATOR_MAPPING[lookup_type] % "TO_TIMESTAMP(%s, 'YYYY-MM-DD HH24:MI:SS')"))
     try:
-        return '%s%s %s' % (table_prefix, field_name, (backend.OPERATOR_MAPPING[lookup_type] % '%s'))
+        return '%s%s %s' % (table_prefix, field_name,
+                            backend.OPERATOR_MAPPING[lookup_type] % cast_sql)
     except KeyError:
         pass
     if lookup_type == 'in':