gis: Added preliminary spatial backend for MySQL (which only supports MBR queries), and added limited test suite for it; updated a few comments in the Oracle backend.

git-svn-id: http://code.djangoproject.com/svn/django/branches/gis@6527 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/gis/db/backend/__init__.py

@@ -43,6 +43,13 @@ elif settings.DATABASE_ENGINE == 'oracle':
          create_spatial_db, get_geo_where_clause, gqn, \
          ASGML, GEOM_SELECT, TRANSFORM, UNION
     SPATIAL_BACKEND = 'oracle'
+elif settings.DATABASE_ENGINE == 'mysql':
+    from django.contrib.gis.db.backend.mysql import \
+        MySQLGeoField as GeoBackendField, \
+        MYSQL_GIS_TERMS as GIS_TERMS, \
+        create_spatial_db, get_geo_where_clause, gqn, \
+        GEOM_SELECT
+    SPATIAL_BACKEND = 'mysql'
 else:
     raise NotImplementedError('No Geographic Backend exists for %s' % settings.DATABASE_ENGINE)
 

django/contrib/gis/db/backend/mysql/__init__.py

@@ -0,0 +1,12 @@
+"""
+ The MySQL spatial database backend module.
+
+ Please note that MySQL only supports bounding box queries, also
+ known as MBRs (Minimum Bounding Rectangles).  Moreover, spatial
+ indices may only be used on MyISAM tables -- if you need 
+ transactions, take a look at PostGIS.
+"""
+
+from django.contrib.gis.db.backend.mysql.creation import create_spatial_db
+from django.contrib.gis.db.backend.mysql.field import MySQLGeoField, gqn
+from django.contrib.gis.db.backend.mysql.query import get_geo_where_clause, MYSQL_GIS_TERMS, GEOM_SELECT

django/contrib/gis/db/backend/mysql/creation.py

@@ -0,0 +1,5 @@
+from django.test.utils import create_test_db
+
+def create_spatial_db(test=True, verbosity=1, autoclobber=False):
+    if not test: raise NotImplementedError('This uses `create_test_db` from test/utils.py')
+    create_test_db(verbosity, autoclobber)

django/contrib/gis/db/backend/mysql/field.py

@@ -0,0 +1,92 @@
+import re
+from types import StringType, UnicodeType
+from django.db import connection
+from django.db.models.fields import Field # Django base Field class
+from django.contrib.gis.geos import GEOSGeometry
+from django.contrib.gis.db.backend.util import GeoFieldSQL
+from django.contrib.gis.db.backend.mysql.query import MYSQL_GIS_TERMS, GEOM_FROM_TEXT
+
+# Quotename & geographic quotename, respectively.
+qn = connection.ops.quote_name
+def gqn(value):
+    if isinstance(value, UnicodeType): value = value.encode('ascii')
+    return "'%s'" % value
+
+class MySQLGeoField(Field):
+    """
+    The backend-specific geographic field for MySQL.
+    """
+
+    def _geom_index(self, style, db_table):
+        """
+        Creates a spatial index for the geometry column.  If MyISAM tables are
+        used an R-Tree index is created, otherwise a B-Tree index is created. 
+        Thus, for best spatial performance, you should use MyISAM tables
+        (which do not support transactions).  For more information, see Ch. 
+        17.6.1 of the MySQL 5.0 documentation.
+        """
+
+        # Getting the index name.
+        idx_name = '%s_%s_id' % (db_table, self.column)
+        
+        sql = style.SQL_KEYWORD('CREATE SPATIAL INDEX ') + \
+              style.SQL_TABLE(qn(idx_name)) + \
+              style.SQL_KEYWORD(' ON ') + \
+              style.SQL_TABLE(qn(db_table)) + '(' + \
+              style.SQL_FIELD(qn(self.column)) + ');'
+        return sql
+
+    def _post_create_sql(self, style, db_table):
+        """
+        Returns SQL that will be executed after the model has been
+        created.
+        """
+        # Getting the geometric index for this Geometry column.
+        if self._index:
+            return (self._geom_index(style, db_table),)
+        else:
+            return ()
+
+    def db_type(self):
+        "The OpenGIS name is returned for the MySQL database column type."
+        return self._geom
+        
+    def get_db_prep_lookup(self, lookup_type, value):
+        """
+        Returns field's value prepared for database lookup, accepts WKT and 
+        GEOS Geometries for the value.
+        """
+        if lookup_type in MYSQL_GIS_TERMS:
+            # special case for isnull lookup
+            if lookup_type == 'isnull': return GeoFieldSQL([], [])
+
+            # When the input is not a GEOS geometry, attempt to construct one
+            # from the given string input.
+            if isinstance(value, GEOSGeometry):
+                pass
+            elif isinstance(value, (StringType, UnicodeType)):
+                try:
+                    value = GEOSGeometry(value)
+                except GEOSException:
+                    raise TypeError("Could not create geometry from lookup value: %s" % str(value))
+            else:
+                raise TypeError('Cannot use parameter of %s type as lookup parameter.' % type(value))
+
+            return GeoFieldSQL(['%s(%%s)' % GEOM_FROM_TEXT], [value])
+
+        else:
+            raise TypeError("Field has invalid lookup: %s" % lookup_type)
+
+    def get_db_prep_save(self, value):
+        "Prepares the value for saving in the database."
+        if not bool(value): return None
+        if isinstance(value, GEOSGeometry):
+            return value
+        else:
+            raise TypeError('Geometry Proxy should only return GEOSGeometry objects.')
+
+    def get_placeholder(self, value):
+        """
+        Nothing special happens here because MySQL does not support transformations.
+        """
+        return '%s(%%s)' % GEOM_FROM_TEXT

django/contrib/gis/db/backend/mysql/query.py

@@ -0,0 +1,54 @@
+"""
+ This module contains the spatial lookup types, and the get_geo_where_clause()
+ routine for MySQL
+"""
+from django.db import connection
+qn = connection.ops.quote_name
+
+# WARNING: MySQL is NOT compliant w/the OpenGIS specification and
+# _every_ one of these lookup types is on the _bounding box_ only.
+MYSQL_GIS_FUNCTIONS = {
+    'bbcontains' : 'MBRContains', # For consistency w/PostGIS API
+    'contained' : 'MBRWithin',    # (ditto)
+    'contains' : 'MBRContains',
+    'disjoint' : 'MBRDisjoint',
+    'equals' : 'MBREqual',
+    'exact' : 'MBREqual',
+    'intersects' : 'MBRIntersects',
+    'overlaps' : 'MBROverlaps',
+    'same_as' : 'MBREqual',
+    'touches' : 'MBRTouches',
+    'within' : 'MBRWithin',
+    }
+
+# This lookup type does not require a mapping.
+MISC_TERMS = ['isnull']
+
+# Assacceptable lookup types for Oracle spatial.
+MYSQL_GIS_TERMS  = MYSQL_GIS_FUNCTIONS.keys()
+MYSQL_GIS_TERMS += MISC_TERMS
+MYSQL_GIS_TERMS = tuple(MYSQL_GIS_TERMS) # Making immutable
+
+def get_geo_where_clause(lookup_type, table_prefix, field_name, value):
+    "Returns the SQL WHERE clause for use in MySQL spatial SQL construction."
+    if table_prefix.endswith('.'):
+        table_prefix = qn(table_prefix[:-1])+'.'
+    field_name = qn(field_name)
+
+    # See if a MySQL Geometry function matches the lookup type next
+    lookup_info = MYSQL_GIS_FUNCTIONS.get(lookup_type, False)
+    if lookup_info:
+        return "%s(%s, %%s)" % (lookup_info, table_prefix + field_name)
+    
+    # Handling 'isnull' lookup type
+    # TODO: Is this needed because MySQL cannot handle NULL
+    # geometries in its spatial indices.
+    if lookup_type == 'isnull':
+        return "%s%s IS %sNULL" % (table_prefix, field_name, (not value and 'NOT ' or ''))
+
+    raise TypeError("Got invalid lookup_type: %s" % repr(lookup_type))
+
+# To ease implementation, WKT is passed to/from MySQL.
+GEOM_FROM_TEXT = 'GeomFromText'
+GEOM_FROM_WKB = 'GeomFromWKB'
+GEOM_SELECT = 'AsText(%s)'

django/contrib/gis/db/backend/oracle/__init__.py

@@ -1,8 +1,10 @@
 """
  The Oracle spatial database backend module.
 
- Please note that WKT support is broken on the XE version, and this will
+ Please note that WKT support is broken on the XE version, and thus
- not work.
+ this backend will not work on such platforms.  Specifically, XE lacks 
+ support for an internal JVM, and Java libraries are required to use 
+ the WKT constructors.
 """
 from django.contrib.gis.db.backend.oracle.creation import create_spatial_db
 from django.contrib.gis.db.backend.oracle.field import OracleSpatialField, gqn

django/contrib/gis/db/backend/oracle/query.py

@@ -35,7 +35,7 @@ def get_geo_where_clause(lookup_type, table_prefix, field_name, value):
         table_prefix = qn(table_prefix[:-1])+'.'
     field_name = qn(field_name)
 
-    # See if a PostGIS Geometry function matches the lookup type next
+    # See if a Oracle Geometry function matches the lookup type next
     lookup_info = ORACLE_GEOMETRY_FUNCTIONS.get(lookup_type, False)
     if lookup_info:
         # Lookup types that are tuples take tuple arguments, e.g., 'relate' and 

django/contrib/gis/tests/__init__.py

@@ -2,6 +2,7 @@ import sys
 from copy import copy
 from unittest import TestSuite, TextTestRunner
 from django.contrib.gis.gdal import HAS_GDAL
+from django.contrib.gis.tests.utils import mysql
 
 # Tests that do not require setting up and tearing down a spatial database.
 test_suite_names = [
@@ -84,8 +85,12 @@ def run_tests(module_list, verbosity=1, interactive=True):
     test_suite = suite()
     for test_model in test_models:
         module_name = 'django.contrib.gis.tests.%s' % test_model
+        if mysql:
+            test_module_name = 'tests_mysql'
+        else:
+            test_module_name = 'tests'
         settings.INSTALLED_APPS.append(module_name)
-        tsuite = getattr(__import__('django.contrib.gis.tests.%s' % test_model, globals(), locals(), ['tests']), 'tests')
+        tsuite = getattr(__import__('django.contrib.gis.tests.%s' % test_model, globals(), locals(), [test_module_name]), test_module_name)
         test_suite.addTest(tsuite.suite())
 
     # Resetting the loaded flag to take into account what we appended to the INSTALLED_APPS

django/contrib/gis/tests/geoapp/models.py

@@ -1,4 +1,8 @@
 from django.contrib.gis.db import models
+from django.contrib.gis.tests.utils import mysql
+
+# MySQL spatial indices can't handle NULL geometries.
+null_flag = not mysql
 
 class Country(models.Model):
     name = models.CharField(max_length=30)
@@ -12,7 +16,7 @@ class City(models.Model):
 
 class State(models.Model):
     name = models.CharField(max_length=30)
-    poly = models.PolygonField(null=True) # Allowing NULL geometries here.
+    poly = models.PolygonField(null=null_flag) # Allowing NULL geometries here.
     objects = models.GeoManager()
 
 class Feature(models.Model):

django/contrib/gis/tests/geoapp/sql/city.mysql.sql

@@ -0,0 +1,8 @@
+INSERT INTO geoapp_city (`name`, `point`) VALUES ('Houston', GeomFromText('POINT (-95.363151 29.763374)'));
+INSERT INTO geoapp_city (`name`, `point`) VALUES ('Dallas', GeomFromText('POINT (-96.801611 32.782057)'));
+INSERT INTO geoapp_city (`name`, `point`) VALUES ('Oklahoma City', GeomFromText('POINT (-97.521157 34.464642)'));
+INSERT INTO geoapp_city (`name`, `point`) VALUES ('Wellington', GeomFromText('POINT (174.783117 -41.315268)'));
+INSERT INTO geoapp_city (`name`, `point`) VALUES ('Pueblo', GeomFromText('POINT (-104.609252 38.255001)'));
+INSERT INTO geoapp_city (`name`, `point`) VALUES ('Lawrence', GeomFromText('POINT (-95.235060 38.971823)'));
+INSERT INTO geoapp_city (`name`, `point`) VALUES ('Chicago', GeomFromText('POINT (-87.650175 41.850385)'));
+INSERT INTO geoapp_city (`name`, `point`) VALUES ('Victoria', GeomFromText('POINT (-123.305196 48.462611)'));

django/contrib/gis/tests/geoapp/tests_mysql.py

@@ -0,0 +1,167 @@
+"""
+ A limited test module is used for a limited spatial database.
+"""
+import os, unittest
+from models import Country, City, State, Feature
+from django.contrib.gis import gdal
+from django.contrib.gis.geos import *
+from django.core.exceptions import ImproperlyConfigured
+
+class GeoModelTest(unittest.TestCase):
+    
+    def test01_initial_sql(self):
+        "Testing geographic initial SQL."
+        # Ensuring that data was loaded from initial SQL.
+        self.assertEqual(2, Country.objects.count())
+        self.assertEqual(8, City.objects.count())
+        self.assertEqual(2, State.objects.count())
+
+    def test02_proxy(self):
+        "Testing Lazy-Geometry support (using the GeometryProxy)."
+        #### Testing on a Point
+        pnt = Point(0, 0)
+        nullcity = City(name='NullCity', point=pnt)
+        nullcity.save()
+
+        # Making sure TypeError is thrown when trying to set with an
+        #  incompatible type.
+        for bad in [5, 2.0, LineString((0, 0), (1, 1))]:
+            try:
+                nullcity.point = bad
+            except TypeError:
+                pass
+            else:
+                self.fail('Should throw a TypeError')
+
+        # Now setting with a compatible GEOS Geometry, saving, and ensuring
+        #  the save took, notice no SRID is explicitly set.
+        new = Point(5, 23)
+        nullcity.point = new
+
+        # Ensuring that the SRID is automatically set to that of the 
+        #  field after assignment, but before saving.
+        self.assertEqual(4326, nullcity.point.srid)
+        nullcity.save()
+
+        # Ensuring the point was saved correctly after saving
+        self.assertEqual(new, City.objects.get(name='NullCity').point)
+
+        # Setting the X and Y of the Point
+        nullcity.point.x = 23
+        nullcity.point.y = 5
+        # Checking assignments pre & post-save.
+        self.assertNotEqual(Point(23, 5), City.objects.get(name='NullCity').point)
+        nullcity.save()
+        self.assertEqual(Point(23, 5), City.objects.get(name='NullCity').point)
+        nullcity.delete()
+
+        #### Testing on a Polygon
+        shell = LinearRing((0, 0), (0, 100), (100, 100), (100, 0), (0, 0))
+        inner = LinearRing((40, 40), (40, 60), (60, 60), (60, 40), (40, 40))
+
+        # Creating a State object using a built Polygon
+        ply = Polygon(shell.clone(), inner.clone())
+        nullstate = State(name='NullState', poly=ply)
+        self.assertEqual(4326, nullstate.poly.srid) # SRID auto-set from None
+        nullstate.save()
+
+        ns = State.objects.get(name='NullState')
+        self.assertEqual(ply, ns.poly)
+        
+        # Testing the `ogr` and `srs` lazy-geometry properties.
+        if gdal.HAS_GDAL:
+            self.assertEqual(True, isinstance(ns.poly.ogr, gdal.OGRGeometry))
+            self.assertEqual(ns.poly.wkb, ns.poly.ogr.wkb)
+            self.assertEqual(True, isinstance(ns.poly.srs, gdal.SpatialReference))
+            self.assertEqual('WGS 84', ns.poly.srs.name)
+
+        # Changing the interior ring on the poly attribute.
+        new_inner = LinearRing((30, 30), (30, 70), (70, 70), (70, 30), (30, 30))
+        nullstate.poly[1] = new_inner.clone()
+        ply[1] = new_inner
+        self.assertEqual(4326, nullstate.poly.srid)
+        nullstate.save()
+        self.assertEqual(ply, State.objects.get(name='NullState').poly)
+        nullstate.delete()
+
+    def test03_contains_contained(self):
+        "Testing the 'contained', 'contains', and 'bbcontains' lookup types."
+        # Getting Texas, yes we were a country -- once ;)
+        texas = Country.objects.get(name='Texas')
+        
+        # Seeing what cities are in Texas, should get Houston and Dallas,
+        #  and Oklahoma City because MySQL 'within' only checks on the
+        #  _bounding box_ of the Geometries.
+        qs = City.objects.filter(point__within=texas.mpoly)
+        self.assertEqual(3, qs.count())
+        cities = ['Houston', 'Dallas', 'Oklahoma City']
+        for c in qs: self.assertEqual(True, c.name in cities)
+
+        # Pulling out some cities.
+        houston = City.objects.get(name='Houston')
+        wellington = City.objects.get(name='Wellington')
+        pueblo = City.objects.get(name='Pueblo')
+        okcity = City.objects.get(name='Oklahoma City')
+        lawrence = City.objects.get(name='Lawrence')
+
+        # Now testing contains on the countries using the points for
+        #  Houston and Wellington.
+        tx = Country.objects.get(mpoly__contains=houston.point) # Query w/GEOSGeometry
+        nz = Country.objects.get(mpoly__contains=wellington.point.hex) # Query w/EWKBHEX
+        ks = State.objects.get(poly__contains=lawrence.point)
+        self.assertEqual('Texas', tx.name)
+        self.assertEqual('New Zealand', nz.name)
+        self.assertEqual('Kansas', ks.name)
+
+        # Pueblo is not contained in Texas or New Zealand.
+        self.assertEqual(0, len(Country.objects.filter(mpoly__contains=pueblo.point))) # Query w/GEOSGeometry object
+
+        # OK City is contained w/in bounding box of Texas.
+        qs = Country.objects.filter(mpoly__bbcontains=okcity.point)
+        self.assertEqual(1, len(qs))
+        self.assertEqual('Texas', qs[0].name)
+
+    def test04_equals(self):
+        "Testing the 'same_as' and 'equals' lookup types."
+        pnt = fromstr('POINT (-95.363151 29.763374)', srid=4326)
+        c1 = City.objects.get(point=pnt)
+        c2 = City.objects.get(point__same_as=pnt)
+        c3 = City.objects.get(point__equals=pnt)
+        for c in [c1, c2, c3]: self.assertEqual('Houston', c.name)
+
+    def test05_geometryfield(self):
+        "Testing GeometryField."
+        f1 = Feature(name='Point', geom=Point(1, 1))
+        f2 = Feature(name='LineString', geom=LineString((0, 0), (1, 1), (5, 5)))
+        f3 = Feature(name='Polygon', geom=Polygon(LinearRing((0, 0), (0, 5), (5, 5), (5, 0), (0, 0))))
+        f4 = Feature(name='GeometryCollection', 
+                     geom=GeometryCollection(Point(2, 2), LineString((0, 0), (2, 2)), 
+                                             Polygon(LinearRing((0, 0), (0, 5), (5, 5), (5, 0), (0, 0)))))
+        f1.save()
+        f2.save()
+        f3.save()
+        f4.save()
+
+        f_1 = Feature.objects.get(name='Point')
+        self.assertEqual(True, isinstance(f_1.geom, Point))
+        self.assertEqual((1.0, 1.0), f_1.geom.tuple)
+        f_2 = Feature.objects.get(name='LineString')
+        self.assertEqual(True, isinstance(f_2.geom, LineString))
+        self.assertEqual(((0.0, 0.0), (1.0, 1.0), (5.0, 5.0)), f_2.geom.tuple)
+
+        f_3 = Feature.objects.get(name='Polygon')
+        self.assertEqual(True, isinstance(f_3.geom, Polygon))
+        f_4 = Feature.objects.get(name='GeometryCollection')
+        self.assertEqual(True, isinstance(f_4.geom, GeometryCollection))
+        self.assertEqual(f_3.geom, f_4.geom[2])
+    
+    def test06_mysql_limitations(self):
+        "Testing that union(), kml(), gml() raise exceptions."
+        self.assertRaises(ImproperlyConfigured, City.objects.union, 'point')
+        self.assertRaises(ImproperlyConfigured, State.objects.all().kml, 'poly')
+        self.assertRaises(ImproperlyConfigured, Country.objects.all().gml, 'mpoly')
+
+def suite():
+    s = unittest.TestSuite()
+    s.addTest(unittest.makeSuite(GeoModelTest))
+    return s

django/contrib/gis/tests/test_spatialrefsys.py

@@ -1,6 +1,7 @@
 import unittest
-from django.contrib.gis.models import SpatialRefSys
+from django.contrib.gis.tests.utils import mysql, no_mysql, oracle, postgis
-from django.contrib.gis.tests.utils import oracle, postgis
+if not mysql:
+    from django.contrib.gis.models import SpatialRefSys
 
 test_srs = ({'srid' : 4326,
              'auth_name' : ('EPSG', True),
@@ -26,6 +27,7 @@ test_srs = ({'srid' : 4326,
 
 class SpatialRefSysTest(unittest.TestCase):
 
+    @no_mysql
     def test01_retrieve(self):
         "Testing retrieval of SpatialRefSys model objects."
         for sd in test_srs:
@@ -46,6 +48,7 @@ class SpatialRefSysTest(unittest.TestCase):
                 self.assertEqual(sd['srtext'], srs.wkt)
                 self.assertEqual(sd['proj4'], srs.proj4text)
 
+    @no_mysql
     def test02_osr(self):
         "Testing getting OSR objects from SpatialRefSys model objects."
         for sd in test_srs:
@@ -61,6 +64,7 @@ class SpatialRefSysTest(unittest.TestCase):
                 self.assertEqual(sd['proj4'], srs.proj4)
                 self.assertEqual(sd['srtext'], srs.wkt)
 
+    @no_mysql
     def test03_ellipsoid(self):
         "Testing the ellipsoid property."
         for sd in test_srs:

django/contrib/gis/tests/utils.py

@@ -14,7 +14,9 @@ def no_backend(test_func, backend):
 # spatial backends.
 def no_oracle(func): return no_backend(func, 'oracle')
 def no_postgis(func): return no_backend(func, 'postgresql_psycopg2')
+def no_mysql(func): return no_backend(func, 'mysql')
 
 # Shortcut booleans to omit only portions of tests.
 oracle  = settings.DATABASE_ENGINE == 'oracle'
 postgis = settings.DATABASE_ENGINE == 'postgresql_psycopg2' 
+mysql   = settings.DATABASE_ENGINE == 'mysql'