gis: LayerMapping refactor

(1) Moved all routines into LayerMapping class (for easier subclassing) and modularized the routines.
 (2) OFTString and OFTReal OGR fields are verified w/the Django fields prior to insertion, thus avoiding invalidating a large transaction.
 (3) Added keyword options for specifying the transaction mode, not performing transformations, and status printing.
 (4) Created unit tests.
Other Changes:
 Updated `ogrinfo` for GDAL refactor and fixed an iterating bug; simplified a few lines in `geoapp` model tests.


git-svn-id: http://code.djangoproject.com/svn/django/branches/gis@6687 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/gis/tests/__init__.py

@@ -2,7 +2,13 @@ import sys
 from copy import copy
 from unittest import TestSuite, TextTestRunner
 from django.contrib.gis.gdal import HAS_GDAL
-from django.contrib.gis.tests.utils import mysql
+try:
+    from django.contrib.gis.tests.utils import mysql
+except:
+    mysql = False
+
+# Tests that require use of a spatial database (e.g., creation of models)
+test_models = ['geoapp']
 
 # Tests that do not require setting up and tearing down a spatial database.
 test_suite_names = [
@@ -10,6 +16,7 @@ test_suite_names = [
     'test_measure',
 ]
 if HAS_GDAL:
+    test_models += ['layermap']
     test_suite_names += [
         'test_gdal_driver',
         'test_gdal_ds',
@@ -21,8 +28,6 @@ if HAS_GDAL:
 else:
     print >>sys.stderr, "GDAL not available - no GDAL tests will be run."
 
-test_models = ['geoapp']
-
 def suite():
     "Builds a test suite for the GIS package."
     s = TestSuite()
@@ -40,8 +45,8 @@ def run_tests(module_list, verbosity=1, interactive=True):
     Run the tests that require creation of a spatial database.
     
     In order to run geographic model tests the DATABASE_USER will require
-     superuser priviliges.  To accomplish this outside the `postgres` user,
+    superuser priviliges.  To accomplish this outside the `postgres` user,
-     create your own PostgreSQL database as a user:
+    create your own PostgreSQL database as a user:
      (1) Initialize database: `initdb -D /path/to/user/db`
      (2) If there's already a Postgres instance on the machine, it will need
          to use a different TCP port than 5432. Edit postgresql.conf (in 
@@ -49,21 +54,21 @@ def run_tests(module_list, verbosity=1, interactive=True):
      (3) Start this database `pg_ctl -D /path/to/user/db start`
 
     On Windows platforms simply use the pgAdmin III utility to add superuser 
-     priviliges to your database user.
+    priviliges to your database user.
 
     Make sure your settings.py matches the settings of the user database. 
-     For example, set the same port number (`DATABASE_PORT=5433`).  
+    For example, set the same port number (`DATABASE_PORT=5433`).  
-     DATABASE_NAME or TEST_DATABSE_NAME must be set, along with DATABASE_USER.
+    DATABASE_NAME or TEST_DATABSE_NAME must be set, along with DATABASE_USER.
       
     In settings.py set TEST_RUNNER='django.contrib.gis.tests.run_tests'.
 
     Finally, this assumes that the PostGIS SQL files (lwpostgis.sql and 
-     spatial_ref_sys.sql) are installed in the directory specified by 
+    spatial_ref_sys.sql) are installed in the directory specified by 
-     `pg_config --sharedir` (and defaults to /usr/local/share if that fails).
+    `pg_config --sharedir` (and defaults to /usr/local/share if that fails).
-     This behavior is overridden if `POSTGIS_SQL_PATH` is in your settings.
+    This behavior is overridden if `POSTGIS_SQL_PATH` is in your settings.
     
-    Windows users should use the POSTGIS_SQL_PATH because the output
+    Windows users should set POSTGIS_SQL_PATH manually because the output
-     of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'.
+    of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'.
 
     Finally, the tests may be run by invoking `./manage.py test`.
     """
@@ -93,9 +98,10 @@ def run_tests(module_list, verbosity=1, interactive=True):
         tsuite = getattr(__import__('django.contrib.gis.tests.%s' % test_model, globals(), locals(), [test_module_name]), test_module_name)
         test_suite.addTest(tsuite.suite())
 
-    # Resetting the loaded flag to take into account what we appended to the INSTALLED_APPS
+    # Resetting the loaded flag to take into account what we appended to 
-    #  (since this routine is invoked through django/core/management, it caches the apps,
+    # the INSTALLED_APPS (since this routine is invoked through 
-    #   this ensures that syncdb will see our appended models)
+    # django/core/management, it caches the apps; this ensures that syncdb 
+    # will see our appended models)
     from django.db.models import loading
     loading._loaded = False
 

django/contrib/gis/tests/geoapp/tests.py

@@ -77,7 +77,7 @@ class GeoModelTest(unittest.TestCase):
         inner = LinearRing((40, 40), (40, 60), (60, 60), (60, 40), (40, 40))
 
         # Creating a State object using a built Polygon
-        ply = Polygon(shell.clone(), inner.clone())
+        ply = Polygon(shell, inner)
         nullstate = State(name='NullState', poly=ply)
         self.assertEqual(4326, nullstate.poly.srid) # SRID auto-set from None
         nullstate.save()
@@ -94,12 +94,12 @@ class GeoModelTest(unittest.TestCase):
 
         # Changing the interior ring on the poly attribute.
         new_inner = LinearRing((30, 30), (30, 70), (70, 70), (70, 30), (30, 30))
-        nullstate.poly[1] = new_inner.clone()
+        ns.poly[1] = new_inner
         ply[1] = new_inner
-        self.assertEqual(4326, nullstate.poly.srid)
+        self.assertEqual(4326, ns.poly.srid)
-        nullstate.save()
+        ns.save()
         self.assertEqual(ply, State.objects.get(name='NullState').poly)
-        nullstate.delete()
+        ns.delete()
 
     @no_oracle # Oracle does not support KML.
     def test03a_kml(self):

django/contrib/gis/tests/layermap/cities/cities.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

django/contrib/gis/tests/layermap/interstates/interstates.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

django/contrib/gis/tests/layermap/models.py

@@ -0,0 +1,29 @@
+from django.contrib.gis.db import models
+
+class City(models.Model):
+    name = models.CharField(max_length=25)
+    population = models.IntegerField()
+    density = models.DecimalField(max_digits=7, decimal_places=1)
+    date = models.DateField()
+    point = models.PointField()
+    objects = models.GeoManager()
+
+class Interstate(models.Model):
+    name = models.CharField(max_length=20)
+    length = models.DecimalField(max_digits=7, decimal_places=2)
+    path = models.LineStringField()
+    objects = models.GeoManager()
+    
+# Mapping dictionary for the City model.
+city_mapping = {'name' : 'Name',
+                'population' : 'Population',
+                'density' : 'Density',
+                'date' : 'Created',
+                'point' : 'POINT',
+                }
+
+# Mapping dictionary for the Interstate model.
+inter_mapping = {'name' : 'Name',
+                 'length' : 'Length',
+                 'path' : 'LINESTRING',
+                 }

django/contrib/gis/tests/layermap/tests.py

@@ -0,0 +1,106 @@
+import os, unittest
+from copy import copy
+from datetime import date
+from decimal import Decimal
+from models import City, Interstate, city_mapping, inter_mapping
+from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError, InvalidDecimal
+from django.contrib.gis.gdal import DataSource
+
+shp_path = os.path.dirname(__file__)
+city_shp = os.path.join(shp_path, 'cities/cities.shp')
+inter_shp = os.path.join(shp_path, 'interstates/interstates.shp')
+
+class LayerMapTest(unittest.TestCase):
+
+    def test01_init(self):
+        "Testing LayerMapping initialization."
+
+        # Model field that does not exist.
+        bad1 = copy(city_mapping)
+        bad1['foobar'] = 'FooField'
+
+        # Shapefile field that does not exist.
+        bad2 = copy(city_mapping)
+        bad2['name'] = 'Nombre'
+
+        # Nonexistent geographic field type.
+        bad3 = copy(city_mapping)
+        bad3['point'] = 'CURVE'
+
+        # Incrementing through the bad mapping dictionaries and
+        # ensuring that a LayerMapError is raised.
+        for bad_map in (bad1, bad2, bad3):
+            try:
+                lm = LayerMapping(City, city_shp, bad_map)
+            except LayerMapError:
+                pass
+            else:
+                self.fail('Expected a LayerMapError.')
+
+        # A LookupError should be thrown for bogus encodings.
+        try:
+            lm = LayerMapping(City, city_shp, city_mapping, encoding='foobar')
+        except LookupError:
+            pass
+        else:
+            self.fail('Expected a LookupError')
+
+    def test02_simple_layermap(self):
+        "Test LayerMapping import of a simple point shapefile."
+
+        # Setting up for the LayerMapping.
+        lm = LayerMapping(City, city_shp, city_mapping)
+        lm.save()
+
+        # There should be three cities in the shape file.
+        self.assertEqual(3, City.objects.count())
+
+        # Opening up the shapefile, and verifying the values in each
+        # of the features made it to the model.
+        ds = DataSource(city_shp)
+        layer = ds[0]
+        for feat in layer:
+            city = City.objects.get(name=feat['Name'].value)
+            self.assertEqual(feat['Population'].value, city.population)
+            self.assertEqual(Decimal(str(feat['Density'])), city.density)
+            self.assertEqual(feat['Created'].value, city.date)
+
+            # Comparing the geometries.
+            pnt1, pnt2 = feat.geom, city.point
+            self.assertAlmostEqual(pnt1.x, pnt2.x, 6)
+            self.assertAlmostEqual(pnt1.y, pnt2.y, 6)
+
+    def test03_layermap_strict(self):
+        "Testing the `strict` keyword, and import of a LineString shapefile."
+
+        # When the `strict` keyword is set an error encountered will force
+        # the importation to stop.
+        try:
+            lm = LayerMapping(Interstate, inter_shp, inter_mapping, 
+                              strict=True, silent=True)
+            lm.save()
+        except InvalidDecimal:
+            pass
+        else:
+            self.fail('Should have failed on strict import with invalid decimal values.')
+
+        # This LayerMapping should work b/c `strict` is not set.
+        lm = LayerMapping(Interstate, inter_shp, inter_mapping, silent=True)
+        lm.save()
+
+        # Only one interstate should have imported correctly.
+        self.assertEqual(1, Interstate.objects.count())
+        
+        # Verifying the values in the single feature w/the model.
+        ds = DataSource(inter_shp)
+        feat = ds[0][0]
+        istate = Interstate.objects.get(name=feat['Name'].value)
+        self.assertEqual(Decimal(str(feat['Length'])), istate.length)
+        for p1, p2 in zip(feat.geom, istate.path):
+            self.assertAlmostEqual(p1[0], p2[0], 6)
+            self.assertAlmostEqual(p1[1], p2[1], 6)
+
+def suite():
+    s = unittest.TestSuite()
+    s.addTest(unittest.makeSuite(LayerMapTest))
+    return s

django/contrib/gis/utils/layermapping.py

@@ -1,21 +1,19 @@
 # LayerMapping -- A Django Model/OGR Layer Mapping Utility
 """
-The LayerMapping class provides a way to map the contents of OGR
+ The LayerMapping class provides a way to map the contents of OGR
  vector files (e.g. SHP files) to Geographic-enabled Django models.
 
-This grew out of my personal needs, specifically the code repetition
+ This grew out of my personal needs, specifically the code repetition
  that went into pulling geometries and fields out of an OGR layer,
  converting to another coordinate system (e.g. WGS84), and then inserting
- into a Geographic Django model.
+ into a GeoDjango model.
 
-This utility is still in early stages of development, so its usage
+ This utility is still in early stages of development, so its usage
  is subject to change -- please report any bugs.
 
-TODO: Unit tests and documentation.
+ Requirements:  OGR C Library (from GDAL) required.
 
-Requirements:  OGR C Library (from GDAL) required.
+ Usage: 
-
-Usage: 
   lm = LayerMapping(model, source_file, mapping) where,
 
   model:
@@ -32,7 +30,7 @@ Usage:
    is a geographic then it should correspond to the OGR
    geometry type, e.g. 'POINT', 'LINESTRING', 'POLYGON'.
 
-Keyword Args:
+ Keyword Args:
   layer:
    The index of the layer to use from the Data Source (defaults to 0)
 
@@ -46,6 +44,26 @@ Keyword Args:
    For example, 'latin-1', 'utf-8', and 'cp437' are all valid
    encoding parameters.
 
+  check:
+   By default, LayerMapping increments through each feature in the
+   layer to ensure that it is compatible with the given model and
+   mapping.  Setting this keyword to False, disables this action,
+   which will speed up execution time for very large files.
+
+  silent:
+   By default, non-fatal error notifications are printed to stdout; this
+   keyword may be set in order to disable these notifications.
+
+  strict:
+   Setting this keyword to True will instruct the save() method to
+   cease execution on the first error encountered.
+
+  transaction_mode:
+   May be 'commit_on_success' (default) or 'autocommit'.
+
+  transform:
+   Setting this to False will disable all coordinate transformations.  
+
 Example:
 
  1. You need a GDAL-supported data source, like a shapefile.
@@ -91,162 +109,86 @@ Example:
   Saved: Name: 3
 
  LayerMapping just transformed the three geometries from the SHP file from their
-  source spatial reference system (WGS84) to the spatial reference system of
+ source spatial reference system (WGS84) to the spatial reference system of
-  the GeoDjango model (NAD83).  If no spatial reference system is defined for
+ the GeoDjango model (NAD83).  If no spatial reference system is defined for
-  the layer, use the `source_srs` keyword with a SpatialReference object to
+ the layer, use the `source_srs` keyword with a SpatialReference object to
-  specify one. Further, data is selectively imported from the given data source 
+ specify one.
-  fields into the model fields.
 """
-from types import StringType, TupleType
+from datetime import date, datetime
-from datetime import datetime
+from decimal import Decimal
-from django.contrib.gis.db.backend import SPATIAL_BACKEND
-from django.contrib.gis.gdal import \
-     OGRGeometry, OGRGeomType, SpatialReference, CoordTransform, \
-     DataSource, OGRException
-from django.contrib.gis.gdal.field import Field, OFTInteger, OFTReal, OFTString, OFTDateTime
-from django.contrib.gis.models import GeometryColumns, SpatialRefSys
-from django.db import connection, transaction
 from django.core.exceptions import ObjectDoesNotExist
+from django.db import connection, transaction
+from django.db.models.fields.related import ForeignKey
+from django.contrib.gis.db.backend import SPATIAL_BACKEND
+from django.contrib.gis.gdal import CoordTransform, DataSource, \
+    OGRException, OGRGeometry, OGRGeomType, SpatialReference
+from django.contrib.gis.gdal.field import OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime
+from django.contrib.gis.models import GeometryColumns, SpatialRefSys
 
-# A mapping of given geometry types to their OGR integer type.
+# LayerMapping exceptions.
-ogc_types = {'POINT' : OGRGeomType('Point'),
+class LayerMapError(Exception): pass
-             'LINESTRING' : OGRGeomType('LineString'),
+class InvalidString(LayerMapError): pass
-             'POLYGON' : OGRGeomType('Polygon'),
+class InvalidDecimal(LayerMapError): pass
-             'MULTIPOINT' : OGRGeomType('MultiPoint'),
-             'MULTILINESTRING' : OGRGeomType('MultiLineString'),
-             'MULTIPOLYGON' : OGRGeomType('MultiPolygon'),
-             'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'),
-             }
 
-# The django.contrib.gis model types.
+class LayerMapping(object):
-gis_fields = {'PointField' : 'POINT',
+    "A class that maps OGR Layers to GeoDjango Models."
-              'LineStringField': 'LINESTRING',
+    
-              'PolygonField': 'POLYGON',
+    # A mapping of given geometry types to their OGR integer type.
-              'MultiPointField' : 'MULTIPOINT',
+    OGC_TYPES = {'POINT' : OGRGeomType('Point'),
-              'MultiLineStringField' : 'MULTILINESTRING',
+                 'LINESTRING' : OGRGeomType('LineString'),
-              'MultiPolygonField' : 'MULTIPOLYGON',
+                 'POLYGON' : OGRGeomType('Polygon'),
-              }
+                 'MULTIPOINT' : OGRGeomType('MultiPoint'),
+                 'MULTILINESTRING' : OGRGeomType('MultiLineString'),
+                 'MULTIPOLYGON' : OGRGeomType('MultiPolygon'),
+                 'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'),
+                 }
 
-# Acceptable 'base' types for a multi-geometry type.
+    # The django.contrib.gis model types.
-multi_types = {'POINT' : OGRGeomType('MultiPoint'),
+    GIS_FIELDS = {'PointField' : 'POINT',
-               'LINESTRING' : OGRGeomType('MultiLineString'),
+                  'LineStringField': 'LINESTRING',
-               'POLYGON' : OGRGeomType('MultiPolygon'),
+                  'PolygonField': 'POLYGON',
-               }
+                  'MultiPointField' : 'MULTIPOINT',
+                  'MultiLineStringField' : 'MULTILINESTRING',
+                  'MultiPolygonField' : 'MULTIPOLYGON',
+                  'GeometryCollectionField' : 'GEOMETRYCOLLECTION',
+                  }
 
-def map_foreign_key(django_field):
+    # Acceptable 'base' types for a multi-geometry type.
-    from django.db.models.fields.related import ForeignKey
+    MULTI_TYPES = {'POINT' : OGRGeomType('MultiPoint'),
+                   'LINESTRING' : OGRGeomType('MultiLineString'),
+                   'POLYGON' : OGRGeomType('MultiPolygon'),
+                   }
 
-    if not django_field.__class__ is ForeignKey:
+    # The acceptable Django field types that map to OGR fields.
-        return django_field.__class__.__name__
+    FIELD_TYPES = {
+        'AutoField' : OFTInteger,
+        'IntegerField' : OFTInteger,
+        'FloatField' : OFTReal,
+        'DateField' : OFTDate,
+        'DateTimeField' : OFTDateTime,
+        'TimeField' : OFTTime,
+        'DecimalField' : OFTReal,
+        'CharField' : OFTString,
+        'TextField' : OFTString,
+        'SmallIntegerField' : OFTInteger,
+        'PositiveSmallIntegerField' : OFTInteger,
+        }
 
-     
+    # The acceptable transaction modes.
-    rf=django_field.rel.get_related_field()
+    TRANSACTION_MODES = {'autocommit' : transaction.autocommit,
+                         'commit_on_success' : transaction.commit_on_success,
+                         }
 
-    return rf.get_internal_type()
+    def __init__(self, model, data, mapping, layer=0, 
-                
+                 source_srs=None, encoding=None, check=True, 
-# The acceptable Django field types that map to OGR fields.
+                 progress=False, interval=1000, strict=False, silent=False,
-field_types = {
+                 transaction_mode='commit_on_success', transform=True):
-               'AutoField' : OFTInteger,
-               'IntegerField' : OFTInteger,
-               'FloatField' : OFTReal,
-               'DateTimeField' : OFTDateTime,
-               'DecimalField' : OFTReal,
-               'CharField' : OFTString,
-               'SmallIntegerField' : OFTInteger,
-               'PositiveSmallIntegerField' : OFTInteger,
-               }
-
-def make_multi(geom_name, model_type):
-    "Determines whether the geometry should be made into a GeometryCollection."
-    if (geom_name in multi_types) and (model_type.startswith('Multi')):
-        return True
-    else:
-        return False
-
-def check_feature(feat, model_fields, mapping):
-    "Checks the OGR layer feature."
-
-    HAS_GEO = False
-
-    # Incrementing through each model_field & ogr_field in the given mapping.
-    for model_field, ogr_field in mapping.items():
-
-        # Making sure the given mapping model field is in the given model fields.
-        if model_field in model_fields:
-            model_type = model_fields[model_field]
-        elif model_field[:-3] in model_fields: #foreign key
-            model_type = model_fields[model_field[:-3]]
-        else:
-            raise Exception('Given mapping field "%s" not in given Model fields!' % model_field)
-
-        ### Handling if we get a geometry in the Field ###
-        if ogr_field in ogc_types:
-            # At this time, no more than one geographic field per model =(
-            if HAS_GEO:
-                raise Exception('More than one geographic field in mapping not allowed (yet).')
-            else:
-                HAS_GEO = ogr_field
-
-            # Making sure this geometry field type is a valid Django GIS field.
-            if not model_type in gis_fields:
-                raise Exception('Unknown Django GIS field type "%s"' % model_type)
-            
-            # Getting the OGRGeometry, it's type (an integer) and it's name (a string)
-            geom  = feat.geom
-            gtype = geom.geom_type
-            gname = geom.geom_name
-
-            if make_multi(gname, model_type):
-                # Do we have to 'upsample' into a Geometry Collection?
-                pass
-            elif gtype == ogc_types[gis_fields[model_type]]:
-                # The geometry type otherwise was expected
-                pass
-            else:
-                raise Exception('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype))
-
-        ## Handling other fields 
-        else:
-            # Making sure the model field is
-            if not model_type in field_types:
-                raise Exception('Django field type "%s" has no OGR mapping (yet).' % model_type)
-
-            # Otherwise, we've got an OGR Field.  Making sure that an
-            # index exists for the mapping OGR field.
-            try:
-                fi = feat.index(ogr_field)
-            except:
-                raise Exception('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field)
-                    
-def check_layer(layer, fields, mapping):
-    "Checks the OGR layer by incrementing through and checking each feature."
-    # Incrementing through each feature in the layer.
-    for feat in layer:
-        check_feature(feat, fields, mapping)
-
-def check_srs(layer, source_srs):
-    "Checks the compatibility of the given spatial reference object."
-    if isinstance(source_srs, SpatialReference):
-        sr = source_srs
-    elif isinstance(source_srs, SpatialRefSys):
-        sr = source_srs.srs
-    elif isinstance(source_srs, (int, str)):
-        sr = SpatialReference(source_srs)
-    else:
-        sr = layer.srs
-    if not sr:
-        raise Exception('No source reference system defined.')
-    else:
-        return sr
-
-class LayerMapping:
-    "A class that maps OGR Layers to Django Models."
-
-    def __init__(self, model, data, mapping, layer=0, source_srs=None, encoding=None):
         "Takes the Django model, the data source, and the mapping (dictionary)"
 
         # Getting the field names and types from the model
-        fields = dict((f.name, map_foreign_key(f)) for f in model._meta.fields)
+        self.fields = dict((f.name, self.map_foreign_key(f)) for f in model._meta.fields)
+        self.field_classes = dict((f.name, f) for f in model._meta.fields)
+
         # Getting the DataSource and its Layer
         if isinstance(data, basestring):
             self.ds = DataSource(data)
@@ -254,120 +196,306 @@ class LayerMapping:
             self.ds = data
         self.layer = self.ds[layer]
 
-        # Checking the layer -- intitialization of the object will fail if
+        # Setting the mapping
-        #  things don't check out before hand.
-        check_layer(self.layer, fields, mapping)
-        
-        # Since the layer checked out, setting the fields and the mapping.
-        self.fields = fields
         self.mapping = mapping
+
+        # Setting the model, and getting the geometry column associated 
+        # with the model (an exception will be raised if there is no 
+        # geometry column).
         self.model = model
-        self.source_srs = check_srs(self.layer, source_srs)
+        self.geo_col = self.geometry_column()
+
+        # Checking the source spatial reference system, and getting
+        # the coordinate transformation object (unless the `transform`
+        # keyword is set to False)
+        self.source_srs = self.check_srs(source_srs)
+        self.transform = transform and self.coord_transform()
+
+        # Checking the layer -- intitialization of the object will fail if
+        # things don't check out before hand.  This may be time-consuming,
+        # and disabled by setting the `check` keyword to False.
+        if check: self.check_layer()
+
+        # The silent, strict, progress, and interval flags.
+        self.silent = silent
+        self.strict = strict
+        self.progress = progress
+        self.interval = interval
 
         # Setting the encoding for OFTString fields, if specified.
         if encoding:
             # Making sure the encoding exists, if not a LookupError
-            #  exception will be thrown.
+            # exception will be thrown.
             from codecs import lookup
             lookup(encoding)
             self.encoding = encoding
         else:
             self.encoding = None
 
-    # Either the import will work, or it won't be committed.
+        # Setting the transaction decorator with the function in the 
-    @transaction.commit_on_success
+        # transaction modes dictionary.
-    def save(self, verbose=False):
+        if transaction_mode in self.TRANSACTION_MODES:
-        "Runs the layer mapping on the given SHP file, and saves to the database."
+            self.transaction_decorator = self.TRANSACTION_MODES[transaction_mode]
+            self.transaction_mode = transaction_mode
+        else:
+            raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode)
 
+    def check_feature(self, feat):
+        "Checks the OGR feature against the model fields and mapping."
+        HAS_GEO = False   
+
+        # Incrementing through each model_field & ogr_field in the given mapping.
+        for model_field, ogr_field in self.mapping.items():
+            # Making sure the given mapping model field is in the given model fields.
+            if model_field in self.fields:
+                model_type = self.fields[model_field]
+            elif model_field[:-3] in self.fields: #foreign key
+                model_type = self.fields[model_field[:-3]]
+            else:
+                raise LayerMapError('Given mapping field "%s" not in given Model fields!' % model_field)
+
+            ### Handling if we get a geometry in the Field ###
+            if ogr_field in self.OGC_TYPES:
+                # At this time, no more than one geographic field per model =(
+                if HAS_GEO:
+                    raise LayerMapError('More than one geographic field in mapping not allowed (yet).')
+                else:
+                    HAS_GEO = ogr_field
+
+                # Making sure this geometry field type is a valid Django GIS field.
+                if not model_type in self.GIS_FIELDS:
+                    raise LayerMapError('Unknown Django GIS field type "%s"' % model_type)
+
+                # Getting the OGRGeometry, it's type (an integer) and it's name (a string)
+                geom  = feat.geom
+                gtype = geom.geom_type
+                gname = geom.geom_name
+
+                if self.make_multi(gname, model_type):
+                    # Do we have to 'upsample' into a Geometry Collection?
+                    pass
+                elif gtype == self.OGC_TYPES[self.GIS_FIELDS[model_type]]:
+                    # The geometry type otherwise was expected
+                    pass
+                else:
+                    raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype))
+            ### Handling other fields ###
+            else:
+                # Making sure the model field is supported.
+                if not model_type in self.FIELD_TYPES:
+                    raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % model_type)
+
+                # Otherwise, we've got an OGR Field.  Making sure that an
+                # index exists for the mapping OGR field.
+                try:
+                    fi = feat.index(ogr_field)
+                except:
+                    raise LayerMapError('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field)
+
+    def check_layer(self):
+        "Checks every feature in this object's layer."
+        for feat in self.layer:
+            self.check_feature(feat)
+
+    def check_srs(self, source_srs):
+        "Checks the compatibility of the given spatial reference object."
+        if isinstance(source_srs, SpatialReference):
+            sr = source_srs
+        elif isinstance(source_srs, SpatialRefSys):
+            sr = source_srs.srs
+        elif isinstance(source_srs, (int, str)):
+            sr = SpatialReference(source_srs)
+        else:
+            # Otherwise just pulling the SpatialReference from the layer
+            sr = self.layer.srs
+            
+        if not sr:
+            raise LayerMapError('No source reference system defined.')
+        else:
+            return sr
+
+    def coord_transform(self):
+        "Returns the coordinate transformation object."
+        try:
+            # Getting the target spatial reference system
+            target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs
+        
+            # Creating the CoordTransform object
+            return CoordTransform(self.source_srs, target_srs)
+        except Exception, msg:
+            raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg)
+
+    def feature_kwargs(self, feat):
+        "Returns the keyword arguments needed for saving a feature."
+        
+        # The keyword arguments for model construction.
+        kwargs = {}
+
+        # The all_prepped flagged, will be set to False if there's a
+        # problem w/a ForeignKey that doesn't exist.
+        all_prepped = True
+
+        # Incrementing through each model field and OGR field in the
+        # dictionary mapping.
+        for model_field, ogr_field in self.mapping.items():
+            is_fk = False
+            try:
+                model_type = self.fields[model_field]
+            except KeyError: #foreign key
+                # The -3 index is b/c foreign keys are appended w/'_id'.
+                model_type = self.fields[model_field[:-3]]
+                is_fk = True
+            
+            if ogr_field in self.OGC_TYPES:
+                # Verify OGR geometry.
+                val = self.verify_geom(feat.geom, model_type)
+            else:
+                # Otherwise, verify OGR Field type.
+                val = self.verify_field(feat[ogr_field], model_field)
+
+            if is_fk:
+                # Handling if foreign key.
+                rel_obj = None
+                field_name = model_field[:-3]
+                try:
+                    # FIXME: refactor to efficiently fetch FKs.
+                    #  Requires significant re-work. :-/
+                    rel = self.model._meta.get_field(field_name).rel
+                    rel_obj = rel.to._default_manager.get(**{('%s__exact' % rel.field_name):val})
+                except ObjectDoesNotExist:
+                    all_prepped = False
+
+                kwargs[model_field[:-3]] = rel_obj
+            else:
+                kwargs[model_field] = val
+            
+        return kwargs, all_prepped
+
+    def verify_field(self, fld, model_field):
+        """
+        Verifies if the OGR Field contents are acceptable to the Django
+        model field.  If they are, the verified value is returned, 
+        otherwise the proper exception is raised.
+        """
+        field_class = self.field_classes[model_field]
+        if isinstance(fld, OFTString):
+            if self.encoding:
+                # The encoding for OGR data sources may be specified here
+                # (e.g., 'cp437' for Census Bureau boundary files).
+                val = unicode(fld.value, self.encoding)
+            else:
+                val = fld.value
+                if len(val) > field_class.max_length:
+                    raise InvalidString('%s model field maximum string length is %s, given %s characters.' %
+                                        (model_field, field_class.max_length, len(val)))
+        elif isinstance(fld, OFTReal):
+            try:
+                # Creating an instance of the Decimal value to use.
+                d = Decimal(str(fld.value))
+            except:
+                raise InvalidDecimal('Could not construct decimal from: %s' % fld)
+            dtup = d.as_tuple()
+            if len(dtup[1]) > field_class.max_digits:
+                raise InvalidDecimal('More than the maximum # of digits encountered.')
+            elif len(dtup[1][dtup[2]:]) > field_class.decimal_places:
+                raise InvalidDecimal('More than the maximum # of decimal places encountered.')
+            val = d
+        else:
+            val = fld.value
+        return val
+
+    def verify_geom(self, geom, model_type):
+        "Verifies the geometry."
+        if self.make_multi(geom.geom_name, model_type):
+            # Constructing a multi-geometry type to contain the single geometry
+            multi_type = self.MULTI_TYPES[geom.geom_name]
+            g = OGRGeometry(multi_type)
+            g.add(geom)
+        else:
+            g = geom
+
+        # Transforming the geometry with our Coordinate Transformation object,
+        # but only if the class variable `transform` is set w/a CoordTransform 
+        # object.
+        if self.transform: g.transform(self.transform)
+        
+        # Returning the WKT of the geometry.
+        return g.wkt
+        
+    def geometry_column(self):
+        "Returns the GeometryColumn model associated with the geographic column."
         # Getting the GeometryColumn object.
         try:
             db_table = self.model._meta.db_table
             if SPATIAL_BACKEND == 'oracle': db_table = db_table.upper()
             gc_kwargs = {GeometryColumns.table_name_col() : db_table}
-            geo_col = GeometryColumns.objects.get(**gc_kwargs)
+            return GeometryColumns.objects.get(**gc_kwargs)
-        except:
-            raise Exception('Geometry column does not exist. (did you run syncdb?)')
-        
-        # Getting the coordinate system needed for transformation (with CoordTransform)  
-        try:
-            # Getting the target spatial reference system
-            target_srs = SpatialRefSys.objects.get(srid=geo_col.srid).srs
-
-            # Creating the CoordTransform object
-            ct = CoordTransform(self.source_srs, target_srs)
         except Exception, msg:
-            raise Exception('Could not translate between the data source and model geometry: %s' % msg)
+            raise LayerMapError('Geometry column does not exist for model. (did you run syncdb?):\n %s' % msg)
 
-        for feat in self.layer:
+    def make_multi(self, geom_name, model_type):
-            # The keyword arguments for model construction
+        "Determines whether the geometry should be made into a GeometryCollection."
-            kwargs = {}
+        return (geom_name in self.MULTI_TYPES) and (model_type.startswith('Multi'))
 
-            # Incrementing through each model field and the OGR field in the mapping
+    def map_foreign_key(self, django_field):
-            all_prepped = True
+        "Handles fields within foreign keys for the given field."
+        if not django_field.__class__ is ForeignKey:
+            # Returning the field's class name.
+            return django_field.__class__.__name__
+        else:
+            # Otherwise, getting the type of the related field's
+            # from the Foreign key.
+            rf = django_field.rel.get_related_field()
+            return rf.get_internal_type()
 
-            for model_field, ogr_field in self.mapping.items():
+    def save(self, verbose=False):
-                is_fk = False
+        "Runs the layer mapping on the given SHP file, and saves to the database."
+        
+        @self.transaction_decorator
+        def _save():
+            num_feat = 0
+            num_saved = 0
+
+            for feat in self.layer:
+                num_feat += 1
+                # Getting the keyword arguments
                 try:
-                    model_type = self.fields[model_field]
+                    kwargs, all_prepped = self.feature_kwargs(feat)
-                except KeyError: #foreign key
+                except LayerMapError, msg:
-                    model_type = self.fields[model_field[:-3]]
+                    # Something borked the validation
-                    is_fk = True
+                    if self.strict: raise
-
+                    elif not self.silent: 
-                if ogr_field in ogc_types:
+                        print 'Ignoring Feature ID %s because: %s' % (feat.fid, msg)
-                    ## Getting the OGR geometry from the field
-                    geom = feat.geom
-
-                    if make_multi(geom.geom_name, model_type):
-                        # Constructing a multi-geometry type to contain the single geometry
-                        multi_type = multi_types[geom.geom_name]
-                        g = OGRGeometry(multi_type)
-                        g.add(geom)
-                    else:
-                        g = geom
-
-                    # Transforming the geometry with our Coordinate Transformation object.
-                    g.transform(ct)
-
-                    # Updating the keyword args with the WKT of the transformed model.
-                    val = g.wkt
                 else:
-                    ## Otherwise, this is an OGR field type
+                    # Constructing the model using the constructed keyword args
-                    fld = feat[ogr_field]
-
-                    if isinstance(fld, OFTString) and self.encoding:
-                        # The encoding for OGR data sources may be specified here
-                        #  (e.g., 'cp437' for Census Bureau boundary files).
-                        val = unicode(fld.value, self.encoding)
-                    else:
-                        val = fld.value
-
-                if is_fk:
-                    rel_obj = None
-                    field_name = model_field[:-3]
-                    try:
-                        #FIXME: refactor to efficiently fetch FKs.
-                        #  Requires significant re-work. :-/
-                        rel = self.model._meta.get_field(field_name).rel
-                        rel_obj = rel.to._default_manager.get(**{('%s__exact' % rel.field_name):val})
-                    except ObjectDoesNotExist:
-                        all_prepped = False
-                    
-                    kwargs[model_field[:-3]] = rel_obj
-                else:
-                    kwargs[model_field] = val
-
-            # Constructing the model using the constructed keyword args
-            if all_prepped:
-                m = self.model(**kwargs)
-
-                # Saving the model
-                try:
                     if all_prepped:
-                        m.save()
+                        m = self.model(**kwargs)
-                        if verbose: print 'Saved: %s' % str(m)                        
+                        try:
+                            m.save()
+                            num_saved += 1
+                            if verbose: print 'Saved: %s' % m
+                        except SystemExit:
+                            raise
+                        except Exception, msg:
+                            if self.transaction_mode == 'autocommit':
+                                # Rolling back the transaction so that other model saves
+                                # will work.
+                                transaction.rollback_unless_managed()
+                            if self.strict: 
+                                # Bailing out if the `strict` keyword is set.
+                                if not self.silent:
+                                    print 'Failed to save the feature (id: %s) into the model with the keyword arguments:' % feat.fid
+                                    print kwargs
+                                raise
+                            elif not self.silent:
+                                print 'Failed to save %s:\n %s\nContinuing' % (kwargs, msg)
                     else:
-                        print "Skipping %s due to missing relation." % kwargs
+                        print 'Skipping %s due to missing relation.' % kwargs
-                except SystemExit:
+
-                    raise
+                # Printing progress information, if requested.
-                except Exception, e:
+                if self.progress and num_feat % self.interval == 0:
-                    print "Failed to save %s\n  Continuing" % kwargs
+                    print 'Processed %d features, saved %d ...' % (num_feat, num_saved)
+               
+        # Calling our defined function, which will use the specified
+        # trasaction mode.
+        _save()

django/contrib/gis/utils/ogrinfo.py

@@ -33,12 +33,12 @@ def ogrinfo(data_source, num_features=10):
 
         width = max(*map(len,layer.fields))
         fmt = " %%%ss: %%s" % width
-        for i, feature in enumerate(layer[:num_features]):
+        for j, feature in enumerate(layer[:num_features]):
-            print "=== Feature %s" % i
+            print "=== Feature %s" % j
-            for field in layer.fields:
+            for fld_name in layer.fields:
-                fld_typ = feature[field].__class__.__name__.replace('OFT', '')
+                type_name = feature[fld_name].type_name
-                output = fmt % (field, fld_typ)
+                output = fmt % (fld_name, type_name)
-                val = feature.get(field)
+                val = feature.get(fld_name)
                 if val:
                     if isinstance(val, str):
                         val_fmt = ' ("%s")'