gis: LayerMapping refactor

(1) Moved all routines into LayerMapping class (for easier subclassing) and modularized the routines.
 (2) OFTString and OFTReal OGR fields are verified w/the Django fields prior to insertion, thus avoiding invalidating a large transaction.
 (3) Added keyword options for specifying the transaction mode, not performing transformations, and status printing.
 (4) Created unit tests.
Other Changes:
 Updated `ogrinfo` for GDAL refactor and fixed an iterating bug; simplified a few lines in `geoapp` model tests.


git-svn-id: http://code.djangoproject.com/svn/django/branches/gis@6687 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/gis/tests/__init__.py

@@ -2,7 +2,13 @@ import sys
 from copy import copy
 from unittest import TestSuite, TextTestRunner
 from django.contrib.gis.gdal import HAS_GDAL
-from django.contrib.gis.tests.utils import mysql
+try:
+    from django.contrib.gis.tests.utils import mysql
+except:
+    mysql = False
+
+# Tests that require use of a spatial database (e.g., creation of models)
+test_models = ['geoapp']
 
 # Tests that do not require setting up and tearing down a spatial database.
 test_suite_names = [
@@ -10,6 +16,7 @@ test_suite_names = [
     'test_measure',
 ]
 if HAS_GDAL:
+    test_models += ['layermap']
     test_suite_names += [
         'test_gdal_driver',
         'test_gdal_ds',
@@ -21,8 +28,6 @@ if HAS_GDAL:
 else:
     print >>sys.stderr, "GDAL not available - no GDAL tests will be run."
 
-test_models = ['geoapp']
-
 def suite():
     "Builds a test suite for the GIS package."
     s = TestSuite()
@@ -62,7 +67,7 @@ def run_tests(module_list, verbosity=1, interactive=True):
     `pg_config --sharedir` (and defaults to /usr/local/share if that fails).
     This behavior is overridden if `POSTGIS_SQL_PATH` is in your settings.
     
-    Windows users should use the POSTGIS_SQL_PATH because the output
+    Windows users should set POSTGIS_SQL_PATH manually because the output
     of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'.
 
     Finally, the tests may be run by invoking `./manage.py test`.
@@ -93,9 +98,10 @@ def run_tests(module_list, verbosity=1, interactive=True):
         tsuite = getattr(__import__('django.contrib.gis.tests.%s' % test_model, globals(), locals(), [test_module_name]), test_module_name)
         test_suite.addTest(tsuite.suite())
 
-    # Resetting the loaded flag to take into account what we appended to the INSTALLED_APPS
-    #  (since this routine is invoked through django/core/management, it caches the apps,
-    #   this ensures that syncdb will see our appended models)
+    # Resetting the loaded flag to take into account what we appended to 
+    # the INSTALLED_APPS (since this routine is invoked through 
+    # django/core/management, it caches the apps; this ensures that syncdb 
+    # will see our appended models)
     from django.db.models import loading
     loading._loaded = False
 

django/contrib/gis/tests/geoapp/tests.py

@@ -77,7 +77,7 @@ class GeoModelTest(unittest.TestCase):
         inner = LinearRing((40, 40), (40, 60), (60, 60), (60, 40), (40, 40))
 
         # Creating a State object using a built Polygon
-        ply = Polygon(shell.clone(), inner.clone())
+        ply = Polygon(shell, inner)
         nullstate = State(name='NullState', poly=ply)
         self.assertEqual(4326, nullstate.poly.srid) # SRID auto-set from None
         nullstate.save()
@@ -94,12 +94,12 @@ class GeoModelTest(unittest.TestCase):
 
         # Changing the interior ring on the poly attribute.
         new_inner = LinearRing((30, 30), (30, 70), (70, 70), (70, 30), (30, 30))
-        nullstate.poly[1] = new_inner.clone()
+        ns.poly[1] = new_inner
         ply[1] = new_inner
-        self.assertEqual(4326, nullstate.poly.srid)
-        nullstate.save()
+        self.assertEqual(4326, ns.poly.srid)
+        ns.save()
         self.assertEqual(ply, State.objects.get(name='NullState').poly)
-        nullstate.delete()
+        ns.delete()
 
     @no_oracle # Oracle does not support KML.
     def test03a_kml(self):

django/contrib/gis/tests/layermap/cities/cities.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

django/contrib/gis/tests/layermap/interstates/interstates.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

django/contrib/gis/tests/layermap/models.py

@@ -0,0 +1,29 @@
+from django.contrib.gis.db import models
+
+class City(models.Model):
+    name = models.CharField(max_length=25)
+    population = models.IntegerField()
+    density = models.DecimalField(max_digits=7, decimal_places=1)
+    date = models.DateField()
+    point = models.PointField()
+    objects = models.GeoManager()
+
+class Interstate(models.Model):
+    name = models.CharField(max_length=20)
+    length = models.DecimalField(max_digits=7, decimal_places=2)
+    path = models.LineStringField()
+    objects = models.GeoManager()
+    
+# Mapping dictionary for the City model.
+city_mapping = {'name' : 'Name',
+                'population' : 'Population',
+                'density' : 'Density',
+                'date' : 'Created',
+                'point' : 'POINT',
+                }
+
+# Mapping dictionary for the Interstate model.
+inter_mapping = {'name' : 'Name',
+                 'length' : 'Length',
+                 'path' : 'LINESTRING',
+                 }

django/contrib/gis/tests/layermap/tests.py

@@ -0,0 +1,106 @@
+import os, unittest
+from copy import copy
+from datetime import date
+from decimal import Decimal
+from models import City, Interstate, city_mapping, inter_mapping
+from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError, InvalidDecimal
+from django.contrib.gis.gdal import DataSource
+
+shp_path = os.path.dirname(__file__)
+city_shp = os.path.join(shp_path, 'cities/cities.shp')
+inter_shp = os.path.join(shp_path, 'interstates/interstates.shp')
+
+class LayerMapTest(unittest.TestCase):
+
+    def test01_init(self):
+        "Testing LayerMapping initialization."
+
+        # Model field that does not exist.
+        bad1 = copy(city_mapping)
+        bad1['foobar'] = 'FooField'
+
+        # Shapefile field that does not exist.
+        bad2 = copy(city_mapping)
+        bad2['name'] = 'Nombre'
+
+        # Nonexistent geographic field type.
+        bad3 = copy(city_mapping)
+        bad3['point'] = 'CURVE'
+
+        # Incrementing through the bad mapping dictionaries and
+        # ensuring that a LayerMapError is raised.
+        for bad_map in (bad1, bad2, bad3):
+            try:
+                lm = LayerMapping(City, city_shp, bad_map)
+            except LayerMapError:
+                pass
+            else:
+                self.fail('Expected a LayerMapError.')
+
+        # A LookupError should be thrown for bogus encodings.
+        try:
+            lm = LayerMapping(City, city_shp, city_mapping, encoding='foobar')
+        except LookupError:
+            pass
+        else:
+            self.fail('Expected a LookupError')
+
+    def test02_simple_layermap(self):
+        "Test LayerMapping import of a simple point shapefile."
+
+        # Setting up for the LayerMapping.
+        lm = LayerMapping(City, city_shp, city_mapping)
+        lm.save()
+
+        # There should be three cities in the shape file.
+        self.assertEqual(3, City.objects.count())
+
+        # Opening up the shapefile, and verifying the values in each
+        # of the features made it to the model.
+        ds = DataSource(city_shp)
+        layer = ds[0]
+        for feat in layer:
+            city = City.objects.get(name=feat['Name'].value)
+            self.assertEqual(feat['Population'].value, city.population)
+            self.assertEqual(Decimal(str(feat['Density'])), city.density)
+            self.assertEqual(feat['Created'].value, city.date)
+
+            # Comparing the geometries.
+            pnt1, pnt2 = feat.geom, city.point
+            self.assertAlmostEqual(pnt1.x, pnt2.x, 6)
+            self.assertAlmostEqual(pnt1.y, pnt2.y, 6)
+
+    def test03_layermap_strict(self):
+        "Testing the `strict` keyword, and import of a LineString shapefile."
+
+        # When the `strict` keyword is set an error encountered will force
+        # the importation to stop.
+        try:
+            lm = LayerMapping(Interstate, inter_shp, inter_mapping, 
+                              strict=True, silent=True)
+            lm.save()
+        except InvalidDecimal:
+            pass
+        else:
+            self.fail('Should have failed on strict import with invalid decimal values.')
+
+        # This LayerMapping should work b/c `strict` is not set.
+        lm = LayerMapping(Interstate, inter_shp, inter_mapping, silent=True)
+        lm.save()
+
+        # Only one interstate should have imported correctly.
+        self.assertEqual(1, Interstate.objects.count())
+        
+        # Verifying the values in the single feature w/the model.
+        ds = DataSource(inter_shp)
+        feat = ds[0][0]
+        istate = Interstate.objects.get(name=feat['Name'].value)
+        self.assertEqual(Decimal(str(feat['Length'])), istate.length)
+        for p1, p2 in zip(feat.geom, istate.path):
+            self.assertAlmostEqual(p1[0], p2[0], 6)
+            self.assertAlmostEqual(p1[1], p2[1], 6)
+
+def suite():
+    s = unittest.TestSuite()
+    s.addTest(unittest.makeSuite(LayerMapTest))
+    return s

django/contrib/gis/utils/layermapping.py

@@ -1,21 +1,19 @@
 # LayerMapping -- A Django Model/OGR Layer Mapping Utility
 """
-The LayerMapping class provides a way to map the contents of OGR
+ The LayerMapping class provides a way to map the contents of OGR
  vector files (e.g. SHP files) to Geographic-enabled Django models.
 
-This grew out of my personal needs, specifically the code repetition
+ This grew out of my personal needs, specifically the code repetition
  that went into pulling geometries and fields out of an OGR layer,
  converting to another coordinate system (e.g. WGS84), and then inserting
- into a Geographic Django model.
+ into a GeoDjango model.
 
-This utility is still in early stages of development, so its usage
+ This utility is still in early stages of development, so its usage
  is subject to change -- please report any bugs.
 
-TODO: Unit tests and documentation.
+ Requirements:  OGR C Library (from GDAL) required.
 
-Requirements:  OGR C Library (from GDAL) required.
-
-Usage: 
+ Usage: 
   lm = LayerMapping(model, source_file, mapping) where,
 
   model:
@@ -32,7 +30,7 @@ Usage:
    is a geographic then it should correspond to the OGR
    geometry type, e.g. 'POINT', 'LINESTRING', 'POLYGON'.
 
-Keyword Args:
+ Keyword Args:
   layer:
    The index of the layer to use from the Data Source (defaults to 0)
 
@@ -46,6 +44,26 @@ Keyword Args:
    For example, 'latin-1', 'utf-8', and 'cp437' are all valid
    encoding parameters.
 
+  check:
+   By default, LayerMapping increments through each feature in the
+   layer to ensure that it is compatible with the given model and
+   mapping.  Setting this keyword to False, disables this action,
+   which will speed up execution time for very large files.
+
+  silent:
+   By default, non-fatal error notifications are printed to stdout; this
+   keyword may be set in order to disable these notifications.
+
+  strict:
+   Setting this keyword to True will instruct the save() method to
+   cease execution on the first error encountered.
+
+  transaction_mode:
+   May be 'commit_on_success' (default) or 'autocommit'.
+
+  transform:
+   Setting this to False will disable all coordinate transformations.  
+
 Example:
 
  1. You need a GDAL-supported data source, like a shapefile.
@@ -94,22 +112,29 @@ Example:
  source spatial reference system (WGS84) to the spatial reference system of
  the GeoDjango model (NAD83).  If no spatial reference system is defined for
  the layer, use the `source_srs` keyword with a SpatialReference object to
-  specify one. Further, data is selectively imported from the given data source 
-  fields into the model fields.
+ specify one.
 """
-from types import StringType, TupleType
-from datetime import datetime
-from django.contrib.gis.db.backend import SPATIAL_BACKEND
-from django.contrib.gis.gdal import \
-     OGRGeometry, OGRGeomType, SpatialReference, CoordTransform, \
-     DataSource, OGRException
-from django.contrib.gis.gdal.field import Field, OFTInteger, OFTReal, OFTString, OFTDateTime
-from django.contrib.gis.models import GeometryColumns, SpatialRefSys
-from django.db import connection, transaction
+from datetime import date, datetime
+from decimal import Decimal
 from django.core.exceptions import ObjectDoesNotExist
+from django.db import connection, transaction
+from django.db.models.fields.related import ForeignKey
+from django.contrib.gis.db.backend import SPATIAL_BACKEND
+from django.contrib.gis.gdal import CoordTransform, DataSource, \
+    OGRException, OGRGeometry, OGRGeomType, SpatialReference
+from django.contrib.gis.gdal.field import OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime
+from django.contrib.gis.models import GeometryColumns, SpatialRefSys
 
-# A mapping of given geometry types to their OGR integer type.
-ogc_types = {'POINT' : OGRGeomType('Point'),
+# LayerMapping exceptions.
+class LayerMapError(Exception): pass
+class InvalidString(LayerMapError): pass
+class InvalidDecimal(LayerMapError): pass
+
+class LayerMapping(object):
+    "A class that maps OGR Layers to GeoDjango Models."
+    
+    # A mapping of given geometry types to their OGR integer type.
+    OGC_TYPES = {'POINT' : OGRGeomType('Point'),
                  'LINESTRING' : OGRGeomType('LineString'),
                  'POLYGON' : OGRGeomType('Polygon'),
                  'MULTIPOINT' : OGRGeomType('MultiPoint'),
@@ -118,135 +143,52 @@ ogc_types = {'POINT' : OGRGeomType('Point'),
                  'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'),
                  }
 
-# The django.contrib.gis model types.
-gis_fields = {'PointField' : 'POINT',
+    # The django.contrib.gis model types.
+    GIS_FIELDS = {'PointField' : 'POINT',
                   'LineStringField': 'LINESTRING',
                   'PolygonField': 'POLYGON',
                   'MultiPointField' : 'MULTIPOINT',
                   'MultiLineStringField' : 'MULTILINESTRING',
                   'MultiPolygonField' : 'MULTIPOLYGON',
+                  'GeometryCollectionField' : 'GEOMETRYCOLLECTION',
                   }
 
-# Acceptable 'base' types for a multi-geometry type.
-multi_types = {'POINT' : OGRGeomType('MultiPoint'),
+    # Acceptable 'base' types for a multi-geometry type.
+    MULTI_TYPES = {'POINT' : OGRGeomType('MultiPoint'),
                    'LINESTRING' : OGRGeomType('MultiLineString'),
                    'POLYGON' : OGRGeomType('MultiPolygon'),
                    }
 
-def map_foreign_key(django_field):
-    from django.db.models.fields.related import ForeignKey
-
-    if not django_field.__class__ is ForeignKey:
-        return django_field.__class__.__name__
-
-     
-    rf=django_field.rel.get_related_field()
-
-    return rf.get_internal_type()
-                
-# The acceptable Django field types that map to OGR fields.
-field_types = {
+    # The acceptable Django field types that map to OGR fields.
+    FIELD_TYPES = {
         'AutoField' : OFTInteger,
         'IntegerField' : OFTInteger,
         'FloatField' : OFTReal,
+        'DateField' : OFTDate,
         'DateTimeField' : OFTDateTime,
+        'TimeField' : OFTTime,
         'DecimalField' : OFTReal,
         'CharField' : OFTString,
+        'TextField' : OFTString,
         'SmallIntegerField' : OFTInteger,
         'PositiveSmallIntegerField' : OFTInteger,
         }
 
-def make_multi(geom_name, model_type):
-    "Determines whether the geometry should be made into a GeometryCollection."
-    if (geom_name in multi_types) and (model_type.startswith('Multi')):
-        return True
-    else:
-        return False
+    # The acceptable transaction modes.
+    TRANSACTION_MODES = {'autocommit' : transaction.autocommit,
+                         'commit_on_success' : transaction.commit_on_success,
+                         }
 
-def check_feature(feat, model_fields, mapping):
-    "Checks the OGR layer feature."
-
-    HAS_GEO = False
-
-    # Incrementing through each model_field & ogr_field in the given mapping.
-    for model_field, ogr_field in mapping.items():
-
-        # Making sure the given mapping model field is in the given model fields.
-        if model_field in model_fields:
-            model_type = model_fields[model_field]
-        elif model_field[:-3] in model_fields: #foreign key
-            model_type = model_fields[model_field[:-3]]
-        else:
-            raise Exception('Given mapping field "%s" not in given Model fields!' % model_field)
-
-        ### Handling if we get a geometry in the Field ###
-        if ogr_field in ogc_types:
-            # At this time, no more than one geographic field per model =(
-            if HAS_GEO:
-                raise Exception('More than one geographic field in mapping not allowed (yet).')
-            else:
-                HAS_GEO = ogr_field
-
-            # Making sure this geometry field type is a valid Django GIS field.
-            if not model_type in gis_fields:
-                raise Exception('Unknown Django GIS field type "%s"' % model_type)
-            
-            # Getting the OGRGeometry, it's type (an integer) and it's name (a string)
-            geom  = feat.geom
-            gtype = geom.geom_type
-            gname = geom.geom_name
-
-            if make_multi(gname, model_type):
-                # Do we have to 'upsample' into a Geometry Collection?
-                pass
-            elif gtype == ogc_types[gis_fields[model_type]]:
-                # The geometry type otherwise was expected
-                pass
-            else:
-                raise Exception('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype))
-
-        ## Handling other fields 
-        else:
-            # Making sure the model field is
-            if not model_type in field_types:
-                raise Exception('Django field type "%s" has no OGR mapping (yet).' % model_type)
-
-            # Otherwise, we've got an OGR Field.  Making sure that an
-            # index exists for the mapping OGR field.
-            try:
-                fi = feat.index(ogr_field)
-            except:
-                raise Exception('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field)
-                    
-def check_layer(layer, fields, mapping):
-    "Checks the OGR layer by incrementing through and checking each feature."
-    # Incrementing through each feature in the layer.
-    for feat in layer:
-        check_feature(feat, fields, mapping)
-
-def check_srs(layer, source_srs):
-    "Checks the compatibility of the given spatial reference object."
-    if isinstance(source_srs, SpatialReference):
-        sr = source_srs
-    elif isinstance(source_srs, SpatialRefSys):
-        sr = source_srs.srs
-    elif isinstance(source_srs, (int, str)):
-        sr = SpatialReference(source_srs)
-    else:
-        sr = layer.srs
-    if not sr:
-        raise Exception('No source reference system defined.')
-    else:
-        return sr
-
-class LayerMapping:
-    "A class that maps OGR Layers to Django Models."
-
-    def __init__(self, model, data, mapping, layer=0, source_srs=None, encoding=None):
+    def __init__(self, model, data, mapping, layer=0, 
+                 source_srs=None, encoding=None, check=True, 
+                 progress=False, interval=1000, strict=False, silent=False,
+                 transaction_mode='commit_on_success', transform=True):
         "Takes the Django model, the data source, and the mapping (dictionary)"
 
         # Getting the field names and types from the model
-        fields = dict((f.name, map_foreign_key(f)) for f in model._meta.fields)
+        self.fields = dict((f.name, self.map_foreign_key(f)) for f in model._meta.fields)
+        self.field_classes = dict((f.name, f) for f in model._meta.fields)
+
         # Getting the DataSource and its Layer
         if isinstance(data, basestring):
             self.ds = DataSource(data)
@@ -254,15 +196,31 @@ class LayerMapping:
             self.ds = data
         self.layer = self.ds[layer]
 
-        # Checking the layer -- intitialization of the object will fail if
-        #  things don't check out before hand.
-        check_layer(self.layer, fields, mapping)
-        
-        # Since the layer checked out, setting the fields and the mapping.
-        self.fields = fields
+        # Setting the mapping
         self.mapping = mapping
+
+        # Setting the model, and getting the geometry column associated 
+        # with the model (an exception will be raised if there is no 
+        # geometry column).
         self.model = model
-        self.source_srs = check_srs(self.layer, source_srs)
+        self.geo_col = self.geometry_column()
+
+        # Checking the source spatial reference system, and getting
+        # the coordinate transformation object (unless the `transform`
+        # keyword is set to False)
+        self.source_srs = self.check_srs(source_srs)
+        self.transform = transform and self.coord_transform()
+
+        # Checking the layer -- intitialization of the object will fail if
+        # things don't check out before hand.  This may be time-consuming,
+        # and disabled by setting the `check` keyword to False.
+        if check: self.check_layer()
+
+        # The silent, strict, progress, and interval flags.
+        self.silent = silent
+        self.strict = strict
+        self.progress = progress
+        self.interval = interval
 
         # Setting the encoding for OFTString fields, if specified.
         if encoding:
@@ -274,78 +232,133 @@ class LayerMapping:
         else:
             self.encoding = None
 
-    # Either the import will work, or it won't be committed.
-    @transaction.commit_on_success
-    def save(self, verbose=False):
-        "Runs the layer mapping on the given SHP file, and saves to the database."
+        # Setting the transaction decorator with the function in the 
+        # transaction modes dictionary.
+        if transaction_mode in self.TRANSACTION_MODES:
+            self.transaction_decorator = self.TRANSACTION_MODES[transaction_mode]
+            self.transaction_mode = transaction_mode
+        else:
+            raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode)
 
-        # Getting the GeometryColumn object.
+    def check_feature(self, feat):
+        "Checks the OGR feature against the model fields and mapping."
+        HAS_GEO = False   
+
+        # Incrementing through each model_field & ogr_field in the given mapping.
+        for model_field, ogr_field in self.mapping.items():
+            # Making sure the given mapping model field is in the given model fields.
+            if model_field in self.fields:
+                model_type = self.fields[model_field]
+            elif model_field[:-3] in self.fields: #foreign key
+                model_type = self.fields[model_field[:-3]]
+            else:
+                raise LayerMapError('Given mapping field "%s" not in given Model fields!' % model_field)
+
+            ### Handling if we get a geometry in the Field ###
+            if ogr_field in self.OGC_TYPES:
+                # At this time, no more than one geographic field per model =(
+                if HAS_GEO:
+                    raise LayerMapError('More than one geographic field in mapping not allowed (yet).')
+                else:
+                    HAS_GEO = ogr_field
+
+                # Making sure this geometry field type is a valid Django GIS field.
+                if not model_type in self.GIS_FIELDS:
+                    raise LayerMapError('Unknown Django GIS field type "%s"' % model_type)
+
+                # Getting the OGRGeometry, it's type (an integer) and it's name (a string)
+                geom  = feat.geom
+                gtype = geom.geom_type
+                gname = geom.geom_name
+
+                if self.make_multi(gname, model_type):
+                    # Do we have to 'upsample' into a Geometry Collection?
+                    pass
+                elif gtype == self.OGC_TYPES[self.GIS_FIELDS[model_type]]:
+                    # The geometry type otherwise was expected
+                    pass
+                else:
+                    raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype))
+            ### Handling other fields ###
+            else:
+                # Making sure the model field is supported.
+                if not model_type in self.FIELD_TYPES:
+                    raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % model_type)
+
+                # Otherwise, we've got an OGR Field.  Making sure that an
+                # index exists for the mapping OGR field.
                 try:
-            db_table = self.model._meta.db_table
-            if SPATIAL_BACKEND == 'oracle': db_table = db_table.upper()
-            gc_kwargs = {GeometryColumns.table_name_col() : db_table}
-            geo_col = GeometryColumns.objects.get(**gc_kwargs)
+                    fi = feat.index(ogr_field)
                 except:
-            raise Exception('Geometry column does not exist. (did you run syncdb?)')
+                    raise LayerMapError('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field)
 
-        # Getting the coordinate system needed for transformation (with CoordTransform)  
+    def check_layer(self):
+        "Checks every feature in this object's layer."
+        for feat in self.layer:
+            self.check_feature(feat)
+
+    def check_srs(self, source_srs):
+        "Checks the compatibility of the given spatial reference object."
+        if isinstance(source_srs, SpatialReference):
+            sr = source_srs
+        elif isinstance(source_srs, SpatialRefSys):
+            sr = source_srs.srs
+        elif isinstance(source_srs, (int, str)):
+            sr = SpatialReference(source_srs)
+        else:
+            # Otherwise just pulling the SpatialReference from the layer
+            sr = self.layer.srs
+            
+        if not sr:
+            raise LayerMapError('No source reference system defined.')
+        else:
+            return sr
+
+    def coord_transform(self):
+        "Returns the coordinate transformation object."
         try:
             # Getting the target spatial reference system
-            target_srs = SpatialRefSys.objects.get(srid=geo_col.srid).srs
+            target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs
         
             # Creating the CoordTransform object
-            ct = CoordTransform(self.source_srs, target_srs)
+            return CoordTransform(self.source_srs, target_srs)
         except Exception, msg:
-            raise Exception('Could not translate between the data source and model geometry: %s' % msg)
+            raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg)
 
-        for feat in self.layer:
-            # The keyword arguments for model construction
+    def feature_kwargs(self, feat):
+        "Returns the keyword arguments needed for saving a feature."
+        
+        # The keyword arguments for model construction.
         kwargs = {}
 
-            # Incrementing through each model field and the OGR field in the mapping
+        # The all_prepped flagged, will be set to False if there's a
+        # problem w/a ForeignKey that doesn't exist.
         all_prepped = True
 
+        # Incrementing through each model field and OGR field in the
+        # dictionary mapping.
         for model_field, ogr_field in self.mapping.items():
             is_fk = False
             try:
                 model_type = self.fields[model_field]
             except KeyError: #foreign key
+                # The -3 index is b/c foreign keys are appended w/'_id'.
                 model_type = self.fields[model_field[:-3]]
                 is_fk = True
             
-                if ogr_field in ogc_types:
-                    ## Getting the OGR geometry from the field
-                    geom = feat.geom
-
-                    if make_multi(geom.geom_name, model_type):
-                        # Constructing a multi-geometry type to contain the single geometry
-                        multi_type = multi_types[geom.geom_name]
-                        g = OGRGeometry(multi_type)
-                        g.add(geom)
+            if ogr_field in self.OGC_TYPES:
+                # Verify OGR geometry.
+                val = self.verify_geom(feat.geom, model_type)
             else:
-                        g = geom
-
-                    # Transforming the geometry with our Coordinate Transformation object.
-                    g.transform(ct)
-
-                    # Updating the keyword args with the WKT of the transformed model.
-                    val = g.wkt
-                else:
-                    ## Otherwise, this is an OGR field type
-                    fld = feat[ogr_field]
-
-                    if isinstance(fld, OFTString) and self.encoding:
-                        # The encoding for OGR data sources may be specified here
-                        #  (e.g., 'cp437' for Census Bureau boundary files).
-                        val = unicode(fld.value, self.encoding)
-                    else:
-                        val = fld.value
+                # Otherwise, verify OGR Field type.
+                val = self.verify_field(feat[ogr_field], model_field)
 
             if is_fk:
+                # Handling if foreign key.
                 rel_obj = None
                 field_name = model_field[:-3]
                 try:
-                        #FIXME: refactor to efficiently fetch FKs.
+                    # FIXME: refactor to efficiently fetch FKs.
                     #  Requires significant re-work. :-/
                     rel = self.model._meta.get_field(field_name).rel
                     rel_obj = rel.to._default_manager.get(**{('%s__exact' % rel.field_name):val})
@@ -356,18 +369,133 @@ class LayerMapping:
             else:
                 kwargs[model_field] = val
             
+        return kwargs, all_prepped
+
+    def verify_field(self, fld, model_field):
+        """
+        Verifies if the OGR Field contents are acceptable to the Django
+        model field.  If they are, the verified value is returned, 
+        otherwise the proper exception is raised.
+        """
+        field_class = self.field_classes[model_field]
+        if isinstance(fld, OFTString):
+            if self.encoding:
+                # The encoding for OGR data sources may be specified here
+                # (e.g., 'cp437' for Census Bureau boundary files).
+                val = unicode(fld.value, self.encoding)
+            else:
+                val = fld.value
+                if len(val) > field_class.max_length:
+                    raise InvalidString('%s model field maximum string length is %s, given %s characters.' %
+                                        (model_field, field_class.max_length, len(val)))
+        elif isinstance(fld, OFTReal):
+            try:
+                # Creating an instance of the Decimal value to use.
+                d = Decimal(str(fld.value))
+            except:
+                raise InvalidDecimal('Could not construct decimal from: %s' % fld)
+            dtup = d.as_tuple()
+            if len(dtup[1]) > field_class.max_digits:
+                raise InvalidDecimal('More than the maximum # of digits encountered.')
+            elif len(dtup[1][dtup[2]:]) > field_class.decimal_places:
+                raise InvalidDecimal('More than the maximum # of decimal places encountered.')
+            val = d
+        else:
+            val = fld.value
+        return val
+
+    def verify_geom(self, geom, model_type):
+        "Verifies the geometry."
+        if self.make_multi(geom.geom_name, model_type):
+            # Constructing a multi-geometry type to contain the single geometry
+            multi_type = self.MULTI_TYPES[geom.geom_name]
+            g = OGRGeometry(multi_type)
+            g.add(geom)
+        else:
+            g = geom
+
+        # Transforming the geometry with our Coordinate Transformation object,
+        # but only if the class variable `transform` is set w/a CoordTransform 
+        # object.
+        if self.transform: g.transform(self.transform)
+        
+        # Returning the WKT of the geometry.
+        return g.wkt
+        
+    def geometry_column(self):
+        "Returns the GeometryColumn model associated with the geographic column."
+        # Getting the GeometryColumn object.
+        try:
+            db_table = self.model._meta.db_table
+            if SPATIAL_BACKEND == 'oracle': db_table = db_table.upper()
+            gc_kwargs = {GeometryColumns.table_name_col() : db_table}
+            return GeometryColumns.objects.get(**gc_kwargs)
+        except Exception, msg:
+            raise LayerMapError('Geometry column does not exist for model. (did you run syncdb?):\n %s' % msg)
+
+    def make_multi(self, geom_name, model_type):
+        "Determines whether the geometry should be made into a GeometryCollection."
+        return (geom_name in self.MULTI_TYPES) and (model_type.startswith('Multi'))
+
+    def map_foreign_key(self, django_field):
+        "Handles fields within foreign keys for the given field."
+        if not django_field.__class__ is ForeignKey:
+            # Returning the field's class name.
+            return django_field.__class__.__name__
+        else:
+            # Otherwise, getting the type of the related field's
+            # from the Foreign key.
+            rf = django_field.rel.get_related_field()
+            return rf.get_internal_type()
+
+    def save(self, verbose=False):
+        "Runs the layer mapping on the given SHP file, and saves to the database."
+        
+        @self.transaction_decorator
+        def _save():
+            num_feat = 0
+            num_saved = 0
+
+            for feat in self.layer:
+                num_feat += 1
+                # Getting the keyword arguments
+                try:
+                    kwargs, all_prepped = self.feature_kwargs(feat)
+                except LayerMapError, msg:
+                    # Something borked the validation
+                    if self.strict: raise
+                    elif not self.silent: 
+                        print 'Ignoring Feature ID %s because: %s' % (feat.fid, msg)
+                else:
                     # Constructing the model using the constructed keyword args
                     if all_prepped:
                         m = self.model(**kwargs)
-
-                # Saving the model
                         try:
-                    if all_prepped:
                             m.save()
-                        if verbose: print 'Saved: %s' % str(m)                        
-                    else:
-                        print "Skipping %s due to missing relation." % kwargs
+                            num_saved += 1
+                            if verbose: print 'Saved: %s' % m
                         except SystemExit:
                             raise
-                except Exception, e:
-                    print "Failed to save %s\n  Continuing" % kwargs
+                        except Exception, msg:
+                            if self.transaction_mode == 'autocommit':
+                                # Rolling back the transaction so that other model saves
+                                # will work.
+                                transaction.rollback_unless_managed()
+                            if self.strict: 
+                                # Bailing out if the `strict` keyword is set.
+                                if not self.silent:
+                                    print 'Failed to save the feature (id: %s) into the model with the keyword arguments:' % feat.fid
+                                    print kwargs
+                                raise
+                            elif not self.silent:
+                                print 'Failed to save %s:\n %s\nContinuing' % (kwargs, msg)
+                    else:
+                        print 'Skipping %s due to missing relation.' % kwargs
+
+                # Printing progress information, if requested.
+                if self.progress and num_feat % self.interval == 0:
+                    print 'Processed %d features, saved %d ...' % (num_feat, num_saved)
+               
+        # Calling our defined function, which will use the specified
+        # trasaction mode.
+        _save()

django/contrib/gis/utils/ogrinfo.py

@@ -33,12 +33,12 @@ def ogrinfo(data_source, num_features=10):
 
         width = max(*map(len,layer.fields))
         fmt = " %%%ss: %%s" % width
-        for i, feature in enumerate(layer[:num_features]):
-            print "=== Feature %s" % i
-            for field in layer.fields:
-                fld_typ = feature[field].__class__.__name__.replace('OFT', '')
-                output = fmt % (field, fld_typ)
-                val = feature.get(field)
+        for j, feature in enumerate(layer[:num_features]):
+            print "=== Feature %s" % j
+            for fld_name in layer.fields:
+                type_name = feature[fld_name].type_name
+                output = fmt % (fld_name, type_name)
+                val = feature.get(fld_name)
                 if val:
                     if isinstance(val, str):
                         val_fmt = ' ("%s")'