diff --git a/django/contrib/gis/gdal/layer.py b/django/contrib/gis/gdal/layer.py index d4d8e1b7c6..a41b28819b 100644 --- a/django/contrib/gis/gdal/layer.py +++ b/django/contrib/gis/gdal/layer.py @@ -5,6 +5,7 @@ from ctypes import byref from django.contrib.gis.gdal.envelope import Envelope, OGREnvelope from django.contrib.gis.gdal.error import OGRException, OGRIndexError, SRSException from django.contrib.gis.gdal.feature import Feature +from django.contrib.gis.gdal.field import FIELD_CLASSES from django.contrib.gis.gdal.geometries import OGRGeomType from django.contrib.gis.gdal.srs import SpatialReference @@ -12,8 +13,8 @@ from django.contrib.gis.gdal.srs import SpatialReference from django.contrib.gis.gdal.prototypes.ds import \ get_extent, get_fd_geom_type, get_fd_name, get_feature, get_feature_count, \ get_field_count, get_field_defn, get_field_name, get_field_precision, \ - get_field_width, get_layer_defn, get_layer_srs, get_next_feature, \ - reset_reading + get_field_width, get_field_type, get_layer_defn, get_layer_srs, \ + get_next_feature, reset_reading from django.contrib.gis.gdal.prototypes.srs import clone_srs # For more information, see the OGR C API source code: @@ -107,10 +108,24 @@ class Layer(object): @property def fields(self): - "Returns a list of the fields available in this Layer." + """ + Returns a list of string names corresponding to each of the Fields + available in this Layer. + """ return [get_field_name(get_field_defn(self._ldefn, i)) for i in xrange(self.num_fields) ] + @property + def field_types(self): + """ + Returns a list of the types of fields in this Layer. For example, + the list [OFTInteger, OFTReal, OFTString] would be returned for + an OGR layer that had an integer, a floating-point, and string + fields. + """ + return [FIELD_CLASSES[get_field_type(get_field_defn(self._ldefn, i))] + for i in xrange(self.num_fields)] + @property def field_widths(self): "Returns a list of the maximum field widths for the features." diff --git a/django/contrib/gis/tests/layermap/counties/counties.dbf b/django/contrib/gis/tests/layermap/counties/counties.dbf index 23b9c18d0c..ccdbb264b9 100644 Binary files a/django/contrib/gis/tests/layermap/counties/counties.dbf and b/django/contrib/gis/tests/layermap/counties/counties.dbf differ diff --git a/django/contrib/gis/tests/layermap/counties/counties.shp b/django/contrib/gis/tests/layermap/counties/counties.shp index e2e867c449..2e7dca5e43 100644 Binary files a/django/contrib/gis/tests/layermap/counties/counties.shp and b/django/contrib/gis/tests/layermap/counties/counties.shp differ diff --git a/django/contrib/gis/tests/layermap/counties/counties.shx b/django/contrib/gis/tests/layermap/counties/counties.shx index 189450fd48..46ed3bb203 100644 Binary files a/django/contrib/gis/tests/layermap/counties/counties.shx and b/django/contrib/gis/tests/layermap/counties/counties.shx differ diff --git a/django/contrib/gis/tests/layermap/models.py b/django/contrib/gis/tests/layermap/models.py index 4b106fc5ba..fe7604f980 100644 --- a/django/contrib/gis/tests/layermap/models.py +++ b/django/contrib/gis/tests/layermap/models.py @@ -1,7 +1,12 @@ from django.contrib.gis.db import models +class State(models.Model): + name = models.CharField(max_length=20) + objects = models.GeoManager() + class County(models.Model): name = models.CharField(max_length=25) + state = models.ForeignKey(State) mpoly = models.MultiPolygonField(srid=4269) # Multipolygon in NAD83 objects = models.GeoManager() @@ -26,6 +31,7 @@ class Interstate(models.Model): # Mapping dictionaries for the models above. co_mapping = {'name' : 'Name', + 'state' : {'name' : 'State'}, # ForeignKey's use another mapping dictionary for the _related_ Model (State in this case). 'mpoly' : 'MULTIPOLYGON', # Will convert POLYGON features into MULTIPOLYGONS. } diff --git a/django/contrib/gis/tests/layermap/tests.py b/django/contrib/gis/tests/layermap/tests.py index 44ff980ed3..382cef56ce 100644 --- a/django/contrib/gis/tests/layermap/tests.py +++ b/django/contrib/gis/tests/layermap/tests.py @@ -2,8 +2,8 @@ import os, unittest from copy import copy from datetime import date from decimal import Decimal -from models import City, County, CountyFeat, Interstate, city_mapping, co_mapping, cofeat_mapping, inter_mapping -from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError, InvalidDecimal +from models import City, County, CountyFeat, Interstate, State, city_mapping, co_mapping, cofeat_mapping, inter_mapping +from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError, InvalidDecimal, MissingForeignKey from django.contrib.gis.gdal import DataSource shp_path = os.path.dirname(__file__) @@ -111,8 +111,8 @@ class LayerMapTest(unittest.TestCase): self.assertAlmostEqual(p1[0], p2[0], 6) self.assertAlmostEqual(p1[1], p2[1], 6) - def test04_layermap_unique_multigeometry(self): - "Testing the `unique`, and `transform` keywords and geometry collection conversion." + def test04_layermap_unique_multigeometry_fk(self): + "Testing the `unique`, and `transform`, geometry collection conversion, and ForeignKey mappings." # All the following should work. try: # Telling LayerMapping that we want no transformations performed on the data. @@ -135,6 +135,23 @@ class LayerMapTest(unittest.TestCase): # No source reference system defined in the shapefile, should raise an error. self.assertRaises(LayerMapError, LayerMapping, County, co_shp, co_mapping) + # Passing in invalid ForeignKey mapping parameters -- must be a dictionary + # mapping for the model the ForeignKey points to. + bad_fk_map1 = copy(co_mapping); bad_fk_map1['state'] = 'name' + bad_fk_map2 = copy(co_mapping); bad_fk_map2['state'] = {'nombre' : 'State'} + self.assertRaises(TypeError, LayerMapping, County, co_shp, bad_fk_map1, transform=False) + self.assertRaises(LayerMapError, LayerMapping, County, co_shp, bad_fk_map2, transform=False) + + # There exist no State models for the ForeignKey mapping to work -- should raise + # a MissingForeignKey exception (this error would be ignored if the `strict` + # keyword is not set). + lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name', silent=True, strict=True) + self.assertRaises(MissingForeignKey, lm.save) + + # Now creating the state models so the ForeignKey mapping may work. + co, hi, tx = State(name='Colorado'), State(name='Hawaii'), State(name='Texas') + co.save(), hi.save(), tx.save() + # If a mapping is specified as a collection, all OGR fields that # are not collections will be converted into them. For example, # a Point column would be converted to MultiPoint. Other things being done @@ -148,21 +165,26 @@ class LayerMapTest(unittest.TestCase): # appended to the geometry collection of the unique model. Thus, # all of the various islands in Honolulu county will be in in one # database record with a MULTIPOLYGON type. - lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name', silent=True) + lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name', silent=True, strict=True) lm.save() # A reference that doesn't use the unique keyword; a new database record will # created for each polygon. - lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False, silent=True) + lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False, silent=True, strict=True) lm.save() # Dictionary to hold what's expected in the shapefile. - exp = {'names' : ('Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo'), - 'num' : (1, 2, 2, 19, 1), # Number of polygons for each. - } - for name, n in zip(exp['names'], exp['num']): - c = County.objects.get(name=name) # Should only be one record. + names = ('Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo') + nums = (1, 2, 1, 19, 1) # Number of polygons for each. + states = ('Texas', 'Texas', 'Texas', 'Hawaii', 'Colorado') + + for name, n, st in zip(names, nums, states): + # Should only be one record b/c of `unique` keyword. + c = County.objects.get(name=name) self.assertEqual(n, len(c.mpoly)) + self.assertEqual(st, c.state.name) # Checking ForeignKey mapping. + + # Multiple records because `unique` was not set. qs = CountyFeat.objects.filter(name=name) self.assertEqual(n, qs.count()) diff --git a/django/contrib/gis/utils/layermapping.py b/django/contrib/gis/utils/layermapping.py index cbc94d8139..ec102f24b6 100644 --- a/django/contrib/gis/utils/layermapping.py +++ b/django/contrib/gis/utils/layermapping.py @@ -45,10 +45,13 @@ encoding parameters. check: - By default, LayerMapping increments through each feature in the - layer to ensure that it is compatible with the given model and - mapping. Setting this keyword to False, disables this action, - which will speed up execution time for very large files. + Due to optimizations, this keyword argument is deprecated and will + be removed in future revisions. + + pipe: + Status information will be written to this file handle. Defaults + to using `sys.stdout`, but any object with a `write` method is + supported. silent: By default, non-fatal error notifications are printed to stdout; this @@ -56,7 +59,8 @@ strict: Setting this keyword to True will instruct the save() method to - cease execution on the first error encountered. + cease execution on the first error encountered. The default behavior + is to attempt to continue even if errors are encountered. transaction_mode: May be 'commit_on_success' (default) or 'autocommit'. @@ -121,64 +125,48 @@ Example: the layer, use the `source_srs` keyword with a SpatialReference object to specify one. """ +import sys from datetime import date, datetime from decimal import Decimal from django.core.exceptions import ObjectDoesNotExist -from django.db import connection, transaction -from django.db.models.fields.related import ForeignKey +from django.contrib.gis.db.models.fields import GeometryField from django.contrib.gis.db.backend import SPATIAL_BACKEND from django.contrib.gis.gdal import CoordTransform, DataSource, \ OGRException, OGRGeometry, OGRGeomType, SpatialReference -from django.contrib.gis.gdal.field import OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime +from django.contrib.gis.gdal.field import \ + OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime from django.contrib.gis.models import GeometryColumns, SpatialRefSys +from django.db import models, transaction # LayerMapping exceptions. class LayerMapError(Exception): pass class InvalidString(LayerMapError): pass class InvalidDecimal(LayerMapError): pass +class MissingForeignKey(LayerMapError): pass class LayerMapping(object): "A class that maps OGR Layers to GeoDjango Models." - # A mapping of given geometry types to their OGR integer type. - OGC_TYPES = {'POINT' : OGRGeomType('Point'), - 'LINESTRING' : OGRGeomType('LineString'), - 'POLYGON' : OGRGeomType('Polygon'), - 'MULTIPOINT' : OGRGeomType('MultiPoint'), - 'MULTILINESTRING' : OGRGeomType('MultiLineString'), - 'MULTIPOLYGON' : OGRGeomType('MultiPolygon'), - 'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'), - } - - # The django.contrib.gis model types. - GIS_FIELDS = {'PointField' : 'POINT', - 'LineStringField': 'LINESTRING', - 'PolygonField': 'POLYGON', - 'MultiPointField' : 'MULTIPOINT', - 'MultiLineStringField' : 'MULTILINESTRING', - 'MultiPolygonField' : 'MULTIPOLYGON', - 'GeometryCollectionField' : 'GEOMETRYCOLLECTION', - } - # Acceptable 'base' types for a multi-geometry type. - MULTI_TYPES = {'POINT' : OGRGeomType('MultiPoint'), - 'LINESTRING' : OGRGeomType('MultiLineString'), - 'POLYGON' : OGRGeomType('MultiPolygon'), + MULTI_TYPES = {1 : OGRGeomType('MultiPoint'), + 2 : OGRGeomType('MultiLineString'), + 3 : OGRGeomType('MultiPolygon'), } - # The acceptable Django field types that map to OGR fields. + # Acceptable Django field types and corresponding acceptable OGR + # counterparts. FIELD_TYPES = { - 'AutoField' : OFTInteger, - 'IntegerField' : OFTInteger, - 'FloatField' : OFTReal, - 'DateField' : OFTDate, - 'DateTimeField' : OFTDateTime, - 'TimeField' : OFTTime, - 'DecimalField' : OFTReal, - 'CharField' : OFTString, - 'TextField' : OFTString, - 'SmallIntegerField' : OFTInteger, - 'PositiveSmallIntegerField' : OFTInteger, + models.AutoField : OFTInteger, + models.IntegerField : (OFTInteger, OFTReal), + models.FloatField : (OFTInteger, OFTReal), + models.DateField : OFTDate, + models.DateTimeField : OFTDateTime, + models.TimeField : OFTTime, + models.DecimalField : (OFTInteger, OFTReal), + models.CharField : OFTString, + models.TextField : OFTString, + models.SmallIntegerField : (OFTInteger, OFTReal), + models.PositiveSmallIntegerField : (OFTInteger, OFTReal), } # The acceptable transaction modes. @@ -187,17 +175,17 @@ class LayerMapping(object): } def __init__(self, model, data, mapping, layer=0, - source_srs=None, encoding=None, check=True, + source_srs=None, encoding=None, check=True, pipe=sys.stdout, progress=False, interval=1000, strict=False, silent=False, transaction_mode='commit_on_success', transform=True, unique=False): - "Takes the Django model, the data source, and the mapping (dictionary)" - - # Getting the field names and types from the model - self.fields = dict((f.name, self.map_foreign_key(f)) for f in model._meta.fields) - self.field_classes = dict((f.name, f) for f in model._meta.fields) - - # Getting the DataSource and its Layer + """ + A LayerMapping object is initialized using the given Model (not an instance), + a DataSource (or string path to an OGR-supported data file), and a mapping + dictionary. See the module level docstring for more details and keyword + argument usage. + """ + # Getting the DataSource and the associated Layer. if isinstance(data, basestring): self.ds = DataSource(data) else: @@ -223,14 +211,16 @@ class LayerMapping(object): self.transform = transform # Checking the layer -- intitialization of the object will fail if - # things don't check out before hand. This may be time-consuming, - # and disabled by setting the `check` keyword to False. - if check: self.check_layer() + # things don't check out before hand. + self.check_layer() - # The silent, strict, progress, and interval flags. - self.silent = silent + # The strict flag -- if it is set, exceptions will be propagated up. self.strict = strict + + # Setting the keyword arguments related to status printing. + self.silent = silent self.progress = progress + self.pipe = pipe self.interval = interval # Setting the encoding for OFTString fields, if specified. @@ -257,63 +247,92 @@ class LayerMapping(object): self.transaction_mode = transaction_mode else: raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode) - - def check_feature(self, feat): - "Checks the OGR feature against the model fields and mapping." - HAS_GEO = False - - # Incrementing through each model_field & ogr_field in the given mapping. - for model_field, ogr_field in self.mapping.items(): - # Making sure the given mapping model field is in the given model fields. - if model_field in self.fields: - model_type = self.fields[model_field] - elif model_field[:-3] in self.fields: #foreign key - model_type = self.fields[model_field[:-3]] - else: - raise LayerMapError('Given mapping field "%s" not in given Model fields!' % model_field) - - ### Handling if we get a geometry in the Field ### - if ogr_field in self.OGC_TYPES: - # At this time, no more than one geographic field per model =( - if HAS_GEO: - raise LayerMapError('More than one geographic field in mapping not allowed (yet).') - else: - HAS_GEO = ogr_field - - # Making sure this geometry field type is a valid Django GIS field. - if not model_type in self.GIS_FIELDS: - raise LayerMapError('Unknown Django GIS field type "%s"' % model_type) - - # Getting the OGRGeometry, it's type (an integer) and it's name (a string) - geom = feat.geom - gtype = geom.geom_type - gname = geom.geom_name - - if self.make_multi(gname, model_type): - # Do we have to 'upsample' into a Geometry Collection? - pass - elif gtype == self.OGC_TYPES[self.GIS_FIELDS[model_type]]: - # The geometry type otherwise was expected - pass - else: - raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype)) - ### Handling other fields ### - else: - # Making sure the model field is supported. - if not model_type in self.FIELD_TYPES: - raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % model_type) - - # Otherwise, we've got an OGR Field. Making sure that an - # index exists for the mapping OGR field. - try: - fi = feat.index(ogr_field) - except: - raise LayerMapError('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field) - + + #### Checking routines used during initialization #### def check_layer(self): - "Checks every feature in this object's layer." - for feat in self.layer: - self.check_feature(feat) + """ + This checks the Layer metadata, and ensures that it is compatible + with the mapping information and model. Unlike previous revisions, + there is no need to increment through each feature in the Layer. + """ + # The geometry field of the model is set here. + # TODO: Support more than one geometry field / model. + self.geom_field = False + self.fields = {} + + # Getting lists of the field names and the field types available in + # the OGR Layer. + ogr_fields = self.layer.fields + ogr_field_types = self.layer.field_types + + # Function for determining if the OGR mapping field is in the Layer. + def check_ogr_fld(ogr_map_fld): + try: + idx = ogr_fields.index(ogr_map_fld) + except ValueError: + raise LayerMapError('Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld) + return idx + + # No need to increment through each feature in the model, simply check + # the Layer metadata against what was given in the mapping dictionary. + for field_name, ogr_name in self.mapping.items(): + # Ensuring that a corresponding field exists in the model + # for the given field name in the mapping. + try: + model_field = self.model._meta.get_field(field_name) + except models.fields.FieldDoesNotExist: + raise LayerMapError('Given mapping field "%s" not in given Model fields.' % field_name) + + # Getting the string name for the Django field class (e.g., 'PointField'). + fld_name = model_field.__class__.__name__ + + if isinstance(model_field, GeometryField): + if self.geom_field: + raise LayerMapError('LayerMapping does not support more than one GeometryField per model.') + + try: + gtype = OGRGeomType(ogr_name) + except OGRException: + raise LayerMapError('Invalid mapping for GeometryField "%s".' % field_name) + + # Making sure that the OGR Layer's Geometry is compatible. + ltype = self.layer.geom_type + if not (gtype == ltype or self.make_multi(ltype, model_field)): + raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s.' % (fld_name, gtype)) + + # Setting the `geom_field` attribute w/the name of the model field + # that is a Geometry. + self.geom_field = field_name + fields_val = model_field + elif isinstance(model_field, models.ForeignKey): + if isinstance(ogr_name, dict): + # Is every given related model mapping field in the Layer? + rel_model = model_field.rel.to + for rel_name, ogr_field in ogr_name.items(): + idx = check_ogr_fld(ogr_field) + try: + rel_field = rel_model._meta.get_field(rel_name) + except models.fields.FieldDoesNotExist: + raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' % + (rel_name, rel_model.__class__.__name__)) + fields_val = rel_model + else: + raise TypeError('ForeignKey mapping must be of dictionary type.') + else: + # Is the model field type supported by LayerMapping? + if not model_field.__class__ in self.FIELD_TYPES: + raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % fld_name) + + # Is the OGR field in the Layer? + idx = check_ogr_fld(ogr_name) + + # Can the OGR field type be mapped to the Django field type? + if not issubclass(ogr_field_types[idx], self.FIELD_TYPES[model_field.__class__]): + raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' % + (ogr_field, ogr_field_types[idx].__name__, fld_name)) + fields_val = model_field + + self.fields[field_name] = fields_val def check_srs(self, source_srs): "Checks the compatibility of the given spatial reference object." @@ -321,12 +340,12 @@ class LayerMapping(object): sr = source_srs elif isinstance(source_srs, SpatialRefSys): sr = source_srs.srs - elif isinstance(source_srs, (int, str)): + elif isinstance(source_srs, (int, basestring)): sr = SpatialReference(source_srs) else: # Otherwise just pulling the SpatialReference from the layer sr = self.layer.srs - + if not sr: raise LayerMapError('No source reference system defined.') else: @@ -334,14 +353,6 @@ class LayerMapping(object): def check_unique(self, unique): "Checks the `unique` keyword parameter -- may be a sequence or string." - # Getting the geometry field; only the first encountered GeometryField - # will be used. - self.geom_field = False - for model_field, ogr_fld in self.mapping.items(): - if ogr_fld in self.OGC_TYPES: - self.geom_field = model_field - break - if isinstance(unique, (list, tuple)): # List of fields to determine uniqueness with for attr in unique: @@ -352,20 +363,14 @@ class LayerMapping(object): else: raise TypeError('Unique keyword argument must be set with a tuple, list, or string.') - def coord_transform(self): - "Returns the coordinate transformation object." - try: - # Getting the target spatial reference system - target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs - - # Creating the CoordTransform object - return CoordTransform(self.source_srs, target_srs) - except Exception, msg: - raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg) - + #### Keyword argument retrieval routines #### def feature_kwargs(self, feat): - "Returns the keyword arguments needed for saving a feature." - + """ + Given an OGR Feature, this will return a dictionary of keyword arguments + for constructing the mapped model. Also returned is the `all_prepped` + flag, which is used to signal that a model corresponding to a ForeignKey + mapping does not exist. + """ # The keyword arguments for model construction. kwargs = {} @@ -375,37 +380,24 @@ class LayerMapping(object): # Incrementing through each model field and OGR field in the # dictionary mapping. - for model_field, ogr_field in self.mapping.items(): - is_fk = False - try: - model_type = self.fields[model_field] - except KeyError: #foreign key - # The -3 index is b/c foreign keys are appended w/'_id'. - model_type = self.fields[model_field[:-3]] - is_fk = True + for field_name, ogr_name in self.mapping.items(): + model_field = self.fields[field_name] - if ogr_field in self.OGC_TYPES: + if isinstance(model_field, GeometryField): # Verify OGR geometry. - val = self.verify_geom(feat.geom, model_type) + val = self.verify_geom(feat.geom, model_field) + elif isinstance(model_field, models.base.ModelBase): + # The related _model_, not a field was passed in -- indicating + # another mapping for the related Model. + val = self.verify_fk(feat, model_field, ogr_name) + if not val: all_prepped = False else: # Otherwise, verify OGR Field type. - val = self.verify_field(feat[ogr_field], model_field) + val = self.verify_ogr_field(feat[ogr_name], model_field) - if is_fk: - # Handling if foreign key. - rel_obj = None - field_name = model_field[:-3] - try: - # FIXME: refactor to efficiently fetch FKs. - # Requires significant re-work. :-/ - rel = self.model._meta.get_field(field_name).rel - rel_obj = rel.to._default_manager.get(**{('%s__exact' % rel.field_name):val}) - except ObjectDoesNotExist: - all_prepped = False - - kwargs[model_field[:-3]] = rel_obj - else: - kwargs[model_field] = val + # Setting the keyword arguments for the field name with the + # value obtained above. + kwargs[field_name] = val return kwargs, all_prepped @@ -420,29 +412,29 @@ class LayerMapping(object): else: return dict((fld, kwargs[fld]) for fld in self.unique) - def verify_field(self, fld, model_field): + #### Verification routines used in constructing model keyword arguments. #### + def verify_ogr_field(self, ogr_field, model_field): """ Verifies if the OGR Field contents are acceptable to the Django model field. If they are, the verified value is returned, otherwise the proper exception is raised. """ - field_class = self.field_classes[model_field] - if isinstance(fld, OFTString): + if isinstance(ogr_field, OFTString): if self.encoding: # The encoding for OGR data sources may be specified here # (e.g., 'cp437' for Census Bureau boundary files). - val = unicode(fld.value, self.encoding) + val = unicode(ogr_field.value, self.encoding) else: - val = fld.value - if len(val) > field_class.max_length: + val = ogr_field.value + if len(val) > model_field.max_length: raise InvalidString('%s model field maximum string length is %s, given %s characters.' % - (model_field, field_class.max_length, len(val))) - elif isinstance(fld, OFTReal): + (model_field.name, model_field.max_length, len(val))) + elif isinstance(ogr_field, OFTReal): try: # Creating an instance of the Decimal value to use. - d = Decimal(str(fld.value)) + d = Decimal(str(ogr_field.value)) except: - raise InvalidDecimal('Could not construct decimal from: %s' % fld) + raise InvalidDecimal('Could not construct decimal from: %s' % ogr_field) # Getting the decimal value as a tuple. dtup = d.as_tuple() @@ -450,7 +442,7 @@ class LayerMapping(object): d_idx = dtup[2] # index where the decimal is # Maximum amount of precision, or digits to the left of the decimal. - max_prec = field_class.max_digits - field_class.decimal_places + max_prec = model_field.max_digits - model_field.decimal_places # Getting the digits to the left of the decimal place for the # given decimal. @@ -463,17 +455,43 @@ class LayerMapping(object): # InvalidDecimal exception. if n_prec > max_prec: raise InvalidDecimal('A DecimalField with max_digits %d, decimal_places %d must round to an absolute value less than 10^%d.' % - (field_class.max_digits, field_class.decimal_places, max_prec)) + (model_field.max_digits, model_field.decimal_places, max_prec)) val = d else: - val = fld.value + val = ogr_field.value return val - def verify_geom(self, geom, model_type): - "Verifies the geometry." - if self.make_multi(geom.geom_name, model_type): + def verify_fk(self, feat, rel_model, rel_mapping): + """ + Given an OGR Feature, the related model and its dictionary mapping, + this routine will retrieve the related model for the ForeignKey + mapping. + """ + # TODO: It is expensive to retrieve a model for every record -- + # explore if an efficient mechanism exists for caching related + # ForeignKey models. + + # Constructing and verifying the related model keyword arguments. + fk_kwargs = {} + for field_name, ogr_name in rel_mapping.items(): + fk_kwargs[field_name] = self.verify_ogr_field(feat[ogr_name], rel_model._meta.get_field(field_name)) + + # Attempting to retrieve and return the related model. + try: + return rel_model.objects.get(**fk_kwargs) + except ObjectDoesNotExist: + if self.strict: raise MissingForeignKey('No %s model found with keyword arguments: %s' % (rel_model.__name__, fk_kwargs)) + else: return None + + def verify_geom(self, geom, model_field): + """ + Verifies the geometry -- will construct and return a GeometryCollection + if necessary (for example if the model field is MultiPolygonField while + the mapped shapefile only contains Polygons). + """ + if self.make_multi(geom.geom_type, model_field): # Constructing a multi-geometry type to contain the single geometry - multi_type = self.MULTI_TYPES[geom.geom_name] + multi_type = self.MULTI_TYPES[geom.geom_type.num] g = OGRGeometry(multi_type) g.add(geom) else: @@ -486,7 +504,19 @@ class LayerMapping(object): # Returning the WKT of the geometry. return g.wkt - + + #### Other model methods #### + def coord_transform(self): + "Returns the coordinate transformation object." + try: + # Getting the target spatial reference system + target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs + + # Creating the CoordTransform object + return CoordTransform(self.source_srs, target_srs) + except Exception, msg: + raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg) + def geometry_column(self): "Returns the GeometryColumn model associated with the geographic column." # Getting the GeometryColumn object. @@ -498,24 +528,23 @@ class LayerMapping(object): except Exception, msg: raise LayerMapError('Geometry column does not exist for model. (did you run syncdb?):\n %s' % msg) - def make_multi(self, geom_name, model_type): - "Determines whether the geometry should be made into a GeometryCollection." - return (geom_name in self.MULTI_TYPES) and (model_type.startswith('Multi')) - - def map_foreign_key(self, django_field): - "Handles fields within foreign keys for the given field." - if not django_field.__class__ is ForeignKey: - # Returning the field's class name. - return django_field.__class__.__name__ - else: - # Otherwise, getting the type of the related field's - # from the Foreign key. - rf = django_field.rel.get_related_field() - return rf.get_internal_type() + def make_multi(self, geom_type, model_field): + """ + Given the OGRGeomType for a geometry and its associated GeometryField, + determine whether the geometry should be turned into a GeometryCollection. + """ + return (geom_type.num in self.MULTI_TYPES and + model_field.__class__.__name__ == 'Multi%s' % geom_type.django) def save(self, verbose=False): - "Runs the layer mapping on the given SHP file, and saves to the database." - + """ + Saves the contents from the OGR DataSource Layer into the database + according to the mapping dictionary given at initialization. If + the `verbose` keyword is set, information will be printed subsequent + to each model save executed on the database. + """ + # Defining the 'real' save method, utilizing the transaction + # decorator created during initialization. @self.transaction_decorator def _save(): num_feat = 0 @@ -530,7 +559,7 @@ class LayerMapping(object): # Something borked the validation if self.strict: raise elif not self.silent: - print 'Ignoring Feature ID %s because: %s' % (feat.fid, msg) + self.pipe.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg)) else: # Constructing the model using the keyword args if all_prepped: @@ -561,7 +590,7 @@ class LayerMapping(object): # Attempting to save. m.save() num_saved += 1 - if verbose: print 'Saved: %s' % m + if verbose: self.pipe.write('Saved: %s\n' % m) except SystemExit: raise except Exception, msg: @@ -572,17 +601,18 @@ class LayerMapping(object): if self.strict: # Bailing out if the `strict` keyword is set. if not self.silent: - print 'Failed to save the feature (id: %s) into the model with the keyword arguments:' % feat.fid - print kwargs + self.pipe.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid) + self.pipe.write('%s\n' % kwargs) raise elif not self.silent: - print 'Failed to save %s:\n %s\nContinuing' % (kwargs, msg) + self.pipe.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg)) else: - print 'Skipping %s due to missing relation.' % kwargs + if not self.silent: self.pipe.write('Skipping due to missing relation:\n%s\n' % kwargs) + # Printing progress information, if requested. if self.progress and num_feat % self.interval == 0: - print 'Processed %d features, saved %d ...' % (num_feat, num_saved) + self.pipe.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved)) # Calling our defined function, which will use the specified # trasaction mode.