From b11172e4e72299d34c27eff635ad034f84a0cdb6 Mon Sep 17 00:00:00 2001 From: Justin Bronn Date: Thu, 3 Jan 2008 19:02:18 +0000 Subject: [PATCH] gis: `LayerMapping`: Improved the internals (i.e., checking every feature in OGR Layer is no longer needed, removed unnecessary class constants); added real support `ForeignKey` model fields; added `field_types` property to `Layer`; fixed county shapefile because of Harris County, Georgia. git-svn-id: http://code.djangoproject.com/svn/django/branches/gis@6992 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/contrib/gis/gdal/layer.py | 21 +- .../gis/tests/layermap/counties/counties.dbf | Bin 2090 -> 3961 bytes .../gis/tests/layermap/counties/counties.shp | Bin 41628 -> 37364 bytes .../gis/tests/layermap/counties/counties.shx | Bin 300 -> 292 bytes django/contrib/gis/tests/layermap/models.py | 6 + django/contrib/gis/tests/layermap/tests.py | 44 +- django/contrib/gis/utils/layermapping.py | 434 ++++++++++-------- 7 files changed, 289 insertions(+), 216 deletions(-) diff --git a/django/contrib/gis/gdal/layer.py b/django/contrib/gis/gdal/layer.py index d4d8e1b7c6..a41b28819b 100644 --- a/django/contrib/gis/gdal/layer.py +++ b/django/contrib/gis/gdal/layer.py @@ -5,6 +5,7 @@ from ctypes import byref from django.contrib.gis.gdal.envelope import Envelope, OGREnvelope from django.contrib.gis.gdal.error import OGRException, OGRIndexError, SRSException from django.contrib.gis.gdal.feature import Feature +from django.contrib.gis.gdal.field import FIELD_CLASSES from django.contrib.gis.gdal.geometries import OGRGeomType from django.contrib.gis.gdal.srs import SpatialReference @@ -12,8 +13,8 @@ from django.contrib.gis.gdal.srs import SpatialReference from django.contrib.gis.gdal.prototypes.ds import \ get_extent, get_fd_geom_type, get_fd_name, get_feature, get_feature_count, \ get_field_count, get_field_defn, get_field_name, get_field_precision, \ - get_field_width, get_layer_defn, get_layer_srs, get_next_feature, \ - reset_reading + get_field_width, get_field_type, get_layer_defn, get_layer_srs, \ + get_next_feature, reset_reading from django.contrib.gis.gdal.prototypes.srs import clone_srs # For more information, see the OGR C API source code: @@ -107,10 +108,24 @@ class Layer(object): @property def fields(self): - "Returns a list of the fields available in this Layer." + """ + Returns a list of string names corresponding to each of the Fields + available in this Layer. + """ return [get_field_name(get_field_defn(self._ldefn, i)) for i in xrange(self.num_fields) ] + @property + def field_types(self): + """ + Returns a list of the types of fields in this Layer. For example, + the list [OFTInteger, OFTReal, OFTString] would be returned for + an OGR layer that had an integer, a floating-point, and string + fields. + """ + return [FIELD_CLASSES[get_field_type(get_field_defn(self._ldefn, i))] + for i in xrange(self.num_fields)] + @property def field_widths(self): "Returns a list of the maximum field widths for the features." diff --git a/django/contrib/gis/tests/layermap/counties/counties.dbf b/django/contrib/gis/tests/layermap/counties/counties.dbf index 23b9c18d0c5267ea9a1e6483a25f0b4dfa6dfbd4..ccdbb264b9365ec7d406c91a322683180177c1ca 100644 GIT binary patch literal 3961 zcmeIxu?hk)3~x zvfgc-bLd{}%g+&SVV-uMyI-uF3YK}{>;Db(H4KSB?R1nV<5e%wW6p6>>j&rXM8!CO zeGANb{qzeSFx|scXjCexF8KEaJ&lxVs!D+1@qsaFrL`O@_$#IfnxIg?Ua=62t(zbt c5e`gH2*%b;kdX)nCMX1B>n6xZgad8CHwdvsO8@`> literal 2090 zcmZRMXP1&>U|?`$2n3Rtz%Ma36)Nfsq65&B@G1n9rY7a&D^L%3Bo-BA7E{kW1*#k9 zlvFgno~+G2P%vjNO@3>hT&)!k`V%<$%c$T KL(?+y!Vmym#5hs_ diff --git a/django/contrib/gis/tests/layermap/counties/counties.shp b/django/contrib/gis/tests/layermap/counties/counties.shp index e2e867c449c7f71d41c61b8a7dc88b7d24b14c6f..2e7dca5e438cfbf386e92bb2a2ce06d6b86907c9 100644 GIT binary patch delta 171 zcmV;c095~+#RBxB0+1R4Ncxc+7zcK_kr%s{SFtK6< z4U@o#rvVO=Xo=?m50g-e_W=--NQF0Tq*g ZkH-NPlSq)40T;92kf#s<7_*3>6aixzKkxtm delta 4466 zcmZA5c|25I0|#&;LX;&!h3et4w%AE=i1|i8>c3HAEBBoTNNl1lKgOI(&(v?an zSz0W46ct4(S}p1Q-S_kMdHTHm7@zO&oI7{UnRCxFUoLQ-xx;yUF{iaf+ENJ~(?^Hb zS1HrW^_u50c&)fOI5;lmGxMIB(Mk0SGV|G{1o6Kbq*nOhk&-}yprH;A^@6|7#R(R? z+H|rq!=Fu+)tD!}n$)6`dkbD9QdZeCkxsz0J-_A@!FS zOX1}8RjFdIb$@dQ<&bN3(cKz!GEli*YY8kaPKMcH>~}$Csh=dD7WOeqfRFo|Gaxdl}ivm ztj@y^qf}2)zUKV1TUM1$o+t~H7l!ZbHM_hV4l2`Dm4)Xes2%$ow$&YDDC#0m?p->p zwFX_b=G+EDcz^2Ds2FUxEqrPNEb!>%M{zh%s6@;^04+@1cm2?6I$4(%`C1TO8o8`u z3+&$c!fe2wAb!01_COw9HlO{l6|NhipM9!AC!P29DN6YfM2mHtLMD7A|70D z(UbQFC=cu>=gfg4s)BDj!e^CD`5Ija;+54+ekWLWg78R#-JYL+I2XRi=!;EBMLIu#SAw}y`aY6m`PX}EXFqeHK=Pp<& z&1m%nM}n9%skUr}muxP+!-lhZhE-eOPMS&g)$IgPRmPT@g&A3oJypvQ9Q*Wb8P`fW zS#Xq*6AUlv)ota3&3zwtyTA{++gqs%W+(0^4>%CSzBO(+94J5VD1YlydkoUgl2(Bz zw@|#2-VXP;@JCTD52({qvqOVAHCdD|gzQg#WkV3^{dQ6QDs%?vSl=|(ZjLv+@%eTL zD)<*j3}teyso&rZKH9>e9(qnpZ}s@_An7jf>a*djlJlt5uXeWIVR7K@V~jit16W-O2qU z01jqZdom;tgvM0qxTDK2eP~%lIrZ>Kug|dWGszd?aPiBl%kOR_h%rW7wGUiz_9L4z zo9ni$vmTvn5Z_tV49^_NRb~C*M+K8glYf{jo0+UxhaM>uO1+Kxo8k)@UeEODWYd#V zYwsZt9aGAmA?W0Iovq(P*z8@Fi`xb|NzF@baC(mIbSP}QShbR}*Oc5~39KX&U(^9l zhq3i64e6wwo#Oh(a8cJvPIse!o!N4wU11ZQeDU1iZ9B|a$1NgiNf0aUyOSR9!d0y*QgFRQ%i(aCr;=}t3Y=A~eJ=s7 zvvas?3?F<}!+8j{ne39WgcZNY>79brT?JBo;XH<~12-E%(rwB3n+TMx2K0*I;=GcM zQTW`WoVpTtv-X_KA8>bq*mi%v+EJL;$@-8C&zDcq$cGuBOf#`~1dqq+`#KS9(_hj=wIDRUaDw0Z-%s4{ z~YYjgEl_++2j@^A3Tc~YtO1GMBrf0?W1Z-5F-Y@`)A;N@jtt+(+Q%4Bi1h+H(Vz2NUaw$ z+218yQkiguYPRtUxK8#_asm7`N7G;y=5geYklsuuyZ4BMdtoLFebN!E3D?GqHbudF zh0lBJVUvXCy|)+$lEh6toe}uRX3F-#AAK#`cfp_EhM7*mwC9b7Vqqs&JeZhzHGBjY zJb~5cp1A7ngw=93`cp4_?!&tAGq6mN>y~DC@u3IYYFHbDf)tK7!B%21$^!M*ECstS9v z+`3S~I~5+-RUKOmuZ&$i^%+)OpmFgT>@Yl59*gC7N75rBbqwt!^E)25f?+wop zL~?w)@DD6`!(O%?=6F(fE7}cR94Z;bhNl-8tSN?<_SoJ}gQI5biu>S;Ti@~~!b6d_ zT{+xweYAmFENrx4aFrtbA*Fm(2&^qS6zdEBZa@=5jy+U?z{tBborwxMCl+yz!Yq%* z!{=a8^4=V2Y(Of$&?sB&V5HLP69N?qWI#izTZ z*9jKq({7i7w~2Q;vf;7t_H#$z^lB_3Fym^QlKMpiYjPBJ-9!+rc;st2tamNt^gWnd zX`z1`7MC!Q?0|&>1jL@gtEBrwTum{`&tlYhVMoOIA|wX349xM>@y6V~N+;naY?L@M zGzh;n(_KQsX=x31!?3Jnw+zf6C4-G4si1JASb?e#5wWqBa$<#H0_6`cm0d?*=>euz zBV03Bo&6CmQqmB501Je~Yw7MLh?knzSG2$fq*o6dhgT+Fyw?n$atZn?a1S;p!*`o0 zch1iZqkMZslGI&Otp9pTCfWpi2;xg*m_2pFm0{Vz%iszxxnQaVhUV3Af59q_;Un2_ z+~K9>XW)%2ZPqDxb;hp29@5eZ+5neB}J^wJg>sMGn zFI>EK+AW@ep#Iyl#vug4l6K?9aBGNBgBdoZ^?pk`G~jhdd;((NxsIyRYVb;*g?lRg zC?DG_r`8SoIW9P;jq)JckGs5n1QEC8pECyV^ou4ZU6}bxckKq4KRxn*39K>O#m(4= zz$3pb%o@R%ebDG^hhv!jaZTq{@H?ZM2bJLi2M6Ny;Pcv&nuhRr(1DLe@Fu|+0Xz6jwts**Y_A@# zu^Uch`gXg)tZW^o4}9T$MvW)@=;%Q)e^~X2P5*8!>bXactfhKLiHa8lG%?CG$K7Mt$M5%-p0vn3;5^fU;r8 zSz|}oI76FBIb(@Hn8P1_BlnAS6BAYLq6Fq1lyhT?NX_-x$I>&s;T@AIZ3*!H0u48> zK&=12q7u2kWBCkq6C86#1=;H}%r9{5`>wMe(855|$dZFFU$TrlN{G#Pxz7o_VwLvn2h0?-X<5yJi7wE?}e>b|Imn z%~CY(${@RaytKKMZ(X`XXgrlV$(!eTcVq33t2)j%`{O~cg!$Ntdfis fr_mO%y7xEJ#96tF44OoxP8?f^CRzFNxG3#E2yxAi diff --git a/django/contrib/gis/tests/layermap/counties/counties.shx b/django/contrib/gis/tests/layermap/counties/counties.shx index 189450fd4825bf0634f538077d23c51df00d9eae..46ed3bb20374af403e57e83dc069b79c31757814 100644 GIT binary patch delta 202 zcmXZVyAHu%7>DtPNF4^TIo^T6AfdEXqe*PUARWaZVi23vs9mfg27`!6BJRQ90$hv# z7pC8PelPJ;d{>S|VWp5r8<4P}wuJN)><@5*mijE9ZOK5+ zWLIA08Jw{k$-5kYoB2DD+H>D>4PO6O_l`1vjz#?!n1dSZn1@^NO|Aj|C42f+f|L&T H=BEDz!vG-p literal 300 zcmZQzQ0HR64yL_eW?*22$TfWON;~jCcfx`nj;0P?=Q?e-Xonu?@eaN9$jR9OyLJ>& zBL)U$42*Llf`P#W field_class.max_length: + val = ogr_field.value + if len(val) > model_field.max_length: raise InvalidString('%s model field maximum string length is %s, given %s characters.' % - (model_field, field_class.max_length, len(val))) - elif isinstance(fld, OFTReal): + (model_field.name, model_field.max_length, len(val))) + elif isinstance(ogr_field, OFTReal): try: # Creating an instance of the Decimal value to use. - d = Decimal(str(fld.value)) + d = Decimal(str(ogr_field.value)) except: - raise InvalidDecimal('Could not construct decimal from: %s' % fld) + raise InvalidDecimal('Could not construct decimal from: %s' % ogr_field) # Getting the decimal value as a tuple. dtup = d.as_tuple() @@ -450,7 +442,7 @@ class LayerMapping(object): d_idx = dtup[2] # index where the decimal is # Maximum amount of precision, or digits to the left of the decimal. - max_prec = field_class.max_digits - field_class.decimal_places + max_prec = model_field.max_digits - model_field.decimal_places # Getting the digits to the left of the decimal place for the # given decimal. @@ -463,17 +455,43 @@ class LayerMapping(object): # InvalidDecimal exception. if n_prec > max_prec: raise InvalidDecimal('A DecimalField with max_digits %d, decimal_places %d must round to an absolute value less than 10^%d.' % - (field_class.max_digits, field_class.decimal_places, max_prec)) + (model_field.max_digits, model_field.decimal_places, max_prec)) val = d else: - val = fld.value + val = ogr_field.value return val - def verify_geom(self, geom, model_type): - "Verifies the geometry." - if self.make_multi(geom.geom_name, model_type): + def verify_fk(self, feat, rel_model, rel_mapping): + """ + Given an OGR Feature, the related model and its dictionary mapping, + this routine will retrieve the related model for the ForeignKey + mapping. + """ + # TODO: It is expensive to retrieve a model for every record -- + # explore if an efficient mechanism exists for caching related + # ForeignKey models. + + # Constructing and verifying the related model keyword arguments. + fk_kwargs = {} + for field_name, ogr_name in rel_mapping.items(): + fk_kwargs[field_name] = self.verify_ogr_field(feat[ogr_name], rel_model._meta.get_field(field_name)) + + # Attempting to retrieve and return the related model. + try: + return rel_model.objects.get(**fk_kwargs) + except ObjectDoesNotExist: + if self.strict: raise MissingForeignKey('No %s model found with keyword arguments: %s' % (rel_model.__name__, fk_kwargs)) + else: return None + + def verify_geom(self, geom, model_field): + """ + Verifies the geometry -- will construct and return a GeometryCollection + if necessary (for example if the model field is MultiPolygonField while + the mapped shapefile only contains Polygons). + """ + if self.make_multi(geom.geom_type, model_field): # Constructing a multi-geometry type to contain the single geometry - multi_type = self.MULTI_TYPES[geom.geom_name] + multi_type = self.MULTI_TYPES[geom.geom_type.num] g = OGRGeometry(multi_type) g.add(geom) else: @@ -486,7 +504,19 @@ class LayerMapping(object): # Returning the WKT of the geometry. return g.wkt - + + #### Other model methods #### + def coord_transform(self): + "Returns the coordinate transformation object." + try: + # Getting the target spatial reference system + target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs + + # Creating the CoordTransform object + return CoordTransform(self.source_srs, target_srs) + except Exception, msg: + raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg) + def geometry_column(self): "Returns the GeometryColumn model associated with the geographic column." # Getting the GeometryColumn object. @@ -498,24 +528,23 @@ class LayerMapping(object): except Exception, msg: raise LayerMapError('Geometry column does not exist for model. (did you run syncdb?):\n %s' % msg) - def make_multi(self, geom_name, model_type): - "Determines whether the geometry should be made into a GeometryCollection." - return (geom_name in self.MULTI_TYPES) and (model_type.startswith('Multi')) - - def map_foreign_key(self, django_field): - "Handles fields within foreign keys for the given field." - if not django_field.__class__ is ForeignKey: - # Returning the field's class name. - return django_field.__class__.__name__ - else: - # Otherwise, getting the type of the related field's - # from the Foreign key. - rf = django_field.rel.get_related_field() - return rf.get_internal_type() + def make_multi(self, geom_type, model_field): + """ + Given the OGRGeomType for a geometry and its associated GeometryField, + determine whether the geometry should be turned into a GeometryCollection. + """ + return (geom_type.num in self.MULTI_TYPES and + model_field.__class__.__name__ == 'Multi%s' % geom_type.django) def save(self, verbose=False): - "Runs the layer mapping on the given SHP file, and saves to the database." - + """ + Saves the contents from the OGR DataSource Layer into the database + according to the mapping dictionary given at initialization. If + the `verbose` keyword is set, information will be printed subsequent + to each model save executed on the database. + """ + # Defining the 'real' save method, utilizing the transaction + # decorator created during initialization. @self.transaction_decorator def _save(): num_feat = 0 @@ -530,7 +559,7 @@ class LayerMapping(object): # Something borked the validation if self.strict: raise elif not self.silent: - print 'Ignoring Feature ID %s because: %s' % (feat.fid, msg) + self.pipe.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg)) else: # Constructing the model using the keyword args if all_prepped: @@ -561,7 +590,7 @@ class LayerMapping(object): # Attempting to save. m.save() num_saved += 1 - if verbose: print 'Saved: %s' % m + if verbose: self.pipe.write('Saved: %s\n' % m) except SystemExit: raise except Exception, msg: @@ -572,17 +601,18 @@ class LayerMapping(object): if self.strict: # Bailing out if the `strict` keyword is set. if not self.silent: - print 'Failed to save the feature (id: %s) into the model with the keyword arguments:' % feat.fid - print kwargs + self.pipe.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid) + self.pipe.write('%s\n' % kwargs) raise elif not self.silent: - print 'Failed to save %s:\n %s\nContinuing' % (kwargs, msg) + self.pipe.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg)) else: - print 'Skipping %s due to missing relation.' % kwargs + if not self.silent: self.pipe.write('Skipping due to missing relation:\n%s\n' % kwargs) + # Printing progress information, if requested. if self.progress and num_feat % self.interval == 0: - print 'Processed %d features, saved %d ...' % (num_feat, num_saved) + self.pipe.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved)) # Calling our defined function, which will use the specified # trasaction mode.