From e88ce426b6133a11df92390806fcde2612a64e99 Mon Sep 17 00:00:00 2001 From: Justin Bronn Date: Sat, 17 Nov 2007 21:57:12 +0000 Subject: [PATCH] gis: LayerMapping refactor (1) Moved all routines into LayerMapping class (for easier subclassing) and modularized the routines. (2) OFTString and OFTReal OGR fields are verified w/the Django fields prior to insertion, thus avoiding invalidating a large transaction. (3) Added keyword options for specifying the transaction mode, not performing transformations, and status printing. (4) Created unit tests. Other Changes: Updated `ogrinfo` for GDAL refactor and fixed an iterating bug; simplified a few lines in `geoapp` model tests. git-svn-id: http://code.djangoproject.com/svn/django/branches/gis@6687 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/contrib/gis/tests/__init__.py | 38 +- django/contrib/gis/tests/geoapp/tests.py | 10 +- django/contrib/gis/tests/layermap/__init__.py | 0 .../gis/tests/layermap/cities/cities.dbf | Bin 0 -> 533 bytes .../gis/tests/layermap/cities/cities.prj | 1 + .../gis/tests/layermap/cities/cities.shp | Bin 0 -> 184 bytes .../gis/tests/layermap/cities/cities.shx | Bin 0 -> 124 bytes .../layermap/interstates/interstates.dbf | Bin 0 -> 307 bytes .../layermap/interstates/interstates.prj | 1 + .../layermap/interstates/interstates.shp | Bin 0 -> 788 bytes .../layermap/interstates/interstates.shx | Bin 0 -> 116 bytes django/contrib/gis/tests/layermap/models.py | 29 + django/contrib/gis/tests/layermap/tests.py | 106 +++ django/contrib/gis/utils/layermapping.py | 616 +++++++++++------- django/contrib/gis/utils/ogrinfo.py | 12 +- 15 files changed, 542 insertions(+), 271 deletions(-) create mode 100644 django/contrib/gis/tests/layermap/__init__.py create mode 100644 django/contrib/gis/tests/layermap/cities/cities.dbf create mode 100644 django/contrib/gis/tests/layermap/cities/cities.prj create mode 100644 django/contrib/gis/tests/layermap/cities/cities.shp create mode 100644 django/contrib/gis/tests/layermap/cities/cities.shx create mode 100644 django/contrib/gis/tests/layermap/interstates/interstates.dbf create mode 100644 django/contrib/gis/tests/layermap/interstates/interstates.prj create mode 100644 django/contrib/gis/tests/layermap/interstates/interstates.shp create mode 100644 django/contrib/gis/tests/layermap/interstates/interstates.shx create mode 100644 django/contrib/gis/tests/layermap/models.py create mode 100644 django/contrib/gis/tests/layermap/tests.py diff --git a/django/contrib/gis/tests/__init__.py b/django/contrib/gis/tests/__init__.py index 0ec2e832f4..57ce089f0c 100644 --- a/django/contrib/gis/tests/__init__.py +++ b/django/contrib/gis/tests/__init__.py @@ -2,7 +2,13 @@ import sys from copy import copy from unittest import TestSuite, TextTestRunner from django.contrib.gis.gdal import HAS_GDAL -from django.contrib.gis.tests.utils import mysql +try: + from django.contrib.gis.tests.utils import mysql +except: + mysql = False + +# Tests that require use of a spatial database (e.g., creation of models) +test_models = ['geoapp'] # Tests that do not require setting up and tearing down a spatial database. test_suite_names = [ @@ -10,6 +16,7 @@ test_suite_names = [ 'test_measure', ] if HAS_GDAL: + test_models += ['layermap'] test_suite_names += [ 'test_gdal_driver', 'test_gdal_ds', @@ -21,8 +28,6 @@ if HAS_GDAL: else: print >>sys.stderr, "GDAL not available - no GDAL tests will be run." -test_models = ['geoapp'] - def suite(): "Builds a test suite for the GIS package." s = TestSuite() @@ -40,8 +45,8 @@ def run_tests(module_list, verbosity=1, interactive=True): Run the tests that require creation of a spatial database. In order to run geographic model tests the DATABASE_USER will require - superuser priviliges. To accomplish this outside the `postgres` user, - create your own PostgreSQL database as a user: + superuser priviliges. To accomplish this outside the `postgres` user, + create your own PostgreSQL database as a user: (1) Initialize database: `initdb -D /path/to/user/db` (2) If there's already a Postgres instance on the machine, it will need to use a different TCP port than 5432. Edit postgresql.conf (in @@ -49,21 +54,21 @@ def run_tests(module_list, verbosity=1, interactive=True): (3) Start this database `pg_ctl -D /path/to/user/db start` On Windows platforms simply use the pgAdmin III utility to add superuser - priviliges to your database user. + priviliges to your database user. Make sure your settings.py matches the settings of the user database. - For example, set the same port number (`DATABASE_PORT=5433`). - DATABASE_NAME or TEST_DATABSE_NAME must be set, along with DATABASE_USER. + For example, set the same port number (`DATABASE_PORT=5433`). + DATABASE_NAME or TEST_DATABSE_NAME must be set, along with DATABASE_USER. In settings.py set TEST_RUNNER='django.contrib.gis.tests.run_tests'. Finally, this assumes that the PostGIS SQL files (lwpostgis.sql and - spatial_ref_sys.sql) are installed in the directory specified by - `pg_config --sharedir` (and defaults to /usr/local/share if that fails). - This behavior is overridden if `POSTGIS_SQL_PATH` is in your settings. + spatial_ref_sys.sql) are installed in the directory specified by + `pg_config --sharedir` (and defaults to /usr/local/share if that fails). + This behavior is overridden if `POSTGIS_SQL_PATH` is in your settings. - Windows users should use the POSTGIS_SQL_PATH because the output - of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'. + Windows users should set POSTGIS_SQL_PATH manually because the output + of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'. Finally, the tests may be run by invoking `./manage.py test`. """ @@ -93,9 +98,10 @@ def run_tests(module_list, verbosity=1, interactive=True): tsuite = getattr(__import__('django.contrib.gis.tests.%s' % test_model, globals(), locals(), [test_module_name]), test_module_name) test_suite.addTest(tsuite.suite()) - # Resetting the loaded flag to take into account what we appended to the INSTALLED_APPS - # (since this routine is invoked through django/core/management, it caches the apps, - # this ensures that syncdb will see our appended models) + # Resetting the loaded flag to take into account what we appended to + # the INSTALLED_APPS (since this routine is invoked through + # django/core/management, it caches the apps; this ensures that syncdb + # will see our appended models) from django.db.models import loading loading._loaded = False diff --git a/django/contrib/gis/tests/geoapp/tests.py b/django/contrib/gis/tests/geoapp/tests.py index 1239b53cb3..0ea6bca141 100644 --- a/django/contrib/gis/tests/geoapp/tests.py +++ b/django/contrib/gis/tests/geoapp/tests.py @@ -77,7 +77,7 @@ class GeoModelTest(unittest.TestCase): inner = LinearRing((40, 40), (40, 60), (60, 60), (60, 40), (40, 40)) # Creating a State object using a built Polygon - ply = Polygon(shell.clone(), inner.clone()) + ply = Polygon(shell, inner) nullstate = State(name='NullState', poly=ply) self.assertEqual(4326, nullstate.poly.srid) # SRID auto-set from None nullstate.save() @@ -94,12 +94,12 @@ class GeoModelTest(unittest.TestCase): # Changing the interior ring on the poly attribute. new_inner = LinearRing((30, 30), (30, 70), (70, 70), (70, 30), (30, 30)) - nullstate.poly[1] = new_inner.clone() + ns.poly[1] = new_inner ply[1] = new_inner - self.assertEqual(4326, nullstate.poly.srid) - nullstate.save() + self.assertEqual(4326, ns.poly.srid) + ns.save() self.assertEqual(ply, State.objects.get(name='NullState').poly) - nullstate.delete() + ns.delete() @no_oracle # Oracle does not support KML. def test03a_kml(self): diff --git a/django/contrib/gis/tests/layermap/__init__.py b/django/contrib/gis/tests/layermap/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django/contrib/gis/tests/layermap/cities/cities.dbf b/django/contrib/gis/tests/layermap/cities/cities.dbf new file mode 100644 index 0000000000000000000000000000000000000000..8b276331b9a3075ae6e58a08bd6505f222287894 GIT binary patch literal 533 zcmZRMXP07TU|?9tPy-|}fnQ>7Dpb@NLt>(!kWnSivW;yeKsP9=nyQ?`whm5el9t@(!w-l!7;Z|badyD2 o9Yqvm3>UIljyrSjFIE8SWrE3JF&F3ph`#v>8}Hg)4~M$}09d3YcK`qY literal 0 HcmV?d00001 diff --git a/django/contrib/gis/tests/layermap/cities/cities.shx b/django/contrib/gis/tests/layermap/cities/cities.shx new file mode 100644 index 0000000000000000000000000000000000000000..6be3fd68e94d52666ec1e39025e496173f45db0c GIT binary patch literal 124 zcmZQzQ0HR64(whqGcYg$<%$kU{`spGb>P9=nyQ?`whm5el9t@(!w-l!7;Z|badyD2 R9Yxd#$OGBq0HysvGyt8x5F-Ep literal 0 HcmV?d00001 diff --git a/django/contrib/gis/tests/layermap/interstates/interstates.dbf b/django/contrib/gis/tests/layermap/interstates/interstates.dbf new file mode 100644 index 0000000000000000000000000000000000000000..7b551461dac65e5f687e0913c2322499335cfe0e GIT binary patch literal 307 zcmZRMXP07PU|>jO$OMv@z%Ma36)Nfsq65&B_@w5gmt=qi{XnDyKdJ&=1y5aLQw6F6 lLlZqSOBA5snOByWlbK>jUAur5nHv}z8yOiJ8ykQv1OOWZCGP+L literal 0 HcmV?d00001 diff --git a/django/contrib/gis/tests/layermap/interstates/interstates.prj b/django/contrib/gis/tests/layermap/interstates/interstates.prj new file mode 100644 index 0000000000..a30c00a55d --- /dev/null +++ b/django/contrib/gis/tests/layermap/interstates/interstates.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] \ No newline at end of file diff --git a/django/contrib/gis/tests/layermap/interstates/interstates.shp b/django/contrib/gis/tests/layermap/interstates/interstates.shp new file mode 100644 index 0000000000000000000000000000000000000000..57de1e7dc5f0cbb6cbf8b0826dbb4f65e62eff31 GIT binary patch literal 788 zcmZQzQ0HR63K+XyFf%X!5yPiTzDWWPA`YM`GjXW3H;vo1)7b%494n8Jfq_v0*(|W$ zPHTsl{zG3iri34WnFUnHpa?YrW-m-_5AC5TSW8UeQz-sHzodcBE7;yk>r;V)xlXkn< z&1De>3T?TrFAlJE&;&^=fSMm?>%aq2Ff-zS_|0?QgPUy~gqjbWTG$$K0Ah}>a*o03F~OaR6-Y0(*xa!9WXKA`VFF-&J9IXzu_v z$1UQ(0x6Jyg9Fgj1-4!h2f+5sa)7wsBjSL%A;%h-TMiCuWTH~6-6Ia{kZXM%XyNEE zYbFpRMjQY|o97-!2a$E|5352V4uHI&VBqACwWnmV1dtC7w`wOy+yMC-iZUj?J>%p6 z3g&JTXn1{aa`*uBt~qDKfkm#{KL0gwb^yoAx$pzkAcMV}A^rpMA65Yc0-YWHto(S% zcT+gTz5-{5fZpkOLaW0M?3)ktP>Ztz*qkNd2SCxhrpFm0Z2*%66A&AK(h+uG;s6mh UaR8+w-0DE8R2)Dgf=1{C0Gp&3O#lD@ literal 0 HcmV?d00001 diff --git a/django/contrib/gis/tests/layermap/interstates/interstates.shx b/django/contrib/gis/tests/layermap/interstates/interstates.shx new file mode 100644 index 0000000000000000000000000000000000000000..a277b9844983a64460f1560161cb688ceb2c6f1e GIT binary patch literal 116 zcmZQzQ0HR64y;}3EWNdgZd4xlPCaj3O7joY=;*#T7?E6<35fl&d7!+^2| E0H09`XaE2J literal 0 HcmV?d00001 diff --git a/django/contrib/gis/tests/layermap/models.py b/django/contrib/gis/tests/layermap/models.py new file mode 100644 index 0000000000..512cdb42af --- /dev/null +++ b/django/contrib/gis/tests/layermap/models.py @@ -0,0 +1,29 @@ +from django.contrib.gis.db import models + +class City(models.Model): + name = models.CharField(max_length=25) + population = models.IntegerField() + density = models.DecimalField(max_digits=7, decimal_places=1) + date = models.DateField() + point = models.PointField() + objects = models.GeoManager() + +class Interstate(models.Model): + name = models.CharField(max_length=20) + length = models.DecimalField(max_digits=7, decimal_places=2) + path = models.LineStringField() + objects = models.GeoManager() + +# Mapping dictionary for the City model. +city_mapping = {'name' : 'Name', + 'population' : 'Population', + 'density' : 'Density', + 'date' : 'Created', + 'point' : 'POINT', + } + +# Mapping dictionary for the Interstate model. +inter_mapping = {'name' : 'Name', + 'length' : 'Length', + 'path' : 'LINESTRING', + } diff --git a/django/contrib/gis/tests/layermap/tests.py b/django/contrib/gis/tests/layermap/tests.py new file mode 100644 index 0000000000..c7e67a63c0 --- /dev/null +++ b/django/contrib/gis/tests/layermap/tests.py @@ -0,0 +1,106 @@ +import os, unittest +from copy import copy +from datetime import date +from decimal import Decimal +from models import City, Interstate, city_mapping, inter_mapping +from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError, InvalidDecimal +from django.contrib.gis.gdal import DataSource + +shp_path = os.path.dirname(__file__) +city_shp = os.path.join(shp_path, 'cities/cities.shp') +inter_shp = os.path.join(shp_path, 'interstates/interstates.shp') + +class LayerMapTest(unittest.TestCase): + + def test01_init(self): + "Testing LayerMapping initialization." + + # Model field that does not exist. + bad1 = copy(city_mapping) + bad1['foobar'] = 'FooField' + + # Shapefile field that does not exist. + bad2 = copy(city_mapping) + bad2['name'] = 'Nombre' + + # Nonexistent geographic field type. + bad3 = copy(city_mapping) + bad3['point'] = 'CURVE' + + # Incrementing through the bad mapping dictionaries and + # ensuring that a LayerMapError is raised. + for bad_map in (bad1, bad2, bad3): + try: + lm = LayerMapping(City, city_shp, bad_map) + except LayerMapError: + pass + else: + self.fail('Expected a LayerMapError.') + + # A LookupError should be thrown for bogus encodings. + try: + lm = LayerMapping(City, city_shp, city_mapping, encoding='foobar') + except LookupError: + pass + else: + self.fail('Expected a LookupError') + + def test02_simple_layermap(self): + "Test LayerMapping import of a simple point shapefile." + + # Setting up for the LayerMapping. + lm = LayerMapping(City, city_shp, city_mapping) + lm.save() + + # There should be three cities in the shape file. + self.assertEqual(3, City.objects.count()) + + # Opening up the shapefile, and verifying the values in each + # of the features made it to the model. + ds = DataSource(city_shp) + layer = ds[0] + for feat in layer: + city = City.objects.get(name=feat['Name'].value) + self.assertEqual(feat['Population'].value, city.population) + self.assertEqual(Decimal(str(feat['Density'])), city.density) + self.assertEqual(feat['Created'].value, city.date) + + # Comparing the geometries. + pnt1, pnt2 = feat.geom, city.point + self.assertAlmostEqual(pnt1.x, pnt2.x, 6) + self.assertAlmostEqual(pnt1.y, pnt2.y, 6) + + def test03_layermap_strict(self): + "Testing the `strict` keyword, and import of a LineString shapefile." + + # When the `strict` keyword is set an error encountered will force + # the importation to stop. + try: + lm = LayerMapping(Interstate, inter_shp, inter_mapping, + strict=True, silent=True) + lm.save() + except InvalidDecimal: + pass + else: + self.fail('Should have failed on strict import with invalid decimal values.') + + # This LayerMapping should work b/c `strict` is not set. + lm = LayerMapping(Interstate, inter_shp, inter_mapping, silent=True) + lm.save() + + # Only one interstate should have imported correctly. + self.assertEqual(1, Interstate.objects.count()) + + # Verifying the values in the single feature w/the model. + ds = DataSource(inter_shp) + feat = ds[0][0] + istate = Interstate.objects.get(name=feat['Name'].value) + self.assertEqual(Decimal(str(feat['Length'])), istate.length) + for p1, p2 in zip(feat.geom, istate.path): + self.assertAlmostEqual(p1[0], p2[0], 6) + self.assertAlmostEqual(p1[1], p2[1], 6) + +def suite(): + s = unittest.TestSuite() + s.addTest(unittest.makeSuite(LayerMapTest)) + return s diff --git a/django/contrib/gis/utils/layermapping.py b/django/contrib/gis/utils/layermapping.py index d25b3fe9a5..9534d86082 100644 --- a/django/contrib/gis/utils/layermapping.py +++ b/django/contrib/gis/utils/layermapping.py @@ -1,21 +1,19 @@ # LayerMapping -- A Django Model/OGR Layer Mapping Utility """ -The LayerMapping class provides a way to map the contents of OGR + The LayerMapping class provides a way to map the contents of OGR vector files (e.g. SHP files) to Geographic-enabled Django models. -This grew out of my personal needs, specifically the code repetition + This grew out of my personal needs, specifically the code repetition that went into pulling geometries and fields out of an OGR layer, converting to another coordinate system (e.g. WGS84), and then inserting - into a Geographic Django model. + into a GeoDjango model. -This utility is still in early stages of development, so its usage + This utility is still in early stages of development, so its usage is subject to change -- please report any bugs. -TODO: Unit tests and documentation. + Requirements: OGR C Library (from GDAL) required. -Requirements: OGR C Library (from GDAL) required. - -Usage: + Usage: lm = LayerMapping(model, source_file, mapping) where, model: @@ -32,7 +30,7 @@ Usage: is a geographic then it should correspond to the OGR geometry type, e.g. 'POINT', 'LINESTRING', 'POLYGON'. -Keyword Args: + Keyword Args: layer: The index of the layer to use from the Data Source (defaults to 0) @@ -46,6 +44,26 @@ Keyword Args: For example, 'latin-1', 'utf-8', and 'cp437' are all valid encoding parameters. + check: + By default, LayerMapping increments through each feature in the + layer to ensure that it is compatible with the given model and + mapping. Setting this keyword to False, disables this action, + which will speed up execution time for very large files. + + silent: + By default, non-fatal error notifications are printed to stdout; this + keyword may be set in order to disable these notifications. + + strict: + Setting this keyword to True will instruct the save() method to + cease execution on the first error encountered. + + transaction_mode: + May be 'commit_on_success' (default) or 'autocommit'. + + transform: + Setting this to False will disable all coordinate transformations. + Example: 1. You need a GDAL-supported data source, like a shapefile. @@ -91,162 +109,86 @@ Example: Saved: Name: 3 LayerMapping just transformed the three geometries from the SHP file from their - source spatial reference system (WGS84) to the spatial reference system of - the GeoDjango model (NAD83). If no spatial reference system is defined for - the layer, use the `source_srs` keyword with a SpatialReference object to - specify one. Further, data is selectively imported from the given data source - fields into the model fields. + source spatial reference system (WGS84) to the spatial reference system of + the GeoDjango model (NAD83). If no spatial reference system is defined for + the layer, use the `source_srs` keyword with a SpatialReference object to + specify one. """ -from types import StringType, TupleType -from datetime import datetime -from django.contrib.gis.db.backend import SPATIAL_BACKEND -from django.contrib.gis.gdal import \ - OGRGeometry, OGRGeomType, SpatialReference, CoordTransform, \ - DataSource, OGRException -from django.contrib.gis.gdal.field import Field, OFTInteger, OFTReal, OFTString, OFTDateTime -from django.contrib.gis.models import GeometryColumns, SpatialRefSys -from django.db import connection, transaction +from datetime import date, datetime +from decimal import Decimal from django.core.exceptions import ObjectDoesNotExist +from django.db import connection, transaction +from django.db.models.fields.related import ForeignKey +from django.contrib.gis.db.backend import SPATIAL_BACKEND +from django.contrib.gis.gdal import CoordTransform, DataSource, \ + OGRException, OGRGeometry, OGRGeomType, SpatialReference +from django.contrib.gis.gdal.field import OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime +from django.contrib.gis.models import GeometryColumns, SpatialRefSys -# A mapping of given geometry types to their OGR integer type. -ogc_types = {'POINT' : OGRGeomType('Point'), - 'LINESTRING' : OGRGeomType('LineString'), - 'POLYGON' : OGRGeomType('Polygon'), - 'MULTIPOINT' : OGRGeomType('MultiPoint'), - 'MULTILINESTRING' : OGRGeomType('MultiLineString'), - 'MULTIPOLYGON' : OGRGeomType('MultiPolygon'), - 'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'), - } +# LayerMapping exceptions. +class LayerMapError(Exception): pass +class InvalidString(LayerMapError): pass +class InvalidDecimal(LayerMapError): pass -# The django.contrib.gis model types. -gis_fields = {'PointField' : 'POINT', - 'LineStringField': 'LINESTRING', - 'PolygonField': 'POLYGON', - 'MultiPointField' : 'MULTIPOINT', - 'MultiLineStringField' : 'MULTILINESTRING', - 'MultiPolygonField' : 'MULTIPOLYGON', - } +class LayerMapping(object): + "A class that maps OGR Layers to GeoDjango Models." + + # A mapping of given geometry types to their OGR integer type. + OGC_TYPES = {'POINT' : OGRGeomType('Point'), + 'LINESTRING' : OGRGeomType('LineString'), + 'POLYGON' : OGRGeomType('Polygon'), + 'MULTIPOINT' : OGRGeomType('MultiPoint'), + 'MULTILINESTRING' : OGRGeomType('MultiLineString'), + 'MULTIPOLYGON' : OGRGeomType('MultiPolygon'), + 'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'), + } -# Acceptable 'base' types for a multi-geometry type. -multi_types = {'POINT' : OGRGeomType('MultiPoint'), - 'LINESTRING' : OGRGeomType('MultiLineString'), - 'POLYGON' : OGRGeomType('MultiPolygon'), - } + # The django.contrib.gis model types. + GIS_FIELDS = {'PointField' : 'POINT', + 'LineStringField': 'LINESTRING', + 'PolygonField': 'POLYGON', + 'MultiPointField' : 'MULTIPOINT', + 'MultiLineStringField' : 'MULTILINESTRING', + 'MultiPolygonField' : 'MULTIPOLYGON', + 'GeometryCollectionField' : 'GEOMETRYCOLLECTION', + } -def map_foreign_key(django_field): - from django.db.models.fields.related import ForeignKey + # Acceptable 'base' types for a multi-geometry type. + MULTI_TYPES = {'POINT' : OGRGeomType('MultiPoint'), + 'LINESTRING' : OGRGeomType('MultiLineString'), + 'POLYGON' : OGRGeomType('MultiPolygon'), + } - if not django_field.__class__ is ForeignKey: - return django_field.__class__.__name__ + # The acceptable Django field types that map to OGR fields. + FIELD_TYPES = { + 'AutoField' : OFTInteger, + 'IntegerField' : OFTInteger, + 'FloatField' : OFTReal, + 'DateField' : OFTDate, + 'DateTimeField' : OFTDateTime, + 'TimeField' : OFTTime, + 'DecimalField' : OFTReal, + 'CharField' : OFTString, + 'TextField' : OFTString, + 'SmallIntegerField' : OFTInteger, + 'PositiveSmallIntegerField' : OFTInteger, + } - - rf=django_field.rel.get_related_field() + # The acceptable transaction modes. + TRANSACTION_MODES = {'autocommit' : transaction.autocommit, + 'commit_on_success' : transaction.commit_on_success, + } - return rf.get_internal_type() - -# The acceptable Django field types that map to OGR fields. -field_types = { - 'AutoField' : OFTInteger, - 'IntegerField' : OFTInteger, - 'FloatField' : OFTReal, - 'DateTimeField' : OFTDateTime, - 'DecimalField' : OFTReal, - 'CharField' : OFTString, - 'SmallIntegerField' : OFTInteger, - 'PositiveSmallIntegerField' : OFTInteger, - } - -def make_multi(geom_name, model_type): - "Determines whether the geometry should be made into a GeometryCollection." - if (geom_name in multi_types) and (model_type.startswith('Multi')): - return True - else: - return False - -def check_feature(feat, model_fields, mapping): - "Checks the OGR layer feature." - - HAS_GEO = False - - # Incrementing through each model_field & ogr_field in the given mapping. - for model_field, ogr_field in mapping.items(): - - # Making sure the given mapping model field is in the given model fields. - if model_field in model_fields: - model_type = model_fields[model_field] - elif model_field[:-3] in model_fields: #foreign key - model_type = model_fields[model_field[:-3]] - else: - raise Exception('Given mapping field "%s" not in given Model fields!' % model_field) - - ### Handling if we get a geometry in the Field ### - if ogr_field in ogc_types: - # At this time, no more than one geographic field per model =( - if HAS_GEO: - raise Exception('More than one geographic field in mapping not allowed (yet).') - else: - HAS_GEO = ogr_field - - # Making sure this geometry field type is a valid Django GIS field. - if not model_type in gis_fields: - raise Exception('Unknown Django GIS field type "%s"' % model_type) - - # Getting the OGRGeometry, it's type (an integer) and it's name (a string) - geom = feat.geom - gtype = geom.geom_type - gname = geom.geom_name - - if make_multi(gname, model_type): - # Do we have to 'upsample' into a Geometry Collection? - pass - elif gtype == ogc_types[gis_fields[model_type]]: - # The geometry type otherwise was expected - pass - else: - raise Exception('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype)) - - ## Handling other fields - else: - # Making sure the model field is - if not model_type in field_types: - raise Exception('Django field type "%s" has no OGR mapping (yet).' % model_type) - - # Otherwise, we've got an OGR Field. Making sure that an - # index exists for the mapping OGR field. - try: - fi = feat.index(ogr_field) - except: - raise Exception('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field) - -def check_layer(layer, fields, mapping): - "Checks the OGR layer by incrementing through and checking each feature." - # Incrementing through each feature in the layer. - for feat in layer: - check_feature(feat, fields, mapping) - -def check_srs(layer, source_srs): - "Checks the compatibility of the given spatial reference object." - if isinstance(source_srs, SpatialReference): - sr = source_srs - elif isinstance(source_srs, SpatialRefSys): - sr = source_srs.srs - elif isinstance(source_srs, (int, str)): - sr = SpatialReference(source_srs) - else: - sr = layer.srs - if not sr: - raise Exception('No source reference system defined.') - else: - return sr - -class LayerMapping: - "A class that maps OGR Layers to Django Models." - - def __init__(self, model, data, mapping, layer=0, source_srs=None, encoding=None): + def __init__(self, model, data, mapping, layer=0, + source_srs=None, encoding=None, check=True, + progress=False, interval=1000, strict=False, silent=False, + transaction_mode='commit_on_success', transform=True): "Takes the Django model, the data source, and the mapping (dictionary)" # Getting the field names and types from the model - fields = dict((f.name, map_foreign_key(f)) for f in model._meta.fields) + self.fields = dict((f.name, self.map_foreign_key(f)) for f in model._meta.fields) + self.field_classes = dict((f.name, f) for f in model._meta.fields) + # Getting the DataSource and its Layer if isinstance(data, basestring): self.ds = DataSource(data) @@ -254,120 +196,306 @@ class LayerMapping: self.ds = data self.layer = self.ds[layer] - # Checking the layer -- intitialization of the object will fail if - # things don't check out before hand. - check_layer(self.layer, fields, mapping) - - # Since the layer checked out, setting the fields and the mapping. - self.fields = fields + # Setting the mapping self.mapping = mapping + + # Setting the model, and getting the geometry column associated + # with the model (an exception will be raised if there is no + # geometry column). self.model = model - self.source_srs = check_srs(self.layer, source_srs) + self.geo_col = self.geometry_column() + + # Checking the source spatial reference system, and getting + # the coordinate transformation object (unless the `transform` + # keyword is set to False) + self.source_srs = self.check_srs(source_srs) + self.transform = transform and self.coord_transform() + + # Checking the layer -- intitialization of the object will fail if + # things don't check out before hand. This may be time-consuming, + # and disabled by setting the `check` keyword to False. + if check: self.check_layer() + + # The silent, strict, progress, and interval flags. + self.silent = silent + self.strict = strict + self.progress = progress + self.interval = interval # Setting the encoding for OFTString fields, if specified. if encoding: # Making sure the encoding exists, if not a LookupError - # exception will be thrown. + # exception will be thrown. from codecs import lookup lookup(encoding) self.encoding = encoding else: self.encoding = None - # Either the import will work, or it won't be committed. - @transaction.commit_on_success - def save(self, verbose=False): - "Runs the layer mapping on the given SHP file, and saves to the database." + # Setting the transaction decorator with the function in the + # transaction modes dictionary. + if transaction_mode in self.TRANSACTION_MODES: + self.transaction_decorator = self.TRANSACTION_MODES[transaction_mode] + self.transaction_mode = transaction_mode + else: + raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode) + def check_feature(self, feat): + "Checks the OGR feature against the model fields and mapping." + HAS_GEO = False + + # Incrementing through each model_field & ogr_field in the given mapping. + for model_field, ogr_field in self.mapping.items(): + # Making sure the given mapping model field is in the given model fields. + if model_field in self.fields: + model_type = self.fields[model_field] + elif model_field[:-3] in self.fields: #foreign key + model_type = self.fields[model_field[:-3]] + else: + raise LayerMapError('Given mapping field "%s" not in given Model fields!' % model_field) + + ### Handling if we get a geometry in the Field ### + if ogr_field in self.OGC_TYPES: + # At this time, no more than one geographic field per model =( + if HAS_GEO: + raise LayerMapError('More than one geographic field in mapping not allowed (yet).') + else: + HAS_GEO = ogr_field + + # Making sure this geometry field type is a valid Django GIS field. + if not model_type in self.GIS_FIELDS: + raise LayerMapError('Unknown Django GIS field type "%s"' % model_type) + + # Getting the OGRGeometry, it's type (an integer) and it's name (a string) + geom = feat.geom + gtype = geom.geom_type + gname = geom.geom_name + + if self.make_multi(gname, model_type): + # Do we have to 'upsample' into a Geometry Collection? + pass + elif gtype == self.OGC_TYPES[self.GIS_FIELDS[model_type]]: + # The geometry type otherwise was expected + pass + else: + raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype)) + ### Handling other fields ### + else: + # Making sure the model field is supported. + if not model_type in self.FIELD_TYPES: + raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % model_type) + + # Otherwise, we've got an OGR Field. Making sure that an + # index exists for the mapping OGR field. + try: + fi = feat.index(ogr_field) + except: + raise LayerMapError('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field) + + def check_layer(self): + "Checks every feature in this object's layer." + for feat in self.layer: + self.check_feature(feat) + + def check_srs(self, source_srs): + "Checks the compatibility of the given spatial reference object." + if isinstance(source_srs, SpatialReference): + sr = source_srs + elif isinstance(source_srs, SpatialRefSys): + sr = source_srs.srs + elif isinstance(source_srs, (int, str)): + sr = SpatialReference(source_srs) + else: + # Otherwise just pulling the SpatialReference from the layer + sr = self.layer.srs + + if not sr: + raise LayerMapError('No source reference system defined.') + else: + return sr + + def coord_transform(self): + "Returns the coordinate transformation object." + try: + # Getting the target spatial reference system + target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs + + # Creating the CoordTransform object + return CoordTransform(self.source_srs, target_srs) + except Exception, msg: + raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg) + + def feature_kwargs(self, feat): + "Returns the keyword arguments needed for saving a feature." + + # The keyword arguments for model construction. + kwargs = {} + + # The all_prepped flagged, will be set to False if there's a + # problem w/a ForeignKey that doesn't exist. + all_prepped = True + + # Incrementing through each model field and OGR field in the + # dictionary mapping. + for model_field, ogr_field in self.mapping.items(): + is_fk = False + try: + model_type = self.fields[model_field] + except KeyError: #foreign key + # The -3 index is b/c foreign keys are appended w/'_id'. + model_type = self.fields[model_field[:-3]] + is_fk = True + + if ogr_field in self.OGC_TYPES: + # Verify OGR geometry. + val = self.verify_geom(feat.geom, model_type) + else: + # Otherwise, verify OGR Field type. + val = self.verify_field(feat[ogr_field], model_field) + + if is_fk: + # Handling if foreign key. + rel_obj = None + field_name = model_field[:-3] + try: + # FIXME: refactor to efficiently fetch FKs. + # Requires significant re-work. :-/ + rel = self.model._meta.get_field(field_name).rel + rel_obj = rel.to._default_manager.get(**{('%s__exact' % rel.field_name):val}) + except ObjectDoesNotExist: + all_prepped = False + + kwargs[model_field[:-3]] = rel_obj + else: + kwargs[model_field] = val + + return kwargs, all_prepped + + def verify_field(self, fld, model_field): + """ + Verifies if the OGR Field contents are acceptable to the Django + model field. If they are, the verified value is returned, + otherwise the proper exception is raised. + """ + field_class = self.field_classes[model_field] + if isinstance(fld, OFTString): + if self.encoding: + # The encoding for OGR data sources may be specified here + # (e.g., 'cp437' for Census Bureau boundary files). + val = unicode(fld.value, self.encoding) + else: + val = fld.value + if len(val) > field_class.max_length: + raise InvalidString('%s model field maximum string length is %s, given %s characters.' % + (model_field, field_class.max_length, len(val))) + elif isinstance(fld, OFTReal): + try: + # Creating an instance of the Decimal value to use. + d = Decimal(str(fld.value)) + except: + raise InvalidDecimal('Could not construct decimal from: %s' % fld) + dtup = d.as_tuple() + if len(dtup[1]) > field_class.max_digits: + raise InvalidDecimal('More than the maximum # of digits encountered.') + elif len(dtup[1][dtup[2]:]) > field_class.decimal_places: + raise InvalidDecimal('More than the maximum # of decimal places encountered.') + val = d + else: + val = fld.value + return val + + def verify_geom(self, geom, model_type): + "Verifies the geometry." + if self.make_multi(geom.geom_name, model_type): + # Constructing a multi-geometry type to contain the single geometry + multi_type = self.MULTI_TYPES[geom.geom_name] + g = OGRGeometry(multi_type) + g.add(geom) + else: + g = geom + + # Transforming the geometry with our Coordinate Transformation object, + # but only if the class variable `transform` is set w/a CoordTransform + # object. + if self.transform: g.transform(self.transform) + + # Returning the WKT of the geometry. + return g.wkt + + def geometry_column(self): + "Returns the GeometryColumn model associated with the geographic column." # Getting the GeometryColumn object. try: db_table = self.model._meta.db_table if SPATIAL_BACKEND == 'oracle': db_table = db_table.upper() gc_kwargs = {GeometryColumns.table_name_col() : db_table} - geo_col = GeometryColumns.objects.get(**gc_kwargs) - except: - raise Exception('Geometry column does not exist. (did you run syncdb?)') - - # Getting the coordinate system needed for transformation (with CoordTransform) - try: - # Getting the target spatial reference system - target_srs = SpatialRefSys.objects.get(srid=geo_col.srid).srs - - # Creating the CoordTransform object - ct = CoordTransform(self.source_srs, target_srs) + return GeometryColumns.objects.get(**gc_kwargs) except Exception, msg: - raise Exception('Could not translate between the data source and model geometry: %s' % msg) + raise LayerMapError('Geometry column does not exist for model. (did you run syncdb?):\n %s' % msg) - for feat in self.layer: - # The keyword arguments for model construction - kwargs = {} + def make_multi(self, geom_name, model_type): + "Determines whether the geometry should be made into a GeometryCollection." + return (geom_name in self.MULTI_TYPES) and (model_type.startswith('Multi')) - # Incrementing through each model field and the OGR field in the mapping - all_prepped = True + def map_foreign_key(self, django_field): + "Handles fields within foreign keys for the given field." + if not django_field.__class__ is ForeignKey: + # Returning the field's class name. + return django_field.__class__.__name__ + else: + # Otherwise, getting the type of the related field's + # from the Foreign key. + rf = django_field.rel.get_related_field() + return rf.get_internal_type() - for model_field, ogr_field in self.mapping.items(): - is_fk = False + def save(self, verbose=False): + "Runs the layer mapping on the given SHP file, and saves to the database." + + @self.transaction_decorator + def _save(): + num_feat = 0 + num_saved = 0 + + for feat in self.layer: + num_feat += 1 + # Getting the keyword arguments try: - model_type = self.fields[model_field] - except KeyError: #foreign key - model_type = self.fields[model_field[:-3]] - is_fk = True - - if ogr_field in ogc_types: - ## Getting the OGR geometry from the field - geom = feat.geom - - if make_multi(geom.geom_name, model_type): - # Constructing a multi-geometry type to contain the single geometry - multi_type = multi_types[geom.geom_name] - g = OGRGeometry(multi_type) - g.add(geom) - else: - g = geom - - # Transforming the geometry with our Coordinate Transformation object. - g.transform(ct) - - # Updating the keyword args with the WKT of the transformed model. - val = g.wkt + kwargs, all_prepped = self.feature_kwargs(feat) + except LayerMapError, msg: + # Something borked the validation + if self.strict: raise + elif not self.silent: + print 'Ignoring Feature ID %s because: %s' % (feat.fid, msg) else: - ## Otherwise, this is an OGR field type - fld = feat[ogr_field] - - if isinstance(fld, OFTString) and self.encoding: - # The encoding for OGR data sources may be specified here - # (e.g., 'cp437' for Census Bureau boundary files). - val = unicode(fld.value, self.encoding) - else: - val = fld.value - - if is_fk: - rel_obj = None - field_name = model_field[:-3] - try: - #FIXME: refactor to efficiently fetch FKs. - # Requires significant re-work. :-/ - rel = self.model._meta.get_field(field_name).rel - rel_obj = rel.to._default_manager.get(**{('%s__exact' % rel.field_name):val}) - except ObjectDoesNotExist: - all_prepped = False - - kwargs[model_field[:-3]] = rel_obj - else: - kwargs[model_field] = val - - # Constructing the model using the constructed keyword args - if all_prepped: - m = self.model(**kwargs) - - # Saving the model - try: + # Constructing the model using the constructed keyword args if all_prepped: - m.save() - if verbose: print 'Saved: %s' % str(m) + m = self.model(**kwargs) + try: + m.save() + num_saved += 1 + if verbose: print 'Saved: %s' % m + except SystemExit: + raise + except Exception, msg: + if self.transaction_mode == 'autocommit': + # Rolling back the transaction so that other model saves + # will work. + transaction.rollback_unless_managed() + if self.strict: + # Bailing out if the `strict` keyword is set. + if not self.silent: + print 'Failed to save the feature (id: %s) into the model with the keyword arguments:' % feat.fid + print kwargs + raise + elif not self.silent: + print 'Failed to save %s:\n %s\nContinuing' % (kwargs, msg) else: - print "Skipping %s due to missing relation." % kwargs - except SystemExit: - raise - except Exception, e: - print "Failed to save %s\n Continuing" % kwargs + print 'Skipping %s due to missing relation.' % kwargs + + # Printing progress information, if requested. + if self.progress and num_feat % self.interval == 0: + print 'Processed %d features, saved %d ...' % (num_feat, num_saved) + + # Calling our defined function, which will use the specified + # trasaction mode. + _save() diff --git a/django/contrib/gis/utils/ogrinfo.py b/django/contrib/gis/utils/ogrinfo.py index fa08f7e729..f9b389c684 100644 --- a/django/contrib/gis/utils/ogrinfo.py +++ b/django/contrib/gis/utils/ogrinfo.py @@ -33,12 +33,12 @@ def ogrinfo(data_source, num_features=10): width = max(*map(len,layer.fields)) fmt = " %%%ss: %%s" % width - for i, feature in enumerate(layer[:num_features]): - print "=== Feature %s" % i - for field in layer.fields: - fld_typ = feature[field].__class__.__name__.replace('OFT', '') - output = fmt % (field, fld_typ) - val = feature.get(field) + for j, feature in enumerate(layer[:num_features]): + print "=== Feature %s" % j + for fld_name in layer.fields: + type_name = feature[fld_name].type_name + output = fmt % (fld_name, type_name) + val = feature.get(fld_name) if val: if isinstance(val, str): val_fmt = ' ("%s")'