Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

gis: `LayerMapping`: Improved the internals (i.e., checking every fea…

…ture in OGR Layer is no longer needed, removed unnecessary class constants); added real support `ForeignKey` model fields; added `field_types` property to `Layer`; fixed county shapefile because of Harris County, Georgia.

git-svn-id: http://code.djangoproject.com/svn/django/branches/gis@6992 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit b11172e4e72299d34c27eff635ad034f84a0cdb6 1 parent 7e322b5
Justin Bronn authored January 03, 2008
21  django/contrib/gis/gdal/layer.py
@@ -5,6 +5,7 @@
5 5
 from django.contrib.gis.gdal.envelope import Envelope, OGREnvelope
6 6
 from django.contrib.gis.gdal.error import OGRException, OGRIndexError, SRSException
7 7
 from django.contrib.gis.gdal.feature import Feature
  8
+from django.contrib.gis.gdal.field import FIELD_CLASSES
8 9
 from django.contrib.gis.gdal.geometries import OGRGeomType
9 10
 from django.contrib.gis.gdal.srs import SpatialReference
10 11
 
@@ -12,8 +13,8 @@
12 13
 from django.contrib.gis.gdal.prototypes.ds import \
13 14
     get_extent, get_fd_geom_type, get_fd_name, get_feature, get_feature_count, \
14 15
     get_field_count, get_field_defn, get_field_name, get_field_precision, \
15  
-    get_field_width, get_layer_defn, get_layer_srs, get_next_feature, \
16  
-    reset_reading
  16
+    get_field_width, get_field_type, get_layer_defn, get_layer_srs, \
  17
+    get_next_feature, reset_reading
17 18
 from django.contrib.gis.gdal.prototypes.srs import clone_srs
18 19
 
19 20
 # For more information, see the OGR C API source code:
@@ -107,10 +108,24 @@ def srs(self):
107 108
 
108 109
     @property
109 110
     def fields(self):
110  
-        "Returns a list of the fields available in this Layer."
  111
+        """
  112
+        Returns a list of string names corresponding to each of the Fields
  113
+        available in this Layer.
  114
+        """
111 115
         return [get_field_name(get_field_defn(self._ldefn, i)) 
112 116
                 for i in xrange(self.num_fields) ]
113 117
     
  118
+    @property
  119
+    def field_types(self):
  120
+        """
  121
+        Returns a list of the types of fields in this Layer.  For example,
  122
+        the list [OFTInteger, OFTReal, OFTString] would be returned for
  123
+        an OGR layer that had an integer, a floating-point, and string
  124
+        fields.
  125
+        """
  126
+        return [FIELD_CLASSES[get_field_type(get_field_defn(self._ldefn, i))]
  127
+                for i in xrange(self.num_fields)]
  128
+
114 129
     @property 
115 130
     def field_widths(self):
116 131
         "Returns a list of the maximum field widths for the features."
BIN  django/contrib/gis/tests/layermap/counties/counties.dbf
Binary file not shown
BIN  django/contrib/gis/tests/layermap/counties/counties.shp
Binary file not shown
BIN  django/contrib/gis/tests/layermap/counties/counties.shx
Binary file not shown
6  django/contrib/gis/tests/layermap/models.py
... ...
@@ -1,7 +1,12 @@
1 1
 from django.contrib.gis.db import models
2 2
 
  3
+class State(models.Model):
  4
+    name = models.CharField(max_length=20)
  5
+    objects = models.GeoManager()
  6
+
3 7
 class County(models.Model):
4 8
     name = models.CharField(max_length=25)
  9
+    state = models.ForeignKey(State)
5 10
     mpoly = models.MultiPolygonField(srid=4269) # Multipolygon in NAD83
6 11
     objects = models.GeoManager()
7 12
 
@@ -26,6 +31,7 @@ class Interstate(models.Model):
26 31
 
27 32
 # Mapping dictionaries for the models above.
28 33
 co_mapping = {'name' : 'Name',
  34
+              'state' : {'name' : 'State'}, # ForeignKey's use another mapping dictionary for the _related_ Model (State in this case).
29 35
               'mpoly' : 'MULTIPOLYGON', # Will convert POLYGON features into MULTIPOLYGONS.
30 36
               }
31 37
 
44  django/contrib/gis/tests/layermap/tests.py
@@ -2,8 +2,8 @@
2 2
 from copy import copy
3 3
 from datetime import date
4 4
 from decimal import Decimal
5  
-from models import City, County, CountyFeat, Interstate, city_mapping, co_mapping, cofeat_mapping, inter_mapping
6  
-from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError, InvalidDecimal
  5
+from models import City, County, CountyFeat, Interstate, State, city_mapping, co_mapping, cofeat_mapping, inter_mapping
  6
+from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError, InvalidDecimal, MissingForeignKey
7 7
 from django.contrib.gis.gdal import DataSource
8 8
 
9 9
 shp_path = os.path.dirname(__file__)
@@ -111,8 +111,8 @@ def test03_layermap_strict(self):
111 111
                 self.assertAlmostEqual(p1[0], p2[0], 6)
112 112
                 self.assertAlmostEqual(p1[1], p2[1], 6)
113 113
 
114  
-    def test04_layermap_unique_multigeometry(self):
115  
-        "Testing the `unique`, and `transform` keywords and geometry collection conversion."
  114
+    def test04_layermap_unique_multigeometry_fk(self):
  115
+        "Testing the `unique`, and `transform`, geometry collection conversion, and ForeignKey mappings."
116 116
         # All the following should work.
117 117
         try:
118 118
             # Telling LayerMapping that we want no transformations performed on the data.
@@ -135,6 +135,23 @@ def test04_layermap_unique_multigeometry(self):
135 135
         # No source reference system defined in the shapefile, should raise an error.
136 136
         self.assertRaises(LayerMapError, LayerMapping, County, co_shp, co_mapping)
137 137
 
  138
+        # Passing in invalid ForeignKey mapping parameters -- must be a dictionary
  139
+        # mapping for the model the ForeignKey points to.
  140
+        bad_fk_map1 = copy(co_mapping); bad_fk_map1['state'] = 'name'
  141
+        bad_fk_map2 = copy(co_mapping); bad_fk_map2['state'] = {'nombre' : 'State'}
  142
+        self.assertRaises(TypeError, LayerMapping, County, co_shp, bad_fk_map1, transform=False)
  143
+        self.assertRaises(LayerMapError, LayerMapping, County, co_shp, bad_fk_map2, transform=False)
  144
+
  145
+        # There exist no State models for the ForeignKey mapping to work -- should raise
  146
+        # a MissingForeignKey exception (this error would be ignored if the `strict`
  147
+        # keyword is not set).
  148
+        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name', silent=True, strict=True)
  149
+        self.assertRaises(MissingForeignKey, lm.save)
  150
+
  151
+        # Now creating the state models so the ForeignKey mapping may work.
  152
+        co, hi, tx = State(name='Colorado'), State(name='Hawaii'), State(name='Texas')
  153
+        co.save(), hi.save(), tx.save()
  154
+
138 155
         # If a mapping is specified as a collection, all OGR fields that
139 156
         # are not collections will be converted into them.  For example,
140 157
         # a Point column would be converted to MultiPoint. Other things being done
@@ -148,21 +165,26 @@ def test04_layermap_unique_multigeometry(self):
148 165
         #    appended to the geometry collection of the unique model.  Thus,
149 166
         #    all of the various islands in Honolulu county will be in in one
150 167
         #    database record with a MULTIPOLYGON type.
151  
-        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name', silent=True)
  168
+        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name', silent=True, strict=True)
152 169
         lm.save()
153 170
 
154 171
         # A reference that doesn't use the unique keyword; a new database record will
155 172
         # created for each polygon.
156  
-        lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False, silent=True)
  173
+        lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False, silent=True, strict=True)
157 174
         lm.save()
158 175
 
159 176
         # Dictionary to hold what's expected in the shapefile.
160  
-        exp = {'names' : ('Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo'),
161  
-               'num' : (1, 2, 2, 19, 1), # Number of polygons for each.
162  
-               }
163  
-        for name, n in zip(exp['names'], exp['num']):
164  
-            c = County.objects.get(name=name) # Should only be one record.
  177
+        names = ('Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo')
  178
+        nums  = (1, 2, 1, 19, 1) # Number of polygons for each.
  179
+        states = ('Texas', 'Texas', 'Texas', 'Hawaii', 'Colorado')
  180
+
  181
+        for name, n, st in zip(names, nums, states):
  182
+            # Should only be one record b/c of `unique` keyword.
  183
+            c = County.objects.get(name=name)
165 184
             self.assertEqual(n, len(c.mpoly))
  185
+            self.assertEqual(st, c.state.name) # Checking ForeignKey mapping.
  186
+
  187
+            # Multiple records because `unique` was not set.
166 188
             qs = CountyFeat.objects.filter(name=name)
167 189
             self.assertEqual(n, qs.count())
168 190
             
428  django/contrib/gis/utils/layermapping.py
@@ -45,10 +45,13 @@
45 45
    encoding parameters.
46 46
 
47 47
   check:
48  
-   By default, LayerMapping increments through each feature in the
49  
-   layer to ensure that it is compatible with the given model and
50  
-   mapping.  Setting this keyword to False, disables this action,
51  
-   which will speed up execution time for very large files.
  48
+   Due to optimizations, this keyword argument is deprecated and will
  49
+   be removed in future revisions.
  50
+
  51
+  pipe:
  52
+   Status information will be written to this file handle.  Defaults
  53
+   to using `sys.stdout`, but any object with a `write` method is 
  54
+   supported.
52 55
 
53 56
   silent:
54 57
    By default, non-fatal error notifications are printed to stdout; this
@@ -56,7 +59,8 @@
56 59
 
57 60
   strict:
58 61
    Setting this keyword to True will instruct the save() method to
59  
-   cease execution on the first error encountered.
  62
+   cease execution on the first error encountered.  The default behavior
  63
+   is to attempt to continue even if errors are encountered.
60 64
 
61 65
   transaction_mode:
62 66
    May be 'commit_on_success' (default) or 'autocommit'.
@@ -121,64 +125,48 @@ def __str__(self):
121 125
  the layer, use the `source_srs` keyword with a SpatialReference object to
122 126
  specify one.
123 127
 """
  128
+import sys
124 129
 from datetime import date, datetime
125 130
 from decimal import Decimal
126 131
 from django.core.exceptions import ObjectDoesNotExist
127  
-from django.db import connection, transaction
128  
-from django.db.models.fields.related import ForeignKey
  132
+from django.contrib.gis.db.models.fields import GeometryField
129 133
 from django.contrib.gis.db.backend import SPATIAL_BACKEND
130 134
 from django.contrib.gis.gdal import CoordTransform, DataSource, \
131 135
     OGRException, OGRGeometry, OGRGeomType, SpatialReference
132  
-from django.contrib.gis.gdal.field import OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime
  136
+from django.contrib.gis.gdal.field import \
  137
+    OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime
133 138
 from django.contrib.gis.models import GeometryColumns, SpatialRefSys
  139
+from django.db import models, transaction
134 140
 
135 141
 # LayerMapping exceptions.
136 142
 class LayerMapError(Exception): pass
137 143
 class InvalidString(LayerMapError): pass
138 144
 class InvalidDecimal(LayerMapError): pass
  145
+class MissingForeignKey(LayerMapError): pass
139 146
 
140 147
 class LayerMapping(object):
141 148
     "A class that maps OGR Layers to GeoDjango Models."
142 149
     
143  
-    # A mapping of given geometry types to their OGR integer type.
144  
-    OGC_TYPES = {'POINT' : OGRGeomType('Point'),
145  
-                 'LINESTRING' : OGRGeomType('LineString'),
146  
-                 'POLYGON' : OGRGeomType('Polygon'),
147  
-                 'MULTIPOINT' : OGRGeomType('MultiPoint'),
148  
-                 'MULTILINESTRING' : OGRGeomType('MultiLineString'),
149  
-                 'MULTIPOLYGON' : OGRGeomType('MultiPolygon'),
150  
-                 'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'),
151  
-                 }
152  
-
153  
-    # The django.contrib.gis model types.
154  
-    GIS_FIELDS = {'PointField' : 'POINT',
155  
-                  'LineStringField': 'LINESTRING',
156  
-                  'PolygonField': 'POLYGON',
157  
-                  'MultiPointField' : 'MULTIPOINT',
158  
-                  'MultiLineStringField' : 'MULTILINESTRING',
159  
-                  'MultiPolygonField' : 'MULTIPOLYGON',
160  
-                  'GeometryCollectionField' : 'GEOMETRYCOLLECTION',
161  
-                  }
162  
-
163 150
     # Acceptable 'base' types for a multi-geometry type.
164  
-    MULTI_TYPES = {'POINT' : OGRGeomType('MultiPoint'),
165  
-                   'LINESTRING' : OGRGeomType('MultiLineString'),
166  
-                   'POLYGON' : OGRGeomType('MultiPolygon'),
  151
+    MULTI_TYPES = {1 : OGRGeomType('MultiPoint'),
  152
+                   2 : OGRGeomType('MultiLineString'),
  153
+                   3 : OGRGeomType('MultiPolygon'),
167 154
                    }
168 155
 
169  
-    # The acceptable Django field types that map to OGR fields.
  156
+    # Acceptable Django field types and corresponding acceptable OGR
  157
+    # counterparts.
170 158
     FIELD_TYPES = {
171  
-        'AutoField' : OFTInteger,
172  
-        'IntegerField' : OFTInteger,
173  
-        'FloatField' : OFTReal,
174  
-        'DateField' : OFTDate,
175  
-        'DateTimeField' : OFTDateTime,
176  
-        'TimeField' : OFTTime,
177  
-        'DecimalField' : OFTReal,
178  
-        'CharField' : OFTString,
179  
-        'TextField' : OFTString,
180  
-        'SmallIntegerField' : OFTInteger,
181  
-        'PositiveSmallIntegerField' : OFTInteger,
  159
+        models.AutoField : OFTInteger,
  160
+        models.IntegerField : (OFTInteger, OFTReal),
  161
+        models.FloatField : (OFTInteger, OFTReal),
  162
+        models.DateField : OFTDate,
  163
+        models.DateTimeField : OFTDateTime,
  164
+        models.TimeField : OFTTime,
  165
+        models.DecimalField : (OFTInteger, OFTReal),
  166
+        models.CharField : OFTString,
  167
+        models.TextField : OFTString,
  168
+        models.SmallIntegerField : (OFTInteger, OFTReal),
  169
+        models.PositiveSmallIntegerField : (OFTInteger, OFTReal),
182 170
         }
183 171
 
184 172
     # The acceptable transaction modes.
@@ -187,17 +175,17 @@ class LayerMapping(object):
187 175
                          }
188 176
 
189 177
     def __init__(self, model, data, mapping, layer=0, 
190  
-                 source_srs=None, encoding=None, check=True, 
  178
+                 source_srs=None, encoding=None, check=True, pipe=sys.stdout,
191 179
                  progress=False, interval=1000, strict=False, silent=False,
192 180
                  transaction_mode='commit_on_success', transform=True,
193 181
                  unique=False):
194  
-        "Takes the Django model, the data source, and the mapping (dictionary)"
195  
-
196  
-        # Getting the field names and types from the model
197  
-        self.fields = dict((f.name, self.map_foreign_key(f)) for f in model._meta.fields)
198  
-        self.field_classes = dict((f.name, f) for f in model._meta.fields)
199  
-
200  
-        # Getting the DataSource and its Layer
  182
+        """
  183
+        A LayerMapping object is initialized using the given Model (not an instance),
  184
+        a DataSource (or string path to an OGR-supported data file), and a mapping
  185
+        dictionary.  See the module level docstring for more details and keyword
  186
+        argument usage.
  187
+        """
  188
+        # Getting the DataSource and the associated Layer.
201 189
         if isinstance(data, basestring):
202 190
             self.ds = DataSource(data)
203 191
         else:
@@ -223,14 +211,16 @@ def __init__(self, model, data, mapping, layer=0,
223 211
             self.transform = transform
224 212
 
225 213
         # Checking the layer -- intitialization of the object will fail if
226  
-        # things don't check out before hand.  This may be time-consuming,
227  
-        # and disabled by setting the `check` keyword to False.
228  
-        if check: self.check_layer()
  214
+        # things don't check out before hand.
  215
+        self.check_layer()
229 216
 
230  
-        # The silent, strict, progress, and interval flags.
231  
-        self.silent = silent
  217
+        # The strict flag -- if it is set, exceptions will be propagated up.
232 218
         self.strict = strict
  219
+
  220
+        # Setting the keyword arguments related to status printing.
  221
+        self.silent = silent
233 222
         self.progress = progress
  223
+        self.pipe = pipe
234 224
         self.interval = interval
235 225
 
236 226
         # Setting the encoding for OFTString fields, if specified.
@@ -257,63 +247,92 @@ def __init__(self, model, data, mapping, layer=0,
257 247
             self.transaction_mode = transaction_mode
258 248
         else:
259 249
             raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode)
  250
+    
  251
+    #### Checking routines used during initialization ####
  252
+    def check_layer(self):
  253
+        """
  254
+        This checks the Layer metadata, and ensures that it is compatible
  255
+        with the mapping information and model.  Unlike previous revisions,
  256
+        there is no need to increment through each feature in the Layer.
  257
+        """
  258
+        # The geometry field of the model is set here.
  259
+        # TODO: Support more than one geometry field / model.
  260
+        self.geom_field = False
  261
+        self.fields = {}
260 262
 
261  
-    def check_feature(self, feat):
262  
-        "Checks the OGR feature against the model fields and mapping."
263  
-        HAS_GEO = False   
264  
-
265  
-        # Incrementing through each model_field & ogr_field in the given mapping.
266  
-        for model_field, ogr_field in self.mapping.items():
267  
-            # Making sure the given mapping model field is in the given model fields.
268  
-            if model_field in self.fields:
269  
-                model_type = self.fields[model_field]
270  
-            elif model_field[:-3] in self.fields: #foreign key
271  
-                model_type = self.fields[model_field[:-3]]
272  
-            else:
273  
-                raise LayerMapError('Given mapping field "%s" not in given Model fields!' % model_field)
  263
+        # Getting lists of the field names and the field types available in
  264
+        # the OGR Layer.
  265
+        ogr_fields = self.layer.fields
  266
+        ogr_field_types = self.layer.field_types
274 267
 
275  
-            ### Handling if we get a geometry in the Field ###
276  
-            if ogr_field in self.OGC_TYPES:
277  
-                # At this time, no more than one geographic field per model =(
278  
-                if HAS_GEO:
279  
-                    raise LayerMapError('More than one geographic field in mapping not allowed (yet).')
280  
-                else:
281  
-                    HAS_GEO = ogr_field
282  
-
283  
-                # Making sure this geometry field type is a valid Django GIS field.
284  
-                if not model_type in self.GIS_FIELDS:
285  
-                    raise LayerMapError('Unknown Django GIS field type "%s"' % model_type)
286  
-
287  
-                # Getting the OGRGeometry, it's type (an integer) and it's name (a string)
288  
-                geom  = feat.geom
289  
-                gtype = geom.geom_type
290  
-                gname = geom.geom_name
291  
-
292  
-                if self.make_multi(gname, model_type):
293  
-                    # Do we have to 'upsample' into a Geometry Collection?
294  
-                    pass
295  
-                elif gtype == self.OGC_TYPES[self.GIS_FIELDS[model_type]]:
296  
-                    # The geometry type otherwise was expected
297  
-                    pass
298  
-                else:
299  
-                    raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype))
300  
-            ### Handling other fields ###
301  
-            else:
302  
-                # Making sure the model field is supported.
303  
-                if not model_type in self.FIELD_TYPES:
304  
-                    raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % model_type)
  268
+        # Function for determining if the OGR mapping field is in the Layer.
  269
+        def check_ogr_fld(ogr_map_fld):
  270
+            try:
  271
+                idx = ogr_fields.index(ogr_map_fld)
  272
+            except ValueError:
  273
+                raise LayerMapError('Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld)
  274
+            return idx
  275
+
  276
+        # No need to increment through each feature in the model, simply check
  277
+        # the Layer metadata against what was given in the mapping dictionary.
  278
+        for field_name, ogr_name in self.mapping.items():
  279
+            # Ensuring that a corresponding field exists in the model
  280
+            # for the given field name in the mapping.
  281
+            try:
  282
+                model_field = self.model._meta.get_field(field_name)
  283
+            except models.fields.FieldDoesNotExist:
  284
+                raise LayerMapError('Given mapping field "%s" not in given Model fields.' % field_name)
305 285
 
306  
-                # Otherwise, we've got an OGR Field.  Making sure that an
307  
-                # index exists for the mapping OGR field.
308  
-                try:
309  
-                    fi = feat.index(ogr_field)
310  
-                except:
311  
-                    raise LayerMapError('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field)
  286
+            # Getting the string name for the Django field class (e.g., 'PointField').
  287
+            fld_name = model_field.__class__.__name__
312 288
 
313  
-    def check_layer(self):
314  
-        "Checks every feature in this object's layer."
315  
-        for feat in self.layer:
316  
-            self.check_feature(feat)
  289
+            if isinstance(model_field, GeometryField):
  290
+                if self.geom_field:
  291
+                    raise LayerMapError('LayerMapping does not support more than one GeometryField per model.')
  292
+
  293
+                try:
  294
+                    gtype = OGRGeomType(ogr_name)
  295
+                except OGRException:
  296
+                    raise LayerMapError('Invalid mapping for GeometryField "%s".' % field_name)
  297
+
  298
+                # Making sure that the OGR Layer's Geometry is compatible.
  299
+                ltype = self.layer.geom_type
  300
+                if not (gtype == ltype or self.make_multi(ltype, model_field)):
  301
+                    raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s.' % (fld_name, gtype))
  302
+
  303
+                # Setting the `geom_field` attribute w/the name of the model field
  304
+                # that is a Geometry.
  305
+                self.geom_field = field_name
  306
+                fields_val = model_field
  307
+            elif isinstance(model_field, models.ForeignKey):
  308
+                if isinstance(ogr_name, dict):
  309
+                    # Is every given related model mapping field in the Layer?
  310
+                    rel_model = model_field.rel.to
  311
+                    for rel_name, ogr_field in ogr_name.items(): 
  312
+                        idx = check_ogr_fld(ogr_field)
  313
+                        try:
  314
+                            rel_field = rel_model._meta.get_field(rel_name)
  315
+                        except models.fields.FieldDoesNotExist:
  316
+                            raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' % 
  317
+                                                (rel_name, rel_model.__class__.__name__))
  318
+                    fields_val = rel_model
  319
+                else:
  320
+                    raise TypeError('ForeignKey mapping must be of dictionary type.')
  321
+            else:
  322
+                # Is the model field type supported by LayerMapping?
  323
+                if not model_field.__class__ in self.FIELD_TYPES:
  324
+                    raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % fld_name)
  325
+
  326
+                # Is the OGR field in the Layer?
  327
+                idx = check_ogr_fld(ogr_name)
  328
+                
  329
+                # Can the OGR field type be mapped to the Django field type?
  330
+                if not issubclass(ogr_field_types[idx], self.FIELD_TYPES[model_field.__class__]):
  331
+                    raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' % 
  332
+                                        (ogr_field, ogr_field_types[idx].__name__, fld_name))
  333
+                fields_val = model_field
  334
+        
  335
+            self.fields[field_name] = fields_val
317 336
 
318 337
     def check_srs(self, source_srs):
319 338
         "Checks the compatibility of the given spatial reference object."
@@ -321,12 +340,12 @@ def check_srs(self, source_srs):
321 340
             sr = source_srs
322 341
         elif isinstance(source_srs, SpatialRefSys):
323 342
             sr = source_srs.srs
324  
-        elif isinstance(source_srs, (int, str)):
  343
+        elif isinstance(source_srs, (int, basestring)):
325 344
             sr = SpatialReference(source_srs)
326 345
         else:
327 346
             # Otherwise just pulling the SpatialReference from the layer
328 347
             sr = self.layer.srs
329  
-            
  348
+        
330 349
         if not sr:
331 350
             raise LayerMapError('No source reference system defined.')
332 351
         else:
@@ -334,14 +353,6 @@ def check_srs(self, source_srs):
334 353
 
335 354
     def check_unique(self, unique):
336 355
         "Checks the `unique` keyword parameter -- may be a sequence or string."
337  
-        # Getting the geometry field; only the first encountered GeometryField
338  
-        # will be used.
339  
-        self.geom_field = False
340  
-        for model_field, ogr_fld in self.mapping.items():
341  
-            if ogr_fld in self.OGC_TYPES:
342  
-                self.geom_field = model_field
343  
-                break
344  
-
345 356
         if isinstance(unique, (list, tuple)):
346 357
             # List of fields to determine uniqueness with
347 358
             for attr in unique: 
@@ -352,20 +363,14 @@ def check_unique(self, unique):
352 363
         else:
353 364
             raise TypeError('Unique keyword argument must be set with a tuple, list, or string.')
354 365
 
355  
-    def coord_transform(self):
356  
-        "Returns the coordinate transformation object."
357  
-        try:
358  
-            # Getting the target spatial reference system
359  
-            target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs
360  
-        
361  
-            # Creating the CoordTransform object
362  
-            return CoordTransform(self.source_srs, target_srs)
363  
-        except Exception, msg:
364  
-            raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg)
365  
-
  366
+    #### Keyword argument retrieval routines ####
366 367
     def feature_kwargs(self, feat):
367  
-        "Returns the keyword arguments needed for saving a feature."
368  
-        
  368
+        """
  369
+        Given an OGR Feature, this will return a dictionary of keyword arguments
  370
+        for constructing the mapped model.  Also returned is the `all_prepped`
  371
+        flag, which is used to signal that a model corresponding to a ForeignKey
  372
+        mapping does not exist.
  373
+        """
369 374
         # The keyword arguments for model construction.
370 375
         kwargs = {}
371 376
 
@@ -375,37 +380,24 @@ def feature_kwargs(self, feat):
375 380
 
376 381
         # Incrementing through each model field and OGR field in the
377 382
         # dictionary mapping.
378  
-        for model_field, ogr_field in self.mapping.items():
379  
-            is_fk = False
380  
-            try:
381  
-                model_type = self.fields[model_field]
382  
-            except KeyError: #foreign key
383  
-                # The -3 index is b/c foreign keys are appended w/'_id'.
384  
-                model_type = self.fields[model_field[:-3]]
385  
-                is_fk = True
  383
+        for field_name, ogr_name in self.mapping.items():
  384
+            model_field = self.fields[field_name]
386 385
             
387  
-            if ogr_field in self.OGC_TYPES:
  386
+            if isinstance(model_field, GeometryField):
388 387
                 # Verify OGR geometry.
389  
-                val = self.verify_geom(feat.geom, model_type)
  388
+                val = self.verify_geom(feat.geom, model_field)
  389
+            elif isinstance(model_field, models.base.ModelBase):
  390
+                # The related _model_, not a field was passed in -- indicating
  391
+                # another mapping for the related Model.
  392
+                val = self.verify_fk(feat, model_field, ogr_name)
  393
+                if not val: all_prepped = False
390 394
             else:
391 395
                 # Otherwise, verify OGR Field type.
392  
-                val = self.verify_field(feat[ogr_field], model_field)
  396
+                val = self.verify_ogr_field(feat[ogr_name], model_field)
393 397
 
394  
-            if is_fk:
395  
-                # Handling if foreign key.
396  
-                rel_obj = None
397  
-                field_name = model_field[:-3]
398  
-                try:
399  
-                    # FIXME: refactor to efficiently fetch FKs.
400  
-                    #  Requires significant re-work. :-/
401  
-                    rel = self.model._meta.get_field(field_name).rel
402  
-                    rel_obj = rel.to._default_manager.get(**{('%s__exact' % rel.field_name):val})
403  
-                except ObjectDoesNotExist:
404  
-                    all_prepped = False
405  
-
406  
-                kwargs[model_field[:-3]] = rel_obj
407  
-            else:
408  
-                kwargs[model_field] = val
  398
+            # Setting the keyword arguments for the field name with the
  399
+            # value obtained above.
  400
+            kwargs[field_name] = val
409 401
             
410 402
         return kwargs, all_prepped
411 403
 
@@ -420,29 +412,29 @@ def unique_kwargs(self, kwargs):
420 412
         else:
421 413
             return dict((fld, kwargs[fld]) for fld in self.unique)
422 414
 
423  
-    def verify_field(self, fld, model_field):
  415
+    #### Verification routines used in constructing model keyword arguments. ####
  416
+    def verify_ogr_field(self, ogr_field, model_field):
424 417
         """
425 418
         Verifies if the OGR Field contents are acceptable to the Django
426 419
         model field.  If they are, the verified value is returned, 
427 420
         otherwise the proper exception is raised.
428 421
         """
429  
-        field_class = self.field_classes[model_field]
430  
-        if isinstance(fld, OFTString):
  422
+        if isinstance(ogr_field, OFTString):
431 423
             if self.encoding:
432 424
                 # The encoding for OGR data sources may be specified here
433 425
                 # (e.g., 'cp437' for Census Bureau boundary files).
434  
-                val = unicode(fld.value, self.encoding)
  426
+                val = unicode(ogr_field.value, self.encoding)
435 427
             else:
436  
-                val = fld.value
437  
-                if len(val) > field_class.max_length:
  428
+                val = ogr_field.value
  429
+                if len(val) > model_field.max_length:
438 430
                     raise InvalidString('%s model field maximum string length is %s, given %s characters.' %
439  
-                                        (model_field, field_class.max_length, len(val)))
440  
-        elif isinstance(fld, OFTReal):
  431
+                                        (model_field.name, model_field.max_length, len(val)))
  432
+        elif isinstance(ogr_field, OFTReal):
441 433
             try:
442 434
                 # Creating an instance of the Decimal value to use.
443  
-                d = Decimal(str(fld.value))
  435
+                d = Decimal(str(ogr_field.value))
444 436
             except:
445  
-                raise InvalidDecimal('Could not construct decimal from: %s' % fld)
  437
+                raise InvalidDecimal('Could not construct decimal from: %s' % ogr_field)
446 438
 
447 439
             # Getting the decimal value as a tuple.
448 440
             dtup = d.as_tuple()
@@ -450,7 +442,7 @@ def verify_field(self, fld, model_field):
450 442
             d_idx = dtup[2] # index where the decimal is
451 443
 
452 444
             # Maximum amount of precision, or digits to the left of the decimal.
453  
-            max_prec = field_class.max_digits - field_class.decimal_places
  445
+            max_prec = model_field.max_digits - model_field.decimal_places
454 446
 
455 447
             # Getting the digits to the left of the decimal place for the 
456 448
             # given decimal.
@@ -463,17 +455,43 @@ def verify_field(self, fld, model_field):
463 455
             # InvalidDecimal exception.
464 456
             if n_prec > max_prec:
465 457
                 raise InvalidDecimal('A DecimalField with max_digits %d, decimal_places %d must round to an absolute value less than 10^%d.' %
466  
-                                     (field_class.max_digits, field_class.decimal_places, max_prec))
  458
+                                     (model_field.max_digits, model_field.decimal_places, max_prec))
467 459
             val = d
468 460
         else:
469  
-            val = fld.value
  461
+            val = ogr_field.value
470 462
         return val
471 463
 
472  
-    def verify_geom(self, geom, model_type):
473  
-        "Verifies the geometry."
474  
-        if self.make_multi(geom.geom_name, model_type):
  464
+    def verify_fk(self, feat, rel_model, rel_mapping):
  465
+        """
  466
+        Given an OGR Feature, the related model and its dictionary mapping,
  467
+        this routine will retrieve the related model for the ForeignKey
  468
+        mapping.
  469
+        """
  470
+        # TODO: It is expensive to retrieve a model for every record --
  471
+        #  explore if an efficient mechanism exists for caching related 
  472
+        #  ForeignKey models.
  473
+
  474
+        # Constructing and verifying the related model keyword arguments.
  475
+        fk_kwargs = {}
  476
+        for field_name, ogr_name in rel_mapping.items():
  477
+            fk_kwargs[field_name] = self.verify_ogr_field(feat[ogr_name], rel_model._meta.get_field(field_name))
  478
+
  479
+        # Attempting to retrieve and return the related model.
  480
+        try:
  481
+            return rel_model.objects.get(**fk_kwargs)
  482
+        except ObjectDoesNotExist:
  483
+            if self.strict: raise MissingForeignKey('No %s model found with keyword arguments: %s' % (rel_model.__name__, fk_kwargs))
  484
+            else: return None
  485
+
  486
+    def verify_geom(self, geom, model_field):
  487
+        """
  488
+        Verifies the geometry -- will construct and return a GeometryCollection
  489
+        if necessary (for example if the model field is MultiPolygonField while
  490
+        the mapped shapefile only contains Polygons).
  491
+        """
  492
+        if self.make_multi(geom.geom_type, model_field):
475 493
             # Constructing a multi-geometry type to contain the single geometry
476  
-            multi_type = self.MULTI_TYPES[geom.geom_name]
  494
+            multi_type = self.MULTI_TYPES[geom.geom_type.num]
477 495
             g = OGRGeometry(multi_type)
478 496
             g.add(geom)
479 497
         else:
@@ -486,7 +504,19 @@ def verify_geom(self, geom, model_type):
486 504
         
487 505
         # Returning the WKT of the geometry.
488 506
         return g.wkt
489  
-        
  507
+
  508
+    #### Other model methods ####
  509
+    def coord_transform(self):
  510
+        "Returns the coordinate transformation object."
  511
+        try:
  512
+            # Getting the target spatial reference system
  513
+            target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs
  514
+
  515
+            # Creating the CoordTransform object
  516
+            return CoordTransform(self.source_srs, target_srs)
  517
+        except Exception, msg:
  518
+            raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg)
  519
+
490 520
     def geometry_column(self):
491 521
         "Returns the GeometryColumn model associated with the geographic column."
492 522
         # Getting the GeometryColumn object.
@@ -498,24 +528,23 @@ def geometry_column(self):
498 528
         except Exception, msg:
499 529
             raise LayerMapError('Geometry column does not exist for model. (did you run syncdb?):\n %s' % msg)
500 530
 
501  
-    def make_multi(self, geom_name, model_type):
502  
-        "Determines whether the geometry should be made into a GeometryCollection."
503  
-        return (geom_name in self.MULTI_TYPES) and (model_type.startswith('Multi'))
504  
-
505  
-    def map_foreign_key(self, django_field):
506  
-        "Handles fields within foreign keys for the given field."
507  
-        if not django_field.__class__ is ForeignKey:
508  
-            # Returning the field's class name.
509  
-            return django_field.__class__.__name__
510  
-        else:
511  
-            # Otherwise, getting the type of the related field's
512  
-            # from the Foreign key.
513  
-            rf = django_field.rel.get_related_field()
514  
-            return rf.get_internal_type()
  531
+    def make_multi(self, geom_type, model_field):
  532
+        """
  533
+        Given the OGRGeomType for a geometry and its associated GeometryField, 
  534
+        determine whether the geometry should be turned into a GeometryCollection.
  535
+        """
  536
+        return (geom_type.num in self.MULTI_TYPES and 
  537
+                model_field.__class__.__name__ == 'Multi%s' % geom_type.django)
515 538
 
516 539
     def save(self, verbose=False):
517  
-        "Runs the layer mapping on the given SHP file, and saves to the database."
518  
-        
  540
+        """
  541
+        Saves the contents from the OGR DataSource Layer into the database
  542
+        according to the mapping dictionary given at initialization. If
  543
+        the `verbose` keyword is set, information will be printed subsequent
  544
+        to each model save executed on the database.
  545
+        """
  546
+        # Defining the 'real' save method, utilizing the transaction 
  547
+        # decorator created during initialization.
519 548
         @self.transaction_decorator
520 549
         def _save():
521 550
             num_feat = 0
@@ -530,7 +559,7 @@ def _save():
530 559
                     # Something borked the validation
531 560
                     if self.strict: raise
532 561
                     elif not self.silent: 
533  
-                        print 'Ignoring Feature ID %s because: %s' % (feat.fid, msg)
  562
+                        self.pipe.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg))
534 563
                 else:
535 564
                     # Constructing the model using the keyword args
536 565
                     if all_prepped:
@@ -561,7 +590,7 @@ def _save():
561 590
                             # Attempting to save.
562 591
                             m.save()
563 592
                             num_saved += 1
564  
-                            if verbose: print 'Saved: %s' % m
  593
+                            if verbose: self.pipe.write('Saved: %s\n' % m)
565 594
                         except SystemExit:
566 595
                             raise
567 596
                         except Exception, msg:
@@ -572,17 +601,18 @@ def _save():
572 601
                             if self.strict: 
573 602
                                 # Bailing out if the `strict` keyword is set.
574 603
                                 if not self.silent:
575  
-                                    print 'Failed to save the feature (id: %s) into the model with the keyword arguments:' % feat.fid
576  
-                                    print kwargs
  604
+                                    self.pipe.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid)
  605
+                                    self.pipe.write('%s\n' % kwargs)
577 606
                                 raise
578 607
                             elif not self.silent:
579  
-                                print 'Failed to save %s:\n %s\nContinuing' % (kwargs, msg)
  608
+                                self.pipe.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg))
580 609
                     else:
581  
-                        print 'Skipping %s due to missing relation.' % kwargs
  610
+                        if not self.silent: self.pipe.write('Skipping due to missing relation:\n%s\n' % kwargs)
  611
+                        
582 612
 
583 613
                 # Printing progress information, if requested.
584 614
                 if self.progress and num_feat % self.interval == 0:
585  
-                    print 'Processed %d features, saved %d ...' % (num_feat, num_saved)
  615
+                    self.pipe.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved))
586 616
                
587 617
         # Calling our defined function, which will use the specified
588 618
         # trasaction mode.

0 notes on commit b11172e

Please sign in to comment.
Something went wrong with that request. Please try again.