Merge 6a9f4b1 into ed1874e

appveyor.yml

@@ -137,7 +137,9 @@ test_script:
   - ps: python -c "import fiona"
 
   # TODO: return to running test_write_gb18030 when GDAL build is updated.
-  - "%CMD_IN_ENV% python -m pytest -k \"not test_write_gb18030\" --cov fiona --cov-report term-missing"
+  # GenericWriting test is skipped on Appveyor because our Windows GDAL doesn't
+  # have iconv support and can't encode field names.
+  - "%CMD_IN_ENV% python -m pytest -k \"not test_write_gb18030 and not GenericWritingTest\" --cov fiona --cov-report term-missing"
 
 matrix:
   allow_failures:

fiona/_err.pxd

@@ -5,5 +5,6 @@ cdef extern from "cpl_vsi.h":
     ctypedef FILE VSILFILE
 
 
+cdef int exc_wrap_int(int retval) except -1
 cdef void *exc_wrap_pointer(void *ptr) except NULL
 cdef VSILFILE *exc_wrap_vsilfile(VSILFILE *f) except NULL

fiona/_err.pyx

@@ -226,6 +226,18 @@ cdef inline object exc_check():
         return
 
 
+cdef int exc_wrap_int(int err) except -1:
+    """Wrap a GDAL/OGR function that returns CPLErr or OGRErr (int)
+
+    Raises a Rasterio exception if a non-fatal error has be set.
+    """
+    if err:
+        exc = exc_check()
+        if exc:
+            raise exc
+    return err
+
+
 cdef void *exc_wrap_pointer(void *ptr) except NULL:
     """Wrap a GDAL/OGR function that returns GDALDatasetH etc (void *)
     Raises a Rasterio exception if a non-fatal error has be set.

fiona/fio/load.py

@@ -2,21 +2,17 @@
 
 
 from functools import partial
-import itertools
-import json
 import logging
 
 import click
 import cligj
 
 import fiona
 from fiona.fio import options
+from fiona.schema import FIELD_TYPES_MAP_REV
 from fiona.transform import transform_geom
 
 
-FIELD_TYPES_MAP_REV = dict([(v, k) for k, v in fiona.FIELD_TYPES_MAP.items()])
-
-
 @click.command(short_help="Load GeoJSON to a dataset in another format.")
 @click.argument('output', required=True)
 @click.option('-f', '--format', '--driver', required=True,

fiona/ogrext.pyx

@@ -19,9 +19,9 @@ from fiona._shim cimport *
 from fiona._geometry cimport (
     GeomBuilder, OGRGeomBuilder, geometry_type_code,
     normalize_geometry_type_code)
-from fiona._err cimport exc_wrap_pointer, exc_wrap_vsilfile
+from fiona._err cimport exc_wrap_int, exc_wrap_pointer, exc_wrap_vsilfile
 
-from fiona._err import cpl_errs, FionaNullPointerError
+from fiona._err import cpl_errs, FionaNullPointerError, CPLE_BaseError
 from fiona._geometry import GEOMETRY_TYPES
 from fiona import compat
 from fiona.errors import (
@@ -30,6 +30,7 @@ from fiona.errors import (
 from fiona.compat import OrderedDict
 from fiona.rfc3339 import parse_date, parse_datetime, parse_time
 from fiona.rfc3339 import FionaDateType, FionaDateTimeType, FionaTimeType
+from fiona.schema import FIELD_TYPES, FIELD_TYPES_MAP, normalize_field_type
 
 from fiona._shim cimport is_field_null
 
@@ -38,40 +39,7 @@ from libc.string cimport strcmp
 from cpython cimport PyBytes_FromStringAndSize, PyBytes_AsString
 
 
-log = logging.getLogger("Fiona")
-
-# Mapping of OGR integer field types to Fiona field type names.
-#
-# Lists are currently unsupported in this version, but might be done as
-# arrays in a future version.
-
-FIELD_TYPES = [
-    'int',          # OFTInteger, Simple 32bit integer
-    None,           # OFTIntegerList, List of 32bit integers
-    'float',        # OFTReal, Double Precision floating point
-    None,           # OFTRealList, List of doubles
-    'str',          # OFTString, String of ASCII chars
-    None,           # OFTStringList, Array of strings
-    None,           # OFTWideString, deprecated
-    None,           # OFTWideStringList, deprecated
-    'bytes',        # OFTBinary, Raw Binary data
-    'date',         # OFTDate, Date
-    'time',         # OFTTime, Time
-    'datetime',     # OFTDateTime, Date and Time
-    'int',          # OFTInteger64, Single 64bit integer
-    None,           # OFTInteger64List, List of 64bit integers
-    ]
-
-# Mapping of Fiona field type names to Python types.
-FIELD_TYPES_MAP = {
-    'int':      int,
-    'float':    float,
-    'str':      text_type,
-    'date':     FionaDateType,
-    'time':     FionaTimeType,
-    'datetime': FionaDateTimeType,
-    'bytes':    bytes,
-   }
+log = logging.getLogger(__name__)
 
 DEFAULT_TRANSACTION_SIZE = 20000
 
@@ -163,8 +131,29 @@ cdef class FeatureBuilder:
 
     cdef build(self, void *feature, encoding='utf-8', bbox=False, driver=None,
                ignore_fields=None, ignore_geometry=False):
-        # The only method anyone ever needs to call
-        cdef void *fdefn
+        """Build a Fiona feature object from an OGR feature
+
+        Parameters
+        ----------
+        feature : void *
+            The OGR feature # TODO: use a real typedef
+        encoding : str
+            The encoding of OGR feature attributes
+        bbox : bool
+            Not used
+        driver : str
+            OGR format driver name like 'GeoJSON'
+        ignore_fields : sequence
+            A sequence of field names that will be ignored and omitted
+            in the Fiona feature properties
+        ignore_geometry : bool
+            Flag for whether the OGR geometry field is to be ignored
+
+        Returns
+        -------
+        dict
+        """
+        cdef void *fdefn = NULL
         cdef int i
         cdef int y = 0
         cdef int m = 0
@@ -173,20 +162,24 @@ cdef class FeatureBuilder:
         cdef int mm = 0
         cdef int ss = 0
         cdef int tz = 0
-        cdef unsigned char *data
+        cdef unsigned char *data = NULL
         cdef int l
         cdef int retval
         cdef int fieldsubtype
         cdef const char *key_c = NULL
+
+        # Skeleton of the feature to be returned.
         fid = OGR_F_GetFID(feature)
         props = OrderedDict()
         fiona_feature = {
             "type": "Feature",
             "id": str(fid),
             "properties": props,
         }
-        if not ignore_fields:
-            ignore_fields = set()
+
+        ignore_fields = set(ignore_fields or [])
+
+        # Iterate over the fields of the OGR feature.
         for i in range(OGR_F_GetFieldCount(feature)):
             fdefn = OGR_F_GetFieldDefnRef(feature, i)
             if fdefn == NULL:
@@ -211,13 +204,22 @@ cdef class FeatureBuilder:
 
             # TODO: other types
             fieldtype = FIELD_TYPES_MAP[fieldtypename]
+
             if is_field_null(feature, i):
                 props[key] = None
+
+            elif fieldtypename is 'int32':
+                if fieldsubtype == OFSTBoolean:
+                    props[key] = bool(OGR_F_GetFieldAsInteger(feature, i))
+                else:
+                    props[key] = OGR_F_GetFieldAsInteger(feature, i)
+
             elif fieldtype is int:
                 if fieldsubtype == OFSTBoolean:
                     props[key] = bool(OGR_F_GetFieldAsInteger64(feature, i))
                 else:
                     props[key] = OGR_F_GetFieldAsInteger64(feature, i)
+
             elif fieldtype is float:
                 props[key] = OGR_F_GetFieldAsDouble(feature, i)
 
@@ -311,7 +313,11 @@ cdef class OGRFeatureBuilder:
             log.debug(
                 "Looking up %s in %s", key, repr(session._schema_mapping))
             ogr_key = session._schema_mapping[key]
-            schema_type = collection.schema['properties'][key]
+
+            schema_type = normalize_field_type(collection.schema['properties'][key])
+
+            log.debug("Normalizing schema type for key %r in schema %r to %r", key, collection.schema['properties'], schema_type)
+
             try:
                 key_bytes = ogr_key.encode(encoding)
             except UnicodeDecodeError:
@@ -328,7 +334,14 @@ cdef class OGRFeatureBuilder:
 
             # Continue over the standard OGR types.
             if isinstance(value, integer_types):
-                OGR_F_SetFieldInteger64(cogr_feature, i, value)
+
+                log.debug("Setting field %r, type %r, to value %r", i, schema_type, value)
+
+                if schema_type == 'int32':
+                    OGR_F_SetFieldInteger(cogr_feature, i, value)
+                else:
+                    OGR_F_SetFieldInteger64(cogr_feature, i, value)
+
             elif isinstance(value, float):
                 OGR_F_SetFieldDouble(cogr_feature, i, value)
             elif (isinstance(value, string_types)
@@ -523,8 +536,8 @@ cdef class Session:
     def get_schema(self):
         cdef int i
         cdef int n
-        cdef void *cogr_featuredefn
-        cdef void *cogr_fielddefn
+        cdef void *cogr_featuredefn = NULL
+        cdef void *cogr_fielddefn = NULL
         cdef const char *key_c
         props = []
 
@@ -539,25 +552,35 @@ cdef class Session:
         cogr_featuredefn = OGR_L_GetLayerDefn(self.cogr_layer)
         if cogr_featuredefn == NULL:
             raise ValueError("Null feature definition")
+
         n = OGR_FD_GetFieldCount(cogr_featuredefn)
+
         for i from 0 <= i < n:
+            cogr_fielddefn == NULL
             cogr_fielddefn = OGR_FD_GetFieldDefn(cogr_featuredefn, i)
             if cogr_fielddefn == NULL:
                 raise ValueError("Null field definition")
+
             key_c = OGR_Fld_GetNameRef(cogr_fielddefn)
             key_b = key_c
+
             if not bool(key_b):
                 raise ValueError("Invalid field name ref: %s" % key)
+
             key = key_b.decode(self.get_internalencoding())
+
             if key in ignore_fields:
+                log.debug("By request, ignoring field %r", key)
                 continue
+
             fieldtypename = FIELD_TYPES[OGR_Fld_GetType(cogr_fielddefn)]
             if not fieldtypename:
                 log.warning(
                     "Skipping field %s: invalid type %s",
                     key,
                     OGR_Fld_GetType(cogr_fielddefn))
                 continue
+
             val = fieldtypename
             if fieldtypename == 'float':
                 fmt = ""
@@ -568,16 +591,16 @@ cdef class Session:
                 if precision: # and precision != 15:
                     fmt += ".%d" % precision
                 val = "float" + fmt
-            elif fieldtypename == 'int':
+            elif fieldtypename in ('int32', 'int64'):
                 fmt = ""
                 width = OGR_Fld_GetWidth(cogr_fielddefn)
-                if width: # and width != 11:
+                if width:
                     fmt = ":%d" % width
-                val = fieldtypename + fmt
+                val = 'int' + fmt
             elif fieldtypename == 'str':
                 fmt = ""
                 width = OGR_Fld_GetWidth(cogr_fielddefn)
-                if width: # and width != 80:
+                if width:
                     fmt = ":%d" % width
                 val = fieldtypename + fmt
 
@@ -973,52 +996,70 @@ cdef class WritingSession(Session):
             # Next, make a layer definition from the given schema properties,
             # which are an ordered dict since Fiona 1.0.1.
             for key, value in collection.schema['properties'].items():
-                log.debug("Creating field: %s %s", key, value)
-
+
+                log.debug("Begin creating field: %r value: %r", key, value)
+
                 field_subtype = OFSTNone
 
                 # Convert 'long' to 'int'. See
                 # https://github.com/Toblerity/Fiona/issues/101.
-                if value == 'long':
-                    value = 'int'
-
+                if GDAL_VERSION_NUM >= 2000000 and value in ('int', 'long'):
+                    value = 'int64'
+                elif value == 'int':
+                    value = 'int32'
+
                 if value == 'bool':
-                    value = 'int'
+                    value = 'int32'
                     field_subtype = OFSTBoolean
 
                 # Is there a field width/precision?
                 width = precision = None
                 if ':' in value:
                     value, fmt = value.split(':')
+
+                    log.debug("Field format parsing, value: %r, fmt: %r", value, fmt)
+
                     if '.' in fmt:
                         width, precision = map(int, fmt.split('.'))
                     else:
                         width = int(fmt)
 
-                field_type = FIELD_TYPES.index(value)
-                if GDAL_VERSION_NUM >= 2000000:
-                    # See https://trac.osgeo.org/gdal/wiki/rfc31_ogr_64
-                    if value == 'int' and (width is not None and width >= 10):
-                        field_type = 12
+                    if value == 'int':
+                        if GDAL_VERSION_NUM >= 2000000 and (width == 0 or width >= 10):
+                            value = 'int64'
+                        else:
+                            value = 'int32'
 
+                field_type = FIELD_TYPES.index(value)
                 encoding = self.get_internalencoding()
                 key_bytes = key.encode(encoding)
 
-                cogr_fielddefn = OGR_Fld_Create(
-                    key_bytes,
-                    field_type)
+                try:
+                    cogr_fielddefn = exc_wrap_pointer(
+                        OGR_Fld_Create(key_bytes, field_type))
+                    log.debug("Creating field %r (%r) with type %r (%r)", key, key_bytes, field_type, value)
+                except Exception:
+                    raise
+
                 if cogr_fielddefn == NULL:
-                    raise ValueError("Null field definition")
+                    raise ValueError("Field {} definition is NULL".format(key))
+
                 if width:
                     OGR_Fld_SetWidth(cogr_fielddefn, width)
                 if precision:
                     OGR_Fld_SetPrecision(cogr_fielddefn, precision)
                 if field_subtype != OFSTNone:
                     # subtypes are new in GDAL 2.x, ignored in 1.x
                     set_field_subtype(cogr_fielddefn, field_subtype)
-                OGR_L_CreateField(self.cogr_layer, cogr_fielddefn, 1)
+
+                try:
+                    exc_wrap_int(OGR_L_CreateField(self.cogr_layer, cogr_fielddefn, 1))
+                except CPLE_BaseError as exc:
+                    raise SchemaError(str(exc))
+
                 OGR_Fld_Destroy(cogr_fielddefn)
-            log.debug("Created fields")
+
+                log.debug("End creating field %r", key)
 
         # Mapping of the Python collection schema to the munged
         # OGR schema.