Skip to content
Permalink
Browse files

Merge pull request #5225 from manisandro/ogr_gdb_subset218

[OGR] Add workaround for OGRSQL not recognizing the column name returned by OGR_L_GetFIDColumn
  • Loading branch information
manisandro committed Sep 21, 2017
2 parents da22be6 + a51ef7d commit 381c8b60d93c6830d61a9f54aaa82a6d1ceba3b7
Showing with 76 additions and 27 deletions.
  1. +14 −9 src/providers/ogr/qgsogrfeatureiterator.cpp
  2. +41 −15 src/providers/ogr/qgsogrprovider.cpp
  3. +19 −1 tests/src/python/test_provider_ogr.py
  4. +2 −2 tests/src/python/test_provider_ogr_sqlite.py
  5. BIN tests/testdata/test_gdb.gdb/a00000001.TablesByName.atx
  6. BIN tests/testdata/test_gdb.gdb/a00000001.gdbindexes
  7. BIN tests/testdata/test_gdb.gdb/a00000001.gdbtable
  8. BIN tests/testdata/test_gdb.gdb/a00000001.gdbtablx
  9. BIN tests/testdata/test_gdb.gdb/a00000002.gdbtable
  10. BIN tests/testdata/test_gdb.gdb/a00000002.gdbtablx
  11. BIN tests/testdata/test_gdb.gdb/a00000003.gdbindexes
  12. BIN tests/testdata/test_gdb.gdb/a00000003.gdbtable
  13. BIN tests/testdata/test_gdb.gdb/a00000003.gdbtablx
  14. BIN tests/testdata/test_gdb.gdb/a00000004.CatItemsByPhysicalName.atx
  15. BIN tests/testdata/test_gdb.gdb/a00000004.CatItemsByType.atx
  16. BIN tests/testdata/test_gdb.gdb/a00000004.FDO_UUID.atx
  17. BIN tests/testdata/test_gdb.gdb/a00000004.gdbindexes
  18. BIN tests/testdata/test_gdb.gdb/a00000004.gdbtable
  19. BIN tests/testdata/test_gdb.gdb/a00000004.gdbtablx
  20. BIN tests/testdata/test_gdb.gdb/a00000004.spx
  21. BIN tests/testdata/test_gdb.gdb/a00000005.CatItemTypesByName.atx
  22. BIN tests/testdata/test_gdb.gdb/a00000005.CatItemTypesByParentTypeID.atx
  23. BIN tests/testdata/test_gdb.gdb/a00000005.CatItemTypesByUUID.atx
  24. BIN tests/testdata/test_gdb.gdb/a00000005.gdbindexes
  25. BIN tests/testdata/test_gdb.gdb/a00000005.gdbtable
  26. BIN tests/testdata/test_gdb.gdb/a00000005.gdbtablx
  27. BIN tests/testdata/test_gdb.gdb/a00000006.CatRelsByDestinationID.atx
  28. BIN tests/testdata/test_gdb.gdb/a00000006.CatRelsByOriginID.atx
  29. BIN tests/testdata/test_gdb.gdb/a00000006.CatRelsByType.atx
  30. BIN tests/testdata/test_gdb.gdb/a00000006.FDO_UUID.atx
  31. BIN tests/testdata/test_gdb.gdb/a00000006.gdbindexes
  32. BIN tests/testdata/test_gdb.gdb/a00000006.gdbtable
  33. BIN tests/testdata/test_gdb.gdb/a00000006.gdbtablx
  34. BIN tests/testdata/test_gdb.gdb/a00000007.CatRelTypesByBackwardLabel.atx
  35. BIN tests/testdata/test_gdb.gdb/a00000007.CatRelTypesByDestItemTypeID.atx
  36. BIN tests/testdata/test_gdb.gdb/a00000007.CatRelTypesByForwardLabel.atx
  37. BIN tests/testdata/test_gdb.gdb/a00000007.CatRelTypesByName.atx
  38. BIN tests/testdata/test_gdb.gdb/a00000007.CatRelTypesByOriginItemTypeID.atx
  39. BIN tests/testdata/test_gdb.gdb/a00000007.CatRelTypesByUUID.atx
  40. BIN tests/testdata/test_gdb.gdb/a00000007.gdbindexes
  41. BIN tests/testdata/test_gdb.gdb/a00000007.gdbtable
  42. BIN tests/testdata/test_gdb.gdb/a00000007.gdbtablx
  43. BIN tests/testdata/test_gdb.gdb/a00000009.gdbindexes
  44. BIN tests/testdata/test_gdb.gdb/a00000009.gdbtable
  45. BIN tests/testdata/test_gdb.gdb/a00000009.gdbtablx
  46. BIN tests/testdata/test_gdb.gdb/a00000009.spx
  47. BIN tests/testdata/test_gdb.gdb/a0000000a.gdbindexes
  48. BIN tests/testdata/test_gdb.gdb/a0000000a.gdbtable
  49. BIN tests/testdata/test_gdb.gdb/a0000000a.gdbtablx
  50. BIN tests/testdata/test_gdb.gdb/a0000000a.spx
  51. BIN tests/testdata/test_gdb.gdb/a0000000b.gdbindexes
  52. BIN tests/testdata/test_gdb.gdb/a0000000b.gdbtable
  53. BIN tests/testdata/test_gdb.gdb/a0000000b.gdbtablx
  54. BIN tests/testdata/test_gdb.gdb/a0000000b.spx
  55. BIN tests/testdata/test_gdb.gdb/gdb
  56. BIN tests/testdata/test_gdb.gdb/timestamps
@@ -172,23 +172,28 @@ bool QgsOgrFeatureIterator::nextFeatureFilterExpression( QgsFeature& f )
bool QgsOgrFeatureIterator::fetchFeatureWithId( QgsFeatureId id, QgsFeature& feature ) const
{
feature.setValid( false );
OGRFeatureH fet;
OGRFeatureH fet = 0;
if ( mOrigFidAdded )
{
OGR_L_ResetReading( ogrLayer );
while (( fet = OGR_L_GetNextFeature( ogrLayer ) ) )
OGRFeatureDefnH fdef = OGR_L_GetLayerDefn( ogrLayer );
int lastField = OGR_FD_GetFieldCount( fdef ) - 1;
if ( lastField >= 0 )
{
if (
while (( fet = OGR_L_GetNextFeature( ogrLayer ) ) )
{
if (
#if defined(GDAL_VERSION_NUM) && GDAL_VERSION_NUM >= 2000000
OGR_F_GetFieldAsInteger64
OGR_F_GetFieldAsInteger64
#else
OGR_F_GetFieldAsInteger
OGR_F_GetFieldAsInteger
#endif
( fet, 0 ) == FID_TO_NUMBER( id ) )
{
break;
( fet, lastField ) == FID_TO_NUMBER( id ) )
{
break;
}
OGR_F_Destroy( fet );
}
OGR_F_Destroy( fet );
}
}
else
@@ -3443,33 +3443,59 @@ OGRLayerH QgsOgrProviderUtils::setSubsetString( OGRLayerH layer, OGRDataSourceH
layerName = encoding->fromUnicode( modifiedLayerName );
}
}
QByteArray sql;
if ( subsetString.startsWith( "SELECT ", Qt::CaseInsensitive ) )
sql = encoding->fromUnicode( subsetString );
OGRLayerH subsetLayer = 0;
if ( subsetString.startsWith( QLatin1String( "SELECT " ), Qt::CaseInsensitive ) )
{
QByteArray sql = encoding->fromUnicode( subsetString );

QgsDebugMsg( QString( "SQL: %1" ).arg( encoding->toUnicode( sql ) ) );
subsetLayer = GDALDatasetExecuteSQL( ds, sql.constData(), nullptr, nullptr );
}
else
{
QByteArray sqlPart1 = "SELECT *";
QByteArray sqlPart3 = " FROM " + quotedIdentifier( layerName, ogrDriverName )
+ " WHERE " + encoding->fromUnicode( subsetString );

origFidAddAttempted = true;

QByteArray fidColumn = OGR_L_GetFIDColumn( layer );
// Fallback to FID if OGR_L_GetFIDColumn returns nothing
if ( fidColumn.isEmpty() )
{
fidColumn = "FID";
}

QByteArray sql = sqlPart1 + ", " + fidColumn + " as orig_ogc_fid" + sqlPart3;
QgsDebugMsg( QString( "SQL: %1" ).arg( encoding->toUnicode( sql ) ) );
subsetLayer = GDALDatasetExecuteSQL( ds, sql.constData(), nullptr, nullptr );

sql = QByteArray( "SELECT " );
if ( !fidColumn.isEmpty() )
// See https://lists.osgeo.org/pipermail/qgis-developer/2017-September/049802.html
// If execute SQL fails because it did not find the fidColumn, retry with hardcoded FID
if ( !subsetLayer )
{
sql += fidColumn + " as orig_ogc_fid, ";
origFidAddAttempted = true;
QByteArray sql = sqlPart1 + ", " + "FID as orig_ogc_fid" + sqlPart3;
QgsDebugMsg( QString( "SQL: %1" ).arg( encoding->toUnicode( sql ) ) );
subsetLayer = GDALDatasetExecuteSQL( ds, sql.constData(), nullptr, nullptr );
}
// If that also fails, just continue without the orig_ogc_fid
if ( !subsetLayer )
{
QByteArray sql = sqlPart1 + sqlPart3;
QgsDebugMsg( QString( "SQL: %1" ).arg( encoding->toUnicode( sql ) ) );
subsetLayer = GDALDatasetExecuteSQL( ds, sql.constData(), nullptr, nullptr );
origFidAddAttempted = false;
}
sql += "* FROM " + quotedIdentifier( layerName, ogrDriverName );
sql += " WHERE " + encoding->fromUnicode( subsetString );
}

QgsDebugMsg( QString( "SQL: %1" ).arg( encoding->toUnicode( sql ) ) );
OGRLayerH subsetLayer = OGR_DS_ExecuteSQL( ds, sql.constData(), nullptr, nullptr );

// Check if first column is orig_ogc_fid
// Check if last column is orig_ogc_fid
if ( origFidAddAttempted && subsetLayer )
{
OGRFeatureDefnH fdef = OGR_L_GetLayerDefn( subsetLayer );
if ( OGR_FD_GetFieldCount( fdef ) > 0 )
int fieldCount = OGR_FD_GetFieldCount( fdef );
if ( fieldCount > 0 )
{
OGRFieldDefnH fldDef = OGR_FD_GetFieldDefn( fdef, 0 );
OGRFieldDefnH fldDef = OGR_FD_GetFieldDefn( fdef, fieldCount - 1 );
origFidAdded = qstrcmp( OGR_Fld_GetNameRef( fldDef ), "orig_ogc_fid" ) == 0;
}
}
@@ -17,7 +17,7 @@
import sys
import tempfile

from qgis.core import QgsVectorLayer, QgsVectorDataProvider, QgsWKBTypes
from qgis.core import QgsVectorLayer, QgsVectorDataProvider, QgsWKBTypes, QgsFeature
from qgis.testing import (
start_app,
unittest
@@ -240,5 +240,23 @@ def testGeometryCollection(self):
os.unlink(datasource)
self.assertFalse(os.path.exists(datasource))

def testGdb(self):
""" Test opening a GDB database layer"""
gdb_path = os.path.join(unitTestDataPath(), 'test_gdb.gdb')
for i in range(3):
l = QgsVectorLayer(gdb_path + '|layerid=' + str(i), 'test', 'ogr')
self.assertTrue(l.isValid())

def testGdbFilter(self):
""" Test opening a GDB database layer with filter"""
gdb_path = os.path.join(unitTestDataPath(), 'test_gdb.gdb')
l = QgsVectorLayer(gdb_path + '|layerid=1|subset="text" = \'shape 2\'', 'test', 'ogr')
self.assertTrue(l.isValid())
it = l.getFeatures()
f = QgsFeature()
while it.nextFeature(f):
self.assertTrue(f.attribute("text") == "shape 2")


if __name__ == '__main__':
unittest.main()
@@ -175,7 +175,7 @@ def testSubsetStringFids(self):

vl = QgsVectorLayer(tmpfile + "|subset=type=2", 'test', 'ogr')
self.assertTrue(vl.isValid())
self.assertTrue(vl.fields().at(0).name() == "orig_ogc_fid")
self.assertTrue(vl.fields().at(vl.fields().count() - 1).name() == "orig_ogc_fid")

req = QgsFeatureRequest()
req.setFilterExpression("value=16")
@@ -186,7 +186,7 @@ def testSubsetStringFids(self):

# Check that subset string is correctly set on reload
vl.reload()
self.assertTrue(vl.fields().at(0).name() == "orig_ogc_fid")
self.assertTrue(vl.fields().at(vl.fields().count() - 1).name() == "orig_ogc_fid")


if __name__ == '__main__':
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.

0 comments on commit 381c8b6

Please sign in to comment.
You can’t perform that action at this time.