Skip to content
Permalink
Browse files

Merge pull request #5787 from nyalldawson/index

Fix OGR provider cannot create attribute or spatial indexes for GeoPackage/SQLite layers
  • Loading branch information
nyalldawson committed Dec 2, 2017
2 parents 31c79da + aaa18e0 commit b8b8c1ae93c8ea4f4774a672c854933bba6d8670
@@ -1966,36 +1966,74 @@ bool QgsOgrProvider::createSpatialIndex()
if ( !doInitialActionsForEdition() )
return false;

if ( mGDALDriverName != QLatin1String( "ESRI Shapefile" ) )
return false;

QByteArray layerName = mOgrOrigLayer->name();
if ( mGDALDriverName == QLatin1String( "ESRI Shapefile" ) )
{
QByteArray sql = QByteArray( "CREATE SPATIAL INDEX ON " ) + quotedIdentifier( layerName ); // quote the layer name so spaces are handled
QgsDebugMsg( QString( "SQL: %1" ).arg( QString::fromUtf8( sql ) ) );
mOgrOrigLayer->ExecuteSQLNoReturn( sql );

QByteArray sql = QByteArray( "CREATE SPATIAL INDEX ON " ) + quotedIdentifier( layerName ); // quote the layer name so spaces are handled
QgsDebugMsg( QString( "SQL: %1" ).arg( QString::fromUtf8( sql ) ) );
mOgrOrigLayer->ExecuteSQLNoReturn( sql );
QFileInfo fi( mFilePath ); // to get the base name
//find out, if the .qix file is there
return QFileInfo::exists( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".qix" ) );
}
else if ( mGDALDriverName == QLatin1String( "GPKG" ) ||
mGDALDriverName == QLatin1String( "SQLite" ) )
{
QMutex *mutex = nullptr;
OGRLayerH layer = mOgrOrigLayer->getHandleAndMutex( mutex );
QByteArray sql = QByteArray( "SELECT CreateSpatialIndex(" + quotedIdentifier( layerName ) + ","
+ quotedIdentifier( OGR_L_GetGeometryColumn( layer ) ) + ") " ); // quote the layer name so spaces are handled
mOgrOrigLayer->ExecuteSQLNoReturn( sql );
return true;
}
return false;
}

QFileInfo fi( mFilePath ); // to get the base name
//find out, if the .qix file is there
QFile indexfile( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".qix" ) );
return indexfile.exists();
QString createIndexName( QString tableName, QString field )
{
QRegularExpression safeExp( QStringLiteral( "[^a-zA-Z0-9]" ) );
tableName.replace( safeExp, QStringLiteral( "_" ) );
field.replace( safeExp, QStringLiteral( "_" ) );
return tableName + "_" + field + "_idx";
}

bool QgsOgrProvider::createAttributeIndex( int field )
{
if ( field < 0 || field >= mAttributeFields.count() )
return false;

if ( !doInitialActionsForEdition() )
return false;

QByteArray quotedLayerName = quotedIdentifier( mOgrOrigLayer->name() );
QByteArray dropSql = "DROP INDEX ON " + quotedLayerName;
mOgrOrigLayer->ExecuteSQLNoReturn( dropSql );
QByteArray createSql = "CREATE INDEX ON " + quotedLayerName + " USING " + textEncoding()->fromUnicode( fields().at( field ).name() );
mOgrOrigLayer->ExecuteSQLNoReturn( createSql );
if ( mGDALDriverName == QLatin1String( "GPKG" ) ||
mGDALDriverName == QLatin1String( "SQLite" ) )
{
if ( field == 0 && mFirstFieldIsFid )
{
// already an index on this field, no need to re-created
return false;
}

QString indexName = createIndexName( mOgrOrigLayer->name(), fields().at( field ).name() );
QByteArray createSql = "CREATE INDEX IF NOT EXISTS " + textEncoding()->fromUnicode( indexName ) + " ON " + quotedLayerName + " (" + textEncoding()->fromUnicode( fields().at( field ).name() ) + ")";
mOgrOrigLayer->ExecuteSQLNoReturn( createSql );
return true;
}
else
{
QByteArray dropSql = "DROP INDEX ON " + quotedLayerName;
mOgrOrigLayer->ExecuteSQLNoReturn( dropSql );
QByteArray createSql = "CREATE INDEX ON " + quotedLayerName + " USING " + textEncoding()->fromUnicode( fields().at( field ).name() );
mOgrOrigLayer->ExecuteSQLNoReturn( createSql );

QFileInfo fi( mFilePath ); // to get the base name
//find out, if the .idm file is there
QFile indexfile( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".idm" ) );
return indexfile.exists();
QFileInfo fi( mFilePath ); // to get the base name
//find out, if the .idm/.ind file is there
QString idmFile( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".idm" ) );
QString indFile( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".ind" ) );
return QFile::exists( idmFile ) || QFile::exists( indFile );
}
}

bool QgsOgrProvider::deleteFeatures( const QgsFeatureIds &id )
@@ -2241,6 +2279,12 @@ void QgsOgrProvider::computeCapabilities()
ability &= ~( AddAttributes | DeleteFeatures );
}
}
else if ( mGDALDriverName == QLatin1String( "GPKG" ) ||
mGDALDriverName == QLatin1String( "SQLite" ) )
{
ability |= CreateSpatialIndex;
ability |= CreateAttributeIndex;
}

/* Curve geometries are available in some drivers starting with GDAL 2.0 */
if ( mOgrLayer->TestCapability( "CurveGeometries" ) )
@@ -34,9 +34,11 @@
QgsPointXY,
QgsProject,
QgsWkbTypes,
QgsDataProvider)
QgsDataProvider,
QgsVectorDataProvider)
from qgis.PyQt.QtCore import QCoreApplication, QVariant
from qgis.testing import start_app, unittest
from qgis.utils import spatialite_connect


def GDAL_COMPUTE_VERSION(maj, min, rev):
@@ -771,6 +773,67 @@ def test_SplitFeature(self):
self.assertEqual([f for f in layer.getFeatures()][0].geometry().asWkt(), 'Polygon ((0.5 0, 0.5 1, 1 1, 1 0, 0.5 0))')
self.assertEqual([f for f in layer.getFeatures()][1].geometry().asWkt(), 'Polygon ((0.5 1, 0.5 0, 0 0, 0 1, 0.5 1))')

def testCreateAttributeIndex(self):
tmpfile = os.path.join(self.basetestpath, 'testGeopackageAttributeIndex.gpkg')
ds = ogr.GetDriverByName('GPKG').CreateDataSource(tmpfile)
lyr = ds.CreateLayer('test', geom_type=ogr.wkbPolygon)
lyr.CreateField(ogr.FieldDefn('str_field', ogr.OFTString))
lyr.CreateField(ogr.FieldDefn('str_field2', ogr.OFTString))
f = None
ds = None

vl = QgsVectorLayer(u'{}'.format(tmpfile) + "|layername=" + "test", 'test', u'ogr')
self.assertTrue(vl.isValid())
self.assertTrue(vl.dataProvider().capabilities() & QgsVectorDataProvider.CreateAttributeIndex)
self.assertFalse(vl.dataProvider().createAttributeIndex(-1))
self.assertFalse(vl.dataProvider().createAttributeIndex(100))

# should not be allowed - there's already a index on the primary key
self.assertFalse(vl.dataProvider().createAttributeIndex(0))

self.assertTrue(vl.dataProvider().createAttributeIndex(1))

con = spatialite_connect(tmpfile, isolation_level=None)
cur = con.cursor()
rs = cur.execute("SELECT * FROM sqlite_master WHERE type='index' AND tbl_name='test'")
res = [row for row in rs]
self.assertEqual(len(res), 1)
index_name = res[0][1]
rs = cur.execute("PRAGMA index_info({})".format(index_name))
res = [row for row in rs]
self.assertEqual(len(res), 1)
self.assertEqual(res[0][2], 'str_field')

# second index
self.assertTrue(vl.dataProvider().createAttributeIndex(2))
rs = cur.execute("SELECT * FROM sqlite_master WHERE type='index' AND tbl_name='test'")
res = [row for row in rs]
self.assertEqual(len(res), 2)
indexed_columns = []
for row in res:
index_name = row[1]
rs = cur.execute("PRAGMA index_info({})".format(index_name))
res = [row for row in rs]
self.assertEqual(len(res), 1)
indexed_columns.append(res[0][2])

self.assertCountEqual(indexed_columns, ['str_field', 'str_field2'])
con.close()

def testCreateSpatialIndex(self):
tmpfile = os.path.join(self.basetestpath, 'testGeopackageSpatialIndex.gpkg')
ds = ogr.GetDriverByName('GPKG').CreateDataSource(tmpfile)
lyr = ds.CreateLayer('test', geom_type=ogr.wkbPolygon, options=['SPATIAL_INDEX=NO'])
lyr.CreateField(ogr.FieldDefn('str_field', ogr.OFTString))
lyr.CreateField(ogr.FieldDefn('str_field2', ogr.OFTString))
f = None
ds = None

vl = QgsVectorLayer(u'{}'.format(tmpfile) + "|layername=" + "test", 'test', u'ogr')
self.assertTrue(vl.isValid())
self.assertTrue(vl.dataProvider().capabilities() & QgsVectorDataProvider.CreateSpatialIndex)
self.assertTrue(vl.dataProvider().createSpatialIndex())


if __name__ == '__main__':
unittest.main()
@@ -584,6 +584,34 @@ def testOpenWithFilter(self):
# force close of data provider
vl.setDataSource('', 'test', 'ogr')

def testCreateAttributeIndex(self):
tmpdir = tempfile.mkdtemp()
self.dirs_to_cleanup.append(tmpdir)
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
for file in glob.glob(os.path.join(srcpath, 'shapefile.*')):
shutil.copy(os.path.join(srcpath, file), tmpdir)
datasource = os.path.join(tmpdir, 'shapefile.shp')

vl = QgsVectorLayer('{}|layerid=0'.format(datasource), 'test', 'ogr')
self.assertTrue(vl.isValid())
self.assertTrue(vl.dataProvider().capabilities() & QgsVectorDataProvider.CreateAttributeIndex)
self.assertFalse(vl.dataProvider().createAttributeIndex(-1))
self.assertFalse(vl.dataProvider().createAttributeIndex(100))
self.assertTrue(vl.dataProvider().createAttributeIndex(1))

def testCreateSpatialIndex(self):
tmpdir = tempfile.mkdtemp()
self.dirs_to_cleanup.append(tmpdir)
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
for file in glob.glob(os.path.join(srcpath, 'shapefile.*')):
shutil.copy(os.path.join(srcpath, file), tmpdir)
datasource = os.path.join(tmpdir, 'shapefile.shp')

vl = QgsVectorLayer('{}|layerid=0'.format(datasource), 'test', 'ogr')
self.assertTrue(vl.isValid())
self.assertTrue(vl.dataProvider().capabilities() & QgsVectorDataProvider.CreateSpatialIndex)
self.assertTrue(vl.dataProvider().createSpatialIndex())


if __name__ == '__main__':
unittest.main()

0 comments on commit b8b8c1a

Please sign in to comment.
You can’t perform that action at this time.