Skip to content

Commit b8b8c1a

Browse files
authored
Merge pull request #5787 from nyalldawson/index
Fix OGR provider cannot create attribute or spatial indexes for GeoPackage/SQLite layers
2 parents 31c79da + aaa18e0 commit b8b8c1a

File tree

3 files changed

+154
-19
lines changed

3 files changed

+154
-19
lines changed

src/providers/ogr/qgsogrprovider.cpp

+62-18
Original file line numberDiff line numberDiff line change
@@ -1966,36 +1966,74 @@ bool QgsOgrProvider::createSpatialIndex()
19661966
if ( !doInitialActionsForEdition() )
19671967
return false;
19681968

1969-
if ( mGDALDriverName != QLatin1String( "ESRI Shapefile" ) )
1970-
return false;
1971-
19721969
QByteArray layerName = mOgrOrigLayer->name();
1970+
if ( mGDALDriverName == QLatin1String( "ESRI Shapefile" ) )
1971+
{
1972+
QByteArray sql = QByteArray( "CREATE SPATIAL INDEX ON " ) + quotedIdentifier( layerName ); // quote the layer name so spaces are handled
1973+
QgsDebugMsg( QString( "SQL: %1" ).arg( QString::fromUtf8( sql ) ) );
1974+
mOgrOrigLayer->ExecuteSQLNoReturn( sql );
19731975

1974-
QByteArray sql = QByteArray( "CREATE SPATIAL INDEX ON " ) + quotedIdentifier( layerName ); // quote the layer name so spaces are handled
1975-
QgsDebugMsg( QString( "SQL: %1" ).arg( QString::fromUtf8( sql ) ) );
1976-
mOgrOrigLayer->ExecuteSQLNoReturn( sql );
1976+
QFileInfo fi( mFilePath ); // to get the base name
1977+
//find out, if the .qix file is there
1978+
return QFileInfo::exists( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".qix" ) );
1979+
}
1980+
else if ( mGDALDriverName == QLatin1String( "GPKG" ) ||
1981+
mGDALDriverName == QLatin1String( "SQLite" ) )
1982+
{
1983+
QMutex *mutex = nullptr;
1984+
OGRLayerH layer = mOgrOrigLayer->getHandleAndMutex( mutex );
1985+
QByteArray sql = QByteArray( "SELECT CreateSpatialIndex(" + quotedIdentifier( layerName ) + ","
1986+
+ quotedIdentifier( OGR_L_GetGeometryColumn( layer ) ) + ") " ); // quote the layer name so spaces are handled
1987+
mOgrOrigLayer->ExecuteSQLNoReturn( sql );
1988+
return true;
1989+
}
1990+
return false;
1991+
}
19771992

1978-
QFileInfo fi( mFilePath ); // to get the base name
1979-
//find out, if the .qix file is there
1980-
QFile indexfile( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".qix" ) );
1981-
return indexfile.exists();
1993+
QString createIndexName( QString tableName, QString field )
1994+
{
1995+
QRegularExpression safeExp( QStringLiteral( "[^a-zA-Z0-9]" ) );
1996+
tableName.replace( safeExp, QStringLiteral( "_" ) );
1997+
field.replace( safeExp, QStringLiteral( "_" ) );
1998+
return tableName + "_" + field + "_idx";
19821999
}
19832000

19842001
bool QgsOgrProvider::createAttributeIndex( int field )
19852002
{
2003+
if ( field < 0 || field >= mAttributeFields.count() )
2004+
return false;
2005+
19862006
if ( !doInitialActionsForEdition() )
19872007
return false;
19882008

19892009
QByteArray quotedLayerName = quotedIdentifier( mOgrOrigLayer->name() );
1990-
QByteArray dropSql = "DROP INDEX ON " + quotedLayerName;
1991-
mOgrOrigLayer->ExecuteSQLNoReturn( dropSql );
1992-
QByteArray createSql = "CREATE INDEX ON " + quotedLayerName + " USING " + textEncoding()->fromUnicode( fields().at( field ).name() );
1993-
mOgrOrigLayer->ExecuteSQLNoReturn( createSql );
2010+
if ( mGDALDriverName == QLatin1String( "GPKG" ) ||
2011+
mGDALDriverName == QLatin1String( "SQLite" ) )
2012+
{
2013+
if ( field == 0 && mFirstFieldIsFid )
2014+
{
2015+
// already an index on this field, no need to re-created
2016+
return false;
2017+
}
2018+
2019+
QString indexName = createIndexName( mOgrOrigLayer->name(), fields().at( field ).name() );
2020+
QByteArray createSql = "CREATE INDEX IF NOT EXISTS " + textEncoding()->fromUnicode( indexName ) + " ON " + quotedLayerName + " (" + textEncoding()->fromUnicode( fields().at( field ).name() ) + ")";
2021+
mOgrOrigLayer->ExecuteSQLNoReturn( createSql );
2022+
return true;
2023+
}
2024+
else
2025+
{
2026+
QByteArray dropSql = "DROP INDEX ON " + quotedLayerName;
2027+
mOgrOrigLayer->ExecuteSQLNoReturn( dropSql );
2028+
QByteArray createSql = "CREATE INDEX ON " + quotedLayerName + " USING " + textEncoding()->fromUnicode( fields().at( field ).name() );
2029+
mOgrOrigLayer->ExecuteSQLNoReturn( createSql );
19942030

1995-
QFileInfo fi( mFilePath ); // to get the base name
1996-
//find out, if the .idm file is there
1997-
QFile indexfile( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".idm" ) );
1998-
return indexfile.exists();
2031+
QFileInfo fi( mFilePath ); // to get the base name
2032+
//find out, if the .idm/.ind file is there
2033+
QString idmFile( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".idm" ) );
2034+
QString indFile( fi.path().append( '/' ).append( fi.completeBaseName() ).append( ".ind" ) );
2035+
return QFile::exists( idmFile ) || QFile::exists( indFile );
2036+
}
19992037
}
20002038

20012039
bool QgsOgrProvider::deleteFeatures( const QgsFeatureIds &id )
@@ -2241,6 +2279,12 @@ void QgsOgrProvider::computeCapabilities()
22412279
ability &= ~( AddAttributes | DeleteFeatures );
22422280
}
22432281
}
2282+
else if ( mGDALDriverName == QLatin1String( "GPKG" ) ||
2283+
mGDALDriverName == QLatin1String( "SQLite" ) )
2284+
{
2285+
ability |= CreateSpatialIndex;
2286+
ability |= CreateAttributeIndex;
2287+
}
22442288

22452289
/* Curve geometries are available in some drivers starting with GDAL 2.0 */
22462290
if ( mOgrLayer->TestCapability( "CurveGeometries" ) )

tests/src/python/test_provider_ogr_gpkg.py

+64-1
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,11 @@
3434
QgsPointXY,
3535
QgsProject,
3636
QgsWkbTypes,
37-
QgsDataProvider)
37+
QgsDataProvider,
38+
QgsVectorDataProvider)
3839
from qgis.PyQt.QtCore import QCoreApplication, QVariant
3940
from qgis.testing import start_app, unittest
41+
from qgis.utils import spatialite_connect
4042

4143

4244
def GDAL_COMPUTE_VERSION(maj, min, rev):
@@ -771,6 +773,67 @@ def test_SplitFeature(self):
771773
self.assertEqual([f for f in layer.getFeatures()][0].geometry().asWkt(), 'Polygon ((0.5 0, 0.5 1, 1 1, 1 0, 0.5 0))')
772774
self.assertEqual([f for f in layer.getFeatures()][1].geometry().asWkt(), 'Polygon ((0.5 1, 0.5 0, 0 0, 0 1, 0.5 1))')
773775

776+
def testCreateAttributeIndex(self):
777+
tmpfile = os.path.join(self.basetestpath, 'testGeopackageAttributeIndex.gpkg')
778+
ds = ogr.GetDriverByName('GPKG').CreateDataSource(tmpfile)
779+
lyr = ds.CreateLayer('test', geom_type=ogr.wkbPolygon)
780+
lyr.CreateField(ogr.FieldDefn('str_field', ogr.OFTString))
781+
lyr.CreateField(ogr.FieldDefn('str_field2', ogr.OFTString))
782+
f = None
783+
ds = None
784+
785+
vl = QgsVectorLayer(u'{}'.format(tmpfile) + "|layername=" + "test", 'test', u'ogr')
786+
self.assertTrue(vl.isValid())
787+
self.assertTrue(vl.dataProvider().capabilities() & QgsVectorDataProvider.CreateAttributeIndex)
788+
self.assertFalse(vl.dataProvider().createAttributeIndex(-1))
789+
self.assertFalse(vl.dataProvider().createAttributeIndex(100))
790+
791+
# should not be allowed - there's already a index on the primary key
792+
self.assertFalse(vl.dataProvider().createAttributeIndex(0))
793+
794+
self.assertTrue(vl.dataProvider().createAttributeIndex(1))
795+
796+
con = spatialite_connect(tmpfile, isolation_level=None)
797+
cur = con.cursor()
798+
rs = cur.execute("SELECT * FROM sqlite_master WHERE type='index' AND tbl_name='test'")
799+
res = [row for row in rs]
800+
self.assertEqual(len(res), 1)
801+
index_name = res[0][1]
802+
rs = cur.execute("PRAGMA index_info({})".format(index_name))
803+
res = [row for row in rs]
804+
self.assertEqual(len(res), 1)
805+
self.assertEqual(res[0][2], 'str_field')
806+
807+
# second index
808+
self.assertTrue(vl.dataProvider().createAttributeIndex(2))
809+
rs = cur.execute("SELECT * FROM sqlite_master WHERE type='index' AND tbl_name='test'")
810+
res = [row for row in rs]
811+
self.assertEqual(len(res), 2)
812+
indexed_columns = []
813+
for row in res:
814+
index_name = row[1]
815+
rs = cur.execute("PRAGMA index_info({})".format(index_name))
816+
res = [row for row in rs]
817+
self.assertEqual(len(res), 1)
818+
indexed_columns.append(res[0][2])
819+
820+
self.assertCountEqual(indexed_columns, ['str_field', 'str_field2'])
821+
con.close()
822+
823+
def testCreateSpatialIndex(self):
824+
tmpfile = os.path.join(self.basetestpath, 'testGeopackageSpatialIndex.gpkg')
825+
ds = ogr.GetDriverByName('GPKG').CreateDataSource(tmpfile)
826+
lyr = ds.CreateLayer('test', geom_type=ogr.wkbPolygon, options=['SPATIAL_INDEX=NO'])
827+
lyr.CreateField(ogr.FieldDefn('str_field', ogr.OFTString))
828+
lyr.CreateField(ogr.FieldDefn('str_field2', ogr.OFTString))
829+
f = None
830+
ds = None
831+
832+
vl = QgsVectorLayer(u'{}'.format(tmpfile) + "|layername=" + "test", 'test', u'ogr')
833+
self.assertTrue(vl.isValid())
834+
self.assertTrue(vl.dataProvider().capabilities() & QgsVectorDataProvider.CreateSpatialIndex)
835+
self.assertTrue(vl.dataProvider().createSpatialIndex())
836+
774837

775838
if __name__ == '__main__':
776839
unittest.main()

tests/src/python/test_provider_shapefile.py

+28
Original file line numberDiff line numberDiff line change
@@ -584,6 +584,34 @@ def testOpenWithFilter(self):
584584
# force close of data provider
585585
vl.setDataSource('', 'test', 'ogr')
586586

587+
def testCreateAttributeIndex(self):
588+
tmpdir = tempfile.mkdtemp()
589+
self.dirs_to_cleanup.append(tmpdir)
590+
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
591+
for file in glob.glob(os.path.join(srcpath, 'shapefile.*')):
592+
shutil.copy(os.path.join(srcpath, file), tmpdir)
593+
datasource = os.path.join(tmpdir, 'shapefile.shp')
594+
595+
vl = QgsVectorLayer('{}|layerid=0'.format(datasource), 'test', 'ogr')
596+
self.assertTrue(vl.isValid())
597+
self.assertTrue(vl.dataProvider().capabilities() & QgsVectorDataProvider.CreateAttributeIndex)
598+
self.assertFalse(vl.dataProvider().createAttributeIndex(-1))
599+
self.assertFalse(vl.dataProvider().createAttributeIndex(100))
600+
self.assertTrue(vl.dataProvider().createAttributeIndex(1))
601+
602+
def testCreateSpatialIndex(self):
603+
tmpdir = tempfile.mkdtemp()
604+
self.dirs_to_cleanup.append(tmpdir)
605+
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
606+
for file in glob.glob(os.path.join(srcpath, 'shapefile.*')):
607+
shutil.copy(os.path.join(srcpath, file), tmpdir)
608+
datasource = os.path.join(tmpdir, 'shapefile.shp')
609+
610+
vl = QgsVectorLayer('{}|layerid=0'.format(datasource), 'test', 'ogr')
611+
self.assertTrue(vl.isValid())
612+
self.assertTrue(vl.dataProvider().capabilities() & QgsVectorDataProvider.CreateSpatialIndex)
613+
self.assertTrue(vl.dataProvider().createSpatialIndex())
614+
587615

588616
if __name__ == '__main__':
589617
unittest.main()

0 commit comments

Comments
 (0)