Skip to content

Commit

Permalink
[OGR provider] Allow opening (GeoPackage) datasets with many layers
Browse files Browse the repository at this point in the history
Currently each time you instanciate a QgsOgrProvider layer, a GDAL dataset is
created. In the case of GeoPackage, this means a SQLite connection and a file
handle. As GDAL enables Spatialite function on GeoPackage connections, we are
bound to Spatialite limits, and Spatialite has a hard limit on a maximum of
64 simultaneous connections. Thus we cannot open more than 64 layers of the
same GeoPackage.
This commits enables sharing of the same GDALDataset object among several
QgsOgrProvider object. Care is made to reuse a GDALDataset object only if the
QgsOgrProvider do not point to the same layer. Mutexes are also taken to
allow safe instanciation and use of QgsOgrProvider objects from multiple
threads (but a same QgsOgrProvider should not be used by more than one thread
at a time)
  • Loading branch information
rouault committed Oct 20, 2017
1 parent e59f1d5 commit 8f3d44d
Show file tree
Hide file tree
Showing 6 changed files with 1,678 additions and 527 deletions.
2 changes: 1 addition & 1 deletion src/providers/ogr/qgsogrconnpool.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ inline void qgsConnectionPool_ConnectionCreate( const QString &connInfo, QgsOgrC
{
c = new QgsOgrConn;
QString filePath = connInfo.left( connInfo.indexOf( QLatin1String( "|" ) ) );
c->ds = GDALOpenEx( filePath.toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr, nullptr );
c->ds = QgsOgrProviderUtils::GDALOpenWrapper( filePath.toUtf8().constData(), false, nullptr, nullptr );
c->path = connInfo;
c->valid = true;
}
Expand Down
6 changes: 3 additions & 3 deletions src/providers/ogr/qgsogrdataitems.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ QgsOgrLayerItem::QgsOgrLayerItem( QgsDataItem *parent,

OGRRegisterAll();
GDALDriverH hDriver;
GDALDatasetH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( mPath.toUtf8().constData(), true, false, &hDriver );
GDALDatasetH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( mPath.toUtf8().constData(), true, nullptr, &hDriver );

if ( hDataSource )
{
Expand Down Expand Up @@ -408,7 +408,7 @@ QVector<QgsDataItem *> QgsOgrDataCollectionItem::createChildren()
QVector<QgsDataItem *> children;

GDALDriverH hDriver;
GDALDatasetH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( mPath.toUtf8().constData(), false, false, &hDriver );
GDALDatasetH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( mPath.toUtf8().constData(), false, nullptr, &hDriver );
if ( !hDataSource )
return children;
int numLayers = GDALDatasetGetLayerCount( hDataSource );
Expand Down Expand Up @@ -639,7 +639,7 @@ QGISEXTERN QgsDataItem *dataItem( QString path, QgsDataItem *parentItem )
// do not print errors, but write to debug
CPLPushErrorHandler( CPLQuietErrorHandler );
CPLErrorReset();
OGRDataSourceH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( path.toUtf8().constData(), false, false, &hDriver );
OGRDataSourceH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( path.toUtf8().constData(), false, nullptr, &hDriver );
CPLPopErrorHandler();

if ( ! hDataSource )
Expand Down
10 changes: 7 additions & 3 deletions src/providers/ogr/qgsogrfeatureiterator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ QgsOgrFeatureIterator::QgsOgrFeatureIterator( QgsOgrFeatureSource *source, bool
, mFilterFids( mRequest.filterFids() )
, mFilterFidsIt( mFilterFids.constBegin() )
{
mConn = QgsOgrConnPool::instance()->acquireConnection( mSource->mDataSource );
//QgsDebugMsg( "Feature iterator of " + mSource->mLayerName + ": acquiring connection");
mConn = QgsOgrConnPool::instance()->acquireConnection( QgsOgrProviderUtils::connectionPoolId( mSource->mDataSource ) );
if ( !mConn->ds )
{
return;
Expand Down Expand Up @@ -316,7 +317,10 @@ bool QgsOgrFeatureIterator::close()
}

if ( mConn )
{
//QgsDebugMsg( "Feature iterator of " + mSource->mLayerName + ": releasing connection");
QgsOgrConnPool::instance()->releaseConnection( mConn );
}

mConn = nullptr;
ogrLayer = nullptr;
Expand Down Expand Up @@ -437,12 +441,12 @@ QgsOgrFeatureSource::QgsOgrFeatureSource( const QgsOgrProvider *p )
{
for ( int i = ( p->mFirstFieldIsFid ) ? 1 : 0; i < mFields.size(); i++ )
mFieldsWithoutFid.append( mFields.at( i ) );
QgsOgrConnPool::instance()->ref( mDataSource );
QgsOgrConnPool::instance()->ref( QgsOgrProviderUtils::connectionPoolId( mDataSource ) );
}

QgsOgrFeatureSource::~QgsOgrFeatureSource()
{
QgsOgrConnPool::instance()->unref( mDataSource );
QgsOgrConnPool::instance()->unref( QgsOgrProviderUtils::connectionPoolId( mDataSource ) );
}

QgsFeatureIterator QgsOgrFeatureSource::getFeatures( const QgsFeatureRequest &request )
Expand Down
Loading

0 comments on commit 8f3d44d

Please sign in to comment.