Fix database locking when editing GeoPackage

Concurrent read and write can lock a GeoPackage database given
the default journaling mode of SQLite (delete). Use WAL when
possible to avoid that.

Fixes #15351

src/providers/ogr/qgsogrconnpool.h

@@ -17,6 +17,7 @@
 #define QGSOGRCONNPOOL_H
 
 #include "qgsconnectionpool.h"
+#include "qgsogrprovider.h"
 #include <ogr_api.h>
 
 
@@ -43,7 +44,7 @@ inline void qgsConnectionPool_ConnectionCreate( QString connInfo, QgsOgrConn*& c
 
 inline void qgsConnectionPool_ConnectionDestroy( QgsOgrConn* c )
 {
-  OGR_DS_Destroy( c->ds );
+  QgsOgrUtils::OGRDestroyWrapper( c->ds );
   delete c;
 }
 

src/providers/ogr/qgsogrfeatureiterator.cpp

@@ -301,6 +301,12 @@ bool QgsOgrFeatureIterator::close()
 
   iteratorClosed();
 
+  // Will for example release SQLite3 statements
+  if ( ogrLayer )
+  {
+    OGR_L_ResetReading( ogrLayer );
+  }
+
   if ( mSubsetStringSet )
   {
     OGR_DS_ReleaseResultSet( mConn->ds, ogrLayer );
@@ -310,6 +316,7 @@ bool QgsOgrFeatureIterator::close()
     QgsOgrConnPool::instance()->releaseConnection( mConn );
 
   mConn = nullptr;
+  ogrLayer = nullptr;
 
   mClosed = true;
   return true;

src/providers/ogr/qgsogrprovider.cpp

@@ -48,6 +48,13 @@ email                : sherman at mrcc.com
 #include "qgsvectorlayerimport.h"
 #include "qgslocalec.h"
 
+#ifdef Q_OS_WIN
+#include <windows.h>
+#endif
+#ifdef Q_OS_LINUX
+#include <sys/vfs.h>
+#endif
+
 static const QString TEXT_PROVIDER_KEY = "ogr";
 static const QString TEXT_PROVIDER_DESCRIPTION =
   QString( "OGR data provider" )
@@ -383,11 +390,14 @@ QgsOgrProvider::QgsOgrProvider( QString const & uri )
 
 QgsOgrProvider::~QgsOgrProvider()
 {
-  close();
   QgsOgrConnPool::instance()->unref( dataSourceUri() );
   // We must also make sure to flush unusef cached connections so that
   // the file can be removed (#15137)
   QgsOgrConnPool::instance()->invalidateConnections( dataSourceUri() );
+
+  // Do that as last step for final cleanup that might be prevented by
+  // still opened datasets.
+  close();
 }
 
 QgsAbstractFeatureSource* QgsOgrProvider::featureSource() const
@@ -2627,6 +2637,152 @@ void QgsOgrProvider::forceReload()
   QgsOgrConnPool::instance()->invalidateConnections( dataSourceUri() );
 }
 
+OGRDataSourceH QgsOgrUtils::OGROpenWrapper( const char* pszPath, bool bUpdate, OGRSFDriverH *phDriver )
+{
+  CPLErrorReset();
+  OGRSFDriverH hDriver = nullptr;
+  OGRDataSourceH hDS = OGROpen( pszPath, bUpdate, &hDriver );
+  if ( phDriver )
+    *phDriver = hDriver;
+  if ( !hDS )
+    return nullptr;
+  // GDAL < 1.11.5 has a crashing bug with GeoPackage databases with curve geometry
+  // types (https://trac.osgeo.org/gdal/ticket/6558)
+#if GDAL_VERSION_MAJOR == 1 && GDAL_VERSION_MINOR == 11 && GDAL_VERSION_MACRO < 5
+  const char* pszLastErrorMsg = CPLGetLastErrorMsg();
+  if ( hDriver == OGRGetDriverByName( "GPKG" ) &&
+       strstr( pszLastErrorMsg, "geometry column" ) &&
+       strstr( pszLastErrorMsg, "of type" ) &&
+       strstr( pszLastErrorMsg, "ignored" ) )
+  {
+    QgsDebugMsg( QString( "Ignoring %1 that is a GeoPackage DB with curve geometries" ).arg( pszPath ) );
+    OGR_DS_Destroy( hDS );
+    hDS = nullptr;
+  }
+#endif
+  return hDS;
+}
+
+static bool IsLocalFile( const QString& path )
+{
+  QString dirName( QFileInfo( path ).absolutePath() );
+  // Start with the OS specific methods since the QT >= 5.4 method just
+  // return a string and not an enumerated type.
+#if defined(Q_OS_WIN)
+  if ( dirName.startsWith( "\\\\" ) )
+    return false;
+  if ( dirName.length() >= 3 && dirName[1] == ':' &&
+       ( dirName[2] == '\\' || dirName[2] == '/' ) )
+  {
+    dirName.resize( 3 );
+    return GetDriveType( dirName.toAscii().constData() ) != DRIVE_REMOTE;
+  }
+  return true;
+#elif defined(Q_OS_LINUX)
+  struct statfs sStatFS;
+  if ( statfs( dirName.toAscii().constData(), &sStatFS ) == 0 )
+  {
+    // Codes from http://man7.org/linux/man-pages/man2/statfs.2.html
+    if ( sStatFS.f_type == 0x6969 /* NFS */ ||
+         sStatFS.f_type == 0x517b /* SMB */ ||
+         sStatFS.f_type == 0xff534d42 /* CIFS */ )
+    {
+      return false;
+    }
+  }
+  return true;
+#elif QT_VERSION >= QT_VERSION_CHECK(5, 4, 0)
+  QStorageInfo info( dirName );
+  QString fileSystem( info.fileSystemType() );
+  QgsDebugMsg( QString( "Filesystem for %1 is %2" ).arg( path ).arg( fileSystem ) );
+  return path != "nfs" && path != "smbfs";
+#else
+  return true;
+#endif
+}
+
+void QgsOgrUtils::OGRDestroyWrapper( OGRDataSourceH ogrDataSource )
+{
+  if ( !ogrDataSource )
+    return;
+  OGRSFDriverH ogrDriver = OGR_DS_GetDriver( ogrDataSource );
+  QString ogrDriverName = OGR_Dr_GetName( ogrDriver );
+  QString datasetName( FROM8( OGR_DS_GetName( ogrDataSource ) ) );
+  if ( ogrDriverName == "GPKG" &&
+       IsLocalFile( datasetName ) &&
+       !CPLGetConfigOption( "OGR_SQLITE_JOURNAL", NULL ) )
+  {
+    // We need to reset all iterators on layers, otherwise we will not
+    // be able to change journal_mode.
+    int layerCount = OGR_DS_GetLayerCount( ogrDataSource );
+    for ( int i = 0; i < layerCount; i ++ )
+    {
+      OGR_L_ResetReading( OGR_DS_GetLayer( ogrDataSource, i ) );
+    }
+
+    CPLPushErrorHandler( CPLQuietErrorHandler );
+    QgsDebugMsg( "GPKG: Trying to return to delete mode" );
+    bool bSuccess = false;
+    OGRLayerH hSqlLyr = OGR_DS_ExecuteSQL( ogrDataSource,
+                                           "PRAGMA journal_mode = delete",
+                                           NULL, NULL );
+    if ( hSqlLyr != NULL )
+    {
+      OGRFeatureH hFeat = OGR_L_GetNextFeature( hSqlLyr );
+      if ( hFeat != NULL )
+      {
+        const char* pszRet = OGR_F_GetFieldAsString( hFeat, 0 );
+        bSuccess = EQUAL( pszRet, "delete" );
+        QgsDebugMsg( QString( "Return: %1" ).arg( pszRet ) );
+        OGR_F_Destroy( hFeat );
+      }
+    }
+    else if ( CPLGetLastErrorType() != CE_None )
+    {
+      QgsDebugMsg( QString( "Return: %1" ).arg( CPLGetLastErrorMsg() ) );
+    }
+    OGR_DS_ReleaseResultSet( ogrDataSource, hSqlLyr );
+    CPLPopErrorHandler();
+    OGR_DS_Destroy( ogrDataSource );
+
+    // This may have not worked if the file was opened in read-only mode,
+    // so retry in update mode
+    if ( !bSuccess )
+    {
+      QgsDebugMsg( "GPKG: Trying again" );
+      CPLSetThreadLocalConfigOption( "OGR_SQLITE_JOURNAL", "DELETE" );
+      ogrDataSource = OGROpen( TO8F( datasetName ), TRUE, NULL );
+      CPLSetThreadLocalConfigOption( "OGR_SQLITE_JOURNAL", NULL );
+      if ( ogrDataSource )
+      {
+#ifdef QGISDEBUG
+        CPLPushErrorHandler( CPLQuietErrorHandler );
+        OGRLayerH hSqlLyr = OGR_DS_ExecuteSQL( ogrDataSource,
+                                               "PRAGMA journal_mode",
+                                               NULL, NULL );
+        CPLPopErrorHandler();
+        if ( hSqlLyr != NULL )
+        {
+          OGRFeatureH hFeat = OGR_L_GetNextFeature( hSqlLyr );
+          if ( hFeat != NULL )
+          {
+            const char* pszRet = OGR_F_GetFieldAsString( hFeat, 0 );
+            QgsDebugMsg( QString( "Return: %1" ).arg( pszRet ) );
+            OGR_F_Destroy( hFeat );
+          }
+          OGR_DS_ReleaseResultSet( ogrDataSource, hSqlLyr );
+        }
+#endif
+        OGR_DS_Destroy( ogrDataSource );
+      }
+    }
+  }
+  else
+  {
+    OGR_DS_Destroy( ogrDataSource );
+  }
+}
+
 QByteArray QgsOgrUtils::quotedIdentifier( QByteArray field, const QString& ogrDriverName )
 {
   if ( ogrDriverName == "MySQL" )
@@ -2855,7 +3011,22 @@ void QgsOgrProvider::open( OpenMode mode )
 
   // first try to open in update mode (unless specified otherwise)
   if ( !openReadOnly )
-    ogrDataSource = OGROpen( TO8F( mFilePath ), true, &ogrDriver );
+  {
+    if ( QFileInfo( mFilePath ).suffix().compare( "gpkg", Qt::CaseInsensitive ) == 0 &&
+         IsLocalFile( mFilePath ) &&
+         !CPLGetConfigOption( "OGR_SQLITE_JOURNAL", NULL ) &&
+         QSettings().value( "/qgis/walForSqlite3", true ).toBool() )
+    {
+      // For GeoPackage, we force opening of the file in WAL (Write Ahead Log)
+      // mode so as to avoid readers blocking writer(s), and vice-versa.
+      // https://www.sqlite.org/wal.html
+      // But only do that on a local file since WAL is advertized not to work
+      // on network shares
+      CPLSetThreadLocalConfigOption( "OGR_SQLITE_JOURNAL", "WAL" );
+    }
+    ogrDataSource = QgsOgrUtils::OGROpenWrapper( TO8F( mFilePath ), true, &ogrDriver );
+    CPLSetThreadLocalConfigOption( "OGR_SQLITE_JOURNAL", NULL );
+  }
 
   mValid = false;
   if ( ogrDataSource )
@@ -2985,7 +3156,7 @@ void QgsOgrProvider::close()
 
   if ( ogrDataSource )
   {
-    OGR_DS_Destroy( ogrDataSource );
+    QgsOgrUtils::OGRDestroyWrapper( ogrDataSource );
   }
   ogrDataSource = nullptr;
   ogrLayer = nullptr;

src/providers/ogr/qgsogrprovider.h

@@ -15,6 +15,9 @@ email                : sherman at mrcc.com
  *                                                                         *
  ***************************************************************************/
 
+#ifndef QGSOGRPROVIDER_H
+#define QGSOGRPROVIDER_H
+
 #include "QTextCodec"
 
 #include "qgsrectangle.h"
@@ -400,4 +403,9 @@ class QgsOgrUtils
     /** Quote a value for placement in a SQL string.
      */
     static QString quotedValue( const QVariant& value );
+
+    static OGRDataSourceH OGROpenWrapper( const char* pszPath, bool bUpdate, OGRSFDriverH *phDriver );
+    static void OGRDestroyWrapper( OGRDataSourceH ogrDataSource );
 };
+
+#endif // QGSOGRPROVIDER_H