Skip to content
Permalink
Browse files

Merge pull request #8827 from m-kuhn/gpkg_transaction_test

sqlite_fetch_and_increment cached mode for transactions
  • Loading branch information
m-kuhn committed Jan 11, 2019
2 parents b49e940 + 5962afd commit 2664737fc2611982dfc8b8d5e5bb3fec293aa65a
@@ -1,16 +1,17 @@
{
"name": "sqlite_fetch_and_increment",
"type": "function",
"description": "Manage autoincrementing values in sqlite databases.<p>SQlite default values can only be applied on insert and not prefetched.</p><p>This makes it impossible to acquire an incremented primary key via AUTO_INCREMENT before creating the row in the database. Sidenote: with postgres, this works via the option <i>evaluate default values</i>.</p><p>When adding new features with relations, it is really nice to be able to already add children for a parent, while the parents form is still open and hence the parent feature uncommitted.</p><p>To get around this limitation, this function can be used to manage sequence values in a separate table on sqlite based formats like gpkg.</p><p>The sequence table will be filtered for a sequence id (filter_attribute and filter_value) and the current value of the id_field will be incremented by 1 ond the incremented value returned.</p><p>If additional columns require values to be specified, the default_value map can be used for this purpose.</p><p><b>Note</b><br/>This function modifies the target sqlite table. It is intended for usage with default value configurations for attributes.</p>",
"description": "Manage autoincrementing values in sqlite databases.<p>SQlite default values can only be applied on insert and not prefetched.</p><p>This makes it impossible to acquire an incremented primary key via AUTO_INCREMENT before creating the row in the database. Sidenote: with postgres, this works via the option <i>evaluate default values</i>.</p><p>When adding new features with relations, it is really nice to be able to already add children for a parent, while the parents form is still open and hence the parent feature uncommitted.</p><p>To get around this limitation, this function can be used to manage sequence values in a separate table on sqlite based formats like gpkg.</p><p>The sequence table will be filtered for a sequence id (filter_attribute and filter_value) and the current value of the id_field will be incremented by 1 ond the incremented value returned.</p><p>If additional columns require values to be specified, the default_value map can be used for this purpose.</p><p><b>Note</b><br/>This function modifies the target sqlite table. It is intended for usage with default value configurations for attributes.</p><p>When the database parameter is a layer and the layer is in transaction mode, the value will only be retrieved once during the lifetime of a transaction and cached and incremented. This makes it unsafe to work on the same database from several processes in parallel.</p>",
"arguments": [
{"arg":"database", "description":"Path to the sqlite file"},
{"arg":"database", "description":"Path to the sqlite file or geopackage layer"},
{"arg":"table", "description":"Name of the table that manages the sequences"},
{"arg":"id_field", "description":"Name of the field that contains the current value"},
{"arg":"filter_attribute", "description":"Name the field that contains a unique identifier for this sequence. Must have a UNIQUE index."},
{"arg":"filter_value", "description":"Name of the sequence to use."},
{"arg":"default_values", "description":"Map with default values for additional columns on the table. The values need to be fully quoted. Functions are allowed.", "optional": true}
],
"examples": [
{ "expression":"sqlite_fetch_and_increment(@layer, 'sequence_table', 'last_unique_id', 'sequence_id', 'global', map('last_change','date(''now'')','user','''' || @user_account_name || ''''))", "returns":"0"},
{ "expression":"sqlite_fetch_and_increment(layer_property(@layer, 'path'), 'sequence_table', 'last_unique_id', 'sequence_id', 'global', map('last_change','date(''now'')','user','''' || @user_account_name || ''''))", "returns":"0"}
]
}
@@ -48,6 +48,8 @@
#include "qgsvectorlayerfeatureiterator.h"
#include "qgsproviderregistry.h"
#include "sqlite3.h"
#include "qgstransaction.h"
#include "qgsthreadingutils.h"

const QString QgsExpressionFunction::helpText() const
{
@@ -1375,78 +1377,163 @@ static QVariant fcnNumSelected( const QVariantList &values, const QgsExpressionC

static QVariant fcnSqliteFetchAndIncrement( const QVariantList &values, const QgsExpressionContext *, QgsExpression *parent, const QgsExpressionNodeFunction * )
{
const QString database = values.at( 0 ).toString();
const QString table = values.at( 1 ).toString();
const QString idColumn = values.at( 2 ).toString();
const QString filterAttribute = values.at( 3 ).toString();
const QVariant filterValue = values.at( 4 ).toString();
const QVariantMap defaultValues = values.at( 5 ).toMap();
static QMap<QString, qlonglong> counterCache;
QVariant functionResult;

std::function<void()> fetchAndIncrementFunc = [ =, &functionResult ]()
{
QString database;
const QgsVectorLayer *layer = QgsExpressionUtils::getVectorLayer( values.at( 0 ), parent );

// read from database
sqlite3_database_unique_ptr sqliteDb;
sqlite3_statement_unique_ptr sqliteStatement;
if ( layer )
{
const QVariantMap decodedUri = QgsProviderRegistry::instance()->decodeUri( layer->providerType(), layer->dataProvider()->dataSourceUri() );
database = decodedUri.value( QStringLiteral( "path" ) ).toString();
if ( database.isEmpty() )
{
parent->setEvalErrorString( QObject::tr( "Could not extract file path from layer `%1`." ).arg( layer->name() ) );
}
}
else
{
database = values.at( 0 ).toString();
}

if ( sqliteDb.open_v2( database, SQLITE_OPEN_READWRITE, nullptr ) != SQLITE_OK )
{
parent->setEvalErrorString( QObject::tr( "Could not open sqlite database %1. Error %2. " ).arg( database, sqliteDb.errorMessage() ) );
return QVariant();
}
const QString table = values.at( 1 ).toString();
const QString idColumn = values.at( 2 ).toString();
const QString filterAttribute = values.at( 3 ).toString();
const QVariant filterValue = values.at( 4 ).toString();
const QVariantMap defaultValues = values.at( 5 ).toMap();

QString currentValSql;
currentValSql = QStringLiteral( "SELECT %1 FROM %2" ).arg( QgsSqliteUtils::quotedIdentifier( idColumn ), QgsSqliteUtils::quotedIdentifier( table ) );
if ( !filterAttribute.isNull() )
{
currentValSql += QStringLiteral( " WHERE %1 = %2" ).arg( QgsSqliteUtils::quotedIdentifier( filterAttribute ), QgsSqliteUtils::quotedValue( filterValue ) );
}
// read from database
sqlite3_database_unique_ptr sqliteDb;
sqlite3_statement_unique_ptr sqliteStatement;

int result;
sqliteStatement = sqliteDb.prepare( currentValSql, result );
if ( result == SQLITE_OK )
{
qlonglong nextId = 0;
if ( sqliteStatement.step() == SQLITE_ROW )
if ( sqliteDb.open_v2( database, SQLITE_OPEN_READWRITE, nullptr ) != SQLITE_OK )
{
nextId = sqliteStatement.columnAsInt64( 0 ) + 1;
parent->setEvalErrorString( QObject::tr( "Could not open sqlite database %1. Error %2. " ).arg( database, sqliteDb.errorMessage() ) );
functionResult = QVariant();
return;
}

QString upsertSql;
upsertSql = QStringLiteral( "INSERT OR REPLACE INTO %1" ).arg( QgsSqliteUtils::quotedIdentifier( table ) );
QStringList cols;
QStringList vals;
cols << QgsSqliteUtils::quotedIdentifier( idColumn );
vals << QgsSqliteUtils::quotedValue( nextId );
QString errorMessage;
QString currentValSql;

qlonglong nextId;
bool cachedMode = false;
bool valueRetrieved = false;

QString cacheString = QStringLiteral( "%1:%2:%3:%4:%5" ).arg( database, table, idColumn, filterAttribute, filterValue.toString() );

if ( !filterAttribute.isNull() )
// Running in transaction mode, check for cached value first
if ( layer && layer->dataProvider() && layer->dataProvider()->transaction() )
{
cols << QgsSqliteUtils::quotedIdentifier( filterAttribute );
vals << QgsSqliteUtils::quotedValue( filterValue );
cachedMode = true;

auto cachedCounter = counterCache.find( cacheString );

if ( cachedCounter != counterCache.end() )
{
qlonglong &cachedValue = cachedCounter.value();
nextId = cachedValue;
nextId += 1;
cachedValue = nextId;
valueRetrieved = true;
}
}

for ( QVariantMap::const_iterator iter = defaultValues.constBegin(); iter != defaultValues.constEnd(); ++iter )
// Either not in cached mode or no cached value found, obtain from DB
if ( !cachedMode || !valueRetrieved )
{
cols << QgsSqliteUtils::quotedIdentifier( iter.key() );
vals << iter.value().toString();
}
int result = SQLITE_ERROR;

upsertSql += QLatin1String( " (" ) + cols.join( ',' ) + ')';
upsertSql += QLatin1String( " VALUES " );
upsertSql += '(' + vals.join( ',' ) + ')';
currentValSql = QStringLiteral( "SELECT %1 FROM %2" ).arg( QgsSqliteUtils::quotedIdentifier( idColumn ), QgsSqliteUtils::quotedIdentifier( table ) );
if ( !filterAttribute.isNull() )
{
currentValSql += QStringLiteral( " WHERE %1 = %2" ).arg( QgsSqliteUtils::quotedIdentifier( filterAttribute ), QgsSqliteUtils::quotedValue( filterValue ) );
}

QString errorMessage;
result = sqliteDb.exec( upsertSql, errorMessage );
if ( result == SQLITE_OK )
{
return nextId;
sqliteStatement = sqliteDb.prepare( currentValSql, result );

if ( result == SQLITE_OK )
{
nextId = 0;
if ( sqliteStatement.step() == SQLITE_ROW )
{
nextId = sqliteStatement.columnAsInt64( 0 ) + 1;
}

// If in cached mode: add value to cache and connect to transaction
if ( cachedMode && result == SQLITE_OK )
{
counterCache.insert( cacheString, nextId );

QObject::connect( layer->dataProvider()->transaction(), &QgsTransaction::destroyed, [cacheString]()
{
counterCache.remove( cacheString );
} );
}
valueRetrieved = true;
}
}
else

if ( valueRetrieved )
{
parent->setEvalErrorString( QStringLiteral( "Could not increment value: SQLite error: \"%1\" (%2)." ).arg( errorMessage, QString::number( result ) ) );
return QVariant();
QString upsertSql;
upsertSql = QStringLiteral( "INSERT OR REPLACE INTO %1" ).arg( QgsSqliteUtils::quotedIdentifier( table ) );
QStringList cols;
QStringList vals;
cols << QgsSqliteUtils::quotedIdentifier( idColumn );
vals << QgsSqliteUtils::quotedValue( nextId );

if ( !filterAttribute.isNull() )
{
cols << QgsSqliteUtils::quotedIdentifier( filterAttribute );
vals << QgsSqliteUtils::quotedValue( filterValue );
}

for ( QVariantMap::const_iterator iter = defaultValues.constBegin(); iter != defaultValues.constEnd(); ++iter )
{
cols << QgsSqliteUtils::quotedIdentifier( iter.key() );
vals << iter.value().toString();
}

upsertSql += QLatin1String( " (" ) + cols.join( ',' ) + ')';
upsertSql += QLatin1String( " VALUES " );
upsertSql += '(' + vals.join( ',' ) + ')';

int result = SQLITE_ERROR;
if ( layer && layer->dataProvider() && layer->dataProvider()->transaction() )
{
QgsTransaction *transaction = layer->dataProvider()->transaction();
if ( transaction->executeSql( upsertSql, errorMessage ) )
{
result = SQLITE_OK;
}
}
else
{
result = sqliteDb.exec( upsertSql, errorMessage );
}
if ( result == SQLITE_OK )
{
functionResult = QVariant( nextId );
return;
}
else
{
parent->setEvalErrorString( QStringLiteral( "Could not increment value: SQLite error: \"%1\" (%2)." ).arg( errorMessage, QString::number( result ) ) );
functionResult = QVariant();
return;
}
}
}

return QVariant(); // really?
functionResult = QVariant();
};

QgsThreadingUtils::runOnMainThread( fetchAndIncrementFunc );

return functionResult;
}

static QVariant fcnConcat( const QVariantList &values, const QgsExpressionContext *, QgsExpression *parent, const QgsExpressionNodeFunction * )
@@ -33,6 +33,7 @@
#include "qgsrasterlayer.h"
#include "qgsproject.h"
#include "qgsexpressionnodeimpl.h"
#include "qgsvectorlayerutils.h"

static void _parseAndEvalExpr( int arg )
{
@@ -3298,6 +3299,63 @@ class TestQgsExpression: public QObject
e = QgsExpression( QStringLiteral( "map('a',1,'bbb',2,'c',3)['b'||'b'||'b']" ) );
QCOMPARE( e.evaluate( &context ).toInt(), 2 );
}


void testSqliteFetchAndIncrementWithTranscationMode()
{
QString testDataDir = QStringLiteral( TEST_DATA_DIR ) + '/';
QTemporaryDir tempDir;
QFile::copy( testDataDir + QStringLiteral( "kbs.qgs" ), tempDir.filePath( "kbs.qgs" ) );
QFile::copy( testDataDir + QStringLiteral( "kbs.gpkg" ), tempDir.filePath( "kbs.gpkg" ) );

QgsProject *project = QgsProject::instance();
QVERIFY( project->read( tempDir.filePath( "kbs.qgs" ) ) );

QgsVectorLayer *zustaendigkeitskataster = project->mapLayer<QgsVectorLayer *>( QStringLiteral( "zustaendigkeitkataster_2b5bb693_3151_4c82_967f_b49d4d348a17" ) );

// There is a default expression setup, dear reader of this test
QVERIFY( zustaendigkeitskataster->defaultValueDefinition( 0 ).expression().contains( "sqlite_fetch_and_increment" ) );

zustaendigkeitskataster->startEditing();

QgsExpressionContext context( QgsExpressionContextUtils::globalProjectLayerScopes( zustaendigkeitskataster ) );
QgsFeature feature = QgsVectorLayerUtils::createFeature( zustaendigkeitskataster, QgsGeometry(), QgsAttributeMap(), &context );
QCOMPARE( feature.attribute( "T_Id" ).toInt(), 0 );
feature.setAttribute( "url_behoerde", "url_behoerde" );
feature.setAttribute( "url_kataster", "url_kataster" );
zustaendigkeitskataster->addFeature( feature );

QgsFeature feature2 = QgsVectorLayerUtils::createFeature( zustaendigkeitskataster, QgsGeometry(), QgsAttributeMap(), &context );
QCOMPARE( feature2.attribute( "T_Id" ).toInt(), 1 );
feature2.setAttribute( "url_behoerde", "url_behoerde_x" );
feature2.setAttribute( "url_kataster", "url_kataster_x" );
zustaendigkeitskataster->addFeature( feature2 );

zustaendigkeitskataster->commitChanges();
QCOMPARE( zustaendigkeitskataster->dataProvider()->featureCount(), 2l );

QCOMPARE( zustaendigkeitskataster->editBuffer(), nullptr );
QCOMPARE( zustaendigkeitskataster->dataProvider()->transaction(), nullptr );

zustaendigkeitskataster->startEditing();
QgsExpressionContext context2( QgsExpressionContextUtils::globalProjectLayerScopes( zustaendigkeitskataster ) );
QgsFeature feature3 = QgsVectorLayerUtils::createFeature( zustaendigkeitskataster, QgsGeometry(), QgsAttributeMap(), &context );
QCOMPARE( feature3.attribute( "T_Id" ).toInt(), 2 );
feature3.setAttribute( "url_behoerde", "url_behoerde" );
feature3.setAttribute( "url_kataster", "url_kataster" );
zustaendigkeitskataster->addFeature( feature3 );

QgsFeature feature4 = QgsVectorLayerUtils::createFeature( zustaendigkeitskataster, QgsGeometry(), QgsAttributeMap(), &context );
QCOMPARE( feature4.attribute( "T_Id" ).toInt(), 3 );
feature4.setAttribute( "url_behoerde", "url_behoerde_x" );
feature4.setAttribute( "url_kataster", "url_kataster_x" );
zustaendigkeitskataster->addFeature( feature4 );

zustaendigkeitskataster->commitChanges();

QCOMPARE( zustaendigkeitskataster->dataProvider()->featureCount(), 4l );
}

};

QGSTEST_MAIN( TestQgsExpression )
@@ -130,6 +130,5 @@ void TestQgsOgrProvider::decodeUri()
QCOMPARE( parts.value( QStringLiteral( "layerName" ) ).toString(), QString( "a_layer" ) );
}


QGSTEST_MAIN( TestQgsOgrProvider )
#include "testqgsogrprovider.moc"
Binary file not shown.

0 comments on commit 2664737

Please sign in to comment.
You can’t perform that action at this time.