Skip to content
Permalink
Browse files

Merge pull request #8722 from m-kuhn/expression_function_sqlite_fetch…

…_and_increment

Expression function sqlite_fetch_and_increment
  • Loading branch information
m-kuhn committed Dec 24, 2018
2 parents 304771a + 36c42d2 commit d2b35753be1a9fa4c0eab33d65541331365577bd
@@ -0,0 +1,16 @@
{
"name": "sqlite_fetch_and_increment",
"type": "function",
"description": "Manage autoincrementing values in sqlite databases.<p>SQlite default values can only be applied on insert and not prefetched.</p><p>This makes it impossible to acquire an incremented primary key via AUTO_INCREMENT before creating the row in the database. Sidenote: with postgres, this works via the option <i>evaluate default values</i>.</p><p>When adding new features with relations, it is really nice to be able to already add children for a parent, while the parents form is still open and hence the parent feature uncommitted.</p><p>To get around this limitation, this function can be used to manage sequence values in a separate table on sqlite based formats like gpkg.</p><p>The sequence table will be filtered for a sequence id (filter_attribute and filter_value) and the current value of the id_field will be incremented by 1 ond the incremented value returned.</p><p>If additional columns require values to be specified, the default_value map can be used for this purpose.</p><p><b>Note</b><br/>This function modifies the target sqlite table. It is intended for usage with default value configurations for attributes.</p>",
"arguments": [
{"arg":"database", "description":"Path to the sqlite file"},
{"arg":"table", "description":"Name of the table that manages the sequences"},
{"arg":"id_field", "description":"Name of the field that contains the current value"},
{"arg":"filter_attribute", "description":"Name the field that contains a unique identifier for this sequence. Must have a UNIQUE index."},
{"arg":"filter_value", "description":"Name of the sequence to use."},
{"arg":"default_values", "description":"Map with default values for additional columns on the table. The values need to be fully quoted. Functions are allowed.", "optional": true}
],
"examples": [
{ "expression":"sqlite_fetch_and_increment(layer_property(@layer, 'path'), 'sequence_table', 'last_unique_id', 'sequence_id', 'global', map('last_change','date(''now'')','user','''' || @user_account_name || ''''))", "returns":"0"}
]
}
@@ -47,6 +47,7 @@
#include "qgsfieldformatter.h"
#include "qgsvectorlayerfeatureiterator.h"
#include "qgsproviderregistry.h"
#include "sqlite3.h"

const QString QgsExpressionFunction::helpText() const
{
@@ -1360,6 +1361,82 @@ static QVariant fcnNumSelected( const QVariantList &values, const QgsExpressionC
return layer->selectedFeatureCount();
}

static QVariant fcnSqliteFetchAndIncrement( const QVariantList &values, const QgsExpressionContext *, QgsExpression *parent, const QgsExpressionNodeFunction * )
{
const QString database = values.at( 0 ).toString();
const QString table = values.at( 1 ).toString();
const QString idColumn = values.at( 2 ).toString();
const QString filterAttribute = values.at( 3 ).toString();
const QVariant filterValue = values.at( 4 ).toString();
const QVariantMap defaultValues = values.at( 5 ).toMap();


// read from database
sqlite3_database_unique_ptr sqliteDb;
sqlite3_statement_unique_ptr sqliteStatement;

if ( sqliteDb.open_v2( database, SQLITE_OPEN_READWRITE, nullptr ) != SQLITE_OK )
{
parent->setEvalErrorString( QObject::tr( "Could not open sqlite database %1. Error %2. " ).arg( database, sqliteDb.errorMessage() ) );
return QVariant();
}

QString currentValSql;
currentValSql = QStringLiteral( "SELECT %1 FROM %2" ).arg( QgsSqliteUtils::quotedIdentifier( idColumn ), QgsSqliteUtils::quotedIdentifier( table ) );
if ( !filterAttribute.isNull() )
{
currentValSql += QStringLiteral( " WHERE %1 = %2" ).arg( QgsSqliteUtils::quotedIdentifier( filterAttribute ), QgsSqliteUtils::quotedValue( filterValue ) );
}

int result;
sqliteStatement = sqliteDb.prepare( currentValSql, result );
if ( result == SQLITE_OK )
{
qlonglong nextId = 0;
if ( sqliteStatement.step() == SQLITE_ROW )
{
nextId = sqliteStatement.columnAsInt64( 0 ) + 1;
}

QString upsertSql;
upsertSql = QStringLiteral( "INSERT OR REPLACE INTO %1" ).arg( QgsSqliteUtils::quotedIdentifier( table ) );
QStringList cols;
QStringList vals;
cols << QgsSqliteUtils::quotedIdentifier( idColumn );
vals << QgsSqliteUtils::quotedValue( nextId );

if ( !filterAttribute.isNull() )
{
cols << QgsSqliteUtils::quotedIdentifier( filterAttribute );
vals << QgsSqliteUtils::quotedValue( filterValue );
}

for ( QVariantMap::const_iterator iter = defaultValues.constBegin(); iter != defaultValues.constEnd(); ++iter )
{
cols << QgsSqliteUtils::quotedIdentifier( iter.key() );
vals << iter.value().toString();
}

upsertSql += QLatin1String( " (" ) + cols.join( ',' ) + ')';
upsertSql += QLatin1String( " VALUES " );
upsertSql += '(' + vals.join( ',' ) + ')';

QString errorMessage;
result = sqliteDb.exec( upsertSql, errorMessage );
if ( result == SQLITE_OK )
{
return nextId;
}
else
{
parent->setEvalErrorString( QStringLiteral( "Could not increment value: SQLite error: \"%1\" (%2)." ).arg( errorMessage, QString::number( result ) ) );
return QVariant();
}
}

return QVariant(); // really?
}

static QVariant fcnConcat( const QVariantList &values, const QgsExpressionContext *, QgsExpression *parent, const QgsExpressionNodeFunction * )
{
QString concat;
@@ -4916,6 +4993,20 @@ const QList<QgsExpressionFunction *> &QgsExpression::Functions()
QSet<QString>()
);

sFunctions
<< new QgsStaticExpressionFunction(
QStringLiteral( "sqlite_fetch_and_increment" ),
QgsExpressionFunction::ParameterList()
<< QgsExpressionFunction::Parameter( QStringLiteral( "database" ) )
<< QgsExpressionFunction::Parameter( QStringLiteral( "table" ) )
<< QgsExpressionFunction::Parameter( QStringLiteral( "id_field" ) )
<< QgsExpressionFunction::Parameter( QStringLiteral( "filter_attribute" ) )
<< QgsExpressionFunction::Parameter( QStringLiteral( "filter_value" ) )
<< QgsExpressionFunction::Parameter( QStringLiteral( "default_values" ), true ),
fcnSqliteFetchAndIncrement,
QStringLiteral( "Record and Attributes" )
);

// **Fields and Values** functions
QgsStaticExpressionFunction *representValueFunc = new QgsStaticExpressionFunction( QStringLiteral( "represent_value" ), QgsExpressionFunction::ParameterList() << QgsExpressionFunction::Parameter( QStringLiteral( "attribute" ) ) << QgsExpressionFunction::Parameter( QStringLiteral( "field_name" ), true ), fcnRepresentValue, QStringLiteral( "Record and Attributes" ) );

@@ -92,6 +92,21 @@ sqlite3_statement_unique_ptr sqlite3_database_unique_ptr::prepare( const QString
return s;
}

int sqlite3_database_unique_ptr::exec( const QString &sql, QString &errorMessage ) const
{
char *errMsg;

int ret = sqlite3_exec( get(), sql.toUtf8(), nullptr, nullptr, &errMsg );

if ( errMsg )
{
errorMessage = QString::fromUtf8( errMsg );
sqlite3_free( errMsg );
}

return ret;
}

QString QgsSqliteUtils::quotedString( const QString &value )
{
if ( value.isNull() )
@@ -21,6 +21,8 @@
#define SIP_NO_FILE

#include "qgis_core.h"
#include "qgis_sip.h"

#include <memory>
#include <QString>

@@ -135,8 +137,17 @@ class CORE_EXPORT sqlite3_database_unique_ptr : public std::unique_ptr< sqlite3,
* Prepares a \a sql statement, returning the result. The \a resultCode
* argument will be filled with the sqlite3 result code.
*/
sqlite3_statement_unique_ptr prepare( const QString &sql, int &resultCode ) const;
sqlite3_statement_unique_ptr prepare( const QString &sql, int &resultCode SIP_OUT ) const;

/**
* Executes the \a sql command in the database. Multiple sql queries can be run within
* one single command.
* Errors are reported to \a errorMessage.
* Returns SQLITE_OK in case of success or an sqlite error code.
*
* \since QGIS 3.6
*/
int exec( const QString &sql, QString &errorMessage SIP_OUT ) const;
};

/**
@@ -1556,6 +1556,47 @@ class TestQgsExpression: public QObject
}
}

void test_sqliteFetchAndIncrement()
{
QTemporaryDir dir;
QString testGpkgName = QStringLiteral( "humanbeings.gpkg" );
QFile::copy( QStringLiteral( TEST_DATA_DIR ) + '/' + testGpkgName, dir.filePath( testGpkgName ) );

QgsExpressionContext context;
QgsExpressionContextScope *scope = new QgsExpressionContextScope();
scope->setVariable( QStringLiteral( "test_database" ), dir.filePath( testGpkgName ) );
scope->setVariable( QStringLiteral( "username" ), "some_username" );
context << scope;

// Test database file does not exist
QgsExpression exp1( QStringLiteral( "sqlite_fetch_and_increment('/path/does/not/exist', 'T_KEY_OBJECT', 'T_LastUniqueId', 'T_Key', 'T_Id')" ) );

exp1.evaluate( &context );
QCOMPARE( exp1.hasEvalError(), true );
const QString evalErrorString1 = exp1.evalErrorString();
QVERIFY2( evalErrorString1.contains( "/path/does/not/exist" ), QStringLiteral( "Path not found in %1" ).arg( evalErrorString1 ).toUtf8().constData() );
QVERIFY2( evalErrorString1.contains( "Error" ), QStringLiteral( "\"Error\" not found in %1" ).arg( evalErrorString1 ).toUtf8().constData() );

// Test default values are not properly quoted
QgsExpression exp2( QStringLiteral( "sqlite_fetch_and_increment(@test_database, 'T_KEY_OBJECT', 'T_LastUniqueId', 'T_Key', 'T_Id', map('T_LastChange','date(''now'')','T_CreateDate','date(''now'')','T_User', @username))" ) );
exp2.evaluate( &context );
QCOMPARE( exp2.hasEvalError(), true );
const QString evalErrorString2 = exp2.evalErrorString();
QVERIFY2( evalErrorString2.contains( "some_username" ), QStringLiteral( "'some_username' not found in '%1'" ).arg( evalErrorString2 ).toUtf8().constData() );

// Test incrementation logic
QgsExpression exp( QStringLiteral( "sqlite_fetch_and_increment(@test_database, 'T_KEY_OBJECT', 'T_LastUniqueId', 'T_Key', 'T_Id', map('T_LastChange','date(''now'')','T_CreateDate','date(''now'')','T_User','''me'''))" ) );
QVariant res = exp.evaluate( &context );
QCOMPARE( res.toInt(), 0 );

res = exp.evaluate( &context );
if ( exp.hasEvalError() )
qDebug() << exp.evalErrorString();
QCOMPARE( exp.hasEvalError(), false );

QCOMPARE( res.toInt(), 1 );
}

void aggregate_data()
{
QTest::addColumn<QString>( "string" );
Binary file not shown.

0 comments on commit d2b3575

Please sign in to comment.
You can’t perform that action at this time.