From 957e92c84707822d97c53bdd5ab54d026eda50d6 Mon Sep 17 00:00:00 2001 From: Blottiere Paul Date: Tue, 23 Jan 2018 07:53:42 +0000 Subject: [PATCH] Add async model for virtual layers --- python/core/core_auto.sip | 1 + python/core/qgsvectordataprovider.sip.in | 3 + python/core/qgsvirtuallayerdefinition.sip.in | 3 + python/core/qgsvirtuallayertask.sip | 46 +++++++++ .../db_plugins/vlayers/data_model.py | 94 ++++++++++++++----- .../db_manager/db_plugins/vlayers/plugin.py | 4 + .../db_manager/dlg_cancel_task_query.py | 1 - python/plugins/db_manager/dlg_sql_window.py | 35 +++---- src/core/CMakeLists.txt | 2 + src/core/qgsvectordataprovider.cpp | 5 + src/core/qgsvectordataprovider.h | 3 + src/core/qgsvirtuallayerdefinition.cpp | 9 ++ src/core/qgsvirtuallayerdefinition.h | 4 + src/core/qgsvirtuallayertask.cpp | 48 ++++++++++ src/core/qgsvirtuallayertask.h | 53 +++++++++++ .../virtual/qgsvirtuallayerprovider.cpp | 36 +++++-- .../virtual/qgsvirtuallayerprovider.h | 3 + 17 files changed, 299 insertions(+), 51 deletions(-) create mode 100644 python/core/qgsvirtuallayertask.sip create mode 100644 src/core/qgsvirtuallayertask.cpp create mode 100644 src/core/qgsvirtuallayertask.h diff --git a/python/core/core_auto.sip b/python/core/core_auto.sip index fb6cce67f6bb..2bfb3c1e0aa9 100644 --- a/python/core/core_auto.sip +++ b/python/core/core_auto.sip @@ -126,6 +126,7 @@ %Include qgsvectorlayerundocommand.sip %Include qgsvectorlayerundopassthroughcommand.sip %Include qgsvectorlayerutils.sip +%Include qgsvirtuallayertask.sip %Include qgsvirtuallayerdefinition.sip %Include qgsvirtuallayerdefinitionutils.sip %Include qgsmapthemecollection.sip diff --git a/python/core/qgsvectordataprovider.sip.in b/python/core/qgsvectordataprovider.sip.in index 82d21f5b513f..b5ba3bffd73d 100644 --- a/python/core/qgsvectordataprovider.sip.in +++ b/python/core/qgsvectordataprovider.sip.in @@ -50,6 +50,7 @@ of feature and attribute information from a spatial datasource. FastTruncate, ReadLayerMetadata, WriteLayerMetadata, + CancelSupport, }; typedef QFlags Capabilities; @@ -231,6 +232,8 @@ Providers with the FastTruncate capability will use an optimised method to trunc .. seealso:: :py:func:`deleteFeatures` %End + virtual bool cancel(); + virtual bool addAttributes( const QList &attributes ); %Docstring Adds new ``attributes`` to the provider. Returns true in case of success and false in case of failure. diff --git a/python/core/qgsvirtuallayerdefinition.sip.in b/python/core/qgsvirtuallayerdefinition.sip.in index 680be0643743..a3a8ff5c644c 100644 --- a/python/core/qgsvirtuallayerdefinition.sip.in +++ b/python/core/qgsvirtuallayerdefinition.sip.in @@ -150,6 +150,9 @@ Get the name of the field with unique identifiers Set the name of the field with unique identifiers %End + void setPostpone( bool postpone ); + bool postpone() const; + QString geometryField() const; %Docstring Get the name of the geometry field. Empty if no geometry field diff --git a/python/core/qgsvirtuallayertask.sip b/python/core/qgsvirtuallayertask.sip new file mode 100644 index 000000000000..efabcba28eb7 --- /dev/null +++ b/python/core/qgsvirtuallayertask.sip @@ -0,0 +1,46 @@ +/************************************************************************ + * This file has been generated automatically from * + * * + * src/core/qgsvirtuallayertask.h * + * * + * Do not edit manually ! Edit header and run scripts/sipify.pl again * + ************************************************************************/ + + + + +class QgsVirtualLayerTask : QgsTask +{ +%Docstring + +Initializes a virtual layer in a separated task. + +.. versionadded:: 3.0 +%End + +%TypeHeaderCode +#include "qgsvirtuallayertask.h" +%End + public: + + QgsVirtualLayerTask( const QgsVirtualLayerDefinition &definition ); + + QgsVectorLayer *layer(); + + QgsVirtualLayerDefinition definition() const; + + virtual bool run(); + + + virtual void cancel(); + + +}; + +/************************************************************************ + * This file has been generated automatically from * + * * + * src/core/qgsvirtuallayertask.h * + * * + * Do not edit manually ! Edit header and run scripts/sipify.pl again * + ************************************************************************/ diff --git a/python/plugins/db_manager/db_plugins/vlayers/data_model.py b/python/plugins/db_manager/db_plugins/vlayers/data_model.py index bc9a33cf6bf3..a0457172e8c3 100644 --- a/python/plugins/db_manager/db_plugins/vlayers/data_model.py +++ b/python/plugins/db_manager/db_plugins/vlayers/data_model.py @@ -19,14 +19,14 @@ ***************************************************************************/ """ -from ..data_model import TableDataModel, BaseTableModel +from ..data_model import TableDataModel, BaseTableModel, SqlResultModelAsync from .connector import VLayerRegistry, getQueryGeometryName from .plugin import LVectorTable -from ..plugin import DbError +from ..plugin import DbError, BaseError from qgis.PyQt.QtCore import QTime, QTemporaryFile -from qgis.core import QgsVectorLayer, QgsWkbTypes, QgsVirtualLayerDefinition +from qgis.core import QgsVectorLayer, QgsWkbTypes, QgsVirtualLayerDefinition, QgsVirtualLayerTask, QgsTask class LTableDataModel(TableDataModel): @@ -63,40 +63,88 @@ def rowCount(self, index=None): return 0 -class LSqlResultModel(BaseTableModel): - # BaseTableModel +class LSqlResultModelTask(QgsTask): + + def __init__(self, subtask, db): + QgsTask.__init__(self) + self.subtask = subtask + self.db = db + self.model = None + + def run(self): + try: + path = self.subtask.definition().filePath() + sql = self.subtask.definition().query() + self.model = LSqlResultModel(self.db, sql, None, self.subtask.layer(), path) + except Exception as e: + self.error = BaseError(str(e)) + return False + return True + + def cancelQuery(self): + self.subtask.cancel() + self.cancel() + + +class LSqlResultModelAsync(SqlResultModelAsync): def __init__(self, db, sql, parent=None): - # create a virtual layer with non-geometry results - t = QTime() - t.start() + SqlResultModelAsync.__init__(self, db, sql, parent) tf = QTemporaryFile() tf.open() - tmp = tf.fileName() + path = tf.fileName() tf.close() df = QgsVirtualLayerDefinition() - df.setFilePath(tmp) - df.setQuery(sql) - p = QgsVectorLayer(df.toString(), "vv", "virtual") - self._secs = t.elapsed() / 1000.0 - - if not p.isValid(): - data = [] - header = [] - raise DbError(p.dataProvider().error().summary(), sql) + df.setFilePath(path) + df.setQuery(self.sql) + + self.subtask = QgsVirtualLayerTask(df) + self.task = LSqlResultModelTask(self.subtask, db) + self.task.addSubTask(self.subtask, [], QgsTask.ParentDependsOnSubTask) + self.task.taskCompleted.connect(self.modelDone) + self.task.taskTerminated.connect(self.modelDone) + + def modelDone(self): + self.status = self.task.status + self.model = self.task.model + self.done.emit() + + +class LSqlResultModel(BaseTableModel): + + def __init__(self, db, sql, parent=None, layer=None, path=None): + t = QTime() + t.start() + + if not layer: + tf = QTemporaryFile() + tf.open() + path = tf.fileName() + tf.close() + + df = QgsVirtualLayerDefinition() + df.setFilePath(path) + df.setQuery(sql) + layer = QgsVectorLayer(df.toString(), "vv", "virtual") + self._secs = t.elapsed() / 1000.0 + + data = [] + header = [] + + if not layer.isValid(): + raise DbError(layer.dataProvider().error().summary(), sql) else: - header = [f.name() for f in p.fields()] + header = [f.name() for f in layer.fields()] has_geometry = False - if p.geometryType() != QgsWkbTypes.NullGeometry: - gn = getQueryGeometryName(tmp) + if layer.geometryType() != QgsWkbTypes.NullGeometry: + gn = getQueryGeometryName(path) if gn: has_geometry = True header += [gn] - data = [] - for f in p.getFeatures(): + for f in layer.getFeatures(): a = f.attributes() if has_geometry: if f.hasGeometry(): diff --git a/python/plugins/db_manager/db_plugins/vlayers/plugin.py b/python/plugins/db_manager/db_plugins/vlayers/plugin.py index 1705433f7f1e..965e20420f89 100644 --- a/python/plugins/db_manager/db_plugins/vlayers/plugin.py +++ b/python/plugins/db_manager/db_plugins/vlayers/plugin.py @@ -98,6 +98,10 @@ def sqlResultModel(self, sql, parent): from .data_model import LSqlResultModel return LSqlResultModel(self, sql, parent) + def sqlResultModelAsync(self, sql, parent): + from .data_model import LSqlResultModelAsync + return LSqlResultModelAsync(self, sql, parent) + def toSqlLayer(self, sql, geomCol, uniqueCol, layerName="QueryLayer", layerType=None, avoidSelectById=False, _filter=""): df = QgsVirtualLayerDefinition() df.setQuery(sql) diff --git a/python/plugins/db_manager/dlg_cancel_task_query.py b/python/plugins/db_manager/dlg_cancel_task_query.py index 7387450ba718..b4583d07341c 100644 --- a/python/plugins/db_manager/dlg_cancel_task_query.py +++ b/python/plugins/db_manager/dlg_cancel_task_query.py @@ -61,6 +61,5 @@ def show(self): super(QDialog, self).show() def hide(self): - self.cancelStatus = False self.mGif.stop() super(QDialog, self).hide() diff --git a/python/plugins/db_manager/dlg_sql_window.py b/python/plugins/db_manager/dlg_sql_window.py index 6bd076cc7bbc..acf0b4ce1e5c 100644 --- a/python/plugins/db_manager/dlg_sql_window.py +++ b/python/plugins/db_manager/dlg_sql_window.py @@ -187,24 +187,25 @@ def executeSqlCanceled(self): def executeSqlCompleted(self): self.dlg_cancel_task.hide() - if self.modelAsync.task.status() == QgsTask.Complete: - model = self.modelAsync.model - cols = [] - quotedCols = [] - - self.viewResult.setModel(model) - self.lblResult.setText(self.tr("{0} rows, {1:.1f} seconds").format(model.affectedRows(), model.secs())) - cols = self.viewResult.model().columnNames() - for col in cols: - quotedCols.append(self.db.connector.quoteId(col)) + with OverrideCursor(Qt.WaitCursor): + if self.modelAsync.task.status() == QgsTask.Complete: + model = self.modelAsync.model + cols = [] + quotedCols = [] + + self.viewResult.setModel(model) + self.lblResult.setText(self.tr("{0} rows, {1:.1f} seconds").format(model.affectedRows(), model.secs())) + cols = self.viewResult.model().columnNames() + for col in cols: + quotedCols.append(self.db.connector.quoteId(col)) - self.setColumnCombos(cols, quotedCols) - self.update() - elif not self.dlg_cancel_task.cancelStatus: - DlgDbError.showError(self.modelAsync.error, self) - self.uniqueModel.clear() - self.geomCombo.clear() - pass + self.setColumnCombos(cols, quotedCols) + self.update() + elif not self.dlg_cancel_task.cancelStatus: + DlgDbError.showError(self.modelAsync.error, self) + self.uniqueModel.clear() + self.geomCombo.clear() + pass def executeSql(self): diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index aea518c95644..8fbcb39f4926 100755 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -297,6 +297,7 @@ SET(QGIS_CORE_SRCS qgsvectorfilewriter.cpp qgsvectorfilewritertask.cpp qgsvectorlayer.cpp + qgsvirtuallayertask.cpp qgsvectorlayerfeaturecounter.cpp qgsvectorlayercache.cpp qgsvectorlayerdiagramprovider.cpp @@ -635,6 +636,7 @@ SET(QGIS_CORE_MOC_HDRS qgsvectorlayereditbuffer.h qgsvectorlayereditpassthrough.h qgsvectorlayer.h + qgsvirtuallayertask.h qgsvectorlayerexporter.h qgsvectorlayerfeaturecounter.h qgsvectorlayerjoinbuffer.h diff --git a/src/core/qgsvectordataprovider.cpp b/src/core/qgsvectordataprovider.cpp index 8de8e06a11e6..e18ed806dd5e 100644 --- a/src/core/qgsvectordataprovider.cpp +++ b/src/core/qgsvectordataprovider.cpp @@ -804,6 +804,11 @@ QTextCodec *QgsVectorDataProvider::textEncoding() const return mEncoding; } +bool QgsVectorDataProvider::cancel() +{ + return false; +} + QStringList QgsVectorDataProvider::sEncodings; QList QgsVectorDataProvider::discoverRelations( const QgsVectorLayer *, const QList & ) const diff --git a/src/core/qgsvectordataprovider.h b/src/core/qgsvectordataprovider.h index bec7b2a44f70..a30df6ddb481 100644 --- a/src/core/qgsvectordataprovider.h +++ b/src/core/qgsvectordataprovider.h @@ -90,6 +90,7 @@ class CORE_EXPORT QgsVectorDataProvider : public QgsDataProvider, public QgsFeat FastTruncate = 1 << 20, //!< Supports fast truncation of the layer (removing all features). Since QGIS 3.0 ReadLayerMetadata = 1 << 21, //!< Provider can read layer metadata from data store. Since QGIS 3.0. See QgsDataProvider::layerMetadata() WriteLayerMetadata = 1 << 22, //!< Provider can write layer metadata to the data store. Since QGIS 3.0. See QgsDataProvider::writeLayerMetadata() + CancelSupport = 1 << 23, //!< Supports interruption of pending queries from a separated thread. Since QGIS 3.0 }; Q_DECLARE_FLAGS( Capabilities, Capability ) @@ -248,6 +249,8 @@ class CORE_EXPORT QgsVectorDataProvider : public QgsDataProvider, public QgsFeat */ virtual bool truncate(); + virtual bool cancel(); + /** * Adds new \a attributes to the provider. Returns true in case of success and false in case of failure. * If attributes are added using this method then QgsVectorLayer::updateFields() must be called diff --git a/src/core/qgsvirtuallayerdefinition.cpp b/src/core/qgsvirtuallayerdefinition.cpp index 262c2182181a..05929797cfc8 100644 --- a/src/core/qgsvirtuallayerdefinition.cpp +++ b/src/core/qgsvirtuallayerdefinition.cpp @@ -157,6 +157,10 @@ QgsVirtualLayerDefinition QgsVirtualLayerDefinition::fromUrl( const QUrl &url ) } } } + else if ( key == QLatin1String( "postpone" ) ) + { + def.setPostpone( true ); + } } def.setFields( fields ); @@ -209,6 +213,11 @@ QUrl QgsVirtualLayerDefinition::toUrl() const url.addQueryItem( QStringLiteral( "field" ), f.name() + ":text" ); } + if ( postpone() ) + { + url.addQueryItem( QStringLiteral( "postpone" ), QLatin1String( "" ) ); + } + return url; } diff --git a/src/core/qgsvirtuallayerdefinition.h b/src/core/qgsvirtuallayerdefinition.h index 5823ec050835..0bdf4f8f0428 100644 --- a/src/core/qgsvirtuallayerdefinition.h +++ b/src/core/qgsvirtuallayerdefinition.h @@ -131,6 +131,9 @@ class CORE_EXPORT QgsVirtualLayerDefinition //! Set the name of the field with unique identifiers void setUid( const QString &uid ) { mUid = uid; } + void setPostpone( bool postpone ) { mPostpone = postpone; } + bool postpone() const { return mPostpone; } + //! Get the name of the geometry field. Empty if no geometry field QString geometryField() const { return mGeometryField; } //! Set the name of the geometry field @@ -174,6 +177,7 @@ class CORE_EXPORT QgsVirtualLayerDefinition QString mGeometryField; QString mFilePath; QgsFields mFields; + bool mPostpone = false; QgsWkbTypes::Type mGeometryWkbType = QgsWkbTypes::Unknown; long mGeometrySrid = 0; }; diff --git a/src/core/qgsvirtuallayertask.cpp b/src/core/qgsvirtuallayertask.cpp new file mode 100644 index 000000000000..cb51b90f996b --- /dev/null +++ b/src/core/qgsvirtuallayertask.cpp @@ -0,0 +1,48 @@ +/*************************************************************************** + qgsvirtuallayertask.cpp - description + ------------------- + begin : Jan 19, 2018 + copyright : (C) 2017 by Paul Blottiere + email : blottiere.paul@gmail.com + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include "qgsvirtuallayertask.h" + +QgsVirtualLayerTask::QgsVirtualLayerTask( const QgsVirtualLayerDefinition &definition ) + : QgsTask() + , mDefinition( definition ) +{ + mDefinition.setPostpone( true ); + mLayer = qgis::make_unique( mDefinition.toString(), "layer", "virtual" ); +} + +bool QgsVirtualLayerTask::run() +{ + mLayer->reload(); // blocking call because the loading is postponed + return mLayer->isValid(); +} + +QgsVirtualLayerDefinition QgsVirtualLayerTask::definition() const +{ + return mDefinition; +} + +QgsVectorLayer *QgsVirtualLayerTask::layer() +{ + return mLayer.get(); +} + +void QgsVirtualLayerTask::cancel() +{ + mLayer->dataProvider()->cancel(); + QgsTask::cancel(); +} diff --git a/src/core/qgsvirtuallayertask.h b/src/core/qgsvirtuallayertask.h new file mode 100644 index 000000000000..790f40f03a2e --- /dev/null +++ b/src/core/qgsvirtuallayertask.h @@ -0,0 +1,53 @@ +/*************************************************************************** + qgsvirtuallayertask.h - description + ------------------- + begin : Jan 19, 2018 + copyright : (C) 2017 by Paul Blottiere + email : blottiere.paul@gmail.com + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#ifndef QGSVIRTUALLAYERTASK_H +#define QGSVIRTUALLAYERTASK_H + +#include "qgsvectorlayer.h" +#include "qgsvirtuallayerdefinition.h" +#include "qgstaskmanager.h" + +/** + * \ingroup core + * + * Initializes a virtual layer in a separated task. + * + * \since QGIS 3.0 + */ +class CORE_EXPORT QgsVirtualLayerTask : public QgsTask +{ + Q_OBJECT + + public: + + QgsVirtualLayerTask( const QgsVirtualLayerDefinition &definition ); + + QgsVectorLayer *layer(); + + QgsVirtualLayerDefinition definition() const; + + bool run() override; + + void cancel() override; + + private: + QgsVirtualLayerDefinition mDefinition; + std::unique_ptr mLayer; +}; + +#endif // QGSVECTORLAYERTASK_H diff --git a/src/providers/virtual/qgsvirtuallayerprovider.cpp b/src/providers/virtual/qgsvirtuallayerprovider.cpp index 4c70ed03c0db..c4ac72e853be 100644 --- a/src/providers/virtual/qgsvirtuallayerprovider.cpp +++ b/src/providers/virtual/qgsvirtuallayerprovider.cpp @@ -76,15 +76,9 @@ QgsVirtualLayerProvider::QgsVirtualLayerProvider( QString const &uri ) { mDefinition = QgsVirtualLayerDefinition::fromUrl( url ); - if ( mDefinition.sourceLayers().empty() && !mDefinition.filePath().isEmpty() && mDefinition.query().isEmpty() ) + if ( !mDefinition.postpone() ) { - // open the file - mValid = openIt(); - } - else - { - // create the file - mValid = createIt(); + reloadData(); } } catch ( std::runtime_error &e ) @@ -100,6 +94,20 @@ QgsVirtualLayerProvider::QgsVirtualLayerProvider( QString const &uri ) } } +void QgsVirtualLayerProvider::reloadData() +{ + if ( mDefinition.sourceLayers().empty() && !mDefinition.filePath().isEmpty() && mDefinition.query().isEmpty() ) + { + // open the file + mValid = openIt(); + } + else + { + // create the file + mValid = createIt(); + } +} + bool QgsVirtualLayerProvider::loadSourceLayers() { Q_FOREACH ( const QgsVirtualLayerDefinition::SourceLayer &layer, mDefinition.sourceLayers() ) @@ -447,6 +455,11 @@ bool QgsVirtualLayerProvider::createIt() return true; } +bool QgsVirtualLayerProvider::cancel() +{ + return mSqlite.interrupt(); +} + void QgsVirtualLayerProvider::resetSqlite() { bool hasSpatialrefsys = false; @@ -563,11 +576,14 @@ bool QgsVirtualLayerProvider::isValid() const QgsVectorDataProvider::Capabilities QgsVirtualLayerProvider::capabilities() const { + QgsVectorDataProvider::Capabilities capabilities = CancelSupport; + if ( !mDefinition.uid().isNull() ) { - return SelectAtId; + capabilities |= SelectAtId; } - return nullptr; + + return capabilities; } QString QgsVirtualLayerProvider::name() const diff --git a/src/providers/virtual/qgsvirtuallayerprovider.h b/src/providers/virtual/qgsvirtuallayerprovider.h index efdb2f4fc4b7..9fe22c40c38c 100644 --- a/src/providers/virtual/qgsvirtuallayerprovider.h +++ b/src/providers/virtual/qgsvirtuallayerprovider.h @@ -53,6 +53,9 @@ class QgsVirtualLayerProvider: public QgsVectorDataProvider QString description() const override; QgsAttributeList pkAttributeIndexes() const override; QSet dependencies() const override; + bool cancel() override; + + void reloadData() override; private: