Skip to content
Permalink
Browse files

Merge pull request #4799 from nyalldawson/m2

Run processing algorithms using task manager
  • Loading branch information
nyalldawson committed Jul 7, 2017
2 parents 29c4fcf + 240dd19 commit 52f4c5e975ef2b54af6f96e67afd4f3d8e7148f0
Showing with 1,323 additions and 515 deletions.
  1. +3 −3 python/analysis/vector/qgsgeometrysnapper.sip
  2. +3 −1 python/core/core.sip
  3. +141 −4 python/core/processing/qgsprocessingalgorithm.sip
  4. +11 −1 python/core/processing/qgsprocessingalgrunnertask.sip
  5. +39 −1 python/core/processing/qgsprocessingcontext.sip
  6. +4 −2 python/core/processing/qgsprocessingmodelalgorithm.sip
  7. +6 −0 python/core/processing/qgsprocessingparameters.sip
  8. +7 −0 python/core/qgsmaplayerstore.sip
  9. +8 −8 python/plugins/processing/algs/qgis/Aspect.py
  10. +57 −47 python/plugins/processing/algs/qgis/ConcaveHull.py
  11. +22 −16 python/plugins/processing/algs/qgis/Delaunay.py
  12. +8 −8 python/plugins/processing/algs/qgis/DropGeometry.py
  13. +0 −4 python/plugins/processing/algs/qgis/ImportIntoPostGIS.py
  14. +0 −2 python/plugins/processing/algs/qgis/ImportIntoSpatialite.py
  15. +0 −7 python/plugins/processing/algs/qgis/PointsLayerFromTable.py
  16. +10 −7 python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py
  17. +3 −0 python/plugins/processing/algs/qgis/QgisAlgorithm.py
  18. +1 −4 python/plugins/processing/algs/qgis/RandomExtract.py
  19. +0 −2 python/plugins/processing/algs/qgis/SaveSelectedFeatures.py
  20. +53 −39 python/plugins/processing/algs/qgis/SnapGeometries.py
  21. +0 −5 python/plugins/processing/algs/qgis/VectorSplit.py
  22. +33 −22 python/plugins/processing/algs/qgis/VoronoiPolygons.py
  23. +39 −31 python/plugins/processing/algs/qgis/ZonalStatistics.py
  24. +13 −1 python/plugins/processing/core/Processing.py
  25. +19 −15 python/plugins/processing/gui/AlgorithmDialog.py
  26. +21 −15 python/plugins/processing/gui/AlgorithmDialogBase.py
  27. +2 −1 python/plugins/processing/gui/HistoryDialog.py
  28. +5 −4 python/plugins/processing/modeler/ModelerDialog.py
  29. +26 −16 python/plugins/processing/script/ScriptAlgorithm.py
  30. +10 −6 python/plugins/processing/tests/AlgorithmsTestBase.py
  31. +5 −0 python/plugins/processing/tests/QgisAlgorithmsTest.py
  32. +142 −142 python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml
  33. +27 −3 python/plugins/processing/tools/general.py
  34. +5 −8 src/analysis/vector/qgsgeometrysnapper.cpp
  35. +4 −4 src/analysis/vector/qgsgeometrysnapper.h
  36. +64 −18 src/core/processing/qgsnativealgorithms.cpp
  37. +18 −9 src/core/processing/qgsnativealgorithms.h
  38. +129 −5 src/core/processing/qgsprocessingalgorithm.cpp
  39. +134 −7 src/core/processing/qgsprocessingalgorithm.h
  40. +21 −13 src/core/processing/qgsprocessingalgrunnertask.cpp
  41. +15 −4 src/core/processing/qgsprocessingalgrunnertask.h
  42. +64 −8 src/core/processing/qgsprocessingcontext.h
  43. +13 −3 src/core/processing/qgsprocessingmodelalgorithm.cpp
  44. +4 −2 src/core/processing/qgsprocessingmodelalgorithm.h
  45. +9 −0 src/core/processing/qgsprocessingparameters.h
  46. +8 −1 src/core/qgsfeedback.h
  47. +17 −0 src/core/qgsmaplayerstore.cpp
  48. +7 −0 src/core/qgsmaplayerstore.h
  49. +1 −1 src/core/qgsspatialindex.cpp
  50. +2 −2 src/gui/qgsmessagebar.cpp
  51. +60 −13 tests/src/core/testqgsprocessing.cpp
  52. +30 −0 tests/src/python/test_qgsmaplayerstore.py
@@ -33,11 +33,11 @@ class QgsGeometrySnapper : QObject
EndPointToEndPoint,
};

QgsGeometrySnapper( QgsVectorLayer *referenceLayer );
QgsGeometrySnapper( QgsFeatureSource *referenceSource );
%Docstring
Constructor for QgsGeometrySnapper. A reference layer which contains geometries to snap to must be
Constructor for QgsGeometrySnapper. A reference feature source which contains geometries to snap to must be
set. It is assumed that all geometries snapped using this object will have the
same CRS as the reference layer (ie, no reprojection is performed).
same CRS as the reference source (ie, no reprojection is performed).
%End

QgsGeometry snapGeometry( const QgsGeometry &geometry, double snapTolerance, SnapMode mode = PreferNodes ) const;
@@ -104,5 +104,7 @@ done:
%Include core_auto.sip

%VirtualErrorHandler processing_exception_handler
throw QgsProcessingException( getTraceback() );
QString trace = getTraceback();
SIP_RELEASE_GIL( sipGILState );
throw QgsProcessingException( trace );
%End
@@ -43,6 +43,13 @@ class QgsProcessingAlgorithm
virtual ~QgsProcessingAlgorithm();



virtual QgsProcessingAlgorithm *create() const = 0 /Factory/;
%Docstring
Creates a copy of the algorithm, ready for execution.
:rtype: QgsProcessingAlgorithm
%End

virtual QString name() const = 0;
%Docstring
Returns the algorithm name, used for identifying the algorithm. This string
@@ -217,7 +224,9 @@ class QgsProcessingAlgorithm
QVariantMap run( const QVariantMap &parameters,
QgsProcessingContext &context, QgsProcessingFeedback *feedback, bool *ok /Out/ = 0 ) const;
%Docstring
Executes the algorithm using the specified ``parameters``.
Executes the algorithm using the specified ``parameters``. This method internally
creates a copy of the algorithm before running it, so it is safe to call
on algorithms directly retrieved from QgsProcessingRegistry and QgsProcessingProvider.

The ``context`` argument specifies the context in which the algorithm is being run.

@@ -227,6 +236,55 @@ class QgsProcessingAlgorithm

:return: A map of algorithm outputs. These may be output layer references, or calculated
values such as statistical calculations.

.. note::

this method can only be called from the main thread. Use prepare(), runPrepared() and postProcess()
if you need to run algorithms from a background thread, or use the QgsProcessingAlgRunnerTask class.
:rtype: QVariantMap
%End

bool prepare( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback );
%Docstring
Prepares the algorithm for execution. This must be run in the main thread, and allows the algorithm
to pre-evaluate input parameters in a thread-safe manner. This must be called before
calling runPrepared() (which is safe to do in any thread).
.. seealso:: runPrepared()
.. seealso:: postProcess()
.. note::

This method modifies the algorithm instance, so it is not safe to call
on algorithms directly retrieved from QgsProcessingRegistry and QgsProcessingProvider. Instead, a copy
of the algorithm should be created with clone() and prepare()/runPrepared() called on the copy.
:rtype: bool
%End

QVariantMap runPrepared( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback );
%Docstring
Runs the algorithm, which has been prepared by an earlier call to prepare().
This method is safe to call from any thread. Returns true if the algorithm was successfully executed.
After runPrepared() has finished, the postProcess() method should be called from the main thread
to allow the algorithm to perform any required cleanup tasks and return its final result.
.. seealso:: prepare()
.. seealso:: postProcess()
.. note::

This method modifies the algorithm instance, so it is not safe to call
on algorithms directly retrieved from QgsProcessingRegistry and QgsProcessingProvider. Instead, a copy
of the algorithm should be created with clone() and prepare()/runPrepared() called on the copy.
:rtype: QVariantMap
%End

QVariantMap postProcess( QgsProcessingContext &context, QgsProcessingFeedback *feedback );
%Docstring
Should be called in the main thread following the completion of runPrepared(). This method
allows the algorithm to perform any required cleanup tasks. The returned variant map
includes the results evaluated by the algorithm.
.. note::

This method modifies the algorithm instance, so it is not safe to call
on algorithms directly retrieved from QgsProcessingRegistry and QgsProcessingProvider. Instead, a copy
of the algorithm should be created with clone() and prepare()/runPrepared() called on the copy.
:rtype: QVariantMap
%End

@@ -298,20 +356,99 @@ class QgsProcessingAlgorithm
:rtype: bool
%End

virtual QVariantMap processAlgorithm( const QVariantMap &parameters,
QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const = 0 /VirtualErrorHandler=processing_exception_handler/;
virtual bool prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) /VirtualErrorHandler=processing_exception_handler/;
%Docstring
Prepares the algorithm to run using the specified ``parameters``. Algorithms should implement
their logic for evaluating parameter values here. The evaluated parameter results should
be stored in member variables ready for a call to processAlgorithm().

The ``context`` argument specifies the context in which the algorithm is being run.

prepareAlgorithm should be used to handle any thread-sensitive preparation which is required
by the algorithm. It will always be called from the same thread that ``context`` has thread
affinity with. While this will generally be the main thread, it is not guaranteed. For instance,
algorithms which are run as a step in a larger model or as a subcomponent of a script-based algorithm
will call prepareAlgorithm from the same thread as that model/script it being executed in.

Note that the processAlgorithm step uses a temporary context with affinity for the thread in
which the algorithm is executed, making it safe for processAlgorithm implementations to load
sources and sinks without issue. Implementing prepareAlgorithm is only required if special
thread safe handling is required by the algorithm.

Algorithm preparation progress should be reported using the supplied ``feedback`` object. Additionally,
well-behaved algorithms should periodically check ``feedback`` to determine whether the
algorithm should be canceled and exited early.

If the preparation was successful algorithms must return true. If a false value is returned
this indicates that the preparation could not be completed, and the algorithm execution
will be canceled.

:return: true if preparation was successful.
.. seealso:: processAlgorithm()
.. seealso:: postProcessAlgorithm()
:rtype: bool
%End

virtual QVariantMap processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) = 0 /VirtualErrorHandler=processing_exception_handler/;
%Docstring
Runs the algorithm using the specified ``parameters``. Algorithms should implement
their custom processing logic here.

The ``context`` argument specifies the context in which the algorithm is being run.
The ``context`` argument gives a temporary context with thread affinity matching the thread
in which the algorithm is being run. This is a cut-back copy of the context passed to
the prepareAlgorithm() and postProcessAlgorithm() steps, but it is generally safe
for most algorithms to utilize this context for loading layers and creating sinks.
Any loaded layers or sinks created within this temporary context will be transferred
back to the main execution context upon successful completion of the processAlgorithm()
step.

Algorithm progress should be reported using the supplied ``feedback`` object. Additionally,
well-behaved algorithms should periodically check ``feedback`` to determine whether the
algorithm should be canceled and exited early.

This method will not be called if the prepareAlgorithm() step failed (returned false).

c++ implementations of processAlgorithm can throw the QgsProcessingException exception
to indicate that a fatal error occurred within the execution. Python based subclasses
should raise GeoAlgorithmExecutionException for the same purpose.

:return: A map of algorithm outputs. These may be output layer references, or calculated
values such as statistical calculations. Unless the algorithm subclass overrides
the postProcessAlgorithm() step this returned map will be used as the output for the
algorithm.

.. seealso:: prepareAlgorithm()
.. seealso:: postProcessAlgorithm()
:rtype: QVariantMap
%End

virtual QVariantMap postProcessAlgorithm( QgsProcessingContext &context, QgsProcessingFeedback *feedback ) /VirtualErrorHandler=processing_exception_handler/;
%Docstring
Allows the algorithm to perform any required cleanup tasks. The returned variant map
includes the results evaluated by the algorithm. These may be output layer references, or calculated
values such as statistical calculations.

The ``context`` argument specifies the context in which the algorithm was run.

Postprocess progress should be reported using the supplied ``feedback`` object. Additionally,
well-behaved algorithms should periodically check ``feedback`` to determine whether the
post processing should be canceled and exited early.

postProcessAlgorithm should be used to handle any thread-sensitive cleanup which is required
by the algorithm. It will always be called from the same thread that ``context`` has thread
affinity with. While this will generally be the main thread, it is not guaranteed. For instance,
algorithms which are run as a step in a larger model or as a subcomponent of a script-based algorithm
will call postProcessAlgorithm from the same thread as that model/script it being executed in.

postProcessAlgorithm will not be called if the prepareAlgorithm() step failed (returned false),
or if an exception was raised by the processAlgorithm() step.

:return: A map of algorithm outputs. These may be output layer references, or calculated
values such as statistical calculations. Implementations which return a non-empty
map will override any results returned by processAlgorithm().

.. seealso:: prepareAlgorithm()
.. seealso:: processAlgorithm()
:rtype: QVariantMap
%End

@@ -24,14 +24,24 @@ class QgsProcessingAlgRunnerTask : QgsTask

QgsProcessingAlgRunnerTask( const QgsProcessingAlgorithm *algorithm,
const QVariantMap &parameters,
QgsProcessingContext &context );
QgsProcessingContext &context,
QgsProcessingFeedback *feedback = 0 );
%Docstring
Constructor for QgsProcessingAlgRunnerTask. Takes an ``algorithm``, algorithm ``parameters``
and processing ``context``.
%End

virtual void cancel();

signals:

void executed( bool successful, const QVariantMap &results );
%Docstring
Emitted when the algorithm has finished execution. If the algorithm completed
execution without errors then ``successful`` will be true. The ``results`` argument
contains the results reported by the algorithm.
%End

protected:

virtual bool run();
@@ -38,6 +38,12 @@ class QgsProcessingContext
%End


void copyThreadSafeSettings( const QgsProcessingContext &other );
%Docstring
Copies all settings which are safe for use across different threads from
``other`` to this context.
%End

QgsProcessingContext::Flags flags() const;
%Docstring
Returns any flags set in the context.
@@ -126,7 +132,6 @@ Destination project
.. seealso:: layersToLoadOnCompletion()
%End


QgsFeatureRequest::InvalidGeometryCheck invalidGeometryCheck() const;
%Docstring
Returns the behavior used for checking invalid geometries in input layers.
@@ -216,6 +221,39 @@ Destination project
.. seealso:: setFeedback()
%End

QThread *thread();
%Docstring
Returns the thread in which the context lives.
.. seealso:: pushToThread()
:rtype: QThread
%End

void pushToThread( QThread *thread );
%Docstring
Pushes the thread affinity for the context (including all layers contained in the temporaryLayerStore()) into
another ``thread``. This method is only safe to call when the current thread matches the existing thread
affinity for the context (see thread()).
.. seealso:: thread()
%End

void takeResultsFrom( QgsProcessingContext &context );
%Docstring
Takes the results from another ``context`` and merges them with the results currently
stored in this context. This includes settings like any layers loaded in the temporaryLayerStore()
and layersToLoadOnCompletion().
This is only safe to call when both this context and the other ``context`` share the same
thread() affinity, and that thread is the current thread.
%End

QgsMapLayer *takeResultLayer( const QString &id ) /TransferBack/;
%Docstring
Takes the result map layer with matching ``id`` from the context and
transfers ownership of it back to the caller. This method can be used
to remove temporary layers which are not required for further processing
from a context.
:rtype: QgsMapLayer
%End

private:
QgsProcessingContext( const QgsProcessingContext &other );
};
@@ -588,6 +588,8 @@ Copies are protected to avoid slicing

virtual QString asPythonCommand( const QVariantMap &parameters, QgsProcessingContext &context ) const;

virtual QgsProcessingModelAlgorithm *create() const /Factory/;


void setName( const QString &name );
%Docstring
@@ -833,8 +835,8 @@ Copies are protected to avoid slicing

protected:

virtual QVariantMap processAlgorithm( const QVariantMap &parameters,
QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const;
virtual QVariantMap processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback );


};

@@ -45,6 +45,12 @@ class QgsProcessingFeatureSourceDefinition
True if only selected features in the source should be used by algorithms.
%End

bool operator==( const QgsProcessingFeatureSourceDefinition &other );

bool operator!=( const QgsProcessingFeatureSourceDefinition &other );
%Docstring
:rtype: bool
%End

operator QVariant() const;
%Docstring
@@ -208,6 +208,13 @@ class QgsMapLayerStore : QObject
.. seealso:: removeMapLayers()
%End

void transferLayersFromStore( QgsMapLayerStore *other );
%Docstring
Transfers all the map layers contained within another map layer store and adds
them to this store.
Note that ``other`` and this store must have the same thread affinity.
%End

signals:

void layersWillBeRemoved( const QStringList &layerIds );
@@ -47,9 +47,9 @@

class Aspect(QgisAlgorithm):

INPUT_LAYER = 'INPUT_LAYER'
INPUT = 'INPUT'
Z_FACTOR = 'Z_FACTOR'
OUTPUT_LAYER = 'OUTPUT_LAYER'
OUTPUT = 'OUTPUT'

def icon(self):
return QIcon(os.path.join(pluginPath, 'images', 'dem.png'))
@@ -60,13 +60,13 @@ def group(self):
def __init__(self):
super().__init__()

self.addParameter(QgsProcessingParameterRasterLayer(self.INPUT_LAYER,
self.addParameter(QgsProcessingParameterRasterLayer(self.INPUT,
self.tr('Elevation layer')))
self.addParameter(QgsProcessingParameterNumber(self.Z_FACTOR,
self.tr('Z factor'), QgsProcessingParameterNumber.Double,
1, False, 1, 999999.99))
self.addParameter(QgsProcessingParameterRasterOutput(self.OUTPUT_LAYER, self.tr('Aspect')))
self.addOutput(QgsProcessingOutputRasterLayer(self.OUTPUT_LAYER, self.tr('Aspect')))
self.addParameter(QgsProcessingParameterRasterOutput(self.OUTPUT, self.tr('Aspect')))
self.addOutput(QgsProcessingOutputRasterLayer(self.OUTPUT, self.tr('Aspect')))

def name(self):
return 'aspect'
@@ -75,15 +75,15 @@ def displayName(self):
return self.tr('Aspect')

def processAlgorithm(self, parameters, context, feedback):
inputFile = exportRasterLayer(self.parameterAsRasterLayer(parameters, self.INPUT_LAYER, context))
inputFile = exportRasterLayer(self.parameterAsRasterLayer(parameters, self.INPUT, context))
zFactor = self.parameterAsDouble(parameters, self.Z_FACTOR, context)

outputFile = self.parameterAsRasterOutputLayer(parameters, self.OUTPUT_LAYER, context)
outputFile = self.parameterAsRasterOutputLayer(parameters, self.OUTPUT, context)

outputFormat = raster.formatShortNameFromFileName(outputFile)

aspect = QgsAspectFilter(inputFile, outputFile, outputFormat)
aspect.setZFactor(zFactor)
aspect.processRaster(feedback)

return {self.OUTPUT_LAYER: outputFile}
return {self.OUTPUT: outputFile}

0 comments on commit 52f4c5e

Please sign in to comment.
You can’t perform that action at this time.