Steps towards executing models

Author: nyalldawson

python/plugins/processing/gui/NumberInputPanel.py

@@ -125,7 +125,7 @@ def showExpressionsBuilder(self):
     def getValue(self):
         value = self.leText.text()
         values = []
-        for param in self.modelParametersDialog.model.parameters:
+        for param in self.modelParametersDialog.model.parameterDefinitions():
             if isinstance(param, QgsProcessingParameterNumber):
                 if "@" + param.name() in value:
                     values.append(ValueFromInput(param.name()))

python/plugins/processing/modeler/ModelerAlgorithm.py

@@ -197,17 +197,6 @@ def prepareAlgorithm(self, alg):
                                                        self.tr("Parameter {0} in algorithm {1} in the model is run with default value! Edit the model to make sure that this is correct.").format(param.name(), alg.displayName()),
                                                        QgsMessageBar.WARNING, 4)
                     value = param.defaultValue()
-                # We allow unexistent filepaths, since that allows
-                # algorithms to skip some conversion routines
-
-                # TODO
-                #if not param.checkValueIsAcceptable(value) and not isinstance(param,
-                #                                                              ParameterDataObject):
-                #    raise GeoAlgorithmExecutionException(
-                #        self.tr('Wrong value {0} for {1} {2}', 'ModelerAlgorithm').format(
-                #            value, param.__class__.__name__, param.name()
-                #        )
-                #    )
 
         # note to self - these are parameters, not outputs
         for out in algInstance.outputDefinitions():
@@ -240,50 +229,6 @@ def resolveValue(self, value, param):
             v = value
         return param.evaluateForModeler(v, self)
 
-    def processAlgorithm(self, parameters, context, feedback):
-        executed = []
-        toExecute = [alg for alg in list(self.childAlgorithms().values()) if alg.isActive()]
-        while len(executed) < len(toExecute):
-            for alg in toExecute:
-                if alg.childId() not in executed:
-                    canExecute = True
-                    required = self.dependsOnChildAlgorithms(alg.childId())
-                    for requiredAlg in required:
-                        if requiredAlg not in executed:
-                            canExecute = False
-                            break
-                    if canExecute:
-                        try:
-                            feedback.pushDebugInfo(
-                                self.tr('Prepare algorithm: {0}', 'ModelerAlgorithm').format(alg.childId()))
-                            self.prepareAlgorithm(alg)
-                            feedback.setProgressText(
-                                self.tr('Running {0} [{1}/{2}]', 'ModelerAlgorithm').format(alg.description, len(executed) + 1, len(toExecute)))
-                            feedback.pushDebugInfo('Parameters: ' + ', '.join([str(p).strip() +
-                                                                               '=' + str(p.value) for p in alg.algorithm.parameters]))
-                            t0 = time.time()
-                            alg.algorithm().execute(parameters, context, feedback)
-                            dt = time.time() - t0
-
-                            # copy algorithm output value(s) back to model in case the algorithm modified those
-                            for out in alg.algorithm().outputs:
-                                if not out.flags() & QgsProcessingParameterDefinition.FlagHidden:
-                                    if out.name() in alg.modelOutputs():
-                                        modelOut = self.getOutputFromName(self.getSafeNameForOutput(alg.childId(), out.name()))
-                                        if modelOut:
-                                            modelOut.value = out.value
-
-                            executed.append(alg.childId())
-                            feedback.pushDebugInfo(
-                                self.tr('OK. Execution took %{0:.3f} ms ({1} outputs).', 'ModelerAlgorithm').format(dt, len(alg.algorithm.modelOutputs())))
-                        except GeoAlgorithmExecutionException as e:
-                            feedback.pushDebugInfo(self.tr('Failed', 'ModelerAlgorithm'))
-                            raise GeoAlgorithmExecutionException(
-                                self.tr('Error executing algorithm {0}\n{1}', 'ModelerAlgorithm').format(alg.description, e.msg))
-
-        feedback.pushDebugInfo(
-            self.tr('Model processed ok. Executed {0} algorithms total', 'ModelerAlgorithm').format(len(executed)))
-
     def asPythonCommand(self, parameters, context):
         if self.descriptionFile:
             return QgsProcessingAlgorithm.asPythonCommand(self, parameters, context)

src/core/processing/qgsprocessingmodelalgorithm.cpp

@@ -284,20 +284,81 @@ QString QgsProcessingModelAlgorithm::svgIconPath() const
   return QgsApplication::iconPath( QStringLiteral( "processingModel.svg" ) );
 }
 
+QVariantMap QgsProcessingModelAlgorithm::parametersForChildAlgorithm( const ChildAlgorithm &child, const QVariantMap &modelParameters, const QMap< QString, QVariantMap > &results ) const
+{
+  QVariantMap childParams;
+  Q_FOREACH ( const QgsProcessingParameterDefinition *def, child.algorithm()->parameterDefinitions() )
+  {
+    if ( def->flags() & QgsProcessingParameterDefinition::FlagHidden )
+      continue;
+
+    if ( !def->isDestination() )
+    {
+      if ( !child.parameterSources().contains( def->name() ) )
+        continue; // use default value
+
+      ChildParameterSource paramSource = child.parameterSources().value( def->name() );
+      switch ( paramSource.source() )
+      {
+        case ChildParameterSource::StaticValue:
+          childParams.insert( def->name(), paramSource.staticValue() );
+          break;
+
+        case ChildParameterSource::ModelParameter:
+          childParams.insert( def->name(), modelParameters.value( paramSource.parameterName() ) );
+          break;
+
+        case ChildParameterSource::ChildOutput:
+        {
+          QVariantMap linkedChildResults = results.value( paramSource.outputChildId() );
+          childParams.insert( def->name(), linkedChildResults.value( paramSource.outputName() ) );
+          break;
+        }
+      }
+    }
+    else
+    {
+      // is destination linked to one of the final outputs from this model?
+      if ( child.modelOutputs().contains( def->name() ) )
+      {
+        QString outputName = child.modelOutputs().value( def->name() ).outputName();
+        QString paramName = child.childId() + ':' + outputName;
+        if ( modelParameters.contains( paramName ) )
+          childParams.insert( def->name(), modelParameters.value( paramName ) );
+      }
+      else
+      {
+        // output is temporary
+        // TODO - skip if this output is optional and not used elsewhere in the model
+        childParams.insert( def->name(), "memory:" );
+
+
+      }
+    }
+  }
+  return childParams;
+}
+
 QVariantMap QgsProcessingModelAlgorithm::processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const
 {
   QSet< QString > toExecute;
   QMap< QString, ChildAlgorithm >::const_iterator childIt = mChildAlgorithms.constBegin();
   for ( ; childIt != mChildAlgorithms.constEnd(); ++childIt )
   {
-    if ( childIt->isActive() )
+    if ( childIt->isActive() && childIt->algorithm() )
       toExecute.insert( childIt->childId() );
   }
 
-  QMap< QString, QVariantMap > resultsMap;
+  QTime totalTime;
+  totalTime.start();
+
+  QMap< QString, QVariantMap > childResults;
+  QVariantMap finalResults;
   QSet< QString > executed;
-  while ( executed.count() < toExecute.count() )
+  bool executedAlg = true;
+  while ( executedAlg && executed.count() < toExecute.count() )
   {
+    executedAlg = false;
     Q_FOREACH ( const QString &childId, toExecute )
     {
       if ( executed.contains( childId ) )
@@ -316,47 +377,47 @@ QVariantMap QgsProcessingModelAlgorithm::processAlgorithm( const QVariantMap &pa
       if ( !canExecute )
         continue;
 
+      executedAlg = true;
       feedback->pushDebugInfo( QObject::tr( "Prepare algorithm: %1" ).arg( childId ) );
 
       const ChildAlgorithm &child = mChildAlgorithms[ childId ];
 
-      // self.prepareAlgorithm( alg )
-
+      QVariantMap childParams = parametersForChildAlgorithm( child, parameters, childResults );
       feedback->setProgressText( QObject::tr( "Running %1 [%2/%3]" ).arg( child.description() ).arg( executed.count() + 1 ).arg( toExecute.count() ) );
       //feedback->pushDebugInfo( "Parameters: " + ', '.join( [str( p ).strip() +
       //           '=' + str( p.value ) for p in alg.algorithm.parameters] ) )
-      //t0 = time.time()
-      QVariantMap results = child.algorithm()->run( parameters, context, feedback );
-      resultsMap.insert( childId, results );
 
-      //dt = time.time() - t0
-#if 0
+      QTime childTime;
+      childTime.start();
 
-# copy algorithm output value(s) back to model in case the algorithm modified those
-    for out in alg.algorithm().outputs :
-      if not out.flags() & QgsProcessingParameterDefinition.FlagHidden:
-          if out.name() in alg.modelOutputs():
-              modelOut = self.getOutputFromName( self.getSafeNameForOutput( alg.childId(), out.name() ) )
-                       if modelOut:
-                         modelOut.value = out.value
-#endif
+      QVariantMap results = child.algorithm()->run( childParams, context, feedback );
+      childResults.insert( childId, results );
 
-                                          executed.insert( childId );
-        //feedback->pushDebugInfo( QObject::tr( "OK. Execution took %1 ms (%2 outputs)." ).arg( dt, len( alg.algorithm.modelOutputs() ) ) )
+      // look through child alg's outputs to determine whether any of these should be copied
+      // to the final model outputs
+      QMap<QString, QgsProcessingModelAlgorithm::ModelOutput> outputs = child.modelOutputs();
+      QMap<QString, QgsProcessingModelAlgorithm::ModelOutput>::const_iterator outputIt = outputs.constBegin();
+      for ( ; outputIt != outputs.constEnd(); ++outputIt )
+      {
+        finalResults.insert( childId + ':' + outputIt->outputName(), results.value( outputIt->outputName() ) );
+      }
+
+      executed.insert( childId );
+      feedback->pushDebugInfo( QObject::tr( "OK. Execution took %1 s (%2 outputs)." ).arg( childTime.elapsed() / 1000.0 ).arg( results.count() ) );
 #if 0
-      except GeoAlgorithmExecutionException as e:
-        feedback.pushDebugInfo( self.tr( 'Failed', 'ModelerAlgorithm' ) )
-        raise GeoAlgorithmExecutionException(
-          self.tr( 'Error executing algorithm {0}\n{1}', 'ModelerAlgorithm' ).format( alg.description, e.msg ) )
+    except GeoAlgorithmExecutionException as e:
+      feedback.pushDebugInfo( self.tr( 'Failed', 'ModelerAlgorithm' ) )
+      raise GeoAlgorithmExecutionException(
+        self.tr( 'Error executing algorithm {0}\n{1}', 'ModelerAlgorithm' ).format( alg.description, e.msg ) )
 #endif
-      }
     }
-    feedback->pushDebugInfo( QObject::tr( "Model processed ok. Executed %1 algorithms total" ).arg( executed.count() ) );
-
-    return QVariantMap();
   }
+  feedback->pushDebugInfo( QObject::tr( "Model processed ok. Executed %1 algorithms total in %2 s." ).arg( executed.count() ).arg( totalTime.elapsed() / 1000.0 ) );
+
+  return finalResults;
+}
 
-  void QgsProcessingModelAlgorithm::setName( const QString &name )
+void QgsProcessingModelAlgorithm::setName( const QString &name )
 {
   mModelName = name;
 }
@@ -434,7 +495,9 @@ void QgsProcessingModelAlgorithm::updateDestinationParameters()
       {
         QgsProcessingOutputDefinition *output = destParam->toOutputDefinition();
         if ( output )
+        {
           addOutput( output );
+        }
       }
     }
   }

src/core/processing/qgsprocessingmodelalgorithm.h

@@ -763,6 +763,8 @@ class CORE_EXPORT QgsProcessingModelAlgorithm : public QgsProcessingAlgorithm
     void dependsOnChildAlgorithmsRecursive( const QString &childId, QSet<QString> &depends ) const;
     void dependentChildAlgorithmsRecursive( const QString &childId, QSet<QString> &depends ) const;
 
+    QVariantMap parametersForChildAlgorithm( const ChildAlgorithm &child, const QVariantMap &modelParameters, const QMap<QString, QVariantMap> &results ) const;
+
     /**
      * Saves this model to a QVariantMap, wrapped in a QVariant.
      * You can use QgsXmlUtils::writeVariant to save it to an XML document.