Skip to content
Permalink
Browse files

Improvementes to SEXTANTE batch processing interface

Better progress handling in SEXTANTE script algorithms
  • Loading branch information
volaya committed Nov 19, 2012
1 parent 6173fa3 commit 8cb001db3a27666ddf6d8035e19f3d21c8e1161b
@@ -16,14 +16,13 @@
* *
***************************************************************************
"""
from sextante.algs.EquivalentNumField import EquivalentNumField

__author__ = 'Victor Olaya'
__date__ = 'August 2012'
__copyright__ = '(C) 2012, Victor Olaya'
# This will get replaced with a git SHA1 when you do a git archive
__revision__ = '$Format:%H$'

from sextante.algs.EquivalentNumField import EquivalentNumField
from sextante.core.AlgorithmProvider import AlgorithmProvider
from sextante.algs.AddTableField import AddTableField
from PyQt4 import QtGui
@@ -258,7 +258,7 @@ def runAlgorithm(algOrName, onFinish, *args):
return
if len(args) != alg.getVisibleParametersCount() + alg.getVisibleOutputsCount():
print ("Error: Wrong number of parameters")
Sextante.alghelp(algOrName)
alghelp(algOrName)
return

alg = alg.getCopy()#copy.deepcopy(alg)
@@ -294,7 +294,14 @@ def runAlgorithm(algOrName, onFinish, *args):

SextanteLog.addToLog(SextanteLog.LOG_ALGORITHM, alg.getAsCommand())

QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
# don't set the wait cursor twice, because then when you restore it
# it will still be a wait cursor
cursor = QApplication.overrideCursor()
if cursor == None or cursor == 0:
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
elif cursor.shape() != Qt.WaitCursor:
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))

if SextanteConfig.getSetting(SextanteConfig.USE_THREADS):
algEx = AlgorithmExecutor(alg)
progress = QProgressDialog()
@@ -26,9 +26,8 @@
import os
import time
import sys

import uuid
from PyQt4.QtCore import *

from qgis.core import *

class SextanteUtils:
@@ -64,9 +63,9 @@ def setTempOutput(out, alg):
ext = out.getDefaultFileExtension(alg)
validChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
safeCmdName = ''.join(c for c in alg.commandLineName() if c in validChars)
filename = SextanteUtils.tempFolder() + os.sep + safeCmdName + str(SextanteUtils.NUM_EXPORTED) + "." + ext
out.value = filename
SextanteUtils.NUM_EXPORTED += 1
uniqueSufix = str(uuid.uuid4()).replace("-","");
filename = SextanteUtils.tempFolder() + os.sep + safeCmdName + uniqueSufix + "." + ext
out.value = filename

@staticmethod
def getTempFilename(ext):
@@ -325,7 +325,7 @@ def startGrassSession():
# End session by removing the temporary GRASS mapset and all the layers.
@staticmethod
def endGrassSession():
#shutil.rmtree(GrassUtils.grassMapsetFolder(), True)
shutil.rmtree(GrassUtils.grassMapsetFolder(), True)
GrassUtils.sessionRunning = False
GrassUtils.sessionLayers = {}

@@ -2,10 +2,10 @@ r.mapcalculator
r.mapcalculator - Calculate new raster map from a r.mapcalc expression.
Raster (r.*)
ParameterRaster|amap|amap|False
ParameterRaster|bmap|bmap|False
ParameterRaster|cmap|cmap|False
ParameterRaster|dmap|dmap|False
ParameterRaster|emap|emap|False
ParameterRaster|fmap|fmap|False
ParameterString|formula|Formula (e.g. A-B or A*C+B|
ParameterRaster|bmap|bmap|True
ParameterRaster|cmap|cmap|True
ParameterRaster|dmap|dmap|True
ParameterRaster|emap|emap|True
ParameterRaster|fmap|fmap|True
ParameterString|formula|Formula (e.g. A-B or A*C+B)|
OutputRaster|outfile|Name for output raster map
@@ -4,7 +4,7 @@ Raster (r.*)
ParameterMultipleInput|input|Name of input raster map(s)|3.0|False
ParameterBoolean|-n|Propagate NULLs|True
ParameterSelection|method|Aggregate operation|average;count;median;mode;minimum;min_raster;maximum;max_raster;stddev;range;sum;threshold;variance;diversity;slope;offset;detcoeff;quart1;quart3;perc90;quantile;skewness;kurtosis
ParameterNumber|quantile|Quantile to calculate for method=quantile|0.0|1.0|0.0
ParameterNumber|threshold|Threshold to calculate for method=threshold|None|None|0.0
ParameterString|range|Ignore values outside this range (lo,hi)|-10000000000,10000000000
*ParameterNumber|quantile|Quantile to calculate for method=quantile|0.0|1.0|0.0
*ParameterNumber|threshold|Threshold to calculate for method=threshold|None|None|0.0
*ParameterString|range|Ignore values outside this range (lo,hi)|-10000000000,10000000000
OutputRaster|output|Name for output raster map
@@ -2,6 +2,9 @@ r.univar
r.univar - Calculates univariate statistics from the non-null cells of a raster map.
Raster (r.*)
ParameterMultipleInput|map|Name of input raster map(s)|3.0|False
ParameterRaster|zones|Raster map used for zoning, must be of type CELL|False
ParameterNumber|percentile|Percentile to calculate (requires extended statistics flag)|0.0|100.0|90
ParameterBoolean|-e|Calculate extended statistics|False
ParameterRaster|zones|Raster map used for zoning, must be of type CELL|True
*ParameterBoolean|-e|Calculate extended statistics|False
*ParameterBoolean|-g|Print the stats in shell script style|False
*ParameterBoolean|-t|Table output format instead of standard output format|False
*ParameterNumber|percentile|Percentile to calculate (requires extended statistics flag)|0.0|100.0|90
OutputFile|output|Name for output text file
@@ -17,6 +17,9 @@
***************************************************************************
"""
from sextante.gui.CrsSelectionPanel import CrsSelectionPanel
from sextante.outputs.OutputNumber import OutputNumber
from sextante.outputs.OutputString import OutputString
from sextante.core.SextanteUtils import SextanteUtils

__author__ = 'Victor Olaya'
__date__ = 'August 2012'
@@ -79,12 +82,14 @@ def __init__(self, alg):
self.deleteRowButton.setText("Delete row")
self.buttonBox.addButton(self.addRowButton, QtGui.QDialogButtonBox.ActionRole)
self.buttonBox.addButton(self.deleteRowButton, QtGui.QDialogButtonBox.ActionRole)
self.table.setColumnCount(len(self.alg.parameters) + len(self.alg.outputs))

self.table.setColumnCount(self.alg.getVisibleParametersCount() + self.alg.getVisibleOutputsCount())
self.setTableContent()
self.table.horizontalHeader().setStretchLastSection(True)
self.table.verticalHeader().setVisible(False)
self.table.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
self.progress = QtGui.QProgressBar()
#self.progress = QtGui.QProgressBar()
#self.progress.setMaximum(100)
self.addRowButton.clicked.connect(self.addRow)
self.deleteRowButton.clicked.connect(self.deleteRow)

@@ -110,13 +115,17 @@ def accept(self):
alg = self.alg.getCopy()#copy.deepcopy(self.alg)
col = 0
for param in alg.parameters:
if param.hidden:
continue
widget = self.table.cellWidget(row, col)
if not self.setParameterValueFromWidget(param, widget, alg):
QMessageBox.critical(self.dialog, "Unable to execute batch process", "Wrong or missing parameter values")
self.algs = None
return
col+=1
for out in alg.outputs:
if out.hidden:
continue
widget = self.table.cellWidget(row, col)
text = widget.getValue()
if text.strip() != "":
@@ -129,18 +138,19 @@ def accept(self):
self.algs.append(alg)

QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
self.progress.setMaximum(len(self.algs))
#self.progress.setMaximum(len(self.algs))
self.table.setEnabled(False)
if SextanteConfig.getSetting(SextanteConfig.USE_THREADS):
self.progress.setValue(0)
#self.progress.setValue(0)
self.nextAlg(0)
else:
i=1
self.progress.setMaximum(len(self.algs))
for alg in self.algs:
if UnthreadedAlgorithmExecutor.runalg(alg, SilentProgress()):
self.progress.setValue(i)
self.loadHTMLResults(alg, i)
self.setBaseText("Processing algorithm " + str(i) + "/" + str(len(self.algs)) + "...")
if UnthreadedAlgorithmExecutor.runalg(alg, self):#SilentProgress()):
#self.progress.setValue(i)
#self.loadHTMLResults(alg, i)
i+=1
else:
QApplication.restoreOverrideCursor()
@@ -164,31 +174,53 @@ def cancel(self):
@pyqtSlot()
def finish(self, i):
i += 1
self.progress.setValue(i)
self.progress.setValue(i)
if len(self.algs) == i:
self.finishAll()
self.algEx = None
else:
self.nextAlg(i)

@pyqtSlot()
def error(self, msg):
QApplication.restoreOverrideCursor()
QMessageBox.critical(self, "Error", msg)
SextanteLog.addToLog(SextanteLog.LOG_ERROR, msg)
self.close()



def nextAlg(self, i):
self.setBaseText("Processing algorithm " + str(i) + "/" + str(len(self.algs)) + "...")
self.algEx = AlgorithmExecutor(self.algs[i]);
self.algEx.percentageChanged.connect(self.setPercentage)
self.algEx.textChanged.connect(self.setText)
self.algEx.error.connect(self.error)
self.algEx.finished.connect(lambda: self.finish(i))
self.algEx.start()

def createSummaryTable(self):
createTable = False
for out in self.algs[0].outputs:
if isinstance(out, (OutputNumber,OutputString)):
createTable = True
break
if not createTable:
return
outputFile = SextanteUtils.getTempFilename("html")
f = open(outputFile, "w")
for alg in self.algs:
for out in alg.outputs:
if isinstance(out, (OutputNumber,OutputString)):
f.write("<p>" + out.description + ": " + str(out.value) + "</p>\n")
f.close()
SextanteResults.addResult(self.algs[0].name + "[summary]", outputFile)

def finishAll(self):
i = 0
for alg in self.algs:
self.loadHTMLResults(alg, i)
i = i + 1
self.createSummaryTable()
QApplication.restoreOverrideCursor()
self.table.setEnabled(True)
QMessageBox.information(self, "Batch processing", "Batch processing successfully completed!")
@@ -266,4 +298,10 @@ def showAdvancedParametersClicked(self):
for param in self.alg.parameters:
if param.isAdvanced:
self.table.setColumnHidden(i, not self.showAdvanced)
i+=1
i+=1

def setText(self, text):
self.progressLabel.setText(self.baseText + " --- [" + text + "]")

def setBaseText(self, text):
self.baseText = text
@@ -56,5 +56,5 @@ def showSelectionDialog(self):
def getValue(self):
s = str(self.text.text())
if SextanteUtils.isWindows():
s = s.replace("/", "\\")
s = s.replace("\\", "/")
return s
@@ -45,6 +45,8 @@
from sextante.parameters.ParameterExtent import ParameterExtent
from sextante.parameters.ParameterFile import ParameterFile
from sextante.outputs.OutputFile import OutputFile
from sextante.parameters.ParameterFactory import ParameterFactory
from sextante.outputs.OutputFactory import OutputFactory
import sys
from sextante.gui.Help2Html import Help2Html

@@ -87,7 +89,6 @@ def defineCharacteristicsFromFile(self):
line = lines.readline()
lines.close()


def defineCharacteristicsFromScript(self):
lines = self.script.split("\n")
self.silentOutputs = []
@@ -107,11 +108,19 @@ def processParameterLine(self,line):
param = None
out = None
line = line.replace("#", "");
# If the line is in the format of the text description files for normal algorithms,
# then process it using parameter and output factories
if '|' in line:
self.processDescriptionParameterLine(line)
return
tokens = line.split("=");
desc = self.createDescriptiveName(tokens[0])
if tokens[1].lower().strip() == "group":
self.group = tokens[0]
return
if tokens[1].lower().strip() == "name":
self.name = tokens[0]
return
if tokens[1].lower().strip() == "raster":
param = ParameterRaster(tokens[0], desc, False)
elif tokens[1].lower().strip() == "vector":
@@ -174,13 +183,27 @@ def processParameterLine(self,line):
self.addOutput(out)
else:
raise WrongScriptException("Could not load script:" + self.descriptionFile + ".\n Problem with line \"" + line + "\"")

def processDescriptionParameterLine(self, line):
try:
if line.startswith("Parameter"):
self.addParameter(ParameterFactory.getFromString(line))
elif line.startswith("*Parameter"):
param = ParameterFactory.getFromString(line[1:])
param.isAdvanced = True
self.addParameter(param)
else:
self.addOutput(OutputFactory.getFromString(line))
except Exception:
raise WrongScriptException("Could not load script:" + self.descriptionFile + ".\n Problem with line \"" + line + "\"")

def processAlgorithm(self, progress):

script = "import sextante\n"

ns = {}

ns['progress'] = progress

for param in self.parameters:
#script += param.name + "=" + param.getValueAsCommandLineParameter() + "\n"
ns[param.name] = param.value
@@ -189,11 +212,8 @@ def processAlgorithm(self, progress):
ns[out.name] = out.value
#script += out.name + "=" + out.getValueAsCommandLineParameter() + "\n"

script+=self.script
redirection = Redirection(progress)
sys.stdout = redirection
exec(script) in ns
sys.stdout = sys.__stdout__
script+=self.script
exec(script) in ns
for out in self.outputs:
out.setValue(ns[out.name])

@@ -205,14 +225,3 @@ def helpFile(self):
else:
return None

class Redirection():
def __init__(self, progress):
self.progress = progress

def write(self, string):
try:
n = int(string)
self.progress.setPercentage(n)
except:
self.progress.setText(string)
pass

0 comments on commit 8cb001d

Please sign in to comment.
You can’t perform that action at this time.