Skip to content

Commit 8cb001d

Browse files
volayavolaya
committed
Improvementes to SEXTANTE batch processing interface
Better progress handling in SEXTANTE script algorithms
1 parent 6173fa3 commit 8cb001d

File tree

10 files changed

+107
-52
lines changed

10 files changed

+107
-52
lines changed

‎python/plugins/sextante/algs/SextanteAlgorithmProvider.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,13 @@
1616
* *
1717
***************************************************************************
1818
"""
19-
from sextante.algs.EquivalentNumField import EquivalentNumField
20-
2119
__author__ = 'Victor Olaya'
2220
__date__ = 'August 2012'
2321
__copyright__ = '(C) 2012, Victor Olaya'
2422
# This will get replaced with a git SHA1 when you do a git archive
2523
__revision__ = '$Format:%H$'
2624

25+
from sextante.algs.EquivalentNumField import EquivalentNumField
2726
from sextante.core.AlgorithmProvider import AlgorithmProvider
2827
from sextante.algs.AddTableField import AddTableField
2928
from PyQt4 import QtGui

‎python/plugins/sextante/core/Sextante.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ def runAlgorithm(algOrName, onFinish, *args):
258258
return
259259
if len(args) != alg.getVisibleParametersCount() + alg.getVisibleOutputsCount():
260260
print ("Error: Wrong number of parameters")
261-
Sextante.alghelp(algOrName)
261+
alghelp(algOrName)
262262
return
263263

264264
alg = alg.getCopy()#copy.deepcopy(alg)
@@ -294,7 +294,14 @@ def runAlgorithm(algOrName, onFinish, *args):
294294

295295
SextanteLog.addToLog(SextanteLog.LOG_ALGORITHM, alg.getAsCommand())
296296

297-
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
297+
# don't set the wait cursor twice, because then when you restore it
298+
# it will still be a wait cursor
299+
cursor = QApplication.overrideCursor()
300+
if cursor == None or cursor == 0:
301+
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
302+
elif cursor.shape() != Qt.WaitCursor:
303+
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
304+
298305
if SextanteConfig.getSetting(SextanteConfig.USE_THREADS):
299306
algEx = AlgorithmExecutor(alg)
300307
progress = QProgressDialog()

‎python/plugins/sextante/core/SextanteUtils.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,8 @@
2626
import os
2727
import time
2828
import sys
29-
29+
import uuid
3030
from PyQt4.QtCore import *
31-
3231
from qgis.core import *
3332

3433
class SextanteUtils:
@@ -64,9 +63,9 @@ def setTempOutput(out, alg):
6463
ext = out.getDefaultFileExtension(alg)
6564
validChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
6665
safeCmdName = ''.join(c for c in alg.commandLineName() if c in validChars)
67-
filename = SextanteUtils.tempFolder() + os.sep + safeCmdName + str(SextanteUtils.NUM_EXPORTED) + "." + ext
68-
out.value = filename
69-
SextanteUtils.NUM_EXPORTED += 1
66+
uniqueSufix = str(uuid.uuid4()).replace("-","");
67+
filename = SextanteUtils.tempFolder() + os.sep + safeCmdName + uniqueSufix + "." + ext
68+
out.value = filename
7069

7170
@staticmethod
7271
def getTempFilename(ext):

‎python/plugins/sextante/grass/GrassUtils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ def startGrassSession():
325325
# End session by removing the temporary GRASS mapset and all the layers.
326326
@staticmethod
327327
def endGrassSession():
328-
#shutil.rmtree(GrassUtils.grassMapsetFolder(), True)
328+
shutil.rmtree(GrassUtils.grassMapsetFolder(), True)
329329
GrassUtils.sessionRunning = False
330330
GrassUtils.sessionLayers = {}
331331

‎python/plugins/sextante/grass/description/r.mapcalculator.txt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ r.mapcalculator
22
r.mapcalculator - Calculate new raster map from a r.mapcalc expression.
33
Raster (r.*)
44
ParameterRaster|amap|amap|False
5-
ParameterRaster|bmap|bmap|False
6-
ParameterRaster|cmap|cmap|False
7-
ParameterRaster|dmap|dmap|False
8-
ParameterRaster|emap|emap|False
9-
ParameterRaster|fmap|fmap|False
10-
ParameterString|formula|Formula (e.g. A-B or A*C+B|
5+
ParameterRaster|bmap|bmap|True
6+
ParameterRaster|cmap|cmap|True
7+
ParameterRaster|dmap|dmap|True
8+
ParameterRaster|emap|emap|True
9+
ParameterRaster|fmap|fmap|True
10+
ParameterString|formula|Formula (e.g. A-B or A*C+B)|
1111
OutputRaster|outfile|Name for output raster map

‎python/plugins/sextante/grass/description/r.series.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ Raster (r.*)
44
ParameterMultipleInput|input|Name of input raster map(s)|3.0|False
55
ParameterBoolean|-n|Propagate NULLs|True
66
ParameterSelection|method|Aggregate operation|average;count;median;mode;minimum;min_raster;maximum;max_raster;stddev;range;sum;threshold;variance;diversity;slope;offset;detcoeff;quart1;quart3;perc90;quantile;skewness;kurtosis
7-
ParameterNumber|quantile|Quantile to calculate for method=quantile|0.0|1.0|0.0
8-
ParameterNumber|threshold|Threshold to calculate for method=threshold|None|None|0.0
9-
ParameterString|range|Ignore values outside this range (lo,hi)|-10000000000,10000000000
7+
*ParameterNumber|quantile|Quantile to calculate for method=quantile|0.0|1.0|0.0
8+
*ParameterNumber|threshold|Threshold to calculate for method=threshold|None|None|0.0
9+
*ParameterString|range|Ignore values outside this range (lo,hi)|-10000000000,10000000000
1010
OutputRaster|output|Name for output raster map

‎python/plugins/sextante/grass/description/r.univar.txt‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ r.univar
22
r.univar - Calculates univariate statistics from the non-null cells of a raster map.
33
Raster (r.*)
44
ParameterMultipleInput|map|Name of input raster map(s)|3.0|False
5-
ParameterRaster|zones|Raster map used for zoning, must be of type CELL|False
6-
ParameterNumber|percentile|Percentile to calculate (requires extended statistics flag)|0.0|100.0|90
7-
ParameterBoolean|-e|Calculate extended statistics|False
5+
ParameterRaster|zones|Raster map used for zoning, must be of type CELL|True
6+
*ParameterBoolean|-e|Calculate extended statistics|False
7+
*ParameterBoolean|-g|Print the stats in shell script style|False
8+
*ParameterBoolean|-t|Table output format instead of standard output format|False
9+
*ParameterNumber|percentile|Percentile to calculate (requires extended statistics flag)|0.0|100.0|90
10+
OutputFile|output|Name for output text file

‎python/plugins/sextante/gui/BatchProcessingDialog.py‎

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
***************************************************************************
1818
"""
1919
from sextante.gui.CrsSelectionPanel import CrsSelectionPanel
20+
from sextante.outputs.OutputNumber import OutputNumber
21+
from sextante.outputs.OutputString import OutputString
22+
from sextante.core.SextanteUtils import SextanteUtils
2023

2124
__author__ = 'Victor Olaya'
2225
__date__ = 'August 2012'
@@ -79,12 +82,14 @@ def __init__(self, alg):
7982
self.deleteRowButton.setText("Delete row")
8083
self.buttonBox.addButton(self.addRowButton, QtGui.QDialogButtonBox.ActionRole)
8184
self.buttonBox.addButton(self.deleteRowButton, QtGui.QDialogButtonBox.ActionRole)
82-
self.table.setColumnCount(len(self.alg.parameters) + len(self.alg.outputs))
85+
86+
self.table.setColumnCount(self.alg.getVisibleParametersCount() + self.alg.getVisibleOutputsCount())
8387
self.setTableContent()
8488
self.table.horizontalHeader().setStretchLastSection(True)
8589
self.table.verticalHeader().setVisible(False)
8690
self.table.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
87-
self.progress = QtGui.QProgressBar()
91+
#self.progress = QtGui.QProgressBar()
92+
#self.progress.setMaximum(100)
8893
self.addRowButton.clicked.connect(self.addRow)
8994
self.deleteRowButton.clicked.connect(self.deleteRow)
9095

@@ -110,13 +115,17 @@ def accept(self):
110115
alg = self.alg.getCopy()#copy.deepcopy(self.alg)
111116
col = 0
112117
for param in alg.parameters:
118+
if param.hidden:
119+
continue
113120
widget = self.table.cellWidget(row, col)
114121
if not self.setParameterValueFromWidget(param, widget, alg):
115122
QMessageBox.critical(self.dialog, "Unable to execute batch process", "Wrong or missing parameter values")
116123
self.algs = None
117124
return
118125
col+=1
119126
for out in alg.outputs:
127+
if out.hidden:
128+
continue
120129
widget = self.table.cellWidget(row, col)
121130
text = widget.getValue()
122131
if text.strip() != "":
@@ -129,18 +138,19 @@ def accept(self):
129138
self.algs.append(alg)
130139

131140
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
132-
self.progress.setMaximum(len(self.algs))
141+
#self.progress.setMaximum(len(self.algs))
133142
self.table.setEnabled(False)
134143
if SextanteConfig.getSetting(SextanteConfig.USE_THREADS):
135-
self.progress.setValue(0)
144+
#self.progress.setValue(0)
136145
self.nextAlg(0)
137146
else:
138147
i=1
139148
self.progress.setMaximum(len(self.algs))
140149
for alg in self.algs:
141-
if UnthreadedAlgorithmExecutor.runalg(alg, SilentProgress()):
142-
self.progress.setValue(i)
143-
self.loadHTMLResults(alg, i)
150+
self.setBaseText("Processing algorithm " + str(i) + "/" + str(len(self.algs)) + "...")
151+
if UnthreadedAlgorithmExecutor.runalg(alg, self):#SilentProgress()):
152+
#self.progress.setValue(i)
153+
#self.loadHTMLResults(alg, i)
144154
i+=1
145155
else:
146156
QApplication.restoreOverrideCursor()
@@ -164,31 +174,53 @@ def cancel(self):
164174
@pyqtSlot()
165175
def finish(self, i):
166176
i += 1
167-
self.progress.setValue(i)
177+
self.progress.setValue(i)
168178
if len(self.algs) == i:
169179
self.finishAll()
170180
self.algEx = None
171181
else:
172182
self.nextAlg(i)
173-
183+
174184
@pyqtSlot()
175185
def error(self, msg):
176186
QApplication.restoreOverrideCursor()
177187
QMessageBox.critical(self, "Error", msg)
178188
SextanteLog.addToLog(SextanteLog.LOG_ERROR, msg)
179189
self.close()
180-
190+
191+
181192
def nextAlg(self, i):
193+
self.setBaseText("Processing algorithm " + str(i) + "/" + str(len(self.algs)) + "...")
182194
self.algEx = AlgorithmExecutor(self.algs[i]);
195+
self.algEx.percentageChanged.connect(self.setPercentage)
196+
self.algEx.textChanged.connect(self.setText)
183197
self.algEx.error.connect(self.error)
184198
self.algEx.finished.connect(lambda: self.finish(i))
185199
self.algEx.start()
186200

201+
def createSummaryTable(self):
202+
createTable = False
203+
for out in self.algs[0].outputs:
204+
if isinstance(out, (OutputNumber,OutputString)):
205+
createTable = True
206+
break
207+
if not createTable:
208+
return
209+
outputFile = SextanteUtils.getTempFilename("html")
210+
f = open(outputFile, "w")
211+
for alg in self.algs:
212+
for out in alg.outputs:
213+
if isinstance(out, (OutputNumber,OutputString)):
214+
f.write("<p>" + out.description + ": " + str(out.value) + "</p>\n")
215+
f.close()
216+
SextanteResults.addResult(self.algs[0].name + "[summary]", outputFile)
217+
187218
def finishAll(self):
188219
i = 0
189220
for alg in self.algs:
190221
self.loadHTMLResults(alg, i)
191222
i = i + 1
223+
self.createSummaryTable()
192224
QApplication.restoreOverrideCursor()
193225
self.table.setEnabled(True)
194226
QMessageBox.information(self, "Batch processing", "Batch processing successfully completed!")
@@ -266,4 +298,10 @@ def showAdvancedParametersClicked(self):
266298
for param in self.alg.parameters:
267299
if param.isAdvanced:
268300
self.table.setColumnHidden(i, not self.showAdvanced)
269-
i+=1
301+
i+=1
302+
303+
def setText(self, text):
304+
self.progressLabel.setText(self.baseText + " --- [" + text + "]")
305+
306+
def setBaseText(self, text):
307+
self.baseText = text

‎python/plugins/sextante/gui/FileSelectionPanel.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,5 +56,5 @@ def showSelectionDialog(self):
5656
def getValue(self):
5757
s = str(self.text.text())
5858
if SextanteUtils.isWindows():
59-
s = s.replace("/", "\\")
59+
s = s.replace("\\", "/")
6060
return s

‎python/plugins/sextante/script/ScriptAlgorithm.py‎

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@
4545
from sextante.parameters.ParameterExtent import ParameterExtent
4646
from sextante.parameters.ParameterFile import ParameterFile
4747
from sextante.outputs.OutputFile import OutputFile
48+
from sextante.parameters.ParameterFactory import ParameterFactory
49+
from sextante.outputs.OutputFactory import OutputFactory
4850
import sys
4951
from sextante.gui.Help2Html import Help2Html
5052

@@ -87,7 +89,6 @@ def defineCharacteristicsFromFile(self):
8789
line = lines.readline()
8890
lines.close()
8991

90-
9192
def defineCharacteristicsFromScript(self):
9293
lines = self.script.split("\n")
9394
self.silentOutputs = []
@@ -107,11 +108,19 @@ def processParameterLine(self,line):
107108
param = None
108109
out = None
109110
line = line.replace("#", "");
111+
# If the line is in the format of the text description files for normal algorithms,
112+
# then process it using parameter and output factories
113+
if '|' in line:
114+
self.processDescriptionParameterLine(line)
115+
return
110116
tokens = line.split("=");
111117
desc = self.createDescriptiveName(tokens[0])
112118
if tokens[1].lower().strip() == "group":
113119
self.group = tokens[0]
114120
return
121+
if tokens[1].lower().strip() == "name":
122+
self.name = tokens[0]
123+
return
115124
if tokens[1].lower().strip() == "raster":
116125
param = ParameterRaster(tokens[0], desc, False)
117126
elif tokens[1].lower().strip() == "vector":
@@ -174,13 +183,27 @@ def processParameterLine(self,line):
174183
self.addOutput(out)
175184
else:
176185
raise WrongScriptException("Could not load script:" + self.descriptionFile + ".\n Problem with line \"" + line + "\"")
186+
187+
def processDescriptionParameterLine(self, line):
188+
try:
189+
if line.startswith("Parameter"):
190+
self.addParameter(ParameterFactory.getFromString(line))
191+
elif line.startswith("*Parameter"):
192+
param = ParameterFactory.getFromString(line[1:])
193+
param.isAdvanced = True
194+
self.addParameter(param)
195+
else:
196+
self.addOutput(OutputFactory.getFromString(line))
197+
except Exception:
198+
raise WrongScriptException("Could not load script:" + self.descriptionFile + ".\n Problem with line \"" + line + "\"")
177199

178200
def processAlgorithm(self, progress):
179201

180202
script = "import sextante\n"
181203

182204
ns = {}
183-
205+
ns['progress'] = progress
206+
184207
for param in self.parameters:
185208
#script += param.name + "=" + param.getValueAsCommandLineParameter() + "\n"
186209
ns[param.name] = param.value
@@ -189,11 +212,8 @@ def processAlgorithm(self, progress):
189212
ns[out.name] = out.value
190213
#script += out.name + "=" + out.getValueAsCommandLineParameter() + "\n"
191214

192-
script+=self.script
193-
redirection = Redirection(progress)
194-
sys.stdout = redirection
195-
exec(script) in ns
196-
sys.stdout = sys.__stdout__
215+
script+=self.script
216+
exec(script) in ns
197217
for out in self.outputs:
198218
out.setValue(ns[out.name])
199219

@@ -205,14 +225,3 @@ def helpFile(self):
205225
else:
206226
return None
207227

208-
class Redirection():
209-
def __init__(self, progress):
210-
self.progress = progress
211-
212-
def write(self, string):
213-
try:
214-
n = int(string)
215-
self.progress.setPercentage(n)
216-
except:
217-
self.progress.setText(string)
218-
pass

0 commit comments

Comments
 (0)