Skip to content
Permalink
Browse files

[processing] Use QgsBlockingProcess instead of subprocess to run GDAL…

… algorithms

This gives us much more flexibility when running the process, as we can
now do things like report feedback messages directly without having
to resort to potentially unsafe event loops regardless of whether the
algorithm is run in the main or a background thread.
  • Loading branch information
nyalldawson committed Jan 8, 2021
1 parent e79922b commit 50b5e563af3f2f0600bd184b1b3973b5aed2f9f8
Showing with 48 additions and 27 deletions.
  1. +48 −27 python/plugins/processing/algs/gdal/GdalUtils.py
@@ -34,6 +34,8 @@
from osgeo import ogr

from qgis.core import (Qgis,
QgsBlockingProcess,
QgsRunProcess,
QgsApplication,
QgsVectorFileWriter,
QgsProcessingFeedback,
@@ -43,9 +45,13 @@
QgsCredentials,
QgsDataSourceUri,
QgsProjUtils,
QgsCoordinateReferenceSystem)
QgsCoordinateReferenceSystem,
QgsProcessingException)

from qgis.PyQt.QtCore import QCoreApplication
from qgis.PyQt.QtCore import (
QCoreApplication,
QProcess
)

from processing.core.ProcessingConfig import ProcessingConfig
from processing.tools.system import isWindows, isMac
@@ -94,32 +100,47 @@ def runGdal(commands, feedback=None):
feedback.pushInfo(GdalUtils.tr('GDAL command:'))
feedback.pushCommandInfo(fused_command)
feedback.pushInfo(GdalUtils.tr('GDAL command output:'))
success = False
retry_count = 0
while not success:
loglines = [GdalUtils.tr('GDAL execution console output')]
try:
with subprocess.Popen(
fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as proc:
for line in proc.stdout:
feedback.pushConsoleInfo(line)
loglines.append(line)
success = True
except IOError as e:
if retry_count < 5:
retry_count += 1
else:
raise IOError(
str(e) + u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}'.format(
len(loglines), u'\n'.join(loglines[-10:])))

QgsMessageLog.logMessage('\n'.join(loglines), 'Processing', Qgis.Info)
loglines = [GdalUtils.tr('GDAL execution console output')]

def on_stdout(ba):
val = ba.data().decode('UTF-8')
on_stdout.buffer += val
if on_stdout.buffer.endswith('\n') or on_stdout.buffer.endswith('\r'):
# flush buffer
feedback.pushConsoleInfo(on_stdout.buffer.rstrip())
loglines.append(on_stdout.buffer.rstrip())
on_stdout.buffer = ''

on_stdout.buffer = ''

def on_stderr(ba):
val = ba.data().decode('UTF-8')
on_stderr.buffer += val

if on_stderr.buffer.endswith('\n') or on_stderr.buffer.endswith('\r'):
# flush buffer
feedback.reportError(on_stderr.buffer.rstrip())
loglines.append(on_stderr.buffer.rstrip())
on_stderr.buffer = ''

on_stderr.buffer = ''

command, *arguments = QgsRunProcess.splitCommand(fused_command)
proc = QgsBlockingProcess(command, arguments)
proc.setStdOutHandler(on_stdout)
proc.setStdErrHandler(on_stderr)

res = proc.run(feedback)
if feedback.isCanceled() and res != 0:
feedback.pushInfo(GdalUtils.tr('Process was canceled and did not complete'))
elif not feedback.isCanceled() and proc.exitStatus() == QProcess.CrashExit:
raise QgsProcessingException(GdalUtils.tr('Process was unexpectedly terminated'))
elif res == 0:
feedback.pushInfo(GdalUtils.tr('Process completed successfully'))
else:
feedback.reportError(GdalUtils.tr('Process returned error code {}').format(res))

return loglines

@staticmethod

0 comments on commit 50b5e56

Please sign in to comment.
You can’t perform that action at this time.