Skip to content

Commit

Permalink
This commits fixes encoding issue on windows.
Browse files Browse the repository at this point in the history 
getWindowsCodePage is taken from Grass7Utils.py

Instead of writing a cli_file at startup, provider now pass all
required env_variables directly to subprocess.popen. This has known to
cause issues when handling with windows path names. subprocess.Popen
handles it correctly depending on platform

Logging of output from otbalgorithm and updating progress bar is
slightly updated.
Algoirthm is now launched directly using otbApplicationLauncherCommandLine
`encoding` (on windows) and env arguments passed to subprocess is
logged in QgsMessageLog
  • Loading branch information
@nyalldawson
Rashad Kanavath authored and nyalldawson committed Mar 20, 2019
1 parent f85d494 commit 07d17a0
Show file tree
Hide file tree
Showing 3 changed files with 138 additions and 158 deletions.
12 changes: 3 additions & 9 deletions python/plugins/processing/algs/otb/OtbAlgorithm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,7 @@

from processing.core.parameters import getParameterFromString
from processing.algs.otb.OtbChoiceWidget import OtbParameterChoice
from processing.algs.otb import OtbUtils

from processing.algs.otb.OtbUtils import OtbUtils

class OtbAlgorithm(QgsProcessingAlgorithm):

Expand Down Expand Up @@ -200,8 +199,8 @@ def preprocessParameters(self, parameters):
return valid_params

def processAlgorithm(self, parameters, context, feedback):
otb_cli_file = OtbUtils.cliPath()
command = '"{}" {} {}'.format(otb_cli_file, self.name(), OtbUtils.appFolder())
app_launcher_path = OtbUtils.getExecutableInPath(OtbUtils.otbFolder(), 'otbApplicationLauncherCommandLine')
command = '"{}" {} {}'.format(app_launcher_path, self.name(), OtbUtils.appFolder())
outputPixelType = None
for k, v in parameters.items():
# if value is None for a parameter we don't have any businees with this key
Expand Down Expand Up @@ -264,11 +263,6 @@ def processAlgorithm(self, parameters, context, feedback):
else:
command += ' -{} "{}"'.format(out.name(), filePath)

QgsMessageLog.logMessage(self.tr('cmd={}'.format(command)), self.tr('Processing'), Qgis.Info)
if not os.path.exists(otb_cli_file) or not os.path.isfile(otb_cli_file):
import errno
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), otb_cli_file)

OtbUtils.executeOtb(command, feedback)

result = {}
Expand Down
78 changes: 6 additions & 72 deletions python/plugins/processing/algs/otb/OtbAlgorithmProvider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,10 @@
from qgis import utils

from processing.core.ProcessingConfig import ProcessingConfig, Setting
from processing.algs.otb import OtbUtils
from processing.algs.otb.OtbUtils import OtbUtils
from processing.algs.otb.OtbSettings import OtbSettings
from processing.algs.otb.OtbAlgorithm import OtbAlgorithm


def otb_exe_file(f):
if os.name == 'nt':
return f + '.exe'
else:
return f


class OtbAlgorithmProvider(QgsProcessingProvider):

def __init__(self):
Expand Down Expand Up @@ -153,63 +145,6 @@ def loadAlgorithms(self):
self.addAlgorithm(a)
self.algs = []

otb_folder = self.normalize_path(OtbUtils.otbFolder())
otb_app_path_env = os.pathsep.join(self.appDirs(OtbUtils.appFolder()))
gdal_data_dir = None
geotiff_csv_dir = None
otbcli_path = OtbUtils.cliPath()
try:
if os.name == 'nt':
app_vargs = " %*"
export_cmd = 'SET '
first_line = ':: Setup environment for OTB package. Generated by QGIS plugin'
otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine.exe')
gdal_data_dir = os.path.join(otb_folder, 'share', 'data')
geotiff_csv_dir = os.path.join(otb_folder, 'share', 'epsg_csv')
else:
app_vargs = " \"$@\""
export_cmd = 'export '
first_line = '#!/bin/sh'
otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine')
lines = None
env_profile = os.path.join(otb_folder, 'otbenv.profile')
if os.path.exists(env_profile):
with open(env_profile) as f:
lines = f.readlines()
lines = [x.strip() for x in lines]
for line in lines:
if not line or line.startswith('#'):
continue
if 'GDAL_DATA=' in line:
gdal_data_dir = line.split("GDAL_DATA=")[1]
if 'GEOTIFF_CSV='in line:
geotiff_csv_dir = line.split("GEOTIFF_CSV=")[1]
with open(otbcli_path, 'w') as otb_cli_file:
otb_cli_file.write(first_line + os.linesep)
otb_cli_file.write(export_cmd + "LC_NUMERIC=C" + os.linesep)
otb_cli_file.write(export_cmd + "GDAL_DRIVER_PATH=disable" + os.linesep)
if gdal_data_dir:
otb_cli_file.write(export_cmd + "GDAL_DATA=" + "\"" + gdal_data_dir + "\"" + os.linesep)
if geotiff_csv_dir:
otb_cli_file.write(export_cmd + "GEOTIFF_CSV=" + "\"" + geotiff_csv_dir + "\"" + os.linesep)
if OtbUtils.loggerLevel():
otb_cli_file.write(export_cmd + "OTB_LOGGER_LEVEL=" + OtbUtils.loggerLevel() + os.linesep)
max_ram_hint = OtbUtils.maxRAMHint()
if max_ram_hint and not int(max_ram_hint) == 128:
otb_cli_file.write(export_cmd + "OTB_MAX_RAM_HINT=" + max_ram_hint + os.linesep)
otb_cli_file.write(export_cmd + "OTB_APPLICATION_PATH=" + "\"" + otb_app_path_env + "\"" + os.linesep)
otb_cli_file.write("\"" + otb_app_launcher + "\"" + app_vargs + os.linesep)

if not os.name == 'nt':
os.chmod(otbcli_path, 0o744)
except BaseException as e:
import traceback
os.remove(otbcli_path)
errmsg = "Cannot write:" + otbcli_path + "\nError:\n" + traceback.format_exc()
QgsMessageLog.logMessage(self.tr(errmsg), self.tr('Processing'), Qgis.Critical)
raise e
QgsMessageLog.logMessage(self.tr("Using otbcli: '{}'.".format(otbcli_path)), self.tr('Processing'), Qgis.Info)

def canBeActivated(self):
if not self.isActive():
return False
Expand Down Expand Up @@ -256,10 +191,8 @@ def validateAppFolders(self, v):
dfile = os.path.join(descr_folder, app_name + '.txt')
isValid = True
if not os.path.exists(dfile):
cmdlist = [os.path.join(
folder, 'bin',
otb_exe_file('otbQgisDescriptor')),
app_name, app_dir, descr_folder + '/']
cmdlist = [OtbUtils.getExecutableInPath(folder, 'otbQgisDescriptor'),
app_name, app_dir, descr_folder + '/']
commands = ' '.join(cmdlist)
QgsMessageLog.logMessage(self.tr(commands), self.tr('Processing'), Qgis.Critical)
OtbUtils.executeOtb(commands, feedback=None)
Expand All @@ -283,9 +216,10 @@ def validateOtbFolder(self, v):
self.setActive(False)
raise ValueError(self.tr("'{}' does not exist. OTB provider will be disabled".format(v)))
path = self.normalize_path(v)
if not os.path.exists(os.path.join(path, 'bin', otb_exe_file('otbApplicationLauncherCommandLine'))):
app_launcher_path = OtbUtils.getExecutableInPath(path, 'otbApplicationLauncherCommandLine')
if not os.path.exists(app_launcher_path):
self.setActive(False)
raise ValueError(self.tr("Cannot find '{}'. OTB will be disabled".format(os.path.join(v, 'bin', otb_exe_file('otbApplicationLauncherCommandLine')))))
raise ValueError(self.tr("Cannot find '{}'. OTB will be disabled".format(app_launcher_path)))

def algsFile(self, d):
return os.path.join(self.descrFolder(d), 'algs.txt')
Expand Down
206 changes: 129 additions & 77 deletions python/plugins/processing/algs/otb/OtbUtils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,92 +40,144 @@
from processing.algs.otb.OtbSettings import OtbSettings


def cliPath():
cli_ext = '.bat' if os.name == 'nt' else ''
return os.path.normpath(os.path.join(QgsApplication.qgisSettingsDirPath(),
'processing', 'qgis_otb_cli' + cli_ext))
class OtbUtils:

@staticmethod
def version():
return ProcessingConfig.getSetting(OtbSettings.VERSION) or '0.0.0'

@staticmethod
def loggerLevel():
return ProcessingConfig.getSetting(OtbSettings.LOGGER_LEVEL) or 'INFO'

@staticmethod
def maxRAMHint():
return ProcessingConfig.getSetting(OtbSettings.MAX_RAM_HINT) or ''

@staticmethod
def otbFolder():
if ProcessingConfig.getSetting(OtbSettings.FOLDER):
return os.path.normpath(os.sep.join(re.split(r'\\|/', ProcessingConfig.getSetting(OtbSettings.FOLDER))))
else:
return None

@staticmethod
def appFolder():
app_folder = ProcessingConfig.getSetting(OtbSettings.APP_FOLDER)
if app_folder:
return os.pathsep.join(app_folder.split(';'))
else:
return None

@staticmethod
def srtmFolder():
return ProcessingConfig.getSetting(OtbSettings.SRTM_FOLDER) or ''

@staticmethod
def geoidFile():
return ProcessingConfig.getSetting(OtbSettings.GEOID_FILE) or ''

@staticmethod
def getExecutableInPath(path, exe):
ext = '.exe' if os.name == 'nt' else ''
return os.path.join(path, 'bin', exe + ext)

@staticmethod
def getAuxiliaryDataDirectories():
gdal_data_dir = None
gtiff_csv_dir = None
otb_folder = OtbUtils.otbFolder()
if os.name == 'nt':
gdal_data_dir = os.path.join(otb_folder, 'share', 'data')
gtiff_csv_dir = os.path.join(otb_folder, 'share', 'epsg_csv')
else:
env_profile = os.path.join(otb_folder, 'otbenv.profile')
try:
if os.path.exists(env_profile):
with open(env_profile) as f:
lines = f.readlines()
lines = [x.strip() for x in lines]
for line in lines:
if not line or line.startswith('#'):
continue
if 'GDAL_DATA=' in line:
gdal_data_dir = line.split("GDAL_DATA=")[1]
if 'GEOTIFF_CSV='in line:
gtiff_csv_dir = line.split("GEOTIFF_CSV=")[1]
except BaseException as exc:
errmsg = "Cannot find gdal and geotiff data directory." + str(exc)
QgsMessageLog.logMessage(errmsg, OtbUtils.tr('Processing'), Qgis.Info)
pass

return gdal_data_dir, gtiff_csv_dir

@staticmethod
def executeOtb(commands, feedback, addToLog=True):
otb_env = {
'LC_NUMERIC': 'C',
'GDAL_DRIVER_PATH': 'disable'
}
gdal_data_dir, gtiff_csv_dir = OtbUtils.getAuxiliaryDataDirectories()
if gdal_data_dir and os.path.exists(gdal_data_dir):
otb_env['GDAL_DATA'] = gdal_data_dir
if gtiff_csv_dir and os.path.exists(gtiff_csv_dir):
otb_env['GEOTIFF_CSV'] = gtiff_csv_dir

otb_env['OTB_LOGGER_LEVEL'] = OtbUtils.loggerLevel()
max_ram_hint = OtbUtils.maxRAMHint()
if max_ram_hint and int(max_ram_hint) > 256:
otb_env['OTB_MAX_RAM_HINT'] = max_ram_hint

kw = {}
kw['env'] = otb_env
if os.name == 'nt' and sys.version_info >= (3, 6):
kw['encoding'] = "cp{}".format(OtbUtils.getWindowsCodePage())

QgsMessageLog.logMessage("{}".format(kw), OtbUtils.tr('Processing'), Qgis.Info)
QgsMessageLog.logMessage("cmd={}".format(commands), OtbUtils.tr('Processing'), Qgis.Info)
with subprocess.Popen(
commands,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
**kw
) as proc:


def version():
return ProcessingConfig.getSetting(OtbSettings.VERSION) or '0.0.0'


def loggerLevel():
return ProcessingConfig.getSetting(OtbSettings.LOGGER_LEVEL) or 'INFO'


def maxRAMHint():
return ProcessingConfig.getSetting(OtbSettings.MAX_RAM_HINT) or ''


def otbFolder():
if ProcessingConfig.getSetting(OtbSettings.FOLDER):
return os.path.normpath(os.sep.join(re.split(r'\\|/', ProcessingConfig.getSetting(OtbSettings.FOLDER))))
else:
return None


def appFolder():
app_folder = ProcessingConfig.getSetting(OtbSettings.APP_FOLDER)
if app_folder:
return os.pathsep.join(app_folder.split(';'))
else:
return None


def srtmFolder():
return ProcessingConfig.getSetting(OtbSettings.SRTM_FOLDER) or ''


def geoidFile():
return ProcessingConfig.getSetting(OtbSettings.GEOID_FILE) or ''


def executeOtb(command, feedback, addToLog=True):
loglines = []
with subprocess.Popen(
[command],
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True
) as proc:
try:
for line in iter(proc.stdout.readline, ''):
line = line.strip()
#'* ]' and ' ]' says its some progress update
#print('line[-3:]',line[-3:])
if line[-3:] == "* ]" or line[-3:] == " ]":
if '% [' in line:
part = line.split(':')[1]
percent = part.split('%')[0]
try:
if int(percent) >= 100:
loglines.append(line)
feedback.pushConsoleInfo(line)
feedback.setProgress(int(percent))
except:
pass
else:
loglines.append(line)
except BaseException as e:
loglines.append(str(e))
pass

for logline in loglines:
if feedback is None:
QgsMessageLog.logMessage(logline, 'Processing', Qgis.Info)
else:
feedback.pushConsoleInfo(logline)

# for logline in loglines:
# if 'INFO' in logline or 'FATAL' in logline:
# if feedback is None:
# QgsMessageLog.logMessage(logline, 'Processing', Qgis.Info)
# else:
# feedback.pushConsoleInfo(logline)


def tr(string, context=''):
if context == '':
context = 'OtbUtils'
return QCoreApplication.translate(context, string)

if feedback is None:
QgsMessageLog.logMessage(line, OtbUtils.tr('Processing'), Qgis.Info)
else:
if any([l in line for l in ['(WARNING)', '(FATAL)', 'ERROR']]):
feedback.reportError(line)
else:
feedback.pushConsoleInfo(line.strip())

@staticmethod
def getWindowsCodePage():
"""
Determines MS-Windows CMD.exe shell codepage.
Used into GRASS exec script under MS-Windows.
"""
from ctypes import cdll
return str(cdll.kernel32.GetACP())

@staticmethod
def tr(string, context=''):
if context == '':
context = 'OtbUtils'
return QCoreApplication.translate(context, string)

0 comments on commit 07d17a0

Please sign in to comment.