Skip to content
Permalink
Browse files

processing: add windows support to exportRasterLayersIntoDirectory (f…

…ixes #20146)
  • Loading branch information
jef-n committed Nov 11, 2018
1 parent 3a4a3af commit b39e5a0937e5e317b42088e8a8ccae327ae8e033
@@ -61,7 +61,6 @@
QgsProcessingParameterFile,
QgsProcessingParameterFolderDestination,
QgsProcessingOutputHtml,
QgsProcessingUtils,
QgsVectorLayer,
QgsProviderRegistry)
from qgis.utils import iface
@@ -73,7 +72,6 @@

from .Grass7Utils import Grass7Utils

#from processing.tools import dataobjects, system
from processing.tools.system import isWindows, getTempFilename

pluginPath = os.path.normpath(os.path.join(
@@ -187,7 +185,7 @@ def initAlgorithm(self, config=None):
"""
for p in self.params:
# We use createOutput argument for automatic output creation
res = self.addParameter(p, True)
self.addParameter(p, True)

def defineCharacteristicsFromFile(self):
"""
@@ -441,7 +439,7 @@ def processInputs(self, parameters, context, feedback):
QgsProcessingParameterMultipleLayers))]
for param in inputs:
paramName = param.name()
if not paramName in parameters:
if paramName not in parameters:
continue
# Handle Null parameter
if parameters[paramName] is None:
@@ -640,7 +638,6 @@ def processCommand(self, parameters, context, feedback, delOutputs=False):
if outName in parameters and parameters[outName] is not None:
# We add an output name to make sure it is unique if the session
# uses this algorithm several times.
#value = self.parameterAsOutputLayer(parameters, outName, context)
uniqueOutputName = outName + self.uniqueSuffix
command += ' {}={}'.format(outName, uniqueOutputName)

@@ -669,7 +666,7 @@ def processOutputs(self, parameters, context, feedback):

for out in self.destinationParameterDefinitions():
outName = out.name()
if not outName in parameters:
if outName not in parameters:
# skipped output
continue

@@ -785,15 +782,22 @@ def exportRasterLayersIntoDirectory(self, name, parameters, context, colorTable=

# Add a loop export from the basename
for cmd in [self.commands, self.outputCommands]:
# TODO Windows support
# TODO Format/options support
cmd.append("for r in $(g.list type=rast pattern='{}*'); do".format(basename))
cmd.append(" r.out.gdal -m{0} input=${{r}} output={1}/${{r}}.tif {2}".format(
' -t' if colorTable else '', outDir,
'--overwrite -c createopt="TFW=YES,COMPRESS=LZW"'
)
)
cmd.append("done")
if isWindows():
cmd.append("if not exist {0} mkdir {0}".format(outDir))
cmd.append("for /F %%r IN ('g.list type^=rast pattern^=\"{0}*\"') do r.out.gdal -m{1} input=%%r output={2}/%%r.tif {3}".format(
basename,
' -t' if colorTable else '',
outDir,
'--overwrite -c createopt="TFW=YES,COMPRESS=LZW"'
))
else:
cmd.append("for r in $(g.list type=rast pattern='{}*'); do".format(basename))
cmd.append(" r.out.gdal -m{0} input=${{r}} output={1}/${{r}}.tif {2}".format(
' -t' if colorTable else '', outDir,
'--overwrite -c createopt="TFW=YES,COMPRESS=LZW"'
))
cmd.append("done")

def loadVectorLayerFromParameter(self, name, parameters, context, feedback, external=False):
"""
@@ -999,7 +1003,6 @@ def setSessionProjectionFromLayer(self, layer):
def convertToHtml(self, fileName):
# Read HTML contents
lines = []
html = False
with open(fileName, 'r', encoding='utf-8') as f:
lines = f.readlines()

@@ -353,7 +353,7 @@ def executeGrass(commands, feedback, outputCommands=None):
loglines.append(Grass7Utils.tr('GRASS GIS 7 execution console output'))
grassOutDone = False
command, grassenv = Grass7Utils.prepareGrassExecution(commands)
#QgsMessageLog.logMessage('exec: {}'.format(command), 'DEBUG', Qgis.Info)
# QgsMessageLog.logMessage('exec: {}'.format(command), 'DEBUG', Qgis.Info)

# For MS-Windows, we need to hide the console window.
if isWindows():
@@ -369,6 +369,7 @@ def executeGrass(commands, feedback, outputCommands=None):
stderr=subprocess.STDOUT,
universal_newlines=True,
env=grassenv,
encoding="cp{}".format(Grass7Utils.getWindowsCodePage()) if isWindows() else None,
startupinfo=si if isWindows() else None
) as proc:
for line in iter(proc.stdout.readline, ''):

7 comments on commit b39e5a0

@EZUSoft

This comment has been minimized.

Copy link

@EZUSoft EZUSoft replied Nov 21, 2018

getWindowsCodePage() return 1252-ansi (only correct for windows itself). But you need/mean the cmd.exe-console and it speaks MSDOS cp850.
https://gis.stackexchange.com/questions/302457/qgis-3-4-error-utf-8-codec-cant-decode-byte-0xe8-in-position-50-invalid-cont/302526?noredirect=1#comment488995_302526

@jef-n

This comment has been minimized.

Copy link
Member Author

@jef-n jef-n replied Nov 22, 2018

corresponds to the chcp in line 256.

@EZUSoft

This comment has been minimized.

Copy link

@EZUSoft EZUSoft replied Nov 22, 2018

Ok, but grass74.bat contains the line
if %ERRORLEVEL% GEQ 1 pause
This Output

"Drücken Sie eine beliebige Taste ..."

is cp850 and the code failed

Traceback (most recent call last):
File "C:/INTERW~1/QGIS 3.4.0/apps/qgis/./python/plugins\processing\algs\grass7\Grass7Algorithm.py", line 411, in processAlgorithm
Grass7Utils.executeGrass(self.commands, feedback, self.outputCommands)
File "C:/INTERW~1/QGIS 3.4.0/apps/qgis/./python/plugins\processing\algs\grass7\Grass7Utils.py", line 375, in executeGrass
for line in iter(proc.stdout.readline, ''):
File "C:\INTERW~1\QGIS 3.4.0\apps\Python37\lib\encodings\cp1252.py", line 23, in decode
return codecs.charmap_decode(input,self.errors,decoding_table)[0]
UnicodeDecodeError: 'charmap' codec can't decode byte 0x81 in position 2: character maps to <undefined>

@jef-n

This comment has been minimized.

Copy link
Member Author

@jef-n jef-n replied Nov 22, 2018

Well, not really sure why the chcp was introduced in 6c81895 - guess to handle filenames with non-ascii characters in the produced batch job - but there's apparently no recoding of filename from UTF-8. After the chcp pause also produces cp1252. Just the message from pause after an error is still cp850 - everything else will be cp1252 (or whatever codepage GetACP() returns). So changing back to cp850 also doesn't seem to be a better choice. Maybe we should start grass74.bat from another batchfile and run chcp upfront (maybe even with 65001 - ie. UTF-8).

@PedroVenancio

This comment has been minimized.

Copy link
Contributor

@PedroVenancio PedroVenancio replied Dec 10, 2018

I'm also getting an error, due to this issue with portuguese regional settings:

Traceback (most recent call last):
File "C:/OSGEO4~1/apps/qgis-rel-dev/./python/plugins\processing\algs\grass7\Grass7Algorithm.py", line 413, in processAlgorithm
Grass7Utils.executeGrass(self.commands, feedback, self.outputCommands)
File "C:/OSGEO4~1/apps/qgis-rel-dev/./python/plugins\processing\algs\grass7\Grass7Utils.py", line 375, in executeGrass
for line in iter(proc.stdout.readline, ''):
File "C:\OSGEO4~1\apps\Python37\lib\codecs.py", line 322, in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'CP_UTF8' codec can't decode byte 0xc7 in position 4: No mapping for the Unicode character exists in the target code page.
@PedroVenancio

This comment has been minimized.

Copy link
Contributor

@PedroVenancio PedroVenancio replied Dec 12, 2018

@EZUSoft

Can you check which Active Code Page do you have in Windows Console?

I tried the updated QGIS 3.4.2-11 in another machine, and it worked without this issue. So I checked the Code Page and it is 850.

In the buggy machine, it was 65001. I had changed the default code page in this machine long time ago (Regional Settings -> Language for non-Unicode programs -> and marked the beta option "Use Unicode UTF-8 for worldwide language support"). Unchecking it, the problem goes away.

@somm-doe

This comment has been minimized.

Copy link

@somm-doe somm-doe replied Jan 10, 2020

In the buggy machine, it was 65001. I had changed the default code page in this machine long time ago (Regional Settings -> Language for non-Unicode programs -> and marked the beta option "Use Unicode UTF-8 for worldwide language support"). Unchecking it, the problem goes away.

I can confirm that removing the "Use Unicode UTF-8 for worldwide language support" option solves the issue.

Please sign in to comment.
You can’t perform that action at this time.