Skip to content
Permalink
Browse files

Merge pull request #8968 from volaya/fix_saga_non_ascii

fixed handling of input filenames with non-ascii characters [processing]
  • Loading branch information
luipir committed Feb 21, 2019
2 parents 5292efd + 52546db commit 5a96fab7755b23d6171477c8a5748f9598204381
@@ -27,6 +27,7 @@
__revision__ = '$Format:%H$'

import os
import shutil
import importlib
from qgis.core import (Qgis,
QgsApplication,
@@ -305,14 +306,22 @@ def processAlgorithm(self, parameters, context, feedback):

output_layers = []
output_files = {}
#If the user has entered an output file that has non-ascii chars, we use a different path with only ascii chars
output_files_nonascii = {}
for out in self.destinationParameterDefinitions():
filePath = self.parameterAsOutputLayer(parameters, out.name(), context)
if isinstance(out, (QgsProcessingParameterRasterDestination, QgsProcessingParameterVectorDestination)):
output_layers.append(filePath)
try:
filePath.encode('ascii')
except UnicodeEncodeError:
nonAsciiFilePath = filePath
filePath = QgsProcessingUtils.generateTempFilename(out.name() + os.path.splitext(filePath)[1])
output_files_nonascii[filePath] = nonAsciiFilePath

output_files[out.name()] = filePath
command += ' -{} "{}"'.format(out.name(), filePath)

commands.append(command)
commands.append(command)

# special treatment for RGB algorithm
# TODO: improve this and put this code somewhere else
@@ -341,6 +350,17 @@ def processAlgorithm(self, parameters, context, feedback):
with open(prjFile, 'w') as f:
f.write(crs.toWkt())

for old, new in output_files_nonascii.items():
oldFolder = os.path.dirname(old)
newFolder = os.path.dirname(new)
newName = os.path.splitext(os.path.basename(new))[0]
files = [f for f in os.listdir(oldFolder)]
for f in files:
ext = os.path.splitext(f)[1]
newPath = os.path.join(newFolder, newName + ext)
oldPath = os.path.join(oldFolder, f)
shutil.move(oldPath, newPath)

result = {}
for o in self.outputDefinitions():
if o.name() in output_files:
@@ -421,8 +441,8 @@ def checkParameterValues(self, parameters, context):

if isinstance(param, QgsProcessingParameterRasterLayer):
raster_layer_params.append(param.name())
elif (isinstance(param, QgsProcessingParameterMultipleLayers) and
param.layerType() == QgsProcessing.TypeRaster):
elif (isinstance(param, QgsProcessingParameterMultipleLayers)
and param.layerType() == QgsProcessing.TypeRaster):
raster_layer_params.extend(param.name())

for layer_param in raster_layer_params:
@@ -97,7 +97,7 @@ def sagaDescriptionPath():

def createSagaBatchJobFileFromSagaCommands(commands):

with open(sagaBatchJobFilename(), 'w') as fout:
with open(sagaBatchJobFilename(), 'w', encoding="utf8") as fout:
if isWindows():
fout.write('set SAGA=' + sagaPath() + '\n')
fout.write('set SAGA_MLB=' + os.path.join(sagaPath(), 'modules') + '\n')
@@ -108,12 +108,7 @@ def createSagaBatchJobFileFromSagaCommands(commands):
else:
pass
for command in commands:
try:
# Python 2
fout.write('saga_cmd ' + command.encode('utf8') + '\n')
except TypeError:
# Python 3
fout.write('saga_cmd ' + command + '\n')
fout.write('saga_cmd ' + command + '\n')

fout.write('exit')

@@ -25,11 +25,22 @@

__revision__ = ':%H$'

import os
import nose2
import shutil
import tempfile

from qgis.core import (QgsProcessingParameterNumber,
QgsProcessingParameterDefinition)
QgsProcessingParameterDefinition,
QgsVectorLayer,
QgsApplication,
QgsFeature,
QgsGeometry,
QgsPointXY,
QgsProcessingContext,
QgsProject,
QgsProcessingFeedback,
QgsProcessingFeatureSourceDefinition)
from qgis.testing import start_app, unittest

from processing.algs.saga.SagaParameters import Parameters, SagaImageOutputParam
@@ -45,6 +56,9 @@ def setUpClass(cls):
Processing.initialize()
cls.cleanup_paths = []

cls.temp_dir = tempfile.mkdtemp()
cls.cleanup_paths.append(cls.temp_dir)

@classmethod
def tearDownClass(cls):
from processing.core.Processing import Processing
@@ -82,6 +96,48 @@ def test_param_line(self):
self.assertEqual(param.defaultFileExtension(), 'tif')
self.assertEqual(param.supportedOutputRasterLayerExtensions(), ['tif'])

def test_non_ascii_output(self):
# create a memory layer and add to project and context
layer = QgsVectorLayer("Point?crs=epsg:3857&field=fldtxt:string&field=fldint:integer",
"testmem", "memory")
self.assertTrue(layer.isValid())
pr = layer.dataProvider()
f = QgsFeature()
f.setAttributes(["test", 123])
f.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(100, 200)))
f2 = QgsFeature()
f2.setAttributes(["test2", 457])
f2.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(110, 200)))
self.assertTrue(pr.addFeatures([f, f2]))
self.assertEqual(layer.featureCount(), 2)
QgsProject.instance().addMapLayer(layer)
context = QgsProcessingContext()
context.setProject(QgsProject.instance())

alg = QgsApplication.processingRegistry().createAlgorithmById('saga:fixeddistancebuffer')
self.assertIsNotNone(alg)

temp_file = os.path.join(self.temp_dir, 'non_ascii_ñññ.shp')
parameters = {'SHAPES': 'testmem',
'DIST_FIELD_DEFAULT': 5,
'NZONES': 1,
'DARC': 5,
'DISSOLVE': False,
'POLY_INNER': False,
'BUFFER': temp_file}
feedback = QgsProcessingFeedback()

results, ok = alg.run(parameters, context, feedback)
self.assertTrue(ok)
self.assertTrue(os.path.exists(temp_file))

# make sure that layer has correct features
res = QgsVectorLayer(temp_file, 'res')
self.assertTrue(res.isValid())
self.assertEqual(res.featureCount(), 2)

QgsProject.instance().removeMapLayer(layer)


if __name__ == '__main__':
nose2.main()
@@ -0,0 +1,58 @@
<?xml version="1.0" encoding="utf-8" ?>
<ogr:FeatureCollection
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation=""
xmlns:ogr="http://ogr.maptools.org/"
xmlns:gml="http://www.opengis.net/gml">
<gml:boundedBy>
<gml:Box>
<gml:coord><gml:X>-1</gml:X><gml:Y>-3</gml:Y></gml:coord>
<gml:coord><gml:X>10</gml:X><gml:Y>6</gml:Y></gml:coord>
</gml:Box>
</gml:boundedBy>

<gml:featureMember>
<ogr:polys2 fid="polys.0">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>-1,-1 -1,3 3,3 3,2 2,2 2,-1 -1,-1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>aaaaa</ogr:name>
<ogr:intval>33</ogr:intval>
<ogr:floatval>44.123456</ogr:floatval>
</ogr:polys2>
</gml:featureMember>
<gml:featureMember>
<ogr:polys2 fid="polys.1">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>5,5 6,4 4,4 5,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>Aaaaa</ogr:name>
<ogr:intval>-33</ogr:intval>
<ogr:floatval>0</ogr:floatval>
</ogr:polys2>
</gml:featureMember>
<gml:featureMember>
<ogr:polys2 fid="polys.2">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>2,5 2,6 3,6 3,5 2,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>bbaaa</ogr:name>
<ogr:floatval>0.123</ogr:floatval>
</ogr:polys2>
</gml:featureMember>
<gml:featureMember>
<ogr:polys2 fid="polys.3">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>6,1 10,1 10,-3 6,-3 6,1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs><gml:innerBoundaryIs><gml:LinearRing><gml:coordinates>7,0 7,-2 9,-2 9,0 7,0</gml:coordinates></gml:LinearRing></gml:innerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>ASDF</ogr:name>
<ogr:intval>0</ogr:intval>
</ogr:polys2>
</gml:featureMember>
<gml:featureMember>
<ogr:polys2 fid="polys.4">
<ogr:intval>120</ogr:intval>
<ogr:floatval>-100291.43213</ogr:floatval>
</ogr:polys2>
</gml:featureMember>
<gml:featureMember>
<ogr:polys2 fid="polys.5">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>3,2 6,1 6,-3 2,-1 2,2 3,2</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>elim</ogr:name>
<ogr:intval>2</ogr:intval>
<ogr:floatval>3.33</ogr:floatval>
</ogr:polys2>
</gml:featureMember>
</ogr:FeatureCollection>
@@ -16,6 +16,20 @@ tests:
geometry:
precision: 7

- name: Centroid with non-ascii input
algorithm: native:centroids
params:
INPUT:
type: vector
name: polys_non_ascii_ñññ.gml
results:
OUTPUT:
type: vector
name: expected/polys_centroid.gml
compare:
geometry:
precision: 7

- name: Aggregate all
algorithm: qgis:aggregate
params:

0 comments on commit 5a96fab

Please sign in to comment.
You can’t perform that action at this time.