Skip to content

Commit a887b7d

Browse files
volayanyalldawson
volaya
authored and
nyalldawson
committed
[processing] Fixed SAGA handling of non-ascii output files
1 parent 46f0e5d commit a887b7d

File tree

5 files changed

+155
-12
lines changed

5 files changed

+155
-12
lines changed

‎python/plugins/processing/algs/saga/SagaAlgorithm.py

+24-4
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
__revision__ = '$Format:%H$'
2828

2929
import os
30+
import shutil
3031
import importlib
3132
from qgis.core import (Qgis,
3233
QgsApplication,
@@ -305,14 +306,22 @@ def processAlgorithm(self, parameters, context, feedback):
305306

306307
output_layers = []
307308
output_files = {}
309+
#If the user has entered an output file that has non-ascii chars, we use a different path with only ascii chars
310+
output_files_nonascii = {}
308311
for out in self.destinationParameterDefinitions():
309312
filePath = self.parameterAsOutputLayer(parameters, out.name(), context)
310313
if isinstance(out, (QgsProcessingParameterRasterDestination, QgsProcessingParameterVectorDestination)):
311314
output_layers.append(filePath)
315+
try:
316+
filePath.encode('ascii')
317+
except UnicodeEncodeError:
318+
nonAsciiFilePath = filePath
319+
filePath = QgsProcessingUtils.generateTempFilename(out.name() + os.path.splitext(filePath)[1])
320+
output_files_nonascii[filePath] = nonAsciiFilePath
321+
312322
output_files[out.name()] = filePath
313323
command += ' -{} "{}"'.format(out.name(), filePath)
314-
315-
commands.append(command)
324+
commands.append(command)
316325

317326
# special treatment for RGB algorithm
318327
# TODO: improve this and put this code somewhere else
@@ -341,6 +350,17 @@ def processAlgorithm(self, parameters, context, feedback):
341350
with open(prjFile, 'w') as f:
342351
f.write(crs.toWkt())
343352

353+
for old, new in output_files_nonascii.items():
354+
oldFolder = os.path.dirname(old)
355+
newFolder = os.path.dirname(new)
356+
newName = os.path.splitext(os.path.basename(new))[0]
357+
files = [f for f in os.listdir(oldFolder)]
358+
for f in files:
359+
ext = os.path.splitext(f)[1]
360+
newPath = os.path.join(newFolder, newName + ext)
361+
oldPath = os.path.join(oldFolder, f)
362+
shutil.move(oldPath, newPath)
363+
344364
result = {}
345365
for o in self.outputDefinitions():
346366
if o.name() in output_files:
@@ -421,8 +441,8 @@ def checkParameterValues(self, parameters, context):
421441

422442
if isinstance(param, QgsProcessingParameterRasterLayer):
423443
raster_layer_params.append(param.name())
424-
elif (isinstance(param, QgsProcessingParameterMultipleLayers) and
425-
param.layerType() == QgsProcessing.TypeRaster):
444+
elif (isinstance(param, QgsProcessingParameterMultipleLayers)
445+
and param.layerType() == QgsProcessing.TypeRaster):
426446
raster_layer_params.extend(param.name())
427447

428448
for layer_param in raster_layer_params:

‎python/plugins/processing/algs/saga/SagaUtils.py

+2-7
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ def sagaDescriptionPath():
9797

9898
def createSagaBatchJobFileFromSagaCommands(commands):
9999

100-
with open(sagaBatchJobFilename(), 'w') as fout:
100+
with open(sagaBatchJobFilename(), 'w', encoding="utf8") as fout:
101101
if isWindows():
102102
fout.write('set SAGA=' + sagaPath() + '\n')
103103
fout.write('set SAGA_MLB=' + os.path.join(sagaPath(), 'modules') + '\n')
@@ -108,12 +108,7 @@ def createSagaBatchJobFileFromSagaCommands(commands):
108108
else:
109109
pass
110110
for command in commands:
111-
try:
112-
# Python 2
113-
fout.write('saga_cmd ' + command.encode('utf8') + '\n')
114-
except TypeError:
115-
# Python 3
116-
fout.write('saga_cmd ' + command + '\n')
111+
fout.write('saga_cmd ' + command + '\n')
117112

118113
fout.write('exit')
119114

‎python/plugins/processing/tests/SagaAlgorithmsTest.py

+57-1
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,22 @@
2525

2626
__revision__ = ':%H$'
2727

28+
import os
2829
import nose2
2930
import shutil
31+
import tempfile
3032

3133
from qgis.core import (QgsProcessingParameterNumber,
32-
QgsProcessingParameterDefinition)
34+
QgsProcessingParameterDefinition,
35+
QgsVectorLayer,
36+
QgsApplication,
37+
QgsFeature,
38+
QgsGeometry,
39+
QgsPointXY,
40+
QgsProcessingContext,
41+
QgsProject,
42+
QgsProcessingFeedback,
43+
QgsProcessingFeatureSourceDefinition)
3344
from qgis.testing import start_app, unittest
3445

3546
from processing.algs.saga.SagaParameters import Parameters, SagaImageOutputParam
@@ -45,6 +56,9 @@ def setUpClass(cls):
4556
Processing.initialize()
4657
cls.cleanup_paths = []
4758

59+
cls.temp_dir = tempfile.mkdtemp()
60+
cls.cleanup_paths.append(cls.temp_dir)
61+
4862
@classmethod
4963
def tearDownClass(cls):
5064
from processing.core.Processing import Processing
@@ -82,6 +96,48 @@ def test_param_line(self):
8296
self.assertEqual(param.defaultFileExtension(), 'tif')
8397
self.assertEqual(param.supportedOutputRasterLayerExtensions(), ['tif'])
8498

99+
def test_non_ascii_output(self):
100+
# create a memory layer and add to project and context
101+
layer = QgsVectorLayer("Point?crs=epsg:3857&field=fldtxt:string&field=fldint:integer",
102+
"testmem", "memory")
103+
self.assertTrue(layer.isValid())
104+
pr = layer.dataProvider()
105+
f = QgsFeature()
106+
f.setAttributes(["test", 123])
107+
f.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(100, 200)))
108+
f2 = QgsFeature()
109+
f2.setAttributes(["test2", 457])
110+
f2.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(110, 200)))
111+
self.assertTrue(pr.addFeatures([f, f2]))
112+
self.assertEqual(layer.featureCount(), 2)
113+
QgsProject.instance().addMapLayer(layer)
114+
context = QgsProcessingContext()
115+
context.setProject(QgsProject.instance())
116+
117+
alg = QgsApplication.processingRegistry().createAlgorithmById('saga:fixeddistancebuffer')
118+
self.assertIsNotNone(alg)
119+
120+
temp_file = os.path.join(self.temp_dir, 'non_ascii_ñññ.shp')
121+
parameters = {'SHAPES': 'testmem',
122+
'DIST_FIELD_DEFAULT': 5,
123+
'NZONES': 1,
124+
'DARC': 5,
125+
'DISSOLVE': False,
126+
'POLY_INNER': False,
127+
'BUFFER': temp_file}
128+
feedback = QgsProcessingFeedback()
129+
130+
results, ok = alg.run(parameters, context, feedback)
131+
self.assertTrue(ok)
132+
self.assertTrue(os.path.exists(temp_file))
133+
134+
# make sure that layer has correct features
135+
res = QgsVectorLayer(temp_file, 'res')
136+
self.assertTrue(res.isValid())
137+
self.assertEqual(res.featureCount(), 2)
138+
139+
QgsProject.instance().removeMapLayer(layer)
140+
85141

86142
if __name__ == '__main__':
87143
nose2.main()

‎python/plugins/processing/tests/testdata/polys_non_ascii_ñññ.gml

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<ogr:FeatureCollection
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation=""
5+
xmlns:ogr="http://ogr.maptools.org/"
6+
xmlns:gml="http://www.opengis.net/gml">
7+
<gml:boundedBy>
8+
<gml:Box>
9+
<gml:coord><gml:X>-1</gml:X><gml:Y>-3</gml:Y></gml:coord>
10+
<gml:coord><gml:X>10</gml:X><gml:Y>6</gml:Y></gml:coord>
11+
</gml:Box>
12+
</gml:boundedBy>
13+
14+
<gml:featureMember>
15+
<ogr:polys2 fid="polys.0">
16+
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>-1,-1 -1,3 3,3 3,2 2,2 2,-1 -1,-1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
17+
<ogr:name>aaaaa</ogr:name>
18+
<ogr:intval>33</ogr:intval>
19+
<ogr:floatval>44.123456</ogr:floatval>
20+
</ogr:polys2>
21+
</gml:featureMember>
22+
<gml:featureMember>
23+
<ogr:polys2 fid="polys.1">
24+
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>5,5 6,4 4,4 5,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
25+
<ogr:name>Aaaaa</ogr:name>
26+
<ogr:intval>-33</ogr:intval>
27+
<ogr:floatval>0</ogr:floatval>
28+
</ogr:polys2>
29+
</gml:featureMember>
30+
<gml:featureMember>
31+
<ogr:polys2 fid="polys.2">
32+
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>2,5 2,6 3,6 3,5 2,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
33+
<ogr:name>bbaaa</ogr:name>
34+
<ogr:floatval>0.123</ogr:floatval>
35+
</ogr:polys2>
36+
</gml:featureMember>
37+
<gml:featureMember>
38+
<ogr:polys2 fid="polys.3">
39+
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>6,1 10,1 10,-3 6,-3 6,1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs><gml:innerBoundaryIs><gml:LinearRing><gml:coordinates>7,0 7,-2 9,-2 9,0 7,0</gml:coordinates></gml:LinearRing></gml:innerBoundaryIs></gml:Polygon></ogr:geometryProperty>
40+
<ogr:name>ASDF</ogr:name>
41+
<ogr:intval>0</ogr:intval>
42+
</ogr:polys2>
43+
</gml:featureMember>
44+
<gml:featureMember>
45+
<ogr:polys2 fid="polys.4">
46+
<ogr:intval>120</ogr:intval>
47+
<ogr:floatval>-100291.43213</ogr:floatval>
48+
</ogr:polys2>
49+
</gml:featureMember>
50+
<gml:featureMember>
51+
<ogr:polys2 fid="polys.5">
52+
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>3,2 6,1 6,-3 2,-1 2,2 3,2</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
53+
<ogr:name>elim</ogr:name>
54+
<ogr:intval>2</ogr:intval>
55+
<ogr:floatval>3.33</ogr:floatval>
56+
</ogr:polys2>
57+
</gml:featureMember>
58+
</ogr:FeatureCollection>

‎python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,20 @@ tests:
1616
geometry:
1717
precision: 7
1818

19+
- name: Centroid with non-ascii input
20+
algorithm: native:centroids
21+
params:
22+
INPUT:
23+
type: vector
24+
name: polys_non_ascii_ñññ.gml
25+
results:
26+
OUTPUT:
27+
type: vector
28+
name: expected/polys_centroid.gml
29+
compare:
30+
geometry:
31+
precision: 7
32+
1933
- name: Aggregate all
2034
algorithm: qgis:aggregate
2135
params:

0 commit comments

Comments
 (0)