Support directory output via QgsProcessingParameterFolderDestination

Author: Médéric RIBREUX

python/plugins/processing/algs/grass7/Grass7Algorithm.py

@@ -57,6 +57,8 @@
                        QgsProcessingParameterVectorDestination,
                        QgsProcessingParameterRasterDestination,
                        QgsProcessingParameterFileDestination,
+                       QgsProcessingParameterFolderDestination,
+                       QgsProcessingOutputFolder,
                        QgsProcessingOutputVectorLayer,
                        QgsProcessingOutputRasterLayer,
                        QgsProcessingOutputHtml,
@@ -265,7 +267,7 @@ def getDefaultCellSize(self):
         layers = [l for l in self.inputLayers if isinstance(l, QgsRasterLayer)]
 
         # Use this function to calculate cell size
-        def cz(l, cellsize): return max(cellsize, (l.extent().xMaximum() - l.extent().xMinimum()) / l.width())
+        cz = lambda l, cellsize: max(cellsize, (l.extent().xMaximum() - l.extent().xMinimum()) / l.width())
 
         for layer in layers:
             cellsize = cz(layer, cellsize)
@@ -533,6 +535,11 @@ def processCommand(self, parameters, context, delOutputs=False):
                             outName,
                             self.parameterAsFileOutput(
                                 parameters, outName, context))
+                # For folders destination
+                if isinstance(out, QgsProcessingParameterFolderDestination):
+                    # We need to add a unique temporary basename
+                    uniqueBasename = outName + self.uniqueSuffix
+                    command += ' {}={}'.format(outName, uniqueBasename)
                 else:
                     # We add an output name to make sure it is unique if the session
                     # uses this algorithm several times.
@@ -572,6 +579,8 @@ def processOutputs(self, parameters, context):
                 self.exportRasterLayerFromParameter(outName, parameters, context)
             elif isinstance(out, QgsProcessingParameterVectorDestination):
                 self.exportVectorLayerFromParameter(outName, parameters, context)
+            elif isinstance(out, QgsProcessingParameterFolderDestination):
+                self.exportRasterLayersIntoDirectory(outName, parameters, context)
 
         QgsMessageLog.logMessage('processOutputs. Commands: {}'.format(self.commands), 'Grass7', QgsMessageLog.INFO)
 
@@ -605,8 +614,9 @@ def exportRasterLayerFromParameter(self, name, parameters, context, colorTable=T
         """
         Creates a dedicated command to export a raster from
         temporary GRASS DB into a file via gdal.
-        :param grassName: name of the parameter
-        :param fileName: file path of raster layer
+        :param name: name of the parameter.
+        :param parameters: Algorithm parameters list.
+        :param context: Algorithm context.
         :param colorTable: preserve color Table.
         """
         fileName = self.parameterAsOutputLayer(parameters, name, context)
@@ -632,6 +642,30 @@ def exportRasterLayer(self, grassName, fileName, colorTable=True):
                 )
             )
 
+    def exportRasterLayersIntoDirectory(self, name, parameters, context, colorTable=True):
+        """
+        Creates a dedicated loop command to export rasters from
+        temporary GRASS DB into a directory gdal.
+        :param name: name of the parameter
+        :param fileName: file path of raster layer
+        :param colorTable: preserve color Table.
+        """
+        # Grab directory name and temporary basename
+        outDir = self.parameterAsString(parameters, name, context)
+        basename = name + self.uniqueSuffix
+
+        # Add a loop export from the basename
+        for cmd in [self.commands, self.outputCommands]:
+            # Adjust region to layer before exporting
+            # TODO: Does-it works under MS-Windows or MacOSX?
+            cmd.append("for r in $(g.list type=rast pattern='{}*'); do".format(basename))
+            cmd.append("  r.out.gdal{0} input=${{r}} output={1}/${{r}}.tif {2}".format(
+                ' -t' if colorTable else '', outDir,
+                '--overwrite -c createopt="TFW=YES,COMPRESS=LZW"'
+            )
+            )
+            cmd.append("done")
+
     def loadVectorLayerFromParameter(self, name, parameters, context, external=None):
         """
         """

python/plugins/processing/algs/grass7/TODO.md

@@ -3,22 +3,19 @@ TODO List for GRASS7 algorithms support into QGIS Processing
 QGIS3 Processing Port
 =====================
 
-* Port to Python3.
-  * print -> print(
-  * unicode -> str
+* Things to do elsewhere
+  * TODO We need Null QgsProcessingParameterNumber!
+  * TODO We need NULL QgsProcessingParameterPoint!
+  * TODO We need a multiple QgsProcessingParameterList!
+  * TODO We need a QgsParameterMultipleInputLayers parameter for minimum and maximum number of layers.
+  * TODO Open all the files in a QgsProcessingOutputFolder at the end of the algorithm.
+* TODO Use GRASS --exec instead of GRASS_BATCH_JOB.
+* TODO Improve Grass Path and Binary detection for all OSs.
+* TODO Review all the methods of QgsProcessingAlgorithm.
+* TODO Make tests under MS-Windows 7 for Utf-8 support.
+* TODO Review Python3 port.
   * dict iteritems
-* TODO Replace all parameters by QgsProcessingParameters.
-  * DONE Review all ParameterFile
-  * TODO We need Null QgsParameterNumber!
-  * TODO We need NULL QgsParameterPoint!
-  * TODO We need a QgsParameterList!
-  * TODO Review all OutputDirectory.
-  * DONE Review all OutputFile
-    * DONE Replace by QgsProcessingParameterFileDestination
-    * DONE QgsProcessingParameterFileDestination should use the file filter in Dialog.
-      Replace fileOut with fileDestination in gui/ParametersUtils.py
-* DONE Re-enable GRASS algorithm by default.
-* Add GRASS 7.2 new algorithms.
+* TODO Add GRASS 7.2 new algorithms.
 * TODO Review all algorithm parameters.
 MOD r.basins.fill
 OK r.blend
@@ -184,18 +181,12 @@ r.watershed
 r.what.color
 r.what
 
-* Improve unit tests.
-* Use some raster/vector layers with spacename into their path.
-* DONE Better support for files output that are HTML.
-  * DONE All html output files will be report outputs.
-  * DONE All html output will come as stdout files by default.
-  * DONE OutputHtml must not be converted to OutputLayerDefinition.
-  * DONE Convert false HTML files to real HTML files.
-  * DONE Opens HTML files in Viewer.
+* TODO Improve unit tests.
+* TODO Use some raster/vector layers with spacename into their path.
 * TODO Use prepareAlgorithm for algorithm preparation.
 * TODO Support ParameterTable.
-* DONE Remove specific algorithms code in Grass7Algorithm.py (move them in ext).
 * TODO Convert all ext scripts.
+  * TODO Review i.py.
   * TODO Force projection in description file?
   * r_rgb.py
   * r_blend_combine.py
@@ -232,17 +223,48 @@ r.what
   * v_net_steiner.py
   * v_net_visibility.py
 
-* TODO Support OutputFolder.
 * TODO Support multiple output raster formats.
 * TODO Support multiple output vector formats.
 * TODO Support multiple input vector formats
   * DONE create a general inputVectorLayer method.
   * TODO Some formats can't be correctly used by v.external:
     * GML.
   * TODO Build a workaround for those formats (use v.in.ogr).
+* DONE Replace all parameters by QgsProcessingParameters.
+  * DONE Review all ParameterFile
+  * DONE Review all OutputDirectory.
+    * DONE Convert all OutputDirectory to QgsProcessingParameterFolderDestination
+    * DONE Default case:
+      * Take the name of the output variable.
+      * create a default value as basename.
+      * export all layers into the directory with a shell loop.
+    * DONE Remove all multipleOutputDir in ext/
+    * r.colors: TODO ext | DONE desc | TODO tests.
+    * r.texture: DONE ext | DONE desc | TODO tests.
+    * r.stats.quantile: DONE ext | DONE desc | TODO tests.
+    * r.series.interp: DONE ext | DONE desc | TODO tests.
+    * r.mapcalc: DONE ext | DONE desc | TODO tests.
+    * i.aster.toar: DONE ext | DONE desc | TODO tests.
+    * i.tasscap: DONE ext | DONE desc | TODO tests.
+    * i.rectify: DONE ext | DONE desc | TODO tests.
+    * i.cca: DONE ext | DONE desc | TODO tests.
+    * i.landsat.toar: DONE ext | DONE desc | TODO tests.
+    * i.pca: DONE ext | DONE desc | TODO tests.
+    * i.topo.corr: DONE ext | DONE desc | TODO tests.
+  * DONE Review all OutputFile
+    * DONE Replace by QgsProcessingParameterFileDestination
+    * DONE QgsProcessingParameterFileDestination should use the file filter in Dialog.
+      Replace fileOut with fileDestination in gui/ParametersUtils.py
+* DONE Remove specific algorithms code in Grass7Algorithm.py (move them in ext).
+* DONE Re-enable GRASS algorithm by default.
 * DONE Support multiple bands input rasters.
-* Review all the methods of QgsProcessingAlgorithm.
-* Make tests under MS-Windows 7 for Utf-8 support.
+* DONE Better support for files output that are HTML.
+  * DONE All html output files will be report outputs.
+  * DONE All html output will come as stdout files by default.
+  * DONE OutputHtml must not be converted to OutputLayerDefinition.
+  * DONE Convert false HTML files to real HTML files.
+  * DONE Opens HTML files in Viewer.
+
 
 Unit tests
 ==========

python/plugins/processing/algs/grass7/description/i.aster.toar.txt

@@ -10,4 +10,4 @@ QgsProcessingParameterBoolean|-b|SWIR is High Gain|False
 QgsProcessingParameterBoolean|-c|VNIR is Low Gain 1|False
 QgsProcessingParameterBoolean|-d|SWIR is Low Gain 1|False
 QgsProcessingParameterBoolean|-e|SWIR is Low Gain 2|False
-OutputDirectory|output|Output Directory
+QgsProcessingParameterFolderDestination|output|Output Directory

python/plugins/processing/algs/grass7/description/i.atcorr.txt

@@ -12,5 +12,3 @@ QgsProcessingParameterRasterDestination|output|Atmospheric correction
 *QgsProcessingParameterBoolean|-r|Input raster map converted to reflectance (default is radiance)|False
 *QgsProcessingParameterBoolean|-a|Input from ETM+ image taken after July 1, 2000|False
 *QgsProcessingParameterBoolean|-b|Input from ETM+ image taken before July 1, 2000|False
-
-

python/plugins/processing/algs/grass7/description/i.cca.txt

@@ -3,4 +3,4 @@ Canonical components analysis (CCA) program for image processing.
 Imagery (i.*)
 QgsProcessingParameterMultipleLayers|input|Input rasters (2 to 8)|3|None|False
 QgsProcessingParameterFile|signature|File containing spectral signatures|0|txt|None|False
-OutputDirectory|output|Output Directory
+QgsProcessingParameterFolderDestination|output|Output Directory

python/plugins/processing/algs/grass7/description/i.colors.enhance.txt

@@ -12,4 +12,3 @@ QgsProcessingParameterNumber|strength|Cropping intensity (upper brightness level
 QgsProcessingParameterRasterDestination|redoutput|Enhanced Red
 QgsProcessingParameterRasterDestination|greenoutput|Enhanced Green
 QgsProcessingParameterRasterDestination|blueoutput|Enhanced Blue
-

python/plugins/processing/algs/grass7/description/i.gensig.txt

@@ -4,4 +4,3 @@ Imagery (i.*)
 QgsProcessingParameterRasterLayer|trainingmap|Ground truth training map|None|False
 QgsProcessingParameterMultipleLayers|input|Input rasters|3|None|False
 QgsProcessingParameterFileDestination|signaturefile|Signature File|Txt files (*.txt)|None|False
-

python/plugins/processing/algs/grass7/description/i.gensigset.txt

@@ -5,4 +5,3 @@ QgsProcessingParameterRasterLayer|trainingmap|Ground truth training map|None|Fal
 QgsProcessingParameterMultipleLayers|input|Input rasters|3|None|False
 QgsProcessingParameterNumber|maxsig|Maximum number of sub-signatures in any class|QgsProcessingParameterNumber.Integer|5|True|0|None
 QgsProcessingParameterFileDestination|signaturefile|Signature File|Txt files (*.txt)|None|False
-

python/plugins/processing/algs/grass7/description/i.group.txt

@@ -3,4 +3,3 @@ Regroup multiple mono-band rasters into a single multiband raster.
 Imagery (i.*)
 QgsProcessingParameterMultipleLayers|input|Input rasters|3|None|False
 QgsProcessingParameterRasterDestination|group|Multiband raster
-

python/plugins/processing/algs/grass7/description/i.ifft.txt

@@ -4,4 +4,3 @@ Imagery (i.*)
 QgsProcessingParameterRasterLayer|real|Name of input raster map (image fft, real part)|None|False
 QgsProcessingParameterRasterLayer|imaginary|Name of input raster map (image fft, imaginary part)|None|False
 QgsProcessingParameterRasterDestination|output|Inverse Fast Fourier Transform
-

python/plugins/processing/algs/grass7/description/i.image.mosaic.txt

@@ -3,4 +3,3 @@ Mosaics several images and extends colormap.
 Imagery (i.*)
 QgsProcessingParameterMultipleLayers|input|Input rasters|3|None|False
 QgsProcessingParameterRasterDestination|output|Mosaic Raster
-

python/plugins/processing/algs/grass7/description/i.in.spotvgt.txt

@@ -4,4 +4,3 @@ Imagery (i.*)
 QgsProcessingParameterRasterLayer|input|Name of input SPOT VGT NDVI HDF file|None|False
 *QgsProcessingParameterBoolean|-a|Also import quality map (SM status map layer) and filter NDVI map|False
 QgsProcessingParameterRasterDestination|output|SPOT NDVI Raster
-

python/plugins/processing/algs/grass7/description/i.landsat.acca.txt

@@ -11,4 +11,3 @@ QgsProcessingParameterNumber|histogram|Number of classes in the cloud temperatur
 *QgsProcessingParameterBoolean|-2|Bypass second-pass processing, and merge warm (not ambiguous) and cold clouds|False
 *QgsProcessingParameterBoolean|-s|Include a category for cloud shadows|False
 QgsProcessingParameterRasterDestination|output|ACCA Raster
-

python/plugins/processing/algs/grass7/description/i.landsat.toar.txt

@@ -15,5 +15,4 @@ QgsProcessingParameterNumber|rayleigh|Rayleigh atmosphere (diffuse sky irradianc
 QgsProcessingParameterNumber|scale|Scale factor for output|QgsProcessingParameterNumber.Double|1.0|True|0.0|None
 *QgsProcessingParameterBoolean|-r|Output at-sensor radiance instead of reflectance for all bands|False
 *QgsProcessingParameterBoolean|-n|Input raster maps use as extension the number of the band instead the code|False
-OutputDirectory|output|Output Directory
-
+QgsProcessingParameterFolderDestination|output|Output Directory

python/plugins/processing/algs/grass7/description/i.modis.qc.txt

@@ -6,4 +6,3 @@ QgsProcessingParameterEnum|productname|Name of MODIS product type|mod09Q1;mod09A
 QgsProcessingParameterEnum|qcname|Name of QC type to extract|adjcorr;atcorr;cloud;data_quality;diff_orbit_from_500m;modland_qa;mandatory_qa_11A1;data_quality_flag_11A1;emis_error_11A1;lst_error_11A1;data_quality_flag_11A2;emis_error_11A2;mandatory_qa_11A2;lst_error_11A2;aerosol_quantity;brdf_correction_performed;cirrus_detected;cloud_shadow;cloud_state;internal_cloud_algorithm;internal_fire_algorithm;internal_snow_mask;land_water;mod35_snow_ice;pixel_adjacent_to_cloud;icm_cloudy;icm_clear;icm_high_clouds;icm_low_clouds;icm_snow;icm_fire;icm_sun_glint;icm_dust;icm_cloud_shadow;icm_pixel_is_adjacent_to_cloud;icm_cirrus;icm_pan_flag;icm_criteria_for_aerosol_retrieval;icm_aot_has_clim_val;modland_qa;vi_usefulness;aerosol_quantity;pixel_adjacent_to_cloud;brdf_correction_performed;mixed_clouds;land_water;possible_snow_ice;possible_shadow;platform;land_water;sun_z_angle_at_local_noon;brdf_correction_performed|False|5
 QgsProcessingParameterString|band|Band number of MODIS product (mod09Q1=[1,2],mod09A1=[1-7],m[o/y]d09CMG=[1-7], mcd43B2q=[1-7])|None|False|True
 QgsProcessingParameterRasterDestination|output|QC Classification
-

python/plugins/processing/algs/grass7/description/i.oif.txt

@@ -5,4 +5,3 @@ QgsProcessingParameterMultipleLayers|input|Name of input raster map(s)|3|None|Fa
 *QgsProcessingParameterBoolean|-g|Print in shell script style|False
 *QgsProcessingParameterBoolean|-s|Process bands serially (default: run in parallel)|False
 QgsProcessingParameterFileDestination|output|OIF File|Txt files (*.txt)|None|False
-

python/plugins/processing/algs/grass7/description/i.pansharpen.txt

@@ -11,4 +11,3 @@ QgsProcessingParameterEnum|method|Method|brovey;ihs;pca|False|1
 QgsProcessingParameterRasterDestination|redoutput|Enhanced Red
 QgsProcessingParameterRasterDestination|greenoutput|Enhanced Green
 QgsProcessingParameterRasterDestination|blueoutput|Enhanced Blue
-

python/plugins/processing/algs/grass7/description/i.pca.txt

@@ -6,5 +6,4 @@ QgsProcessingParameterRange|rescale|Rescaling range for output maps. For no resc
 QgsProcessingParameterNumber|percent|Cumulative percent importance for filtering|QgsProcessingParameterNumber.Integer|99|True|50|99
 *QgsProcessingParameterBoolean|-n|Normalize (center and scale) input maps|False
 *QgsProcessingParameterBoolean|-f|Output will be filtered input bands|False
-OutputDirectory|output|Output Directory
-
+QgsProcessingParameterFolderDestination|output|Output Directory

python/plugins/processing/algs/grass7/description/i.rectify.txt

@@ -10,5 +10,4 @@ QgsProcessingParameterEnum|method|Interpolation method to use|nearest;linear;cub
 QgsProcessingParameterCrs|crs|Destination CRS|None|False
 Hardcoded|extension=rectified
 *QgsProcessingParameterBoolean|-t|Use thin plate spline|False
-OutputDirectory|output|Output Directory
-
+QgsProcessingParameterFolderDestination|output|Output Directory

python/plugins/processing/algs/grass7/description/i.tasscap.txt

@@ -3,4 +3,4 @@ Performs Tasseled Cap (Kauth Thomas) transformation.
 Imagery (i.*)
 QgsProcessingParameterMultipleLayers|input|Input rasters. Landsat4-7: bands 1,2,3,4,5,7; Landsat8: bands 2,3,4,5,6,7; MODIS: bands 1,2,3,4,5,6,7|3|None|False
 QgsProcessingParameterEnum|sensor|Satellite sensor|landsat4_tm;landsat5_tm;landsat7_etm;landsat8_oli;modis|False|0
-OutputDirectory|output|Output Directory
+QgsProcessingParameterFolderDestination|output|Output Directory

python/plugins/processing/algs/grass7/description/i.topo.corr.txt

@@ -6,4 +6,4 @@ QgsProcessingParameterRasterLayer|basemap|Name of illumination input base raster
 QgsProcessingParameterNumber|zenith|Solar zenith in degrees|QgsProcessingParameterNumber.Double|0.0|False|0.0|360.0
 QgsProcessingParameterEnum|method|Topographic correction method|cosine;minnaert;c-factor;percent|False|0
 *QgsProcessingParameterBoolean|-s|Scale output to input and copy color rules|False
-OutputDirectory|output|Output Directory
+QgsProcessingParameterFolderDestination|output|Output Directory

python/plugins/processing/algs/grass7/description/i.zc.txt

@@ -6,4 +6,3 @@ QgsProcessingParameterNumber|width|x-y extent of the Gaussian filter|QgsProcessi
 QgsProcessingParameterNumber|threshold|Sensitivity of Gaussian filter|QgsProcessingParameterNumber.Double|10.0|False|None|0
 QgsProcessingParameterNumber|orientations|Number of azimuth directions categorized|QgsProcessingParameterNumber.Double|1|False|None|0
 QgsProcessingParameterRasterDestination|output|Zero crossing
-

python/plugins/processing/algs/grass7/description/r.blend.rgb.txt

@@ -7,4 +7,3 @@ QgsProcessingParameterNumber|percent|Percentage weight of first map for color bl
 QgsProcessingParameterRasterDestination|output_red|Blended Red
 QgsProcessingParameterRasterDestination|output_green|Blended Green
 QgsProcessingParameterRasterDestination|output_blue|Blended Blue
-

python/plugins/processing/algs/grass7/description/r.colors.txt

@@ -12,5 +12,4 @@ QgsProcessingParameterBoolean|-n|Invert colors|False
 QgsProcessingParameterBoolean|-g|Logarithmic scaling|False
 QgsProcessingParameterBoolean|-a|Logarithmic-absolute scaling|False
 QgsProcessingParameterBoolean|-e|Histogram equalization|False
-OutputDirectory|output_dir|Output Directory
-
+QgsProcessingParameterFolderDestination|output_dir|Output Directory

python/plugins/processing/algs/grass7/description/r.mapcalc.txt

@@ -6,4 +6,4 @@ QgsProcessingParameterString|expression|Expression to evaluate. Syntax e.g. `ras
 QgsProcessingParameterFile|file|File containing expression(s) to evaluate (same rule for raster names than above)|0|txt|None|True
 QgsProcessingParameterString|seed|Integer seed for rand() function|None|False|True
 *QgsProcessingParameterBoolean|-s|Generate random seed (result is non-deterministic)|False
-OutputDirectory|output_dir|Results Directory
+QgsProcessingParameterFolderDestination|output_dir|Results Directory

python/plugins/processing/algs/grass7/description/r.series.interp.txt

@@ -4,8 +4,8 @@ Raster (r.*)
 QgsProcessingParameterMultipleLayers|input|Input raster layer(s)|3|None|False
 QgsProcessingParameterString|datapos|Data point position for each input map|None|True|True
 QgsProcessingParameterFile|infile|Input file with one input raster map name and data point position per line, field separator between name and sample point is 'pipe'|0|txt|None|True
-QgsProcessingParameterString|output|Name for output raster map (comma separated list if multiple|None|False|True
+QgsProcessingParameterString|output|Name for output raster map (comma separated list if multiple)|None|False|True
 QgsProcessingParameterString|samplingpos|Sampling point position for each output map (comma separated list)|None|True|True
 QgsProcessingParameterFile|outfile|Input file with one output raster map name and sample point position per line, field separator between name and sample point is 'pipe'|0|txt|NoneTrue
 QgsProcessingParameterEnum|method|Interpolation method, currently only linear interpolation is supported|linear|False|0
-OutputDirectory|output_dir|Interpolated rasters
+QgsProcessingParameterFolderDestination|output_dir|Interpolated rasters

python/plugins/processing/algs/grass7/description/r.spread.txt

@@ -18,4 +18,3 @@ QgsProcessingParameterNumber|lag|Simulating time duration LAG (fill the region)
 QgsProcessingParameterRasterDestination|output|Spread Time
 QgsProcessingParameterRasterDestination|x_output|X Back Coordinates
 QgsProcessingParameterRasterDestination|y_output|Y Back Coordinates
-

python/plugins/processing/algs/grass7/description/r.stats.quantile.rast.txt

@@ -7,5 +7,5 @@ QgsProcessingParameterNumber|quantiles|Number of quantiles|QgsProcessingParamete
 QgsProcessingParameterString|percentiles|List of percentiles|None|False|True
 QgsProcessingParameterNumber|bins|Number of bins to use|QgsProcessingParameterNumber.Integer|1000|True|0|None
 *QgsProcessingParameterBoolean|-r|Create reclass map with statistics as category labels|False
-OutputDirectory|output_dir|Output Directory
+QgsProcessingParameterFolderDestination|output_dir|Output Directory