added tests & removed pyramid subcommand

CHANGELOG.rst

@@ -10,6 +10,7 @@ Changelog
 
   * output drivers must now provide `OutputDataWriter` and `OutputDataReader` classes instead of a single `OutputData` class
   * `OutputDataWriter.close()` method must accept `exc_type=None, exc_value=None, exc_traceback=None` keywords
+  * `mapchete pyramid` CLI was removed and is replaced by the more versatile `mapchete convert`
 
 * fixed `area_at_zoom()` when using input groups (#181)
 * fixed single GeoTIFF output bounds should use process area (#182)

mapchete/_processing.py

@@ -387,6 +387,7 @@ def __init__(self, func, fargs=None, fkwargs=None):
 
     def result(self):
         if self._exception:
+            logger.exception(self._exception)
             raise self._exception
         else:
             return self._result

mapchete/cli/default/convert.py

@@ -1,8 +1,11 @@
 import click
 import logging
+from multiprocessing import cpu_count
 import os
+from pprint import pformat
 import rasterio
 from rasterio.dtypes import dtype_ranges
+from shapely.geometry import box
 import sys
 import tilematrix
 
@@ -12,6 +15,7 @@
     driver_from_file, available_output_formats, available_input_formats
 )
 from mapchete.io import read_json, get_best_zoom_level
+from mapchete.io.vector import reproject_geometry
 from mapchete.tile import BufferedTilePyramid
 
 logger = logging.getLogger(__name__)
@@ -26,23 +30,23 @@
 @utils.opt_point
 @utils.opt_wkt_geometry
 @click.option(
-    "--clip_geometry", "-c", type=click.Path(exists=True),
+    "--clip-geometry", "-c", type=click.Path(exists=True),
     help="Clip output by geometry"
 )
 @click.option(
-    "--output_pyramid", "-op", type=click.Choice(tilematrix._conf.PYRAMID_PARAMS.keys()),
+    "--output-pyramid", type=click.Choice(tilematrix._conf.PYRAMID_PARAMS.keys()),
     help="Output pyramid to write to."
 )
 @click.option(
-    "--output_metatiling", "-m", type=click.INT, default=1,
+    "--output-metatiling", "-m", type=click.INT,
     help="Output metatiling.",
 )
 @click.option(
     "--output_format", type=click.Choice(available_output_formats()),
     help="Output format."
 )
 @click.option(
-    "--output_dtype", type=click.Choice(dtype_ranges.keys()),
+    "--output-dtype", type=click.Choice(dtype_ranges.keys()),
     help="Output data type (for raster output only)."
 )
 @click.option(
@@ -57,7 +61,10 @@
 @utils.opt_verbose
 @utils.opt_no_pbar
 @utils.opt_debug
+@utils.opt_multi
 @utils.opt_logfile
+@utils.opt_vrt
+@utils.opt_idx_out_dir
 def convert(
     input_,
     output,
@@ -77,6 +84,9 @@ def convert(
     verbose=False,
     no_pbar=False,
     debug=False,
+    multi=None,
+    vrt=False,
+    idx_out_dir=None
 ):
     input_info = _get_input_info(input_)
     output_info = _get_output_info(output)
@@ -86,36 +96,58 @@ def convert(
         process="mapchete.processes.convert",
         input=dict(raster=input_, clip=clip_geometry),
         pyramid=(
-            dict(grid=output_pyramid, metatiling=output_metatiling)
+            dict(
+                grid=output_pyramid,
+                metatiling=(
+                    output_metatiling or
+                    (
+                        input_info["pyramid"].get("metatiling", 1)
+                        if input_info["pyramid"]
+                        else 1
+                    )
+                ),
+                pixelbuffer=(
+                    input_info["pyramid"].get("pixelbuffer", 0)
+                    if input_info["pyramid"]
+                    else 0
+                )
+            )
             if output_pyramid
             else input_info["pyramid"]
         ),
         output=dict(
-            input_info["output_params"],
+            {
+                k: v
+                for k, v in input_info["output_params"].items()
+                if k not in ["delimiters", "bounds", "mode"]
+            },
             path=output,
             format=(
                 output_format or
                 output_info["driver"] or
                 input_info["output_params"]["format"]
-            )
+            ),
+            dtype=output_dtype or input_info["output_params"].get("dtype")
         ),
         config_dir=os.getcwd(),
-        zoom_levels=zoom or input_info["zoom_levels"]
+        zoom_levels=zoom or input_info["zoom_levels"],
+        scale_ratio=scale_ratio,
+        scale_offset=scale_offset
     )
 
+    # assert all required information is there
     if mapchete_config["output"]["format"] is None:
         raise click.BadOptionUsage("output_format", "Output format required.")
     output_type = OUTPUT_FORMATS[mapchete_config["output"]["format"]]["data_type"]
-
     if mapchete_config["pyramid"] is None:
         raise click.BadOptionUsage("output_pyramid", "Output pyramid required.")
     elif mapchete_config["zoom_levels"] is None:
         try:
             mapchete_config.update(
-                zoom=list(range(
-                    0,
-                    get_best_zoom_level(input_, mapchete_config["pyramid"]["grid"]) + 1
-                ))
+                zoom_levels=dict(
+                    min=0,
+                    max=get_best_zoom_level(input_, mapchete_config["pyramid"]["grid"])
+                )
             )
         except:
             raise click.BadOptionUsage("zoom", "Zoom levels required.")
@@ -127,6 +159,22 @@ def convert(
             )
         )
 
+    # determine process bounds
+    out_pyramid = BufferedTilePyramid.from_dict(mapchete_config["pyramid"])
+    mapchete_config.update(
+        bounds=(
+            bounds or
+            reproject_geometry(
+                box(*input_info["bounds"]),
+                src_crs=input_info["crs"],
+                dst_crs=out_pyramid.crs
+            ).bounds
+            if input_info["bounds"]
+            else out_pyramid.bounds
+        )
+    )
+    logger.debug("temporary config generated: %s", pformat(mapchete_config))
+
     utils._process_area(
         debug=debug,
         mapchete_config=mapchete_config,
@@ -135,14 +183,17 @@ def convert(
         wkt_geometry=wkt_geometry,
         point=point,
         bounds=bounds,
+        multi=multi or cpu_count(),
         verbose_dst=open(os.devnull, 'w') if debug or not verbose else sys.stdout,
         no_pbar=no_pbar,
+        vrt=vrt,
+        idx_out_dir=idx_out_dir
     )
 
 
 def _get_input_info(input_):
+    # single file input can be a mapchete file or a rasterio/fiona file
     if os.path.isfile(input_):
-        # single file input can be a mapchete file or a rasterio/fiona file
         driver = driver_from_file(input_)
 
         if driver == "Mapchete":
@@ -159,8 +210,8 @@ def _get_input_info(input_):
             logger.debug("input is vector_file")
             raise NotImplementedError()
 
+    # assuming tile directory
     else:
-        # assuming tile directory
         logger.debug("input is tile directory")
         input_info = _input_tile_directory_info(input_)
 
@@ -177,7 +228,8 @@ def _input_mapchete_info(input_):
         crs=pyramid.crs,
         zoom_levels=get_zoom_levels(process_zoom_levels=conf["zoom_levels"]),
         pixel_size=None,
-        input_type=OUTPUT_FORMATS[output_params["format"]]["data_type"]
+        input_type=OUTPUT_FORMATS[output_params["format"]]["data_type"],
+        bounds=conf.get("bounds")
     )
 
 
@@ -193,7 +245,8 @@ def _input_rasterio_info(input_):
             crs=src.crs,
             zoom_levels=None,
             pixel_size=src.transform[0],
-            input_type="raster"
+            input_type="raster",
+            bounds=src.bounds
         )
 
 
@@ -206,7 +259,8 @@ def _input_tile_directory_info(input_):
         crs=pyramid.crs,
         zoom_levels=None,
         pixel_size=None,
-        input_type=OUTPUT_FORMATS[conf["driver"]["format"]]["data_type"]
+        input_type=OUTPUT_FORMATS[conf["driver"]["format"]]["data_type"],
+        bounds=None
     )
 
 
@@ -222,10 +276,5 @@ def _get_output_info(output):
             type="SingleFile",
             driver="GTiff"
         )
-    elif file_ext == ".png":
-        return dict(
-            type="SingleFile",
-            driver="PNG"
-        )
     else:
         raise TypeError("Output file extension not recognized: %s", file_ext)