dev commit

mapchete/cli/default/serve.py

@@ -77,7 +77,7 @@ def create_app(
 
     mp = next(iter(mapchete_processes.values()))
     pyramid_type = mp.config.process_pyramid.grid
-    pyramid_srid = mp.config.process_pyramid.srid
+    pyramid_srid = mp.config.process_pyramid.crs.to_epsg()
     process_bounds = ",".join([str(i) for i in mp.config.bounds_at_zoom()])
     grid = "g" if pyramid_srid == 3857 else "WGS84"
     web_pyramid = BufferedTilePyramid(pyramid_type)

mapchete/config.py

@@ -267,7 +267,7 @@ def output(self):
         """Output object of driver."""
         output_params = dict(
             self._raw["output"],
-            type=self.output_pyramid.grid,
+            grid=self.output_pyramid.grid,
             pixelbuffer=self.output_pyramid.pixelbuffer,
             metatiling=self.output_pyramid.metatiling
         )
@@ -285,7 +285,6 @@ def output(self):
                     output_params["format"], str(available_output_formats())
                 )
             )
-
         writer = load_output_writer(output_params)
         try:
             writer.is_valid_with_config(output_params)
@@ -527,48 +526,52 @@ def bounds_at_zoom(self, zoom=None):
     @cached_property
     def crs(self):
         """Deprecated."""
-        warnings.warn("self.crs is now self.process_pyramid.crs.")
+        warnings.warn(DeprecationWarning("self.crs is now self.process_pyramid.crs."))
         return self.process_pyramid.crs
 
     @cached_property
     def metatiling(self):
         """Deprecated."""
         warnings.warn(
-            "self.metatiling is now self.process_pyramid.metatiling.")
+            DeprecationWarning("self.metatiling is now self.process_pyramid.metatiling.")
+        )
         return self.process_pyramid.metatiling
 
     @cached_property
     def pixelbuffer(self):
         """Deprecated."""
         warnings.warn(
-            "self.pixelbuffer is now self.process_pyramid.pixelbuffer.")
+            DeprecationWarning(
+                "self.pixelbuffer is now self.process_pyramid.pixelbuffer."
+            )
+        )
         return self.process_pyramid.pixelbuffer
 
     @cached_property
     def inputs(self):
         """Deprecated."""
-        warnings.warn("self.inputs renamed to self.input.")
+        warnings.warn(DeprecationWarning("self.inputs renamed to self.input."))
         return self.input
 
     @cached_property
     def process_file(self):
         """Deprecated."""
-        warnings.warn("'self.process_file' is deprecated")
+        warnings.warn(DeprecationWarning("'self.process_file' is deprecated"))
         return os.path.join(self._raw["config_dir"], self._raw["process"])
 
     def at_zoom(self, zoom):
         """Deprecated."""
-        warnings.warn("Method renamed to self.params_at_zoom(zoom).")
+        warnings.warn(DeprecationWarning("Method renamed to self.params_at_zoom(zoom)."))
         return self.params_at_zoom(zoom)
 
     def process_area(self, zoom=None):
         """Deprecated."""
-        warnings.warn("Method renamed to self.area_at_zoom(zoom).")
+        warnings.warn(DeprecationWarning("Method renamed to self.area_at_zoom(zoom)."))
         return self.area_at_zoom(zoom)
 
     def process_bounds(self, zoom=None):
         """Deprecated."""
-        warnings.warn("Method renamed to self.bounds_at_zoom(zoom).")
+        warnings.warn(DeprecationWarning("Method renamed to self.bounds_at_zoom(zoom)."))
         return self.bounds_at_zoom(zoom)
 
 
@@ -685,9 +688,7 @@ def raw_conf(mapchete_file):
     -------
     dictionary
     """
-    return _map_to_new_config(
-        yaml.load(open(mapchete_file, "r").read())
-    )
+    return _map_to_new_config(yaml.load(open(mapchete_file, "r").read()))
 
 
 def raw_conf_process_pyramid(raw_conf):
@@ -954,16 +955,29 @@ def _unflatten_tree(flat):
 
 
 def _map_to_new_config(config):
+    try:
+        validate_values(config, [("output", dict)])
+    except Exception as e:
+        raise MapcheteConfigError(e)
+    if "type" in config["output"]:
+        warnings.warn(DeprecationWarning("'type' is deprecated and should be 'grid'"))
+        if "grid" not in config["output"]:
+            config["output"]["grid"] = config["output"].pop("type")
     if "pyramid" not in config:
-        warnings.warn("'pyramid' needs to be defined in root config element.")
+        warnings.warn(
+            DeprecationWarning("'pyramid' needs to be defined in root config element.")
+        )
         config["pyramid"] = dict(
-            grid=config["output"]["type"],
+            grid=config["output"]["grid"],
             metatiling=config.get("metatiling", 1),
             pixelbuffer=config.get("pixelbuffer", 0))
     if "zoom_levels" not in config:
         warnings.warn(
-            "use new config element 'zoom_levels' instead of 'process_zoom', "
-            "'process_minzoom' and 'process_maxzoom'")
+            DeprecationWarning(
+                "use new config element 'zoom_levels' instead of 'process_zoom', "
+                "'process_minzoom' and 'process_maxzoom'"
+            )
+        )
         if "process_zoom" in config:
             config["zoom_levels"] = config["process_zoom"]
         elif all([
@@ -978,22 +992,28 @@ def _map_to_new_config(config):
     if "bounds" not in config:
         if "process_bounds" in config:
             warnings.warn(
-                "'process_bounds' are deprecated and renamed to 'bounds'")
+                DeprecationWarning(
+                    "'process_bounds' are deprecated and renamed to 'bounds'"
+                )
+            )
             config["bounds"] = config["process_bounds"]
         else:
             config["bounds"] = None
     if "input" not in config:
         if "input_files" in config:
             warnings.warn(
-                "'input_files' are deprecated and renamed to 'input'")
+                DeprecationWarning("'input_files' are deprecated and renamed to 'input'")
+            )
             config["input"] = config["input_files"]
         else:
             raise MapcheteConfigError("no 'input' found")
     elif "input_files" in config:
         raise MapcheteConfigError(
             "'input' and 'input_files' are not allowed at the same time")
     if "process_file" in config:
-        warnings.warn("'process_file' is deprecated and renamed to 'process'")
+        warnings.warn(
+            DeprecationWarning("'process_file' is deprecated and renamed to 'process'")
+        )
         config["process"] = config.pop("process_file")
 
     return config

mapchete/formats/__init__.py

@@ -7,9 +7,12 @@
 import logging
 import os
 import pkg_resources
+from pprint import pformat
 import warnings
 
-from mapchete import errors
+from mapchete.errors import MapcheteConfigError, MapcheteDriverError
+from mapchete.io import read_json, write_json
+from mapchete.tile import BufferedTilePyramid
 
 DRIVERS_ENTRY_POINT = "mapchete.formats.drivers"
 
@@ -29,8 +32,10 @@ def _file_ext_to_driver():
             _driver = v.load()
             if not hasattr(_driver, "METADATA"):
                 warnings.warn(
-                    "driver %s cannot be loaded, METADATA is missing" % (
-                        str(v).split(" ")[-1]
+                    DeprecationWarning(
+                        "driver %s cannot be loaded, METADATA is missing" % (
+                            str(v).split(" ")[-1]
+                        )
                     )
                 )
                 continue
@@ -46,7 +51,7 @@ def _file_ext_to_driver():
             except Exception:
                 pass
         if not _FILE_EXT_TO_DRIVER:
-            raise errors.MapcheteDriverError("no drivers could be found")
+            raise MapcheteDriverError("no drivers could be found")
         return _FILE_EXT_TO_DRIVER
 
 
@@ -107,9 +112,7 @@ def load_output_writer(output_params, readonly=False):
             _driver.METADATA["driver_name"] == driver_name
         ):
             return _driver.OutputData(output_params, readonly=readonly)
-    raise errors.MapcheteDriverError(
-        "no loader for driver '%s' could be found." % driver_name
-    )
+    raise MapcheteDriverError("no loader for driver '%s' could be found." % driver_name)
 
 
 def load_input_reader(input_params, readonly=False):
@@ -134,15 +137,14 @@ def load_input_reader(input_params, readonly=False):
             logger.debug("%s is a directory", input_params["path"])
             driver_name = "TileDirectory"
     else:
-        raise errors.MapcheteDriverError("invalid input parameters %s" % input_params)
+        raise MapcheteDriverError("invalid input parameters %s" % input_params)
     for v in pkg_resources.iter_entry_points(DRIVERS_ENTRY_POINT):
         driver_ = v.load()
         if hasattr(driver_, "METADATA") and (
             driver_.METADATA["driver_name"] == driver_name
         ):
             return v.load().InputData(input_params, readonly=readonly)
-    raise errors.MapcheteDriverError(
-        "no loader for driver '%s' could be found." % driver_name)
+    raise MapcheteDriverError("no loader for driver '%s' could be found." % driver_name)
 
 
 def driver_from_file(input_file):
@@ -156,12 +158,93 @@ def driver_from_file(input_file):
     """
     file_ext = os.path.splitext(input_file)[1].split(".")[1]
     if file_ext not in _file_ext_to_driver():
-        raise errors.MapcheteDriverError(
+        raise MapcheteDriverError(
             "no driver could be found for file extension %s" % file_ext
         )
     driver = _file_ext_to_driver()[file_ext]
     if len(driver) > 1:
         warnings.warn(
-            "more than one driver for file found, taking %s" % driver[0]
+            DeprecationWarning(
+                "more than one driver for file found, taking %s" % driver[0]
+            )
         )
     return driver[0]
+
+
+def params_to_dump(params):
+    # in case GridDefinition was not yet initialized
+    return dict(
+        pyramid=BufferedTilePyramid(
+            grid=params["grid"],
+            tile_size=params.get("tile_size", 256),
+            metatiling=params.get("metatiling", 1),
+            pixelbuffer=params.get("pixelbuffer", 0),
+        ).to_dict(),
+        driver={
+           k: v
+           for k, v in params.items()
+           if k not in ["path", "grid", "pixelbuffer", "metatiling"]
+        }
+    )
+
+
+def read_output_metadata(metadata_json):
+    params = read_json(metadata_json)
+    grid = params["pyramid"]["grid"]
+    if grid["type"] == "geodetic" and grid["shape"] == [2, 1]:
+        warnings.warn(
+            DeprecationWarning(
+                """Deprecated grid shape ordering found. """
+                """Please change grid shape from [2, 1] to [1, 2] in %s."""
+                % metadata_json
+            )
+        )
+        params["pyramid"]["grid"]["shape"] = [1, 2]
+    params.update(
+        pyramid=BufferedTilePyramid(
+            params["pyramid"]["grid"],
+            metatiling=params["pyramid"].get("metatiling", 1),
+            tile_size=params["pyramid"].get("tile_size", 256),
+            pixelbuffer=params["pyramid"].get("pixelbuffer", 0)
+        )
+    )
+    return params
+
+
+def write_output_metadata(output_params):
+    """Dump output JSON and verify parameters if output metadata exist."""
+    if "path" in output_params:
+        metadata_path = os.path.join(output_params["path"], "metadata.json")
+        logger.debug("check for output %s", metadata_path)
+        try:
+            existing_params = read_output_metadata(metadata_path)
+            logger.debug("%s exists", metadata_path)
+            logger.debug("existing output parameters: %s", pformat(existing_params))
+            existing_tp = existing_params["pyramid"]
+            current_params = params_to_dump(output_params)
+            logger.debug("current output parameters: %s", pformat(current_params))
+            current_tp = BufferedTilePyramid(**current_params["pyramid"])
+            if existing_tp != current_tp:
+                raise MapcheteConfigError(
+                    "pyramid definitions between existing and new output do not match: "
+                    "%s != %s" % (existing_tp, current_tp)
+                )
+            existing_format = existing_params["driver"]["format"]
+            current_format = current_params["driver"]["format"]
+            if existing_format != current_format:
+                raise MapcheteConfigError(
+                    "existing output format does not match new output format: "
+                    "%s != %s" % (
+                        (existing_format, current_format)
+                    )
+                )
+        except FileNotFoundError:
+            logger.debug("%s does not exist", metadata_path)
+            dump_params = params_to_dump(output_params)
+            # dump output metadata
+            try:
+                write_json(metadata_path, dump_params)
+            except Exception as e:
+                logger.warning("failed to write %s: %s", metadata_path, e)
+    else:
+        logger.debug("no path parameter found")

mapchete/formats/base.py

@@ -5,8 +5,11 @@
 respective interfaces.
 """
 
-from mapchete.io import path_exists, write_output_metadata
 from tilematrix import TilePyramid
+import warnings
+
+from mapchete.formats import write_output_metadata
+from mapchete.io import path_exists
 
 
 class InputData(object):
@@ -26,8 +29,6 @@ class InputData(object):
         output ``TilePyramid``
     crs : ``rasterio.crs.CRS``
         object describing the process coordinate reference system
-    srid : string
-        spatial reference ID of CRS (e.g. "{'init': 'epsg:4326'}")
     """
 
     METADATA = {
@@ -41,7 +42,6 @@ def __init__(self, input_params, **kwargs):
         self.pyramid = input_params["pyramid"]
         self.pixelbuffer = input_params["pixelbuffer"]
         self.crs = self.pyramid.crs
-        self.srid = self.pyramid.srid
 
     def open(self, tile, **kwargs):
         """
@@ -150,8 +150,6 @@ class OutputData(object):
         output ``TilePyramid``
     crs : ``rasterio.crs.CRS``
         object describing the process coordinate reference system
-    srid : string
-        spatial reference ID of CRS (e.g. "{'init': 'epsg:4326'}")
     """
 
     METADATA = {
@@ -163,11 +161,15 @@ class OutputData(object):
     def __init__(self, output_params, readonly=False):
         """Initialize."""
         self.pixelbuffer = output_params["pixelbuffer"]
+        if "type" in output_params:
+            warnings.warn(DeprecationWarning("'type' is deprecated and should be 'grid'"))
+            if "grid" not in output_params:
+                output_params["grid"] = output_params.pop("type")
         self.pyramid = TilePyramid(
-            output_params["type"], metatiling=output_params["metatiling"]
+            grid=output_params["grid"],
+            metatiling=output_params["metatiling"]
         )
         self.crs = self.pyramid.crs
-        self.srid = self.pyramid.srid
         self._bucket = None
         if not readonly:
             write_output_metadata(output_params)

mapchete/formats/default/geojson.py

@@ -81,7 +81,7 @@ def __init__(self, output_params, **kwargs):
         self.output_params = output_params
         self._bucket = self.path.split("/")[2] if self.path.startswith("s3://") else None
 
-    def read(self, output_tile):
+    def read(self, output_tile, **kwargs):
         """
         Read existing process output.
 
@@ -255,7 +255,7 @@ def __init__(self, tile, process):
         self.process = process
         self._cache = {}
 
-    def read(self, validity_check=True, no_neighbors=False):
+    def read(self, validity_check=True, no_neighbors=False, **kwargs):
         """
         Read data from process output.