pass on output parameters to mapchete process (#215, fixes #214)

mapchete/_processing.py

@@ -29,6 +29,7 @@ class TileProcess():
     """
 
     def __init__(self, tile=None, config=None, skip=False):
+        """Set attributes depending on baselevels or not."""
         self.tile = (
             config.process_pyramid.tile(*tile) if isinstance(tile, tuple) else tile
         )
@@ -38,10 +39,11 @@ def __init__(self, tile=None, config=None, skip=False):
         self.process_path = None if skip else config.process_path
         self.config_dir = None if skip else config.config_dir
         if skip or self.tile.zoom not in self.config_zoom_levels:
-            self.input, self.process_func_params = {}, {}
+            self.input, self.process_func_params, self.output_params = {}, {}, {}
         else:
             self.input = config.get_inputs_for_tile(tile)
             self.process_func_params = config.get_process_func_params(tile.zoom)
+            self.output_params = config.output_reader.output_params
         self.mode = None if skip else config.mode
         self.output_reader = (
             None if skip or not config.baselevels else config.output_reader
@@ -92,7 +94,8 @@ def _execute(self):
                     MapcheteProcess(
                         tile=self.tile,
                         params=self.process_func_params,
-                        input=self.input
+                        input=self.input,
+                        output_params=self.output_params
                     ),
                     **self.process_func_params
                 )
@@ -130,7 +133,7 @@ def _interpolate_from_baselevel(self, baselevel=None):
                     in_affine=parent_tile.affine,
                     out_tile=self.tile,
                     resampling=self.config_baselevels["higher"],
-                    nodataval=self.output_reader.nodata
+                    nodataval=self.output_reader.output_params["nodata"]
                 )
             # resample from children tiles
             elif baselevel == "lower":
@@ -143,7 +146,7 @@ def _interpolate_from_baselevel(self, baselevel=None):
                     in_affine=mosaic.affine,
                     out_tile=self.tile,
                     resampling=self.config_baselevels["lower"],
-                    nodataval=self.output_reader.nodata
+                    nodataval=self.output_reader.output_params["nodata"]
                 )
         logger.debug((self.tile.id, "generated from baselevel", str(t)))
         return process_data
@@ -185,7 +188,14 @@ class MapcheteProcess(object):
         process parameters
     """
 
-    def __init__(self, tile=None, params=None, input=None, config=None):
+    def __init__(
+        self,
+        tile=None,
+        params=None,
+        input=None,
+        output_params=None,
+        config=None
+    ):
         """Initialize Mapchete process."""
         self.identifier = ""
         self.title = ""
@@ -199,6 +209,7 @@ def __init__(self, tile=None, params=None, input=None, config=None):
             params = config.params_at_zoom(tile.zoom)
         self.params = dict(params, input=input)
         self.input = input
+        self.output_params = output_params
 
     def write(self, data, **kwargs):
         """Deprecated."""
@@ -313,24 +324,23 @@ def clip(
         """
         return commons_clip.clip_array_with_vector(
             array, self.tile.affine, geometries,
-            inverted=inverted, clip_buffer=clip_buffer*self.tile.pixel_x_size
+            inverted=inverted, clip_buffer=clip_buffer * self.tile.pixel_x_size
         )
 
 
 #############################################################
 # wrappers helping to abstract multiprocessing and billiard #
 #############################################################
-
 class Executor():
-    """
-    Wrapper class to be used with multiprocessing or billiard.
-    """
+    """Wrapper class to be used with multiprocessing or billiard."""
+
     def __init__(
         self,
         start_method="spawn",
         max_workers=None,
         multiprocessing_module=multiprocessing
     ):
+        """Set attributes."""
         self.start_method = start_method
         self.max_workers = max_workers or os.cpu_count()
         self.multiprocessing_module = multiprocessing_module
@@ -347,6 +357,7 @@ def as_completed(
         fkwargs=None,
         chunksize=1
     ):
+        """Yield finished tasks."""
         fargs = fargs or []
         fkwargs = fkwargs or {}
         if self.max_workers == 1:
@@ -373,10 +384,10 @@ def as_completed(
 
 
 class FinishedTask():
-    """
-    Wrapper class to encapsulate exceptions.
-    """
+    """Wrapper class to encapsulate exceptions."""
+
     def __init__(self, func, fargs=None, fkwargs=None):
+        """Set attributes."""
         fargs = fargs or []
         fkwargs = fkwargs or {}
         try:
@@ -385,21 +396,24 @@ def __init__(self, func, fargs=None, fkwargs=None):
             self._result, self._exception = None, e
 
     def result(self):
+        """Return task result."""
         if self._exception:
             logger.exception(self._exception)
             raise self._exception
         else:
             return self._result
 
     def exception(self):
+        """Raise task exception if any."""
         return self._exception
 
     def __repr__(self):
+        """Return string representation."""
         return "FinishedTask(result=%s, exception=%s)" % (self._result, self._exception)
 
 
 def _exception_wrapper(func, fargs, fkwargs, i):
-    """Wraps function around FinishedTask object."""
+    """Wrap function around FinishedTask object."""
     return FinishedTask(func, list(chain([i], fargs)), fkwargs)
 
 
@@ -469,12 +483,12 @@ def _filter_skipable(process=None, tiles=None, todo=None, target_set=None):
     for tile, skip in process.skip_tiles(tiles=tiles):
         if skip and tile not in target_set:
             yield ProcessInfo(
-                    tile=tile,
-                    processed=False,
-                    process_msg="output already exists",
-                    written=False,
-                    write_msg="nothing written"
-                )
+                tile=tile,
+                processed=False,
+                process_msg="output already exists",
+                written=False,
+                write_msg="nothing written"
+            )
         else:
             todo.add(tile)
 

mapchete/commons/hillshade.py

@@ -11,12 +11,12 @@
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
+modification, are permitted provided that the followg conditions are met:
 
 - Redistributions of source code must retain the above copyright notice,
-  this list of conditions and the following disclaimer.
+  this list of conditions and the followg disclaimer.
 - Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
+  this list of conditions and the followg disclaimer in the documentation
   and/or other materials provided with the distribution.
 - Neither the name of the project nor the names of its contributors may be
   used to endorse or promote products derived from this software without
@@ -73,26 +73,20 @@ def calculate_slope_aspect(elevation, xres, yres, z=1.0, scale=1.0):
     z = float(z)
     scale = float(scale)
     height, width = elevation.shape[0] - 2, elevation.shape[1] - 2
-    window = [
+    w = [
         z * elevation[row:(row + height), col:(col + width)]
         for (row, col) in product(range(3), range(3))
     ]
-    x = (
-        (window[0] + window[3] + window[3] + window[6])
-        - (window[2] + window[5] + window[5] + window[8])
-        ) / (8.0 * xres * scale)
-    y = (
-        (window[6] + window[7] + window[7] + window[8])
-        - (window[0] + window[1] + window[1] + window[2])
-        ) / (8.0 * yres * scale)
+    x = ((w[0] + w[3] + w[3] + w[6]) - (w[2] + w[5] + w[5] + w[8])) / (8.0 * xres * scale)
+    y = ((w[6] + w[7] + w[7] + w[8]) - (w[0] + w[1] + w[1] + w[2])) / (8.0 * yres * scale)
     # in radians, from 0 to pi/2
-    slope = math.pi/2 - np.arctan(np.sqrt(x*x + y*y))
+    slope = math.pi / 2 - np.arctan(np.sqrt(x * x + y * y))
     # in radians counterclockwise, from -pi at north back to pi
     aspect = np.arctan2(x, y)
     return slope, aspect
 
 
-def hillshade(elevation, tile, azimuth=315.0, altitude=45.0, z=1.0, scale=1.0):
+def hillshade(elevation, tile, azimuth=315.0, altitude=45.0, z=1.0, scale=1.0, ):
     """
     Return hillshaded numpy array.
 
@@ -108,21 +102,27 @@ def hillshade(elevation, tile, azimuth=315.0, altitude=45.0, z=1.0, scale=1.0):
         scale factor of pixel size units versus height units (insert 112000
         when having elevation values in meters in a geodetic projection)
     """
+    elevation = elevation[0] if elevation.ndim == 3 else elevation
     azimuth = float(azimuth)
     altitude = float(altitude)
     z = float(z)
     scale = float(scale)
     xres = tile.tile.pixel_x_size
     yres = -tile.tile.pixel_y_size
     slope, aspect = calculate_slope_aspect(
-        elevation, xres, yres, z=z, scale=scale)
+        elevation,
+        xres,
+        yres,
+        z=z,
+        scale=scale
+    )
     deg2rad = math.pi / 180.0
     shaded = np.sin(altitude * deg2rad) * np.sin(slope) \
         + np.cos(altitude * deg2rad) * np.cos(slope) \
         * np.cos((azimuth - 90.0) * deg2rad - aspect)
     # shaded now has values between -1.0 and +1.0
     # stretch to 0 - 255 and invert
-    shaded = (((shaded+1.0)/2)*-255.0).astype("uint8")
+    shaded = (((shaded + 1.0) / 2) * -255.0).astype("uint8")
     # add one pixel padding using the edge values
     return ma.masked_array(
         data=np.pad(shaded, 1, mode='edge'), mask=elevation.mask

mapchete/formats/base.py

@@ -215,7 +215,7 @@ def extract_subset(self, input_data_tiles=None, out_tile=None):
             return extract_from_array(
                 in_raster=prepare_array(
                     mosaic.data,
-                    nodata=self.nodata,
+                    nodata=self.output_params["nodata"],
                     dtype=self.output_params["dtype"]
                 ),
                 in_affine=mosaic.affine,

mapchete/formats/default/gtiff.py

@@ -189,7 +189,10 @@ def for_web(self, data):
         """
         return memory_file(
             prepare_array(
-                data, masked=True, nodata=self.nodata, dtype=self.profile()["dtype"]
+                data,
+                masked=True,
+                nodata=self.output_params["nodata"],
+                dtype=self.profile()["dtype"]
             ),
             self.profile()
         ), "image/tiff"
@@ -230,8 +233,10 @@ def is_valid_with_config(self, config):
     def _set_attributes(self, output_params):
         self.path = output_params["path"]
         self.file_extension = ".tif"
-        self.output_params = output_params
-        self.nodata = output_params.get("nodata", GTIFF_DEFAULT_PROFILE["nodata"])
+        self.output_params = dict(
+            output_params,
+            nodata=output_params.get("nodata", GTIFF_DEFAULT_PROFILE["nodata"])
+        )
         self._bucket = self.path.split("/")[2] if self.path.startswith("s3://") else None
 
 
@@ -304,7 +309,7 @@ def profile(self, tile=None):
             count=self.output_params["bands"],
             dtype=self.output_params["dtype"],
             driver="GTiff",
-            nodata=self.output_params.get("nodata", GTIFF_DEFAULT_PROFILE["nodata"])
+            nodata=self.output_params["nodata"]
         )
         dst_metadata.pop("transform", None)
         if tile is not None:
@@ -355,7 +360,7 @@ def write(self, process_tile, data):
         data = prepare_array(
             data,
             masked=True,
-            nodata=self.nodata,
+            nodata=self.output_params["nodata"],
             dtype=self.profile(process_tile)["dtype"]
         )
 
@@ -527,7 +532,7 @@ def write(self, process_tile, data):
         data = prepare_array(
             data,
             masked=True,
-            nodata=self.nodata,
+            nodata=self.output_params["nodata"],
             dtype=self.profile(process_tile)["dtype"]
         )
 

mapchete/formats/default/png.py

@@ -82,9 +82,11 @@ def __init__(self, output_params, **kwargs):
         super().__init__(output_params)
         self.path = output_params["path"]
         self.file_extension = ".png"
-        self.output_params = output_params
-        self.output_params["dtype"] = PNG_DEFAULT_PROFILE["dtype"]
-        self.nodata = output_params.get("nodata", PNG_DEFAULT_PROFILE["nodata"])
+        self.output_params = dict(
+            output_params,
+            nodata=output_params.get("nodata", PNG_DEFAULT_PROFILE["nodata"]),
+            dtype=PNG_DEFAULT_PROFILE["dtype"]
+        )
         self._bucket = self.path.split("/")[2] if self.path.startswith("s3://") else None
 
     def read(self, output_tile, **kwargs):
@@ -158,7 +160,7 @@ def for_web(self, data):
         web data : array
         """
         rgba = self._prepare_array_for_png(data)
-        data = ma.masked_where(rgba == self.nodata, rgba)
+        data = ma.masked_where(rgba == self.output_params["nodata"], rgba)
         return memory_file(data, self.profile()), 'image/png'
 
     def empty(self, process_tile):
@@ -193,15 +195,15 @@ def _prepare_array_for_png(self, data):
             rgba = np.stack((
                 data[0], data[0], data[0],
                 np.where(
-                    data[0].data == self.nodata, 0, 255)
+                    data[0].data == self.output_params["nodata"], 0, 255)
                 .astype("uint8")
             ))
         elif len(data) == 2:
             rgba = np.stack((data[0], data[0], data[0], data[1]))
         elif len(data) == 3:
             rgba = np.stack((
                 data[0], data[1], data[2], np.where(
-                    data[0].data == self.nodata, 0, 255
+                    data[0].data == self.output_params["nodata"], 0, 255
                 ).astype("uint8", copy=False)
             ))
         elif len(data) == 4:
@@ -225,7 +227,7 @@ def write(self, process_tile, data):
             must be member of process ``TilePyramid``
         """
         rgba = self._prepare_array_for_png(data)
-        data = ma.masked_where(rgba == self.nodata, rgba)
+        data = ma.masked_where(rgba == self.output_params["nodata"], rgba)
 
         if data.mask.all():
             logger.debug("data empty, nothing to write")

mapchete/formats/default/png_hillshade.py

@@ -84,9 +84,12 @@ def __init__(self, output_params, **kwargs):
         super().__init__(output_params)
         self.path = output_params["path"]
         self.file_extension = ".png"
-        self.output_params = output_params
+        self.output_params = dict(
+            output_params,
+            nodata=output_params.get("nodata", PNG_DEFAULT_PROFILE["nodata"]),
+            dtype=PNG_DEFAULT_PROFILE["dtype"]
+        )
         self._profile = dict(PNG_DEFAULT_PROFILE)
-        self.nodata = self._profile["nodata"]
         try:
             self.old_band_num = output_params["old_band_num"]
             self._profile.update(count=4)

mapchete/index.py

@@ -421,7 +421,7 @@ def close(self):
         # get VRT attributes
         vrt_affine, vrt_shape = raster.tiles_to_affine_shape(list(all_entries.keys()))
         vrt_dtype = _gdal_typename(self._output.profile()["dtype"])
-        vrt_nodata = self._output.nodata
+        vrt_nodata = self._output.output_params["nodata"]
 
         # build XML
         E = ElementMaker()