Merge pull request #457 from ungarj/fix_454_path_exists_calls

better handle non-existing files for rasterio and fiona

.pre-commit-config.yaml

@@ -1,9 +1,9 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 21.5b2
+    rev: 22.3.0
     hooks:
     - id: black
-      language_version: python3.8
+      language_version: python3
       exclude: test/testdata/syntax_error.py
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v1.2.3

doc/source/conf.py

@@ -45,9 +45,9 @@
 master_doc = "index"
 
 # General information about the project.
-project = u"Mapchete"
-copyright = u"2015, 2016, 2017, 2018, EOX IT Services"
-author = u"Joachim Ungar"
+project = "Mapchete"
+copyright = "2015, 2016, 2017, 2018, EOX IT Services"
+author = "Joachim Ungar"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -127,15 +127,15 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, "Mapchete.tex", u"Mapchete Documentation", u"Joachim Ungar", "manual"),
+    (master_doc, "Mapchete.tex", "Mapchete Documentation", "Joachim Ungar", "manual"),
 ]
 
 
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [(master_doc, "mapchete", u"Mapchete Documentation", [author], 1)]
+man_pages = [(master_doc, "mapchete", "Mapchete Documentation", [author], 1)]
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -147,7 +147,7 @@
     (
         master_doc,
         "Mapchete",
-        u"Mapchete Documentation",
+        "Mapchete Documentation",
         author,
         "Mapchete",
         "Mapchete processes raster and vector geodata.",

mapchete/formats/base.py

@@ -665,19 +665,22 @@ def _read_as_tiledir(
                 [path for _, path in tiles_paths],
                 out_tile,
                 validity_check=validity_check,
+                skip_missing_files=True,
             )
-        else:
+        else:  # pragma: no cover
             return []
     elif data_type == "raster":
         if tiles_paths:
             return read_raster_window(
                 [path for _, path in tiles_paths],
                 out_tile,
-                indexes=indexes,
+                indexes=indexes or list(range(1, profile["count"] + 1)),
                 resampling=resampling,
                 src_nodata=profile["nodata"],
                 dst_nodata=dst_nodata,
                 gdal_opts=gdal_opts,
+                skip_missing_files=True,
+                dst_dtype=profile["dtype"],
             )
         else:
             bands = len(indexes) if indexes else profile["count"]

mapchete/formats/default/gtiff.py

@@ -476,7 +476,7 @@ def prepare(self, process_area=None, **kwargs):
             self.overviews_levels = self.output_params.get(
                 "overviews_levels",
                 [
-                    2 ** i
+                    2**i
                     for i in range(
                         1,
                         get_maximum_overview_level(

mapchete/formats/default/tile_directory.py

@@ -191,7 +191,9 @@ def bbox(self, out_crs=None):
         )
 
 
-def _get_tiles_paths(basepath=None, ext=None, pyramid=None, bounds=None, zoom=None):
+def _get_tiles_paths(
+    basepath=None, ext=None, pyramid=None, bounds=None, zoom=None, exists_check=False
+):
     return [
         (_tile, _path)
         for _tile, _path in [
@@ -205,7 +207,7 @@ def _get_tiles_paths(basepath=None, ext=None, pyramid=None, bounds=None, zoom=No
             )
             for t in pyramid.tiles_from_bounds(bounds, zoom)
         ]
-        if path_exists(_path)
+        if not exists_check or (exists_check and path_exists(_path))
     ]
 
 
@@ -405,10 +407,9 @@ def _get_tiles_paths(
                     pyramid=self._td_pyramid,
                     bounds=td_bounds,
                     zoom=zoom,
+                    exists_check=True,
                 )
-                logger.debug(
-                    "%s existing tiles found at zoom %s", len(tiles_paths), zoom
-                )
+                logger.debug("%s potential tiles at zoom %s", len(tiles_paths), zoom)
                 zoom -= 1
         else:
             tiles_paths = _get_tiles_paths(
@@ -418,6 +419,6 @@ def _get_tiles_paths(
                 bounds=td_bounds,
                 zoom=zoom,
             )
-            logger.debug("%s existing tiles found at zoom %s", len(tiles_paths), zoom)
+            logger.debug("%s potential tiles at zoom %s", len(tiles_paths), zoom)
 
         return tiles_paths

mapchete/io/_geometry_operations.py

@@ -76,7 +76,6 @@ def reproject_geometry(
     -------
     geometry : ``shapely.geometry``
     """
-    logger.debug("reproject geometry from %s to %s", src_crs, dst_crs)
     src_crs = validate_crs(src_crs)
     dst_crs = validate_crs(dst_crs)
 

mapchete/io/_path.py

@@ -43,6 +43,7 @@ def path_exists(path, fs=None, **kwargs):
     """
     fs = fs or fs_from_path(path, **kwargs)
     fs.invalidate_cache(path=path)
+    logger.debug("check if path exists: %s", path)
     return fs.exists(path)