Merge pull request #486 from ungarj/optimize_tests

make slowest tests faster; reuse DaskExecutor() where possible

test/conftest.py

@@ -12,6 +12,7 @@
 from tilematrix import Bounds, GridDefinition
 
 from mapchete.cli.default.serve import create_app
+from mapchete._executor import DaskExecutor
 from mapchete.io import fs_from_path
 from mapchete.testing import ProcessFixture, dict_from_mapchete
 
@@ -745,3 +746,10 @@ def stac_metadata():
 def s3_example_tile(gtiff_s3):
     """Example tile for fixture."""
     return (5, 15, 32)
+
+
+@pytest.fixture(scope="package")
+def dask_executor():
+    """DaskExecutor()"""
+    with DaskExecutor() as executor:
+        yield executor

test/test_cli.py

@@ -711,6 +711,11 @@ def test_convert_mapchete(cleantopo_br, mp_tmpdir):
             "-d",
             "--concurrency",
             "none",
+            "--bounds",
+            "168.75",
+            "-90.0",
+            "180.0",
+            "-78.75",
         ]
     )
     for zoom, row, col in [(4, 15, 31), (3, 7, 15), (2, 3, 7), (1, 1, 3)]:
@@ -742,6 +747,11 @@ def test_convert_tiledir(cleantopo_br, mp_tmpdir):
             "-d",
             "--concurrency",
             "none",
+            "--bounds",
+            "168.75",
+            "-90.0",
+            "180.0",
+            "-78.75",
         ]
     )
     for zoom, row, col in [(4, 15, 31), (3, 7, 15), (2, 3, 7), (1, 1, 3)]:
@@ -793,6 +803,11 @@ def test_convert_geobuf(landpoly, mp_tmpdir):
             "Geobuf",
             "--concurrency",
             "none",
+            "--bounds",
+            "-101.25",
+            "67.5",
+            "-90.0",
+            "90.0",
         ]
     )
     for (zoom, row, col), control in zip([(4, 0, 7), (4, 1, 7)], [9, 32]):
@@ -819,6 +834,11 @@ def test_convert_geobuf(landpoly, mp_tmpdir):
             "GeoJSON",
             "--concurrency",
             "none",
+            "--bounds",
+            "-101.25",
+            "67.5",
+            "-90.0",
+            "90.0",
         ]
     )
     for (zoom, row, col), control in zip([(4, 0, 7), (4, 1, 7)], [9, [31, 32]]):

test/test_commands.py

@@ -406,18 +406,18 @@ def test_convert_zoom_maxmin(cleantopo_br_tif, mp_tmpdir):
 
 def test_convert_mapchete(cleantopo_br, mp_tmpdir):
     # prepare data
-    job = execute(cleantopo_br.path, zoom=[1, 4])
+    job = execute(cleantopo_br.path, zoom=[1, 3])
     assert len(job)
 
     job = convert(
         cleantopo_br.path,
         mp_tmpdir,
         output_pyramid="geodetic",
         output_metatiling=1,
-        zoom=[1, 4],
+        zoom=[1, 3],
     )
     assert len(job)
-    for zoom, row, col in [(4, 15, 31), (3, 7, 15), (2, 3, 7), (1, 1, 3)]:
+    for zoom, row, col in [(3, 7, 15), (2, 3, 7), (1, 1, 3)]:
         out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"])
         with rasterio.open(out_file, "r") as src:
             assert src.meta["driver"] == "GTiff"

test/test_config.py

@@ -103,18 +103,18 @@ def test_read_zoom_level(zoom_mapchete):
     assert 5 in config.zoom_levels
 
 
-def test_minmax_zooms(minmax_zoom):
+@pytest.mark.parametrize("zoom", [7, 8, 9, 10])
+def test_minmax_zooms(minmax_zoom, zoom):
     """Read min/max zoom levels from config file."""
     config = MapcheteConfig(minmax_zoom.dict)
-    for zoom in [7, 8, 9, 10]:
-        assert zoom in config.zoom_levels
+    assert zoom in config.zoom_levels
 
 
-def test_override_zoom_levels(minmax_zoom):
+@pytest.mark.parametrize("zoom", [7, 8])
+def test_override_zoom_levels(minmax_zoom, zoom):
     """Override zoom levels when constructing configuration."""
     config = MapcheteConfig(minmax_zoom.dict, zoom=[7, 8])
-    for zoom in [7, 8]:
-        assert zoom in config.zoom_levels
+    assert zoom in config.zoom_levels
 
 
 def test_read_bounds(zoom_mapchete):

test/test_executor.py

@@ -186,24 +186,13 @@ def test_dask_executor_as_completed_skip():
         assert items == count
 
 
-def test_dask_executor_as_completed_max_tasks():
+@pytest.mark.parametrize("max_submitted_tasks", [1, None])
+def test_dask_executor_as_completed_max_tasks(max_submitted_tasks):
     items = 100
     with Executor(concurrency="dask") as executor:
         # abort
         for future in executor.as_completed(
-            _dummy_process, range(items), max_submitted_tasks=1
-        ):
-            assert future.result()
-
-        assert not executor.running_futures
-
-
-def test_dask_executor_as_completed_unlimited_max_tasks():
-    items = 100
-    with Executor(concurrency="dask") as executor:
-        # abort
-        for future in executor.as_completed(
-            _dummy_process, range(items), max_submitted_tasks=None
+            _dummy_process, range(items), max_submitted_tasks=max_submitted_tasks
         ):
             assert future.result()
 

test/test_io_raster.py

@@ -543,7 +543,8 @@ def test_create_mosaic_errors():
         create_mosaic(tiles=[])
 
 
-def test_create_mosaic():
+@pytest.mark.parametrize("pixelbuffer", [0, 10])
+def test_create_mosaic(pixelbuffer):
     """Create mosaic from tiles."""
     tp = BufferedTilePyramid("geodetic")
     # quick return mosaic if there is just one tile
@@ -555,48 +556,46 @@ def test_create_mosaic():
     assert tile.affine == mosaic.affine
     zoom = 5
     # multiple tiles on top left corner of tile matrix
-    for pixelbuffer in [0, 10]:
-        tp = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
-        tiles = [
-            (tp.tile(zoom, row, col), np.ones(tp.tile(zoom, row, col).shape))
-            for row, col in product(range(4), range(4))
-        ]
-        # 4x4 top left tiles from zoom 5 equal top left tile from zoom 3
-        # also use tile generator
-        mosaic = create_mosaic((t for t in tiles))
-        assert isinstance(mosaic, ReferencedRaster)
-        assert np.all(np.where(mosaic.data == 1, True, False))
-        mosaic_bbox = box(
-            mosaic.affine[2],
-            mosaic.affine[5] + mosaic.data.shape[1] * mosaic.affine[4],
-            mosaic.affine[2] + mosaic.data.shape[2] * mosaic.affine[0],
-            mosaic.affine[5],
-        )
-        control_bbox = box(*unary_union([t.bbox for t, _ in tiles]).bounds)
-        assert mosaic_bbox.equals(control_bbox)
+    tp = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
+    tiles = [
+        (tp.tile(zoom, row, col), np.ones(tp.tile(zoom, row, col).shape))
+        for row, col in product(range(4), range(4))
+    ]
+    # 4x4 top left tiles from zoom 5 equal top left tile from zoom 3
+    # also use tile generator
+    mosaic = create_mosaic((t for t in tiles))
+    assert isinstance(mosaic, ReferencedRaster)
+    assert np.all(np.where(mosaic.data == 1, True, False))
+    mosaic_bbox = box(
+        mosaic.affine[2],
+        mosaic.affine[5] + mosaic.data.shape[1] * mosaic.affine[4],
+        mosaic.affine[2] + mosaic.data.shape[2] * mosaic.affine[0],
+        mosaic.affine[5],
+    )
+    control_bbox = box(*unary_union([t.bbox for t, _ in tiles]).bounds)
+    assert mosaic_bbox.equals(control_bbox)
     # multiple tiles on bottom right corner of tile matrix
-    for pixelbuffer in [0, 10]:
-        tp = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
-        tiles = [
-            (tp.tile(zoom, row, col), np.ones(tp.tile(zoom, row, col).shape))
-            for row, col in product(
-                range(tp.matrix_height(zoom) - 4, tp.matrix_height(zoom)),
-                range(tp.matrix_width(zoom) - 4, tp.matrix_width(zoom)),
-            )
-        ]
-        # 4x4 top left tiles from zoom 5 equal top left tile from zoom 3
-        # also use tile generator
-        mosaic = create_mosaic((t for t in tiles))
-        assert isinstance(mosaic, ReferencedRaster)
-        assert np.all(np.where(mosaic.data == 1, True, False))
-        mosaic_bbox = box(
-            mosaic.affine[2],
-            mosaic.affine[5] + mosaic.data.shape[1] * mosaic.affine[4],
-            mosaic.affine[2] + mosaic.data.shape[2] * mosaic.affine[0],
-            mosaic.affine[5],
+    tp = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
+    tiles = [
+        (tp.tile(zoom, row, col), np.ones(tp.tile(zoom, row, col).shape))
+        for row, col in product(
+            range(tp.matrix_height(zoom) - 4, tp.matrix_height(zoom)),
+            range(tp.matrix_width(zoom) - 4, tp.matrix_width(zoom)),
         )
-        control_bbox = box(*unary_union([t.bbox for t, _ in tiles]).bounds)
-        assert mosaic_bbox.equals(control_bbox)
+    ]
+    # 4x4 top left tiles from zoom 5 equal top left tile from zoom 3
+    # also use tile generator
+    mosaic = create_mosaic((t for t in tiles))
+    assert isinstance(mosaic, ReferencedRaster)
+    assert np.all(np.where(mosaic.data == 1, True, False))
+    mosaic_bbox = box(
+        mosaic.affine[2],
+        mosaic.affine[5] + mosaic.data.shape[1] * mosaic.affine[4],
+        mosaic.affine[2] + mosaic.data.shape[2] * mosaic.affine[0],
+        mosaic.affine[5],
+    )
+    control_bbox = box(*unary_union([t.bbox for t, _ in tiles]).bounds)
+    assert mosaic_bbox.equals(control_bbox)
 
 
 def test_create_mosaic_antimeridian():

test/test_io_vector.py

@@ -29,36 +29,47 @@
 from mapchete.tile import BufferedTilePyramid
 
 
-def test_read_vector_window(geojson, landpoly_3857):
+def test_read_vector_window(geojson):
     """Read vector data from read_vector_window."""
     zoom = 4
     config = MapcheteConfig(geojson.dict)
     vectorfile = config.params_at_zoom(zoom)["input"]["file1"]
     pixelbuffer = 5
     tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
-    tiles = tile_pyramid.tiles_from_geom(vectorfile.bbox(), zoom)
-    feature_count = 0
+    tiles = tile_pyramid.tiles_from_geom(
+        vectorfile.bbox(out_crs=tile_pyramid.crs), zoom
+    )
     for tile in tiles:
-        for feature in read_vector_window(vectorfile.path, tile):
-            assert "properties" in feature
-            assert shape(feature["geometry"]).is_valid
-            feature_count += 1
-    assert feature_count
-    # into different CRS
+        features = read_vector_window(vectorfile.path, tile)
+        if features:
+            for feature in features:
+                assert "properties" in feature
+                assert shape(feature["geometry"]).is_valid
+            break
+    else:
+        raise RuntimeError("no features read!")
+
+
+def test_read_vector_window_reproject(geojson, landpoly_3857):
+    zoom = 4
     raw_config = geojson.dict
     raw_config["input"].update(file1=landpoly_3857)
     config = MapcheteConfig(raw_config)
     vectorfile = config.params_at_zoom(zoom)["input"]["file1"]
     pixelbuffer = 5
     tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
-    tiles = tile_pyramid.tiles_from_geom(vectorfile.bbox(), zoom)
-    feature_count = 0
+    tiles = tile_pyramid.tiles_from_geom(
+        vectorfile.bbox(out_crs=tile_pyramid.crs), zoom
+    )
     for tile in tiles:
-        for feature in read_vector_window(vectorfile.path, tile):
-            assert "properties" in feature
-            assert shape(feature["geometry"]).is_valid
-            feature_count += 1
-    assert feature_count
+        features = read_vector_window(vectorfile.path, tile)
+        if features:
+            for feature in features:
+                assert "properties" in feature
+                assert shape(feature["geometry"]).is_valid
+            break
+    else:
+        raise RuntimeError("no features read!")
 
 
 def test_read_vector_window_errors(invalid_geojson):
@@ -409,7 +420,10 @@ def test_bounds_intersect():
     assert bounds_intersect(b1, b7)
     assert bounds_intersect(b1, b8)
 
-    intersecting = [
+
+@pytest.mark.parametrize(
+    "intersecting",
+    [
         (-121.2451171875, 75.7177734375, -120.849609375, 76.025390625),
         (-119.3994140625, 75.498046875, -117.4658203125, 76.11328125),
         (-122.958984375, 75.8056640625, -115.400390625, 77.5634765625),
@@ -458,9 +472,10 @@ def test_bounds_intersect():
         (-91.1865234375, 77.2119140625, -89.9560546875, 77.6513671875),
         (-96.7236328125, 78.134765625, -89.9560546875, 81.38671875),
         (-168.134765625, 13.6669921875, -89.9560546875, 74.1796875),
-    ]
-    for b in intersecting:
-        assert bounds_intersect(b, (-135.0, 60, -90, 80))
+    ],
+)
+def test_bounds_intersect_custom(intersecting):
+    assert bounds_intersect(intersecting, (-135.0, 60, -90, 80))
 
 
 def test_bounds_not_intersect():