retry failed elevation tile download

CHANGELOG.md

@@ -21,6 +21,7 @@
    * FIXED: lots of issues with CostMatrix (primarily deadend logic) with a complete refactor modeling things very close to bidir A*, also to prepare for a unification of the two [#4372](https://github.com/valhalla/valhalla/pull/4372)
    * FIXED: diff_names check was missing for Graphfilter and Shortcutbuilder for AddEdgeInfo call.  [#4436](https://github.com/valhalla/valhalla/pull/4436)
    * FIXED: updated timezone database and added code to keep compatibility with old servers/new data and vice versa [#4446](https://github.com/valhalla/valhalla/pull/4446)
+   * FIXED: retry elevation tile download if the download failed for some reason or the downloaded tile was corrupt [#4461](https://github.com/valhalla/valhalla/pull/4461)
 * **Enhancement**
    * UPDATED: French translations, thanks to @xlqian [#4159](https://github.com/valhalla/valhalla/pull/4159)
    * CHANGED: -j flag for multithreaded executables to override mjolnir.concurrency [#4168](https://github.com/valhalla/valhalla/pull/4168)

scripts/valhalla_build_elevation

@@ -13,9 +13,10 @@ from multiprocessing.dummy import Pool as ThreadPool
 import os
 from pathlib import Path
 import sys
+from time import sleep
 from typing import List, Iterable, Set
 from urllib import request
-from urllib.error import URLError
+from urllib.error import URLError, HTTPError
 
 # hack so ArgumentParser can accept negative numbers
 # see https://github.com/valhalla/valhalla/issues/3426
@@ -53,6 +54,12 @@ class TileCompression(Enum):
             return ".lz4"
 
 
+class DownloadStatus(Enum):
+    OK = 1
+    FAILED = 2
+    CORRUPTED = 3
+
+
 parser = argparse.ArgumentParser(description=description)
 method = parser.add_mutually_exclusive_group()
 method.add_argument(
@@ -225,7 +232,7 @@ def get_tiles_with_graph(graph_dir: Path) -> Set[Tile]:
     return tile_infos
 
 
-def download(tile: Tile, output_dir, compression: TileCompression):
+def download(tile: Tile, output_dir, compression: TileCompression) -> bool:
     dest_directory = Path(output_dir, tile.dir)
     dest_directory.mkdir(parents=True, exist_ok=True)
 
@@ -237,39 +244,71 @@ def download(tile: Tile, output_dir, compression: TileCompression):
     url = f"https://elevation-tiles-prod.s3.us-east-1.amazonaws.com/skadi/{tile.dir}/{tile.name}.gz"
 
     LOGGER.info(f"Downloading tile {tile.name}")
-    try:
-        with request.urlopen(url) as res, open(filepath, "wb") as f:
-            if compression is TileCompression.GZIP:
-                f.write(res.read())
-            else:
-                with gzip.GzipFile(fileobj=res, mode="rb") as gz:
-                    uncompressed = gz.read()
-                    if compression is TileCompression.UNCOMPRESSED:
-                        f.write(uncompressed)
-                    elif compression is TileCompression.LZ4:
-                        # Compression level 6 was chosen after some benchmarking as the approx efficient frontier
-                        # between compression time and space savings (decompression time is roughly constant regardless
-                        # of level). The end result is larger than the maximally gzipped tiles from AWS, but only
-                        # by around 12%.
-                        import lz4.frame
-
-                        with lz4.frame.LZ4FrameCompressor(
-                            block_size=lz4.frame.BLOCKSIZE_MAX4MB, compression_level=6
-                        ) as compressor:
-                            # Optimization: we know the exact size of every uncompressed hgt file
-                            f.write(compressor.begin(25934402))
-                            f.write(compressor.compress(uncompressed))
-                            f.write(compressor.flush())
-
-            LOGGER.debug(f"Successfully downloaded tile {tile.name}")
-
-        return True
-    except URLError as e:
-        LOGGER.critical(f"Download failed of elevation tile {tile.dir}/{tile.name}: {e.reason}")
+
+    download_status = DownloadStatus.FAILED
+    for i in range(5):
+        try:
+            LOGGER.debug(f"Downloading tile {tile.dir}/{tile.name} for the {i}th time.")
+
+            # tries up to 24 mins with exponentially increasing sleeps in between, starting with 5 seconds
+            sleep((i**2) / 2 * 30)
+            with request.urlopen(url) as res, open(filepath, "wb") as f:
+
+                if compression is TileCompression.GZIP:
+                    f.write(res.read())
+                else:
+                    with gzip.GzipFile(fileobj=res, mode="rb") as gz:
+                        try:
+                            uncompressed = gz.read()
+                        except Exception as e:
+                            download_status = DownloadStatus.CORRUPTED
+                            LOGGER.error(
+                                f"Decompression error on tile {tile.dir}/{tile.name}: {e}. Likely a corrupted download."
+                            )
+                            continue
+                        if compression is TileCompression.UNCOMPRESSED:
+                            f.write(uncompressed)
+                        elif compression is TileCompression.LZ4:
+                            # Compression level 6 was chosen after some benchmarking as the approx efficient frontier
+                            # between compression time and space savings (decompression time is roughly constant regardless
+                            # of level). The end result is larger than the maximally gzipped tiles from AWS, but only
+                            # by around 12%.
+                            import lz4.frame
+
+                            with lz4.frame.LZ4FrameCompressor(
+                                block_size=lz4.frame.BLOCKSIZE_MAX4MB, compression_level=6
+                            ) as compressor:
+                                # Optimization: we know the exact size of every uncompressed hgt file
+                                f.write(compressor.begin(25934402))
+                                f.write(compressor.compress(uncompressed))
+                                f.write(compressor.flush())
+
+            download_status = DownloadStatus.OK
+            LOGGER.debug(f"Successfully downloaded tile {tile.dir}/{tile.name}")
+            break
+        except HTTPError as e:
+            LOGGER.error(f"Download failed with HTTP error {e.code}: {e.reason}.\nTrying again...")
+            continue
+        except URLError as e:
+            LOGGER.error(
+                f"Download failed of elevation tile {tile.dir}/{tile.name}: {e.reason}.\nTrying again.."
+            )
+            continue
+        except ImportError:
+            LOGGER.critical(
+                "Could not import lz4. Please install lz4 or use another compression format."
+            )
+            sys.exit(1)
+
+    if download_status == DownloadStatus.CORRUPTED:
+        LOGGER.error(f"Tile {tile.dir}/{tile.name} was corrupted, removing it...")
+        filepath.unlink()
         return False
-    except ImportError:
-        LOGGER.critical("Could not import lz4. Please install lz4 or use another compression format.")
-        sys.exit(1)
+    elif download_status == DownloadStatus.FAILED:
+        LOGGER.error(f"Tile {tile.dir}/{tile.name} couldn't be downloaded...")
+        return False
+
+    return True
 
 
 if __name__ == "__main__":