pathprof.srtm: warn instead of except when tile not available on disk

notebooks/03d_more_on_srtm_data.ipynb

@@ -148,7 +148,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "('srtm_dir', 'download', 'server', 'interp', 'spline_opts')\n"
+      "('srtm_dir', 'download', 'server', 'interp', 'spline_opts', 'tile_size', 'hgt_res')\n"
      ]
     }
    ],
@@ -196,7 +196,7 @@
    "source": [
     "### Download missing tiles\n",
     "\n",
-    "By default, pycraf won't download missing SRTM data:"
+    "By default, pycraf won't download missing SRTM data, but set the tile heights to Zero:"
    ]
   },
   {
@@ -207,10 +207,23 @@
    },
    "outputs": [
     {
-     "name": "stdout",
+     "name": "stderr",
      "output_type": "stream",
      "text": [
-      "No hgt-file found for (7d, 45d), was looking for N45E007.hgt\n"
+      "/home/bwinkel/local/miniconda/envs/pycraf3.6/lib/python3.6/site-packages/pycraf/pathprof/srtm.py:482: TileNotAvailableOnDiskWarning: \n",
+      "No hgt-file found for (7d, 45d) - was looking for file N45E007.hgt\n",
+      "in directory: /tmp/tmp7vue3h7o\n",
+      "Will set terrain heights in this area to zero. Note, you can have pycraf\n",
+      "download missing tiles automatically - just use \"pycraf.pathprof.SrtmConf\"\n",
+      "(see its documentation).\n",
+      "  stacklevel=1,\n",
+      "/home/bwinkel/local/miniconda/envs/pycraf3.6/lib/python3.6/site-packages/pycraf/pathprof/srtm.py:482: TileNotAvailableOnDiskWarning: \n",
+      "No hgt-file found for (7d, 46d) - was looking for file N46E007.hgt\n",
+      "in directory: /tmp/tmp7vue3h7o\n",
+      "Will set terrain heights in this area to zero. Note, you can have pycraf\n",
+      "download missing tiles automatically - just use \"pycraf.pathprof.SrtmConf\"\n",
+      "(see its documentation).\n",
+      "  stacklevel=1,\n"
      ]
     }
    ],
@@ -221,11 +234,7 @@
     "lons = np.linspace(bbox[0], bbox[1], 501) * u.deg\n",
     "lats = np.linspace(bbox[2], bbox[3], 501)[:, np.newaxis] * u.deg\n",
     "\n",
-    "try:\n",
-    "    heights = pp.srtm_height_data(lons, lats)\n",
-    "except OSError as e:\n",
-    "    # avoid printing full traceback\n",
-    "    print(e.args[0].split('\\n')[0])"
+    "heights = pp.srtm_height_data(lons, lats)"
    ]
   },
   {
@@ -370,7 +379,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "To use them, we first have to clear the interpolator cache and then delete the old tile data in our temporary directory."
+    "To use them, we need to change the server option accordingly. Unfortunately, the 'viewpano' tiles are stored into a sub-directory, such that pycraf would find the tile twice and raise an error. Therefore, we first have to delete the old tile data in our temporary directory."
    ]
   },
   {
@@ -381,7 +390,6 @@
    },
    "outputs": [],
    "source": [
-    "pp.srtm.get_tile_interpolator.cache_clear()\n",
     "shutil.rmtree(temp_srtm_dir)\n",
     "os.mkdir(temp_srtm_dir)"
    ]
@@ -406,7 +414,7 @@
     "    # technically, one could omit the context in this case\n",
     "    # as 'linear' is the default\n",
     "    heights_linear_vp = pp.srtm_height_data(lons, lats)\n",
-    "    \n",
+    "\n",
     "# since above we downloaded the missing tile already,\n",
     "# we don't need the 'download' option below anymore\n",
     "with pp.SrtmConf.set(interp='nearest'):\n",

pycraf/pathprof/srtm.py

@@ -27,6 +27,7 @@
 
 # from functools import partial, lru_cache
 import os
+import warnings
 import shutil
 from zipfile import ZipFile
 import re
@@ -40,7 +41,13 @@
 from .. import utils
 
 
-__all__ = ['SrtmConf', 'srtm_height_data']
+__all__ = [
+    'TileNotAvailableOnServerError',
+    'TileNotAvailableOnDiskError',
+    'TileNotAvailableOnDiskWarning',
+    'TilesSizeError',
+    'SrtmConf', 'srtm_height_data'
+    ]
 
 
 _NASA_JSON_NAME = get_pkg_data_filename('data/nasa.json')
@@ -52,7 +59,17 @@
 VIEWPANO_TILES = np.load(_VIEWPANO_NAME)
 
 
-class TileNotAvailableError(Exception):
+class TileNotAvailableOnServerError(Exception):
+
+    pass
+
+
+class TileNotAvailableOnDiskError(Exception):
+
+    pass
+
+
+class TileNotAvailableOnDiskWarning(UserWarning):
 
     pass
 
@@ -221,6 +238,18 @@ def hook(cls, **kwargs):
             if kwargs['srtm_dir'] != cls.srtm_dir:
                 get_tile_interpolator.cache_clear()
 
+        if 'download' in kwargs or 'server' in kwargs:
+            # check if 'download' strategy was changed and clear cache
+            # this is necessary, because missing tiles will lead to
+            # zero heights in the tile cache (for that tile) and if user
+            # later sets the option to download missing tiles, the reading
+            # routine needs to run again
+            if (
+                    kwargs['download'] != cls.download or
+                    kwargs['server'] != cls.server
+                    ):
+                get_tile_interpolator.cache_clear()
+
 
 def _hgt_filename(ilon, ilat):
     # construct proper hgt-file name
@@ -246,7 +275,7 @@ def _check_availability(ilon, ilat):
             if tile_name in tiles:
                 break
         else:
-            raise TileNotAvailableError(
+            raise TileNotAvailableOnServerError(
                 'No tile found for ({}d, {}d) in list of available '
                 'tiles.'.format(
                     ilon, ilat
@@ -260,7 +289,7 @@ def _check_availability(ilon, ilat):
         idx = np.where(tiles == tile_name)
 
         if len(tiles[idx]) == 0:
-            raise TileNotAvailableError(
+            raise TileNotAvailableOnServerError(
                 'No tile found for ({}d, {}d) in list of available '
                 'tiles.'.format(
                     ilon, ilat
@@ -402,7 +431,7 @@ def get_hgt_file(ilon, ilat):
 
     hgt_file = _get_hgt_diskpath(tile_name)
     if hgt_file is None:
-        raise IOError(
+        raise TileNotAvailableOnDiskError(
             'No hgt-file found for ({}d, {}d), was looking for {}\n'
             'in directory: {}'.format(
                 ilon, ilat, tile_name, srtm_dir
@@ -429,12 +458,30 @@ def get_tile_data(ilon, ilat):
         tile = tile.astype(np.float32)
         tile[bad_mask] = np.nan
 
-    except TileNotAvailableError:
+    except TileNotAvailableOnServerError:
         # always use very small tile size for zero tiles
         # (just enough to make spline interpolation work)
         tile_size = 5
         tile = np.zeros((tile_size, tile_size), dtype=np.float32)
 
+    except TileNotAvailableOnDiskError:
+        # also set to zero, but raise a warning
+        tile_size = 5
+        tile = np.zeros((tile_size, tile_size), dtype=np.float32)
+
+        tile_name = _hgt_filename(ilon, ilat)
+        srtm_dir = SrtmConf.srtm_dir
+        warnings.warn(
+            '''
+No hgt-file found for ({}d, {}d) - was looking for file {}
+in directory: {}
+Will set terrain heights in this area to zero. Note, you can have pycraf
+download missing tiles automatically - just use "pycraf.pathprof.SrtmConf"
+(see its documentation).'''.format(ilon, ilat, tile_name, srtm_dir),
+            category=TileNotAvailableOnDiskWarning,
+            stacklevel=1,
+            )
+
     dx = dy = 1. / (tile_size - 1)
     x, y = np.ogrid[0:tile_size, 0:tile_size]
     lons, lats = x * dx + ilon, y * dy + ilat
@@ -513,6 +560,14 @@ def srtm_height_data(lons, lats):
     heights : `~astropy.units.Quantity`
         SRTM heights [m]
 
+    Raises
+    ------
+    TileNotAvailableOnDiskWarning : UserWarning
+        If a tile is requested that should exist on the chosen server
+        but is not available on disk (at least not in the search path)
+        a warning is raised. In this case, the tile height data is set
+        to Zeros.
+
     Notes
     -----
     - `SRTM <https://www2.jpl.nasa.gov/srtm/>`_ data tiles (`*.hgt`) need

pycraf/pathprof/tests/test_srtm.py

@@ -199,7 +199,7 @@ def test_check_availability_nasa():
     for ilon, ilat, name in nasa_cases:
 
         if name is None:
-            with pytest.raises(srtm.TileNotAvailableError):
+            with pytest.raises(srtm.TileNotAvailableOnServerError):
                 srtm._check_availability(ilon, ilat)
         else:
             assert srtm._check_availability(ilon, ilat) == name
@@ -237,7 +237,7 @@ def test_check_availability_pano():
         for ilon, ilat, name in pano_cases:
 
             if name is None:
-                with pytest.raises(srtm.TileNotAvailableError):
+                with pytest.raises(srtm.TileNotAvailableOnServerError):
                     srtm._check_availability(ilon, ilat)
             else:
                 assert srtm._check_availability(ilon, ilat) == name
@@ -293,7 +293,7 @@ def test_get_hgt_diskpath(srtm_temp_dir):
         open(os.path.join(srtm_temp_dir, 'd1', 'foo.hgt'), 'w').close()
         open(os.path.join(srtm_temp_dir, 'd2', 'foo.hgt'), 'w').close()
 
-        with pytest.raises(IOError):
+        with pytest.raises(IOError, match=r'.* exists .* times in .*'):
             srtm._get_hgt_diskpath('foo.hgt')
 
         # cleaning up
@@ -322,7 +322,10 @@ def test_get_hgt_file_download_never(srtm_temp_dir):
         ilon, ilat = 12, 50
         tile_name = srtm._hgt_filename(ilon, ilat)
 
-        with pytest.raises(IOError):
+        with pytest.raises(
+                srtm.TileNotAvailableOnDiskError,
+                match=r'.*No hgt-file found for .*'
+                ):
             srtm.get_hgt_file(ilon, ilat)
 
 
@@ -407,6 +410,24 @@ def test_get_tile_zero(srtm_temp_dir):
         assert_allclose(tile, np.zeros((5, 5), dtype=np.float32))
 
 
+def test_get_tile_warning(srtm_temp_dir):
+
+    with srtm.SrtmConf.set(srtm_dir=srtm_temp_dir):
+
+        # ilon, ilat = 6, 54
+        ilon, ilat = 15, 50
+        with pytest.warns(srtm.TileNotAvailableOnDiskWarning):
+            lons, lats, tile = srtm.get_tile_data(ilon, ilat)
+
+        assert_allclose(lons[:, 0], np.array([
+            15., 15.25, 15.5, 15.75, 16.
+            ]))
+        assert_allclose(lats[0, :], np.array([
+            50., 50.25, 50.5, 50.75, 51.
+            ]))
+        assert_allclose(tile, np.zeros((5, 5), dtype=np.float32))
+
+
 @remote_data(source='any')
 def test_srtm_height_data_linear(srtm_temp_dir):