From d3971a8714b0de14383f7815914a9e1766ae6334 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Thu, 8 Sep 2022 17:08:43 +0200 Subject: [PATCH 1/3] Fix uncessary error when ocean tiles are hit by COPDEM --- oggm/utils/_downloads.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/oggm/utils/_downloads.py b/oggm/utils/_downloads.py index 5ade34fae..1c3b9fabe 100644 --- a/oggm/utils/_downloads.py +++ b/oggm/utils/_downloads.py @@ -608,13 +608,14 @@ def _dlf(cache_path): return _verified_download_helper(cache_obj_name, _dlf, reset) -def file_downloader(www_path, retry_max=5, cache_name=None, - reset=False, auth=None, timeout=None): +def file_downloader(www_path, retry_max=3, sleep_on_retry=5, + cache_name=None, reset=False, auth=None, + timeout=None): """A slightly better downloader: it tries more than once.""" local_path = None retry_counter = 0 - while retry_counter <= retry_max: + while retry_counter < retry_max: # Try to download try: retry_counter += 1 @@ -629,18 +630,19 @@ def file_downloader(www_path, retry_max=5, cache_name=None, # Ok so this *should* be an ocean tile return None elif err.code >= 500 and err.code < 600: - logger.info("Downloading %s failed with HTTP error %s, " - "retrying in 10 seconds... %s/%s" % - (www_path, err.code, retry_counter, retry_max)) - time.sleep(10) + logger.info(f"Downloading {www_path} failed with " + f"HTTP error {err.code}, " + f"retrying in {sleep_on_retry} seconds... " + f"{retry_counter}/{retry_max}") + time.sleep(sleep_on_retry) continue else: raise except HttpContentTooShortError as err: logger.info("Downloading %s failed with ContentTooShortError" - " error %s, retrying in 10 seconds... %s/%s" % - (www_path, err.code, retry_counter, retry_max)) - time.sleep(10) + " error %s, retrying in %s seconds... %s/%s" % + (www_path, err.code, sleep_on_retry, retry_counter, retry_max)) + time.sleep(sleep_on_retry) continue except DownloadVerificationFailedException as err: if (cfg.PATHS['dl_cache_dir'] and @@ -670,14 +672,14 @@ def file_downloader(www_path, retry_max=5, cache_name=None, else: # in other cases: try again logger.info("Downloading %s failed with ConnectionError, " - "retrying in 10 seconds... %s/%s" % - (www_path, retry_counter, retry_max)) - time.sleep(10) + "retrying in %s seconds... %s/%s" % + (www_path, sleep_on_retry, retry_counter, retry_max)) + time.sleep(sleep_on_retry) continue except FTPSDownloadError as err: logger.info("Downloading %s failed with FTPSDownloadError" - " error: '%s', retrying in 10 seconds... %s/%s" % - (www_path, err.orgerr, retry_counter, retry_max)) + " error: '%s', retrying in %s seconds... %s/%s" % + (www_path, err.orgerr, sleep_on_retry, retry_counter, retry_max)) time.sleep(10) continue @@ -1562,14 +1564,14 @@ def copdem_zone(lon_ex, lat_ex, source): ew = 'W' if lon < 0 else 'E' lat_str = '{}{:02.0f}'.format(ns, abs(lat)) lon_str = '{}{:03.0f}'.format(ew, abs(lon)) - # COPDEM is global, if we miss tiles it is worth an error try: filename = df.loc[(df['Long'] == lon_str) & (df['Lat'] == lat_str)]['CPP filename'].iloc[0] flist.append((filename, 'Copernicus_DSM_{}_{}_00_{}_00'.format(asec, lat_str, lon_str))) except IndexError: - raise InvalidDEMError('Could not find a matching Copernicus DEM file.') + # COPDEM is global, if we miss tiles it is probably in the ocean + pass return flist From 33147c902e90d8c4d1b88fa5f04608984947f040 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Thu, 8 Sep 2022 17:10:00 +0200 Subject: [PATCH 2/3] Fix uncessary error when ocean tiles are hit by COPDEM --- oggm/utils/_downloads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oggm/utils/_downloads.py b/oggm/utils/_downloads.py index 1c3b9fabe..deb57d67f 100644 --- a/oggm/utils/_downloads.py +++ b/oggm/utils/_downloads.py @@ -680,7 +680,7 @@ def file_downloader(www_path, retry_max=3, sleep_on_retry=5, logger.info("Downloading %s failed with FTPSDownloadError" " error: '%s', retrying in %s seconds... %s/%s" % (www_path, err.orgerr, sleep_on_retry, retry_counter, retry_max)) - time.sleep(10) + time.sleep(sleep_on_retry) continue # See if we managed (fail is allowed) From 5dbb1cdcb79b426c229107cdea47b3a91a22c0c2 Mon Sep 17 00:00:00 2001 From: Fabien Maussion Date: Thu, 8 Sep 2022 17:48:01 +0200 Subject: [PATCH 3/3] fix test --- oggm/tests/test_utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/oggm/tests/test_utils.py b/oggm/tests/test_utils.py index cd5f18db1..62b6b9496 100644 --- a/oggm/tests/test_utils.py +++ b/oggm/tests/test_utils.py @@ -2492,10 +2492,6 @@ def test_copdemzone(self): self.assertTrue('Copernicus_DSM_10_N46_00_E007_00' in [z[0][1], z[1][1]]) - # we want an error if copdem does not find all or any - self.assertRaises(InvalidDEMError, utils.copdem_zone, - [0, 1], [0, 1], 'COPDEM90') - def test_is_dem_source_available(self): assert utils.is_dem_source_available('SRTM', [11, 11], [47, 47]) assert utils.is_dem_source_available('GIMP', [-25, -25], [71, 71])