Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix uncessary error when ocean tiles are hit by COPDEM #1470

Merged
merged 3 commits into from
Sep 9, 2022
Merged

Fix uncessary error when ocean tiles are hit by COPDEM #1470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d3971a8
Fix uncessary error when ocean tiles are hit by COPDEM
fmaussion Sep 8, 2022
33147c9
Fix uncessary error when ocean tiles are hit by COPDEM
fmaussion Sep 8, 2022
5dbb1cd
fix test
fmaussion Sep 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions oggm/tests/test_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2492,10 +2492,6 @@ def test_copdemzone(self):
self.assertTrue('Copernicus_DSM_10_N46_00_E007_00' in
[z[0][1], z[1][1]])

# we want an error if copdem does not find all or any
self.assertRaises(InvalidDEMError, utils.copdem_zone,
[0, 1], [0, 1], 'COPDEM90')

def test_is_dem_source_available(self):
assert utils.is_dem_source_available('SRTM', [11, 11], [47, 47])
assert utils.is_dem_source_available('GIMP', [-25, -25], [71, 71])
Expand Down
38 changes: 20 additions & 18 deletions oggm/utils/_downloads.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -608,13 +608,14 @@ def _dlf(cache_path):
return _verified_download_helper(cache_obj_name, _dlf, reset)


def file_downloader(www_path, retry_max=5, cache_name=None,
reset=False, auth=None, timeout=None):
def file_downloader(www_path, retry_max=3, sleep_on_retry=5,
cache_name=None, reset=False, auth=None,
timeout=None):
"""A slightly better downloader: it tries more than once."""

local_path = None
retry_counter = 0
while retry_counter <= retry_max:
while retry_counter < retry_max:
# Try to download
try:
retry_counter += 1
Expand All @@ -629,18 +630,19 @@ def file_downloader(www_path, retry_max=5, cache_name=None,
# Ok so this *should* be an ocean tile
return None
elif err.code >= 500 and err.code < 600:
logger.info("Downloading %s failed with HTTP error %s, "
"retrying in 10 seconds... %s/%s" %
(www_path, err.code, retry_counter, retry_max))
time.sleep(10)
logger.info(f"Downloading {www_path} failed with "
f"HTTP error {err.code}, "
f"retrying in {sleep_on_retry} seconds... "
f"{retry_counter}/{retry_max}")
time.sleep(sleep_on_retry)
continue
else:
raise
except HttpContentTooShortError as err:
logger.info("Downloading %s failed with ContentTooShortError"
" error %s, retrying in 10 seconds... %s/%s" %
(www_path, err.code, retry_counter, retry_max))
time.sleep(10)
" error %s, retrying in %s seconds... %s/%s" %
(www_path, err.code, sleep_on_retry, retry_counter, retry_max))
time.sleep(sleep_on_retry)
continue
except DownloadVerificationFailedException as err:
if (cfg.PATHS['dl_cache_dir'] and
Expand Down Expand Up @@ -670,15 +672,15 @@ def file_downloader(www_path, retry_max=5, cache_name=None,
else:
# in other cases: try again
logger.info("Downloading %s failed with ConnectionError, "
"retrying in 10 seconds... %s/%s" %
(www_path, retry_counter, retry_max))
time.sleep(10)
"retrying in %s seconds... %s/%s" %
(www_path, sleep_on_retry, retry_counter, retry_max))
time.sleep(sleep_on_retry)
continue
except FTPSDownloadError as err:
logger.info("Downloading %s failed with FTPSDownloadError"
" error: '%s', retrying in 10 seconds... %s/%s" %
(www_path, err.orgerr, retry_counter, retry_max))
time.sleep(10)
" error: '%s', retrying in %s seconds... %s/%s" %
(www_path, err.orgerr, sleep_on_retry, retry_counter, retry_max))
time.sleep(sleep_on_retry)
continue

# See if we managed (fail is allowed)
Expand Down Expand Up @@ -1562,14 +1564,14 @@ def copdem_zone(lon_ex, lat_ex, source):
ew = 'W' if lon < 0 else 'E'
lat_str = '{}{:02.0f}'.format(ns, abs(lat))
lon_str = '{}{:03.0f}'.format(ew, abs(lon))
# COPDEM is global, if we miss tiles it is worth an error
try:
filename = df.loc[(df['Long'] == lon_str) &
(df['Lat'] == lat_str)]['CPP filename'].iloc[0]
flist.append((filename,
'Copernicus_DSM_{}_{}_00_{}_00'.format(asec, lat_str, lon_str)))
except IndexError:
raise InvalidDEMError('Could not find a matching Copernicus DEM file.')
# COPDEM is global, if we miss tiles it is probably in the ocean
pass
return flist


Expand Down