Skip to content

Commit

Permalink
Change the COPDEM server to prism-dem-open.copernicus.eu (#1684)
Browse files Browse the repository at this point in the history 
* change the COPDEM server to prism-dem-open.copernicus.eu; update the tests

* Change to 2023 version, no tile list required

---------

Co-authored-by: Fabien Maussion <fabien.maussion@bristol.ac.uk>
  • Loading branch information
@dcodrut @fmaussion
dcodrut and fmaussion committed Apr 18, 2024
1 parent fc233f4 commit cd4f53c
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 52 deletions.
3 changes: 3 additions & 0 deletions docs/whats-new.rst
View file
Expand Up @@ -43,6 +43,9 @@ Bug fixes
different CRS. If they do, all files are converted to a single, common CRS
to make sure they can be merged without issues (:pull:`1698`).
By `Patrick Schmitt <https://github.com/pat-schmitt>`_
- COPDEM data is now downloaded from public http server (no credentials
required) (:pull:`1684`).
By `Codruț Diaconu <https://github.com/dcodrut>`_


v1.6.1 (August 27, 2023)
Expand Down
40 changes: 16 additions & 24 deletions oggm/tests/test_utils.py
View file
Expand Up @@ -2230,8 +2230,7 @@ def test_copdem90(self):

# Make a fake topo file
deep_path = os.path.join(self.dldir,
'DEM1_SAR_DGE_90_20110517T170701_20140817T170857'
'_ADS_000000_3682.DEM',
'COP-DEM_GLO-90-DGED__2023_1',
'Copernicus_DSM_30_N46_00_E010_00', 'DEM')
utils.mkdir(deep_path)
upper_path = os.path.dirname(os.path.dirname(deep_path))
Expand All @@ -2242,14 +2241,13 @@ def test_copdem90(self):
archive='tar', extension='.tar')

def down_check(url, *args, **kwargs):
expected = ('ftps://cdsdata.copernicus.eu:990/datasets/'
'COP-DEM_GLO-90-DGED/2022_1/'
'DEM1_SAR_DGE_90_20110517T170701_20140817T170857_ADS_'
'000000_3682.DEM.tar')
expected = ('https://prism-dem-open.copernicus.eu/pd-desk-open-access/prismDownload/'
'COP-DEM_GLO-90-DGED__2023_1/'
'Copernicus_DSM_30_N46_00_E010_00.tar')
self.assertEqual(expected, url)
return tf

with FakeDownloadManager('download_with_authentication', down_check):
with FakeDownloadManager('file_downloader', down_check):
of, source = utils.get_topo_file([10.5, 10.8], [46.6, 46.8],
source='COPDEM90')

Expand All @@ -2260,8 +2258,7 @@ def test_copdem30(self):

# Make a fake topo file
deep_path = os.path.join(self.dldir,
'DEM1_SAR_DGE_30_20110517T170701_20140817T170857'
'_ADS_000000_bma2.DEM',
'COP-DEM_GLO-30-DGED__2023_1',
'Copernicus_DSM_10_N46_00_E010_00', 'DEM')
utils.mkdir(deep_path)
upper_path = os.path.dirname(os.path.dirname(deep_path))
Expand All @@ -2272,14 +2269,13 @@ def test_copdem30(self):
archive='tar', extension='.tar')

def down_check(url, *args, **kwargs):
expected = ('ftps://cdsdata.copernicus.eu:990/datasets/'
'COP-DEM_GLO-30-DGED/2022_1/'
'DEM1_SAR_DGE_30_20110517T170701_20140817T170857_ADS_'
'000000_bma2.DEM.tar')
expected = ('https://prism-dem-open.copernicus.eu/pd-desk-open-access/prismDownload/'
'COP-DEM_GLO-30-DGED__2023_1/'
'Copernicus_DSM_10_N46_00_E010_00.tar')
self.assertEqual(expected, url)
return tf

with FakeDownloadManager('download_with_authentication', down_check):
with FakeDownloadManager('file_downloader', down_check):
of, source = utils.get_topo_file([10.5, 10.8], [46.6, 46.8],
source='COPDEM30')

Expand Down Expand Up @@ -2762,29 +2758,25 @@ def test_dem3_viewpano_zone(self):
def test_copdemzone(self):
z = utils.copdem_zone([-77.6, -77.3], [-9.8, -9.5], 'COPDEM90')
self.assertTrue(len(z) == 1)
self.assertEqual(('DEM1_SAR_DGE_90_20110427T104941_20140819T105300_'
'ADS_000000_7913.DEM.tar'), z[0][0])
self.assertEqual('Copernicus_DSM_30_S10_00_W078_00', z[0][1])
self.assertEqual('Copernicus_DSM_30_S10_00_W078_00', z[0])

z = utils.copdem_zone([-77.6, -77.3], [-9.8, -9.5], 'COPDEM30')
self.assertTrue(len(z) == 1)
self.assertEqual(('DEM1_SAR_DGE_30_20110427T104941_20140819T105300_'
'ADS_000000_elyt.DEM.tar'), z[0][0])
self.assertEqual('Copernicus_DSM_10_S10_00_W078_00', z[0][1])
self.assertEqual('Copernicus_DSM_10_S10_00_W078_00', z[0])

z = utils.copdem_zone([7.89, 8.12], [46.37, 46.59], 'COPDEM90')
self.assertTrue(len(z) == 2)
self.assertTrue('Copernicus_DSM_30_N46_00_E008_00' in
[z[0][1], z[1][1]])
[z[0], z[1]])
self.assertTrue('Copernicus_DSM_30_N46_00_E007_00' in
[z[0][1], z[1][1]])
[z[0], z[1]])

z = utils.copdem_zone([7.89, 8.12], [46.37, 46.59], 'COPDEM30')
self.assertTrue(len(z) == 2)
self.assertTrue('Copernicus_DSM_10_N46_00_E008_00' in
[z[0][1], z[1][1]])
[z[0], z[1]])
self.assertTrue('Copernicus_DSM_10_N46_00_E007_00' in
[z[0][1], z[1][1]])
[z[0], z[1]])

def test_is_dem_source_available(self):
assert utils.is_dem_source_available('SRTM', [11, 11], [47, 47])
Expand Down
53 changes: 25 additions & 28 deletions oggm/utils/_downloads.py
View file
Expand Up @@ -1137,12 +1137,12 @@ def _download_topo_file_from_cluster_unlocked(fname):
return outpath


def _download_copdem_file(cppfile, tilename, source):
def _download_copdem_file(tilename, source):
with get_lock():
return _download_copdem_file_unlocked(cppfile, tilename, source)
return _download_copdem_file_unlocked(tilename, source)


def _download_copdem_file_unlocked(cppfile, tilename, source):
def _download_copdem_file_unlocked(tilename, source):
"""Checks if Copernicus DEM file is in the directory, if not download it.
cppfile : name of the tarfile to download
Expand All @@ -1164,12 +1164,9 @@ def _download_copdem_file_unlocked(cppfile, tilename, source):
return demfile

# Did we download it yet?
ftpfile = ('ftps://cdsdata.copernicus.eu:990/' +
'datasets/COP-DEM_GLO-{}-DGED/2022_1/'.format(source[-2:]) +
cppfile)

dest_file = download_with_authentication(ftpfile,
'spacedata.copernicus.eu')
url = (f"https://prism-dem-open.copernicus.eu/pd-desk-open-access/prismDownload"
f"/COP-DEM_GLO-{source[-2:]}-DGED__2023_1/{tilename}.tar")
dest_file = file_downloader(url)

# None means we tried hard but we couldn't find it
if not dest_file:
Expand Down Expand Up @@ -1563,7 +1560,21 @@ def alaska_dem_zone(lon_ex, lat_ex):


def copdem_zone(lon_ex, lat_ex, source):
"""Returns a list of Copernicus DEM tarfile and tilename tuples
"""Returns a list of Copernicus DEM tilenames.
New:
We now go for the PRISM server download, which has a slightly different
API.
Parse available datasets:
curl -k -H "accept: csv" https://prism-dem-open.copernicus.eu/pd-desk-open-access/publicDemURLs
Parse available tiles:
curl -k -H "accept: csv" https://prism-dem-open.copernicus.eu/pd-desk-open-access/publicDemURLs/COP-DEM_GLO-30-DGED__2023_1
But I'm not sure why parsing the tiles is necessary, we can just use the
tilename as is and try to download it. If it doesn't exist, it's probably
OK to skip it.
"""

# because we use both meters and arc secs in our filenames...
Expand All @@ -1574,13 +1585,6 @@ def copdem_zone(lon_ex, lat_ex, source):
else:
raise InvalidDEMError('COPDEM Version not valid.')

# either reuse or load lookup table
if source in cfg.DATA:
df = cfg.DATA[source]
else:
df = pd.read_csv(get_demo_file('{}_2022_1.csv'.format(source.lower())))
cfg.DATA[source] = df

# adding small buffer for unlikely case where one lon/lat_ex == xx.0
lons = np.arange(np.floor(lon_ex[0]-1e-9), np.ceil(lon_ex[1]+1e-9))
lats = np.arange(np.floor(lat_ex[0]-1e-9), np.ceil(lat_ex[1]+1e-9))
Expand All @@ -1594,14 +1598,7 @@ def copdem_zone(lon_ex, lat_ex, source):
ew = 'W' if lon < 0 else 'E'
lat_str = '{}{:02.0f}'.format(ns, abs(lat))
lon_str = '{}{:03.0f}'.format(ew, abs(lon))
try:
filename = df.loc[(df['Long'] == lon_str) &
(df['Lat'] == lat_str)]['CPP filename'].iloc[0]
flist.append((filename,
'Copernicus_DSM_{}_{}_00_{}_00'.format(asec, lat_str, lon_str)))
except IndexError:
# COPDEM is global, if we miss tiles it is probably in the ocean
pass
flist.append('Copernicus_DSM_{}_{}_00_{}_00'.format(asec, lat_str, lon_str))
return flist


Expand Down Expand Up @@ -2480,9 +2477,9 @@ def get_topo_file(lon_ex=None, lat_ex=None, gdir=None, *,
files.append(_download_srtm_file(z))

if source in ['COPDEM30', 'COPDEM90']:
filetuple = copdem_zone(lon_ex, lat_ex, source)
for cpp, eop in filetuple:
files.append(_download_copdem_file(cpp, eop, source))
tilenames = copdem_zone(lon_ex, lat_ex, source)
for tilename in tilenames:
files.append(_download_copdem_file(tilename, source))

if source == 'NASADEM':
zones = nasadem_zone(lon_ex, lat_ex)
Expand Down

0 comments on commit cd4f53c

Please sign in to comment.