Skip to content

Commit

Permalink
Fix #66
Browse files Browse the repository at this point in the history
Concatenate 'title' and 'id' of a product to build it's name.
Add a 'sanitize' method to remove spaces, punctuation,.... in product name
  • Loading branch information
saimeCS committed Nov 5, 2018
1 parent 6f7cb0e commit 0a280d1
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 2 deletions.
9 changes: 7 additions & 2 deletions eodag/plugins/download/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

from eodag.plugins.download.base import Download
from eodag.plugins.search.base import MisconfiguredError

from eodag.utils import sanitize

logger = logging.getLogger('eodag.plugins.download.http')

Expand Down Expand Up @@ -68,7 +68,12 @@ def download(self, product, auth=None, progress_callback=None):
# Strong asumption made here: all products downloaded will be zip files
# If they are not, the '.zip' extension will be removed when they are downloaded and returned as is
prefix = os.path.abspath(self.config['outputs_prefix'])
fs_path = os.path.join(prefix, '{}.zip'.format(product.properties['title']))
sanitized_title = sanitize(product.properties['title'])
collision_avoidance_suffix = '' \
if sanitized_title == product.properties['title'] \
else ('-' + sanitize(product.properties['id']))
fs_path = os.path.join(prefix, '{}{}.zip'.format(sanitize(product.properties['title']),
collision_avoidance_suffix))
download_records_dir = os.path.join(prefix, '.downloaded')
try:
os.makedirs(download_records_dir)
Expand Down
25 changes: 25 additions & 0 deletions eodag/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from __future__ import unicode_literals

import re
import string
import sys
import types
import unicodedata
Expand All @@ -28,6 +29,7 @@
from rasterio.crs import CRS
from requests.auth import AuthBase
from tqdm import tqdm, tqdm_notebook
from unidecode import unidecode


# All modules using these should import them from utils package
Expand Down Expand Up @@ -119,6 +121,29 @@ def utf8_everywhere(mapping):
)


def sanitize(value):
"""Sanitize string to be used as a name of a directory.
>>> sanitize('productName')
'productName'
>>> sanitize('name with multiple spaces')
'name_with_multiple_spaces'
>>> sanitize('âtre fête île alcôve bûche çà génèse où Noël ovoïde capharnaüm')
'atre_fete_ile_alcove_buche_ca_genese_ou_Noel_ovoide_capharnaum'
>>> sanitize('replace,ponctuation:;signs!?byunderscorekeeping-hyphen.dot_and_underscore')
'replace_ponctuation_signs_byunderscorekeeping-hyphen.dot_and_underscore'
"""
# remove accents
rv = unidecode(value)
# replace punctuation signs and spaces by underscore
## keep hyphen, dot and underscore from punctuation
tobereplaced = re.sub('[-_.]', '', string.punctuation)
## add spaces to be removed
tobereplaced += '\s'

rv = re.sub('[' + tobereplaced + ']+', '_', rv)
return str(rv)


def mutate_dict_in_place(func, mapping):
"""Apply func to values of mapping.
Expand Down
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
'jsonpath-rw',
'lxml',
'xarray',
'unidecode == 1.0.22',
],
extras_require={
'dev': [
Expand Down

0 comments on commit 0a280d1

Please sign in to comment.