Skip to content

Commit

Permalink
Download To Temporary Files
Browse files Browse the repository at this point in the history
This patch makes sentinel5dl use a temporary file for downloading data,
moving the file to its intended location only after the download is
complete. This clearly marks half-finished downloads which otherwise
could be easily confused with fully downloaded files.

This fixes #52
This fixes #53
  • Loading branch information
lkiesow authored and tibroc committed Jan 9, 2020
1 parent 8df231d commit dee594b
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions sentinel5dl/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ def __http_request(path, filename=None, retries=9):
url = API + path.lstrip('/')
logger.debug('Requesting %s', url)
try:
with open(filename, 'wb') if filename else io.BytesIO() as f:
with open(f'{filename}.tmp', 'wb') if filename else io.BytesIO() as f:
curl = pycurl.Curl()
curl.setopt(curl.URL, url.encode('ascii', 'ignore'))
curl.setopt(curl.USERPWD, f'{USER}:{PASS}')
Expand All @@ -109,11 +109,16 @@ def __http_request(path, filename=None, retries=9):
curl.perform()
curl.close()

if not filename:
if filename:
os.rename(f'{filename}.tmp', filename)
else:
return f.getvalue()

except pycurl.error as err:
if not retries:
if filename:
logger.info('Removing temporary file %s.tmp', filename)
os.remove(f'{filename}.tmp')
raise err
logger.warning('Retrying failed HTTP request. %s', err)
time.sleep(1)
Expand Down

0 comments on commit dee594b

Please sign in to comment.