From d4413d4fa8ab3a6ffe6c3312ae0a60620d4bdfd6 Mon Sep 17 00:00:00 2001 From: Lars Kiesow Date: Thu, 2 Jan 2020 14:36:51 +0100 Subject: [PATCH] Download To Temporary Files This patch makes sentinel5dl use a temporary file for downloading data, moving the file to its intended location only after the download is complete. This clearly marks half-finished downloads which otherwise could be easily confused with fully downloaded files. This fixes #52 This fixes #53 --- sentinel5dl/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sentinel5dl/__init__.py b/sentinel5dl/__init__.py index 4cc4013..4748b35 100644 --- a/sentinel5dl/__init__.py +++ b/sentinel5dl/__init__.py @@ -91,7 +91,7 @@ def __http_request(path, filename=None, retries=9): url = API + path.lstrip('/') logger.debug('Requesting %s', url) try: - with open(filename, 'wb') if filename else io.BytesIO() as f: + with open(f'{filename}.tmp', 'wb') if filename else io.BytesIO() as f: curl = pycurl.Curl() curl.setopt(curl.URL, url.encode('ascii', 'ignore')) curl.setopt(curl.USERPWD, f'{USER}:{PASS}') @@ -109,11 +109,16 @@ def __http_request(path, filename=None, retries=9): curl.perform() curl.close() - if not filename: + if filename: + os.rename(f'{filename}.tmp', filename) + else: return f.getvalue() except pycurl.error as err: if not retries: + if filename: + logger.info('Removing temporary file %s.tmp', filename) + os.remove(f'{filename}.tmp') raise err logger.warning('Retrying failed HTTP request. %s', err) time.sleep(1)