From 03f4148332a9b7cba0b8ea420aa0fb885d234694 Mon Sep 17 00:00:00 2001
From: Lars Kiesow <lkiesow@uos.de>
Date: Fri, 18 Oct 2019 01:31:48 +0200
Subject: [PATCH] Simplify HTTP Requests

This patch simplifies the HTTP requests by combining the different
methods which were already using pycURL in mostly the same manner
anyway.
---
 sentinel5dl/__init__.py | 43 +++++++++++++----------------------------
 tests/__main__.py       | 18 ++++++++---------
 2 files changed, 21 insertions(+), 40 deletions(-)

diff --git a/sentinel5dl/__init__.py b/sentinel5dl/__init__.py
index bf7cfdf..278956d 100644
--- a/sentinel5dl/__init__.py
+++ b/sentinel5dl/__init__.py
@@ -38,39 +38,20 @@ def __md5(filename):
     return hash_md5.hexdigest().upper()
 
 
-def __http_request(path):
-    '''Make an HTTP request to a given URL with optional parameters.
-
-    :param path: Request path relative to the base API
-    :returns: The response body
+def __http_request(path, filename=None):
+    '''Make an HTTP request to the API via HTTP, optionally downloading the
+    response.
+
+    :param path: Request path relative to the base API.
+    :param filename: Optional output file name. Note that this file will be
+                     overwritten if it already exists. If no filename is
+                     provided, the response will be returned.
+    :returns: The response body or None if a filename is provided.
     '''
-    buf = io.BytesIO()
-    curl = pycurl.Curl()
     url = API + path.lstrip('/')
     logger.debug(f'Requesting {url}')
-    curl.setopt(curl.URL, url.encode('ascii', 'ignore'))
-
-    curl.setopt(curl.USERPWD, f'{USER}:{PASS}')
-    curl.setopt(curl.WRITEFUNCTION, buf.write)
-    curl.setopt(curl.FAILONERROR, True)
-    curl.perform()
-    curl.close()
-    result = buf.getvalue()
-    buf.close()
-    return result
-
-
-def __http_download(path, filename):
-    '''Download a file from the API via HTTP.
-
-    :param path: Request path relative to the base API
-    :param filename: Output file name. Note that this file will be overwritten
-                     if it already exists.
-    '''
-    with open(filename, 'wb') as f:
+    with open(filename, 'wb') if filename else io.BytesIO() as f:
         curl = pycurl.Curl()
-        url = API + path.lstrip('/')
-        logger.debug(f'Requesting {url} for download')
         curl.setopt(curl.URL, url.encode('ascii', 'ignore'))
 
         curl.setopt(curl.USERPWD, f'{USER}:{PASS}')
@@ -78,6 +59,8 @@ def __http_download(path, filename):
         curl.setopt(curl.FAILONERROR, True)
         curl.perform()
         curl.close()
+        if not filename:
+            return f.getvalue()
 
 
 def _search(polygon, begin_ts, end_ts, product, processing_level, offset,
@@ -173,4 +156,4 @@ def download(products, output_dir='.'):
             logger.info(f'Overriding {filename} since md5 hash differs.')
 
         # Download file
-        __http_download(path, filename)
+        __http_request(path, filename)
diff --git a/tests/__main__.py b/tests/__main__.py
index 6ddda8f..a3472df 100644
--- a/tests/__main__.py
+++ b/tests/__main__.py
@@ -9,9 +9,16 @@
 
 class TestSentinel5dl(unittest.TestCase):
 
-    def _mock_http_request(self, path):
+    def _mock_http_request(self, path, filename=None):
         '''Mock HTTP requests to the ESA API
         '''
+        if filename is not None:
+            self._count_download += 1
+            with open(filename, 'wb') as f:
+                f.write(b'123')
+            return
+
+        # no nownload
         self._count_request += 1
         if path.startswith('/api/stub/products?'):
             with open(os.path.join(testpath, 'products.json'), 'rb') as f:
@@ -20,20 +27,11 @@ def _mock_http_request(self, path):
             # MD5 checksum for string `123`
             return b'202CB962AC59075B964B07152D234B70'
 
-    def _mock_http_download(self, path, filename):
-        '''Mock downloads from the ESA API.
-        The content of all files downloaded will be `123`
-        '''
-        self._count_download += 1
-        with open(filename, 'wb') as f:
-            f.write(b'123')
-
     def setUp(self):
         '''Patch cURL based operation in sentinel5dl so that we do not really
         make any HTTP requests and reset the request counters.
         '''
         setattr(sentinel5dl, '__http_request', self._mock_http_request)
-        setattr(sentinel5dl, '__http_download', self._mock_http_download)
         self._count_request = 0
         self._count_download = 0