Simplify HTTP Requests

This patch simplifies the HTTP requests by combining the different
methods which were already using pycURL in mostly the same manner
anyway.

sentinel5dl/__init__.py

@@ -38,46 +38,29 @@ def __md5(filename):
     return hash_md5.hexdigest().upper()
 
 
-def __http_request(path):
-    '''Make an HTTP request to a given URL with optional parameters.
-
-    :param path: Request path relative to the base API
-    :returns: The response body
+def __http_request(path, filename=None):
+    '''Make an HTTP request to the API via HTTP, optionally downloading the
+    response.
+
+    :param path: Request path relative to the base API.
+    :param filename: Optional output file name. Note that this file will be
+                     overwritten if it already exists. If no filename is
+                     provided, the response will be returned.
+    :returns: The response body or None if a filename is provided.
     '''
-    buf = io.BytesIO()
-    curl = pycurl.Curl()
     url = API + path.lstrip('/')
     logger.debug(f'Requesting {url}')
-    curl.setopt(curl.URL, url.encode('ascii', 'ignore'))
-
-    curl.setopt(curl.USERPWD, f'{USER}:{PASS}')
-    curl.setopt(curl.WRITEFUNCTION, buf.write)
-    curl.setopt(curl.FAILONERROR, True)
-    curl.perform()
-    curl.close()
-    result = buf.getvalue()
-    buf.close()
-    return result
-
-
-def __http_download(path, filename):
-    '''Download a file from the API via HTTP.
-
-    :param path: Request path relative to the base API
-    :param filename: Output file name. Note that this file will be overwritten
-                     if it already exists.
-    '''
-    with open(filename, 'wb') as f:
+    with open(filename, 'wb') if filename else io.BytesIO() as f:
         curl = pycurl.Curl()
-        url = API + path.lstrip('/')
-        logger.debug(f'Requesting {url} for download')
         curl.setopt(curl.URL, url.encode('ascii', 'ignore'))
 
         curl.setopt(curl.USERPWD, f'{USER}:{PASS}')
         curl.setopt(curl.WRITEDATA, f)
         curl.setopt(curl.FAILONERROR, True)
         curl.perform()
         curl.close()
+        if not filename:
+            return f.getvalue()
 
 
 def _search(polygon, begin_ts, end_ts, product, processing_level, offset,
@@ -173,4 +156,4 @@ def download(products, output_dir='.'):
             logger.info(f'Overriding {filename} since md5 hash differs.')
 
         # Download file
-        __http_download(path, filename)
+        __http_request(path, filename)

tests/__main__.py

@@ -9,9 +9,16 @@
 
 class TestSentinel5dl(unittest.TestCase):
 
-    def _mock_http_request(self, path):
+    def _mock_http_request(self, path, filename=None):
         '''Mock HTTP requests to the ESA API
         '''
+        if filename is not None:
+            self._count_download += 1
+            with open(filename, 'wb') as f:
+                f.write(b'123')
+            return
+
+        # no nownload
         self._count_request += 1
         if path.startswith('/api/stub/products?'):
             with open(os.path.join(testpath, 'products.json'), 'rb') as f:
@@ -20,20 +27,11 @@ def _mock_http_request(self, path):
             # MD5 checksum for string `123`
             return b'202CB962AC59075B964B07152D234B70'
 
-    def _mock_http_download(self, path, filename):
-        '''Mock downloads from the ESA API.
-        The content of all files downloaded will be `123`
-        '''
-        self._count_download += 1
-        with open(filename, 'wb') as f:
-            f.write(b'123')
-
     def setUp(self):
         '''Patch cURL based operation in sentinel5dl so that we do not really
         make any HTTP requests and reset the request counters.
         '''
         setattr(sentinel5dl, '__http_request', self._mock_http_request)
-        setattr(sentinel5dl, '__http_download', self._mock_http_download)
         self._count_request = 0
         self._count_download = 0