fix: end-to-end tests, fixes #119, fixes #98

tests/__init__.py

@@ -40,6 +40,7 @@
 
 TEST_RESOURCES_PATH = jp(dirn(__file__), "resources")
 RESOURCES_PATH = jp(dirn(__file__), "..", "eodag", "resources")
+TESTS_DOWNLOAD_PATH = "/tmp/eodag_tests"
 
 
 class EODagTestCase(unittest.TestCase):

tests/test_end_to_end.py

@@ -17,11 +17,12 @@
 # limitations under the License.
 from __future__ import unicode_literals
 
+import glob
 import multiprocessing
 import os
 import unittest
 
-from tests import TEST_RESOURCES_PATH
+from tests import TEST_RESOURCES_PATH, TESTS_DOWNLOAD_PATH
 from tests.context import EODataAccessGateway
 
 
@@ -30,25 +31,47 @@ class TestEODagEndToEnd(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls.eodag = EODataAccessGateway(
-            user_conf_file_path=os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
-        )
+
+        # use tests/resources/user_conf.yml if exists else default file ~/.config/eodag/eodag.yml
+        tests_user_conf = os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
+        if os.path.isfile(tests_user_conf):
+            cls.eodag = EODataAccessGateway(
+                user_conf_file_path=os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
+            )
+        else:
+            cls.eodag = EODataAccessGateway()
+
+        # create TESTS_DOWNLOAD_PATH is not exists
+        if not os.path.exists(TESTS_DOWNLOAD_PATH):
+            os.makedirs(TESTS_DOWNLOAD_PATH)
+
         # Ensure that we will be able to make search requests on only one provider, by
         # setting partial to False by
         # default for all the supported product types of all the providers
         for provider, conf in cls.eodag.providers_config.items():
-            for product_type, pt_conf in conf["products"].items():
+            for product_type, pt_conf in conf.products.items():
                 pt_conf["partial"] = False
+            # Change download directory to TESTS_DOWNLOAD_PATH for tests
+            if hasattr(conf, "download") and hasattr(conf.download, "outputs_prefix"):
+                conf.download.outputs_prefix = TESTS_DOWNLOAD_PATH
+            elif hasattr(conf, "api") and hasattr(conf.api, "outputs_prefix"):
+                conf.api.outputs_prefix = TESTS_DOWNLOAD_PATH
+            else:
+                # no outputs_prefix found for provider
+                pass
             # Force all providers implementing RestoSearch and defining how to retrieve
             # products by specifying the
             # location scheme to use https, enabling actual downloading of the product
-            if conf.get("search", {}).get("product_location_scheme", "https") == "file":
-                conf["search"]["product_location_scheme"] = "https"
+            if (
+                getattr(getattr(conf, "search", {}), "product_location_scheme", "https")
+                == "file"
+            ):
+                conf.search.product_location_scheme = "https"
             # Disable extraction
             try:  # Case HTTPDownload plugin
-                conf["download"]["extract"] = False
-            except KeyError:  # case api plugin
-                conf["api"]["extract"] = False
+                conf.download.extract = False
+            except (KeyError, AttributeError):  # case api plugin
+                conf.api.extract = False
 
     def setUp(self):
         self.downloaded_file_path = ""
@@ -78,7 +101,7 @@ def execute_search(self, provider, product_type, start, end, bbox):
             },
         }
         self.eodag.set_preferred_provider(provider)
-        results = self.eodag.search(product_type, **search_criteria)
+        results, _ = self.eodag.search(productType=product_type, **search_criteria)
         one_product = results[0]
         self.assertEqual(one_product.provider, provider)
         return one_product
@@ -88,6 +111,7 @@ def execute_download(self, product, expected_filename):
         download, then do some checks and delete the downloaded result from the
         filesystem.
         """
+
         dl_process = multiprocessing.Process(
             target=self.eodag.download, args=(product,)
         )
@@ -97,30 +121,40 @@ def execute_download(self, product, expected_filename):
             # Megabytes of data from provider
             # Consider changing this to fit a lower internet bandwidth
             dl_process.join(timeout=5)
+            # added this timeout loop, to handle long download start times
+            max_wait_time = 120
+            while (
+                dl_process.is_alive()
+                and not glob.glob("%s/[!quicklooks]*" % TESTS_DOWNLOAD_PATH)
+                and max_wait_time > 0
+            ):
+                dl_process.join(timeout=5)
+                max_wait_time -= 5
             if dl_process.is_alive():  # The process has timed out
                 dl_process.terminate()
         except KeyboardInterrupt:
             while dl_process.is_alive():
                 dl_process.terminate()
+
         self.assertIn(
-            expected_filename, os.listdir(self.eodag.user_config["outputs_prefix"])
+            expected_filename, os.listdir(product.downloader.config.outputs_prefix)
         )
         self.downloaded_file_path = os.path.join(
-            self.eodag.user_config["outputs_prefix"], expected_filename
+            product.downloader.config.outputs_prefix, expected_filename
         )
         # The partially downloaded file should be greater or equal to 1 MB
         self.assertGreaterEqual(os.stat(self.downloaded_file_path).st_size, 2 ** 20)
 
     def test_end_to_end_search_download_usgs(self):
         product = self.execute_search(
-            "USGS", "L8_OLI_TIRS_C1L1", "2017-03-01", "2017-03-15", (50, 50, 50.3, 50.3)
+            "usgs", "L8_OLI_TIRS_C1L1", "2017-03-01", "2017-03-15", (50, 50, 50.3, 50.3)
         )
         expected_filename = "{}.tar.bz".format(product.properties["title"])
         self.execute_download(product, expected_filename)
 
     def test_end_to_end_search_download_airbus(self):
         product = self.execute_search(
-            "airbus-ds",
+            "sobloo",
             "S2_MSI_L1C",
             "2018-05-01",
             "2018-06-01",
@@ -136,7 +170,7 @@ def test_end_to_end_search_download_airbus(self):
 
     @unittest.skip(
         "Download of S2_MSI_L1C Products on peps provider before 2016/12/05 is known "
-        "o be asynchronous and this feature is not present in eodag at the moment"
+        "to be asynchronous and this feature is not present in eodag at the moment"
     )
     def test_end_to_end_search_download_peps_before_20161205(self):
         product = self.execute_search(
@@ -153,26 +187,25 @@ def test_end_to_end_search_download_peps_after_20161205(self):
         product = self.execute_search(
             "peps",
             "S2_MSI_L1C",
-            "2018-06-05",
-            "2018-06-16",
-            (137.772897, -37.134202, 153.749135, 73.885986),
+            "2019-11-08",
+            "2019-11-16",
+            (137.772897, 13.134202, 153.749135, 23.885986),
         )
         expected_filename = "{}.zip".format(product.properties["title"])
         self.execute_download(product, expected_filename)
 
-    def test_get_quiclook_peps(self):
+    def test_get_quicklook_peps(self):
         product = self.execute_search(
-            "peps", "S2_MSI_L1C", "2014-03-01", "2017-03-15", (50, 50, 50.3, 50.3)
+            "peps", "S2_MSI_L1C", "2019-03-01", "2019-03-15", (50, 50, 50.3, 50.3)
         )
         quicklook_file_path = product.get_quicklook(filename="peps_quicklook")
+        # TearDown will remove quicklook_file_path on end
+        self.downloaded_file_path = quicklook_file_path
 
         self.assertNotEqual(quicklook_file_path, "")
         self.assertEqual(os.path.basename(quicklook_file_path), "peps_quicklook")
         self.assertEqual(
             os.path.dirname(quicklook_file_path),
-            os.path.join(self.eodag.user_config["outputs_prefix"], "quicklooks"),
+            os.path.join(product.downloader.config.outputs_prefix, "quicklooks"),
         )
         self.assertGreaterEqual(os.stat(quicklook_file_path).st_size, 2 ** 5)
-
-        # Teardown
-        os.remove(quicklook_file_path)