feat: restore pruned providers when configuration is updated (#844)

Co-authored-by: Sylvain Brunato <sylvain.brunato@c-s.fr>

eodag/api/core.py

@@ -134,6 +134,8 @@ def __init__(self, user_conf_file_path=None, locations_conf_path=None):
         # use updated and checked providers_config
         self.providers_config = self._plugins_manager.providers_config
 
+        # store pruned providers configs
+        self._pruned_providers_config = {}
         # filter out providers needing auth that have no credentials set
         self._prune_providers_list()
 
@@ -301,6 +303,18 @@ def update_providers_config(self, yaml_conf):
         :type yaml_conf: str
         """
         conf_update = yaml.safe_load(yaml_conf)
+
+        # restore the pruned configuration
+        for provider in list(self._pruned_providers_config.keys()):
+            if provider in conf_update:
+                logger.info(
+                    "%s: provider restored from the pruned configurations",
+                    provider,
+                )
+                self.providers_config[provider] = self._pruned_providers_config.pop(
+                    provider
+                )
+
         # check if metada-mapping as already been built as jsonpath in providers_config
         for provider, provider_conf in conf_update.items():
             if (
@@ -335,6 +349,7 @@ def update_providers_config(self, yaml_conf):
                 )
 
         override_config_from_mapping(self.providers_config, conf_update)
+
         stac_provider_config = load_stac_provider_config()
         for provider in conf_update.keys():
             provider_config_init(self.providers_config[provider], stac_provider_config)
@@ -356,7 +371,9 @@ def _prune_providers_list(self):
                 )
                 if not credentials_exist:
                     # credentials needed but not found
-                    del self.providers_config[provider]
+                    self._pruned_providers_config[provider] = self.providers_config.pop(
+                        provider
+                    )
                     update_needed = True
                     logger.info(
                         "%s: provider needing auth for search has been pruned because no crendentials could be found",
@@ -365,7 +382,9 @@ def _prune_providers_list(self):
             elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
                 if not hasattr(conf, "auth"):
                     # credentials needed but no auth plugin was found
-                    del self.providers_config[provider]
+                    self._pruned_providers_config[provider] = self.providers_config.pop(
+                        provider
+                    )
                     update_needed = True
                     logger.info(
                         "%s: provider needing auth for search has been pruned because no auth plugin could be found",
@@ -380,15 +399,19 @@ def _prune_providers_list(self):
                 )
                 if not credentials_exist:
                     # credentials needed but not found
-                    del self.providers_config[provider]
+                    self._pruned_providers_config[provider] = self.providers_config.pop(
+                        provider
+                    )
                     update_needed = True
                     logger.info(
                         "%s: provider needing auth for search has been pruned because no crendentials could be found",
                         provider,
                     )
             elif not hasattr(conf, "api") and not hasattr(conf, "search"):
                 # provider should have at least an api or search plugin
-                del self.providers_config[provider]
+                self._pruned_providers_config[provider] = self.providers_config.pop(
+                    provider
+                )
                 logger.info(
                     "%s: provider has been pruned because no api or search plugin could be found",
                     provider,

eodag/api/product/metadata_mapping.py

@@ -1013,6 +1013,17 @@ def mtd_cfg_as_conversion_and_querypath(src_dict, dest_dict={}, result_type="jso
     :returns: dest_dict
     :rtype: dict
     """
+    # check if the configuration has already been converted
+    some_configured_value = (
+        next(iter(dest_dict.values())) if dest_dict else next(iter(src_dict.values()))
+    )
+    if (
+        isinstance(some_configured_value, list)
+        and isinstance(some_configured_value[1], tuple)
+        or isinstance(some_configured_value, tuple)
+    ):
+        return dest_dict or src_dict
+
     if not dest_dict:
         dest_dict = deepcopy(src_dict)
     for metadata in src_dict:

tests/integration/test_core_config.py

@@ -119,6 +119,55 @@ def test_core_providers_config_update(
             self.dag.providers_config["foo_provider"].auth.type, "GenericAuth"
         )
 
+        # update pruned provider with credentials
+        self.assertNotIn("usgs", self.dag.providers_config)
+        self.dag.update_providers_config(
+            """
+            usgs:
+                api:
+                    credentials:
+                        username: foo
+                        password: bar
+            """
+        )
+        self.assertIsInstance(self.dag.providers_config["usgs"].api, PluginConfig)
+        self.assertEqual(
+            self.dag.providers_config["usgs"].api.credentials["username"], "foo"
+        )
+
+        # add new provider that requires auth but without credentials
+        self.dag.update_providers_config(
+            """
+            bar_provider:
+                search:
+                    type: StacSearch
+                    api_endpoint: https://foo.bar/search
+                    need_auth: True
+                products:
+                    GENERIC_PRODUCT_TYPE:
+                        productType: '{productType}'
+            """
+        )
+        self.assertIn("bar_provider", self.dag.providers_config)
+
+        # update provider with credentials
+        self.dag.update_providers_config(
+            """
+            bar_provider:
+                auth:
+                    credentials:
+                        username: bar
+                        password: foo
+            """
+        )
+        self.assertIsInstance(
+            self.dag.providers_config["bar_provider"].auth, PluginConfig
+        )
+        self.assertEqual(
+            self.dag.providers_config["bar_provider"].auth.credentials["username"],
+            "bar",
+        )
+
 
 class TestCoreProductTypesConfig(TestCase):
     def setUp(self):