HDXDSYS-589 kwargs take precedence over environment variables (#64)

* Make kwargs take precedence over environment variables which take precedence over configuration dictionary or file

* Fix test

documentation/main.md

@@ -52,6 +52,12 @@ The library has detailed API documentation which can be found in the menu at the
 
 
 ## Breaking Changes
+From 6.2.6, kwargs take preference over environment variables which take
+preference over configuration files
+
+From 6.2.5, environment variables take preference over kwargs which take
+preference over configuration files
+
 From 6.1.5, any method or parameter with "reference_period" in it is renamed
 to "time_period" and  any method or parameter with "file_type" in it is renamed
 to "format"

requirements.txt

@@ -251,7 +251,7 @@ urllib3==2.2.1
     # via
     #   libhxl
     #   requests
-validators==0.22.0
+validators==0.23.1
     # via frictionless
 virtualenv==20.25.1
     # via pre-commit

src/hdx/api/configuration.py

@@ -34,7 +34,7 @@ class Configuration(UserDict):
         user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yaml.
         user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
         hdx_url (str): HDX url to use. Overrides hdx_site.
-        hdx_site (str): HDX site to use eg. prod, test.
+        hdx_site (str): HDX site to use eg. prod, test. Defaults to stage.
         hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
         hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
         hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
@@ -198,18 +198,22 @@ def __init__(self, **kwargs: Any) -> None:
                     prefix=Configuration.prefix, **self.data
                 )
 
-        hdx_url = os.getenv(
-            "HDX_URL", kwargs.get("hdx_url", self.data.get("hdx_url"))
-        )
+        hdx_url = kwargs.get("hdx_url")
+        if not hdx_url:
+            hdx_url = os.getenv("HDX_URL")
+            if not hdx_url:
+                hdx_url = self.data.get("hdx_url")
         if hdx_url:
             hdx_site = "custom"
             self.hdx_site = "hdx_custom_site"
             hdx_url = hdx_url.rstrip("/")
             self.data[self.hdx_site] = {"url": hdx_url}
         else:
-            hdx_site = os.getenv(
-                "HDX_SITE", kwargs.get("hdx_site", self.data.get("hdx_site"))
-            )
+            hdx_site = kwargs.get("hdx_site")
+            if not hdx_site:
+                hdx_site = os.getenv("HDX_SITE")
+                if not hdx_site:
+                    hdx_site = self.data.get("hdx_site")
             if not hdx_site:
                 hdx_site = "stage"
             self.hdx_site = f"hdx_{hdx_site}_site"
@@ -228,16 +232,16 @@ def __init__(self, **kwargs: Any) -> None:
         if self.hdx_site == "hdx_custom_site":
             hdx_key = None
         else:
-            hdx_key = os.getenv(f"HDX_KEY_{hdx_site.upper()}")
+            hdx_key_site = f"hdx_key_{hdx_site}"
+            hdx_key = kwargs.get(hdx_key_site)
             if not hdx_key:
-                hdx_key_site = f"hdx_key_{hdx_site}"
-                hdx_key = kwargs.get(hdx_key_site)
+                hdx_key = os.getenv(f"HDX_KEY_{hdx_site.upper()}")
                 if not hdx_key:
                     hdx_key = self.data.get(hdx_key_site)
         if not hdx_key:
-            hdx_key = os.getenv("HDX_KEY")
+            hdx_key = kwargs.get("hdx_key")
             if not hdx_key:
-                hdx_key = kwargs.get("hdx_key")
+                hdx_key = os.getenv("HDX_KEY")
                 if not hdx_key:
                     hdx_key = self.data.get("hdx_key")
         if hdx_key:
@@ -529,7 +533,7 @@ def setup(
             user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yaml.
             user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
             hdx_url (str): HDX url to use. Overrides hdx_site.
-            hdx_site (str): HDX site to use eg. prod, test.
+            hdx_site (str): HDX site to use eg. prod, test. Defaults to stage.
             hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
             hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
             hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
@@ -569,7 +573,7 @@ def _create(
             user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yaml.
             user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
             hdx_url (str): HDX url to use. Overrides hdx_site.
-            hdx_site (str): HDX site to use eg. prod, test.
+            hdx_site (str): HDX site to use eg. prod, test.  Defaults to stage.
             hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
             hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
             hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
@@ -608,7 +612,7 @@ def create(
             user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yaml.
             user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
             hdx_url (str): HDX url to use. Overrides hdx_site.
-            hdx_site (str): HDX site to use eg. prod, test.
+            hdx_site (str): HDX site to use eg. prod, test. Defaults to stage.
             hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
             hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
             hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR

tests/hdx/api/test_configuration.py

@@ -62,10 +62,26 @@ def test_init(
         Configuration.default_hdx_config_yaml = "NOT EXIST"
         with pytest.raises(UserAgentError):
             Configuration()
+        with pytest.raises(ConfigurationError):
+            Configuration(user_agent="test")
+        configuration = Configuration(user_agent="test", hdx_key="test")
+        assert configuration.hdx_read_only is False
+        assert (
+            configuration.get_hdx_site_url()
+            == "https://stage.data-humdata-org.ahconu.org"
+        )
+        assert (
+            configuration.get_user_agent()
+            == f"HDXPythonLibrary/{__version__}-test"
+        )
+        configuration = Configuration(full_agent="test", hdx_key="test")
+        assert configuration.get_user_agent() == "test"
         Configuration.default_hdx_config_yaml = default_config_file
 
         Configuration.default_hdx_config_yaml = hdx_config_yaml
-        assert Configuration(user_agent="test").get_api_key() == "12345"
+        configuration = Configuration(user_agent="test")
+        assert configuration.get_api_key() == "12345"
+
         Configuration.default_hdx_config_yaml = default_config_file
 
         with pytest.raises(IOError):

tests/hdx/facades/test_keyword_arguments.py

@@ -90,10 +90,10 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
         assert testresult.actual_result == my_test_key
         UserAgent.clear_global()
         testresult.actual_result = None
-        monkeypatch.setenv("HDX_KEY", my_test_key)
+        monkeypatch.setenv("HDX_KEY", "aaaa")
         facade(
             my_testfnkw,
-            hdx_key="aaaa",
+            hdx_key=my_test_key,
             user_agent=my_user_agent,
             hdx_config_yaml=hdx_config_yaml,
             project_config_yaml=project_config_yaml,
@@ -117,10 +117,10 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
         )
         UserAgent.clear_global()
         testresult.actual_result = None
-        monkeypatch.setenv("HDX_SITE", my_test_hdxsite)
+        monkeypatch.setenv("HDX_SITE", "feature")
         facade(
             my_testfnkw,
-            hdx_site="feature",
+            hdx_site=my_test_hdxsite,
             user_agent=my_user_agent,
             hdx_config_yaml=hdx_config_yaml,
             project_config_yaml=project_config_yaml,
@@ -143,11 +143,11 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
         )
         assert testresult.actual_result == my_test_hdxurl
         UserAgent.clear_global()
-        my_test_hdxurl2 = "http://other-data.humdata.org/"
+        my_test_hdxurl2 = "http://other2-data.humdata.org/"
         monkeypatch.setenv("HDX_URL", my_test_hdxurl2)
         facade(
             my_testfnkw,
-            hdx_site="feature",
+            hdx_url=my_test_hdxurl,
             user_agent=my_user_agent,
             hdx_config_yaml=hdx_config_yaml,
             project_config_yaml=project_config_yaml,

tests/hdx/facades/test_simple.py

@@ -84,10 +84,10 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
         assert testresult.actual_result == my_test_key
         UserAgent.clear_global()
         testresult.actual_result = None
-        monkeypatch.setenv("HDX_KEY", my_test_key)
+        monkeypatch.setenv("HDX_KEY", "aaaa")
         facade(
             my_testkeyfn,
-            hdx_key="aaaa",
+            hdx_key=my_test_key,
             user_agent=my_user_agent,
             hdx_config_yaml=hdx_config_yaml,
             project_config_yaml=project_config_yaml,
@@ -109,10 +109,10 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
         )
         UserAgent.clear_global()
         testresult.actual_result = None
-        monkeypatch.setenv("HDX_SITE", my_test_hdxsite)
+        monkeypatch.setenv("HDX_SITE", "feature")
         facade(
             my_testfn,
-            hdx_site="feature",
+            hdx_site=my_test_hdxsite,
             user_agent=my_user_agent,
             hdx_config_yaml=hdx_config_yaml,
             project_config_yaml=project_config_yaml,
@@ -133,11 +133,11 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
         )
         assert testresult.actual_result == my_test_hdxurl
         UserAgent.clear_global()
-        my_test_hdxurl2 = "http://other-data.humdata.org/"
+        my_test_hdxurl2 = "http://other2-data.humdata.org/"
         monkeypatch.setenv("HDX_URL", my_test_hdxurl2)
         facade(
             my_testfn,
-            hdx_site="feature",
+            hdx_url=my_test_hdxurl,
             user_agent=my_user_agent,
             hdx_config_yaml=hdx_config_yaml,
             project_config_yaml=project_config_yaml,