Add support for env var HDX_KEY_STAGE, HDX_KEY_DEV etc.

Update pre-commit config

.config/pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.14
+    rev: v0.3.2
     hooks:
       # Run the linter.
       - id: ruff
@@ -18,7 +18,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/jazzband/pip-tools
-    rev: 7.3.0
+    rev: 7.4.1
     hooks:
       - id: pip-compile
         name: pip-compile requirements.txt

requirements.txt

@@ -30,9 +30,7 @@ click==8.1.7
 colorama==0.4.6
     # via typer
 coverage[toml]==7.4.3
-    # via
-    #   coverage
-    #   pytest-cov
+    # via pytest-cov
 cryptography==42.0.5
     # via pyopenssl
 defopt==6.4.0
@@ -55,15 +53,15 @@ filelock==3.13.1
     # via virtualenv
 frictionless==5.16.1
     # via hdx-python-utilities
-google-auth==2.28.1
+google-auth==2.28.2
     # via
     #   google-auth-oauthlib
     #   gspread
 google-auth-oauthlib==1.2.0
     # via gspread
 gspread==6.0.2
     # via hdx-python-api (pyproject.toml)
-hdx-python-country==3.6.4
+hdx-python-country==3.6.8
     # via hdx-python-api (pyproject.toml)
 hdx-python-utilities==3.6.5
     # via
@@ -121,7 +119,7 @@ oauthlib==3.2.2
     # via requests-oauthlib
 openpyxl==3.1.2
     # via hdx-python-utilities
-packaging==23.2
+packaging==24.0
     # via pytest
 petl==1.7.14
     # via frictionless
@@ -155,15 +153,15 @@ pydantic-core==2.16.3
     # via pydantic
 pygments==2.17.2
     # via rich
-pyopenssl==24.0.0
+pyopenssl==24.1.0
     # via
     #   hdx-python-api (pyproject.toml)
     #   ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
 pyrsistent==0.20.0
     # via jsonschema
-pytest==8.0.2
+pytest==8.1.1
     # via
     #   hdx-python-api (pyproject.toml)
     #   pytest-cov
@@ -199,7 +197,7 @@ requests==2.31.0
     #   requests-oauthlib
 requests-file==2.0.0
     # via hdx-python-utilities
-requests-oauthlib==1.3.1
+requests-oauthlib==1.4.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
@@ -237,9 +235,7 @@ tabulate==0.9.0
 text-unidecode==1.3
     # via python-slugify
 typer[all]==0.9.0
-    # via
-    #   frictionless
-    #   typer
+    # via frictionless
 typing-extensions==4.10.0
     # via
     #   frictionless

src/hdx/api/configuration.py

@@ -1,4 +1,5 @@
 """Configuration for HDX"""
+
 import logging
 import os
 from base64 import b64decode
@@ -389,16 +390,23 @@ def _environment_variables(**kwargs: Any) -> Any:
 
         """
 
-        hdx_key = os.getenv("HDX_KEY")
-        if hdx_key is not None:
-            kwargs["hdx_key"] = hdx_key
         hdx_url = os.getenv("HDX_URL")
         if hdx_url is not None:
             kwargs["hdx_url"] = hdx_url
         else:
             hdx_site = os.getenv("HDX_SITE")
             if hdx_site is not None:
                 kwargs["hdx_site"] = hdx_site
+        hdx_key = os.getenv("HDX_KEY")
+        if hdx_key is None:
+            hdx_site = kwargs.get("hdx_site")
+            if hdx_site is not None:
+                hdx_key_site = f"HDX_KEY_{hdx_site.upper()}"
+                hdx_key = os.getenv(hdx_key_site)
+                if hdx_key:
+                    kwargs["hdx_key"] = hdx_key
+        else:
+            kwargs["hdx_key"] = hdx_key
         return kwargs
 
     @classmethod

src/hdx/api/locations.py

@@ -1,4 +1,5 @@
 """Locations in HDX"""
+
 from typing import Dict, List, Optional, Tuple
 
 from hdx.api.configuration import Configuration

src/hdx/api/remotehdx.py

@@ -1,6 +1,7 @@
 """Connection to HDX with rate limiting
 Currently unused awaiting HDX server to add Retry-After header
 """
+
 import logging
 from time import sleep
 

src/hdx/data/dataset.py

@@ -1,5 +1,5 @@
-"""Dataset class containing all logic for creating, checking, and updating datasets and associated resources.
-"""
+"""Dataset class containing all logic for creating, checking, and updating datasets and associated resources."""
+
 import json
 import logging
 import sys
@@ -599,9 +599,9 @@ def _prepare_hdx_call(self, data: Dict, kwargs: Any) -> None:
         else:
             scriptinfo = self.configuration.get_user_agent()
         # No need to output timezone info here
-        data[
-            "updated_by_script"
-        ] = f"{scriptinfo} ({datetime.now(timezone.utc).replace(tzinfo=None).isoformat(timespec='microseconds')})"
+        data["updated_by_script"] = (
+            f"{scriptinfo} ({datetime.now(timezone.utc).replace(tzinfo=None).isoformat(timespec='microseconds')})"
+        )
         batch = kwargs.get("batch")
         if batch:
             if not is_valid_uuid(batch):

src/hdx/data/dataset_title_helper.py

@@ -1,5 +1,5 @@
-"""Helper to the Dataset class for handling processing dataset titles.
-"""
+"""Helper to the Dataset class for handling processing dataset titles."""
+
 import logging
 import re
 from datetime import datetime, timedelta

src/hdx/data/date_helper.py

@@ -1,5 +1,5 @@
-"""Helper to the Dataset and Resource classes for handling HDX dates.
-"""
+"""Helper to the Dataset and Resource classes for handling HDX dates."""
+
 from collections.abc import Iterable
 from datetime import datetime, timezone
 from typing import Dict, List, Optional, Tuple, Union

src/hdx/data/filestore_helper.py

@@ -1,5 +1,5 @@
-"""Helper to the Dataset class for handling resources with filestores.
-"""
+"""Helper to the Dataset class for handling resources with filestores."""
+
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any, Dict
 

src/hdx/data/hdxobject.py

@@ -1,6 +1,7 @@
 """HDXObject abstract class containing helper functions for creating, checking, and updating HDX objects.
 New HDX objects should extend this in similar fashion to Resource for example.
 """
+
 import copy
 import logging
 from abc import ABC, abstractmethod

src/hdx/data/organization.py

@@ -1,4 +1,5 @@
 """Organization class containing all logic for creating, checking, and updating organizations."""
+
 import logging
 from os.path import join
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union

src/hdx/data/resource.py

@@ -1,4 +1,5 @@
 """Resource class containing all logic for creating, checking, and updating resources."""
+
 import logging
 import warnings
 from datetime import datetime, timezone

src/hdx/data/resource_matcher.py

@@ -1,5 +1,5 @@
-"""Helper to the Dataset class for handling matching resources.
-"""
+"""Helper to the Dataset class for handling matching resources."""
+
 import collections
 from typing import TYPE_CHECKING, List, Optional, Tuple
 

src/hdx/data/resource_view.py

@@ -1,4 +1,5 @@
 """Resource view class containing all logic for creating, checking, and updating resource views."""
+
 import logging
 from os.path import join
 from typing import Any, Dict, List, Optional, Union

src/hdx/data/showcase.py

@@ -1,4 +1,5 @@
 """Showcase class containing all logic for creating, checking, and updating showcases."""
+
 import logging
 import sys
 from os.path import join

src/hdx/data/user.py

@@ -1,4 +1,5 @@
 """User class containing all logic for creating, checking, and updating users."""
+
 import logging
 from os.path import join
 from typing import Any, Dict, List, Optional

src/hdx/data/vocabulary.py

@@ -1,4 +1,5 @@
 """Vocabulary class containing all logic for creating, checking, and updating vocabularies."""
+
 import logging
 from collections import OrderedDict
 from os.path import join
@@ -118,9 +119,9 @@ def get_all_vocabularies(
         """
 
         vocabulary = Vocabulary(configuration=configuration)
-        vocabulary[
-            "id"
-        ] = "all vocabulary names"  # only for error message if produced
+        vocabulary["id"] = (
+            "all vocabulary names"  # only for error message if produced
+        )
         vocabularies = []
         for vocabularydict in vocabulary._write_to_hdx("list", {}):
             vocabularies.append(

src/hdx/facades/infer_arguments.py

@@ -1,4 +1,5 @@
 """Facade to simplify project setup that calls project main function with kwargs"""
+
 import logging
 import sys
 from inspect import getdoc

src/hdx/facades/keyword_arguments.py

@@ -1,4 +1,5 @@
 """Facade to simplify project setup that calls project main function with kwargs"""
+
 import logging
 from typing import Any, Callable
 

src/hdx/facades/simple.py

@@ -1,4 +1,5 @@
 """Facade to simplify project setup that calls project main function"""
+
 import logging
 from typing import Any, Callable
 

tests/hdx/api/test_ckan.py

@@ -2,6 +2,7 @@
 Unit tests for the freshness class.
 
 """
+
 import json
 import logging
 import random

tests/hdx/api/test_configuration.py

@@ -1,4 +1,5 @@
 """Configuration Tests"""
+
 from os.path import join
 
 import pytest
@@ -793,6 +794,31 @@ def test_set_hdx_key_value(self, project_config_yaml):
         configuration.set_read_only(False)
         assert configuration.get_api_key() == "NEW API KEY"
 
+    def test_env_vars(self, monkeypatch):
+        hdx_url = "https://testurl"
+        hdx_key = "TEST_HDX_KEY"
+        monkeypatch.setenv("HDX_URL", hdx_url)
+        monkeypatch.setenv("HDX_KEY", hdx_key)
+        Configuration._create(
+            user_agent="test",
+            hdx_base_config_dict={},
+        )
+        configuration = Configuration.read()
+        assert configuration.get_hdx_site_url() == hdx_url
+        assert configuration.get_api_key() == hdx_key
+        monkeypatch.delenv("HDX_URL")
+        monkeypatch.delenv("HDX_KEY")
+
+        hdx_key = "TEST_HDX_KEY_STAGE"
+        monkeypatch.setenv("HDX_KEY_STAGE", hdx_key)
+        Configuration._create(
+            user_agent="test",
+            hdx_site="stage",
+            hdx_base_config_dict={},
+        )
+        configuration = Configuration.read()
+        assert configuration.get_api_key() == hdx_key
+
     def test_create_set_configuration(self, project_config_yaml):
         Configuration._create(
             user_agent="test",

tests/hdx/api/test_locations.py

@@ -1,4 +1,5 @@
 """HDX Location Tests"""
+
 from hdx.api.configuration import Configuration
 from hdx.api.locations import Locations
 from hdx.location.country import Country

tests/hdx/conftest.py

@@ -1,4 +1,5 @@
 """Global fixtures"""
+
 import re
 import smtplib
 from os.path import join

tests/hdx/data/test_dataset_core.py

@@ -1,4 +1,5 @@
 """Dataset Tests (core methods)"""
+
 import copy
 import json
 import re

tests/hdx/data/test_dataset_noncore.py

@@ -1,4 +1,5 @@
 """Dataset Tests (noncore methods)"""
+
 import copy
 import json
 from datetime import datetime, timezone

tests/hdx/data/test_dataset_title_helper.py

@@ -1,4 +1,5 @@
 """Dataset Title Helper Tests"""
+
 from datetime import datetime, timezone
 
 import pytest

tests/hdx/data/test_organization.py

@@ -1,4 +1,5 @@
 """Organization Tests"""
+
 import copy
 import json
 from os.path import join

tests/hdx/data/test_resource.py

@@ -1,4 +1,5 @@
 """Resource Tests"""
+
 import copy
 import json
 import os
@@ -393,9 +394,9 @@ def post(url, data, headers, files, allow_redirects, auth=None):
                         resultdictcopy["url_type"] = "upload"
                         resultdictcopy["resource_type"] = "file.upload"
                         filename = os.path.basename(files["upload"].name)
-                        resultdictcopy[
-                            "url"
-                        ] = f"http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/{filename}"
+                        resultdictcopy["url"] = (
+                            f"http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/{filename}"
+                        )
                     resultdictcopy["state"] = datadict["state"]
 
                     result = json.dumps(resultdictcopy)
@@ -456,9 +457,9 @@ def post(url, data, headers, files, allow_redirects, auth=None):
                         resultdictcopy["url_type"] = "upload"
                         resultdictcopy["resource_type"] = "file.upload"
                         filename = os.path.basename(files["upload"].name)
-                        resultdictcopy[
-                            "url"
-                        ] = f"http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/{filename}"
+                        resultdictcopy["url"] = (
+                            f"http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/{filename}"
+                        )
                     result = json.dumps(resultdictcopy)
                     return MockResponse(
                         200,

tests/hdx/data/test_resource_view.py

@@ -1,4 +1,5 @@
 """Resource view Tests"""
+
 import copy
 import json
 from os.path import join

tests/hdx/data/test_showcase.py

@@ -1,4 +1,5 @@
 """Showcase Tests"""
+
 import copy
 import json
 from os.path import join