diff --git a/.config/pre-commit-config.yaml b/.config/pre-commit-config.yaml index 497359d..e24df73 100644 --- a/.config/pre-commit-config.yaml +++ b/.config/pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: exclude: test_scraper_.*\.json - id: check-ast - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.14 + rev: v0.3.2 hooks: # Run the linter. - id: ruff @@ -18,7 +18,7 @@ repos: - id: ruff-format args: [--config, .config/ruff.toml] - repo: https://github.com/jazzband/pip-tools - rev: 7.3.0 + rev: 7.4.1 hooks: - id: pip-compile name: pip-compile requirements.txt diff --git a/requirements.txt b/requirements.txt index d126c35..d9c905c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -30,9 +30,7 @@ click==8.1.7 colorama==0.4.6 # via typer coverage[toml]==7.4.3 - # via - # coverage - # pytest-cov + # via pytest-cov cryptography==42.0.5 # via pyopenssl defopt==6.4.0 @@ -55,7 +53,7 @@ filelock==3.13.1 # via virtualenv frictionless==5.16.1 # via hdx-python-utilities -google-auth==2.28.1 +google-auth==2.28.2 # via # google-auth-oauthlib # gspread @@ -63,7 +61,7 @@ google-auth-oauthlib==1.2.0 # via gspread gspread==6.0.2 # via hdx-python-api (pyproject.toml) -hdx-python-country==3.6.4 +hdx-python-country==3.6.8 # via hdx-python-api (pyproject.toml) hdx-python-utilities==3.6.5 # via @@ -121,7 +119,7 @@ oauthlib==3.2.2 # via requests-oauthlib openpyxl==3.1.2 # via hdx-python-utilities -packaging==23.2 +packaging==24.0 # via pytest petl==1.7.14 # via frictionless @@ -155,7 +153,7 @@ pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via rich -pyopenssl==24.0.0 +pyopenssl==24.1.0 # via # hdx-python-api (pyproject.toml) # ndg-httpsclient @@ -163,7 +161,7 @@ pyphonetics==0.5.3 # via hdx-python-country pyrsistent==0.20.0 # via jsonschema -pytest==8.0.2 +pytest==8.1.1 # via # hdx-python-api (pyproject.toml) # pytest-cov @@ -199,7 +197,7 @@ requests==2.31.0 # requests-oauthlib requests-file==2.0.0 # via hdx-python-utilities -requests-oauthlib==1.3.1 +requests-oauthlib==1.4.0 # via google-auth-oauthlib rfc3986==2.0.0 # via frictionless @@ -237,9 +235,7 @@ tabulate==0.9.0 text-unidecode==1.3 # via python-slugify typer[all]==0.9.0 - # via - # frictionless - # typer + # via frictionless typing-extensions==4.10.0 # via # frictionless diff --git a/src/hdx/api/configuration.py b/src/hdx/api/configuration.py index d2d2c1f..2da22a2 100755 --- a/src/hdx/api/configuration.py +++ b/src/hdx/api/configuration.py @@ -1,4 +1,5 @@ """Configuration for HDX""" + import logging import os from base64 import b64decode @@ -389,9 +390,6 @@ def _environment_variables(**kwargs: Any) -> Any: """ - hdx_key = os.getenv("HDX_KEY") - if hdx_key is not None: - kwargs["hdx_key"] = hdx_key hdx_url = os.getenv("HDX_URL") if hdx_url is not None: kwargs["hdx_url"] = hdx_url @@ -399,6 +397,16 @@ def _environment_variables(**kwargs: Any) -> Any: hdx_site = os.getenv("HDX_SITE") if hdx_site is not None: kwargs["hdx_site"] = hdx_site + hdx_key = os.getenv("HDX_KEY") + if hdx_key is None: + hdx_site = kwargs.get("hdx_site") + if hdx_site is not None: + hdx_key_site = f"HDX_KEY_{hdx_site.upper()}" + hdx_key = os.getenv(hdx_key_site) + if hdx_key: + kwargs["hdx_key"] = hdx_key + else: + kwargs["hdx_key"] = hdx_key return kwargs @classmethod diff --git a/src/hdx/api/locations.py b/src/hdx/api/locations.py index 741d268..e28bfe2 100755 --- a/src/hdx/api/locations.py +++ b/src/hdx/api/locations.py @@ -1,4 +1,5 @@ """Locations in HDX""" + from typing import Dict, List, Optional, Tuple from hdx.api.configuration import Configuration diff --git a/src/hdx/api/remotehdx.py b/src/hdx/api/remotehdx.py index a5d5b8b..6c4e167 100644 --- a/src/hdx/api/remotehdx.py +++ b/src/hdx/api/remotehdx.py @@ -1,6 +1,7 @@ """Connection to HDX with rate limiting Currently unused awaiting HDX server to add Retry-After header """ + import logging from time import sleep diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index 06ffd60..5e98499 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -1,5 +1,5 @@ -"""Dataset class containing all logic for creating, checking, and updating datasets and associated resources. -""" +"""Dataset class containing all logic for creating, checking, and updating datasets and associated resources.""" + import json import logging import sys @@ -599,9 +599,9 @@ def _prepare_hdx_call(self, data: Dict, kwargs: Any) -> None: else: scriptinfo = self.configuration.get_user_agent() # No need to output timezone info here - data[ - "updated_by_script" - ] = f"{scriptinfo} ({datetime.now(timezone.utc).replace(tzinfo=None).isoformat(timespec='microseconds')})" + data["updated_by_script"] = ( + f"{scriptinfo} ({datetime.now(timezone.utc).replace(tzinfo=None).isoformat(timespec='microseconds')})" + ) batch = kwargs.get("batch") if batch: if not is_valid_uuid(batch): diff --git a/src/hdx/data/dataset_title_helper.py b/src/hdx/data/dataset_title_helper.py index 1aee0e7..f5a3eea 100755 --- a/src/hdx/data/dataset_title_helper.py +++ b/src/hdx/data/dataset_title_helper.py @@ -1,5 +1,5 @@ -"""Helper to the Dataset class for handling processing dataset titles. -""" +"""Helper to the Dataset class for handling processing dataset titles.""" + import logging import re from datetime import datetime, timedelta diff --git a/src/hdx/data/date_helper.py b/src/hdx/data/date_helper.py index 09e3fc9..31ac31c 100755 --- a/src/hdx/data/date_helper.py +++ b/src/hdx/data/date_helper.py @@ -1,5 +1,5 @@ -"""Helper to the Dataset and Resource classes for handling HDX dates. -""" +"""Helper to the Dataset and Resource classes for handling HDX dates.""" + from collections.abc import Iterable from datetime import datetime, timezone from typing import Dict, List, Optional, Tuple, Union diff --git a/src/hdx/data/filestore_helper.py b/src/hdx/data/filestore_helper.py index 0e014be..dbbee9f 100755 --- a/src/hdx/data/filestore_helper.py +++ b/src/hdx/data/filestore_helper.py @@ -1,5 +1,5 @@ -"""Helper to the Dataset class for handling resources with filestores. -""" +"""Helper to the Dataset class for handling resources with filestores.""" + from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Dict diff --git a/src/hdx/data/hdxobject.py b/src/hdx/data/hdxobject.py index 39c3e86..6575b79 100755 --- a/src/hdx/data/hdxobject.py +++ b/src/hdx/data/hdxobject.py @@ -1,6 +1,7 @@ """HDXObject abstract class containing helper functions for creating, checking, and updating HDX objects. New HDX objects should extend this in similar fashion to Resource for example. """ + import copy import logging from abc import ABC, abstractmethod diff --git a/src/hdx/data/organization.py b/src/hdx/data/organization.py index 44ebdd9..e3fbbac 100755 --- a/src/hdx/data/organization.py +++ b/src/hdx/data/organization.py @@ -1,4 +1,5 @@ """Organization class containing all logic for creating, checking, and updating organizations.""" + import logging from os.path import join from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union diff --git a/src/hdx/data/resource.py b/src/hdx/data/resource.py index fbdc5f3..e7e6dff 100755 --- a/src/hdx/data/resource.py +++ b/src/hdx/data/resource.py @@ -1,4 +1,5 @@ """Resource class containing all logic for creating, checking, and updating resources.""" + import logging import warnings from datetime import datetime, timezone diff --git a/src/hdx/data/resource_matcher.py b/src/hdx/data/resource_matcher.py index 6e4e66c..ab27b75 100755 --- a/src/hdx/data/resource_matcher.py +++ b/src/hdx/data/resource_matcher.py @@ -1,5 +1,5 @@ -"""Helper to the Dataset class for handling matching resources. -""" +"""Helper to the Dataset class for handling matching resources.""" + import collections from typing import TYPE_CHECKING, List, Optional, Tuple diff --git a/src/hdx/data/resource_view.py b/src/hdx/data/resource_view.py index cb8cacc..a9b8fd9 100755 --- a/src/hdx/data/resource_view.py +++ b/src/hdx/data/resource_view.py @@ -1,4 +1,5 @@ """Resource view class containing all logic for creating, checking, and updating resource views.""" + import logging from os.path import join from typing import Any, Dict, List, Optional, Union diff --git a/src/hdx/data/showcase.py b/src/hdx/data/showcase.py index a5ec69c..7f3b2b2 100755 --- a/src/hdx/data/showcase.py +++ b/src/hdx/data/showcase.py @@ -1,4 +1,5 @@ """Showcase class containing all logic for creating, checking, and updating showcases.""" + import logging import sys from os.path import join diff --git a/src/hdx/data/user.py b/src/hdx/data/user.py index 63ca5d6..2d13112 100755 --- a/src/hdx/data/user.py +++ b/src/hdx/data/user.py @@ -1,4 +1,5 @@ """User class containing all logic for creating, checking, and updating users.""" + import logging from os.path import join from typing import Any, Dict, List, Optional diff --git a/src/hdx/data/vocabulary.py b/src/hdx/data/vocabulary.py index 34c77fb..6860ba6 100755 --- a/src/hdx/data/vocabulary.py +++ b/src/hdx/data/vocabulary.py @@ -1,4 +1,5 @@ """Vocabulary class containing all logic for creating, checking, and updating vocabularies.""" + import logging from collections import OrderedDict from os.path import join @@ -118,9 +119,9 @@ def get_all_vocabularies( """ vocabulary = Vocabulary(configuration=configuration) - vocabulary[ - "id" - ] = "all vocabulary names" # only for error message if produced + vocabulary["id"] = ( + "all vocabulary names" # only for error message if produced + ) vocabularies = [] for vocabularydict in vocabulary._write_to_hdx("list", {}): vocabularies.append( diff --git a/src/hdx/facades/infer_arguments.py b/src/hdx/facades/infer_arguments.py index 3dd6995..34af66f 100755 --- a/src/hdx/facades/infer_arguments.py +++ b/src/hdx/facades/infer_arguments.py @@ -1,4 +1,5 @@ """Facade to simplify project setup that calls project main function with kwargs""" + import logging import sys from inspect import getdoc diff --git a/src/hdx/facades/keyword_arguments.py b/src/hdx/facades/keyword_arguments.py index 65fa471..425591c 100755 --- a/src/hdx/facades/keyword_arguments.py +++ b/src/hdx/facades/keyword_arguments.py @@ -1,4 +1,5 @@ """Facade to simplify project setup that calls project main function with kwargs""" + import logging from typing import Any, Callable diff --git a/src/hdx/facades/simple.py b/src/hdx/facades/simple.py index 8413648..6835170 100755 --- a/src/hdx/facades/simple.py +++ b/src/hdx/facades/simple.py @@ -1,4 +1,5 @@ """Facade to simplify project setup that calls project main function""" + import logging from typing import Any, Callable diff --git a/tests/hdx/api/test_ckan.py b/tests/hdx/api/test_ckan.py index 3693d16..1370be6 100644 --- a/tests/hdx/api/test_ckan.py +++ b/tests/hdx/api/test_ckan.py @@ -2,6 +2,7 @@ Unit tests for the freshness class. """ + import json import logging import random diff --git a/tests/hdx/api/test_configuration.py b/tests/hdx/api/test_configuration.py index 73f292a..64ffec1 100755 --- a/tests/hdx/api/test_configuration.py +++ b/tests/hdx/api/test_configuration.py @@ -1,4 +1,5 @@ """Configuration Tests""" + from os.path import join import pytest @@ -793,6 +794,31 @@ def test_set_hdx_key_value(self, project_config_yaml): configuration.set_read_only(False) assert configuration.get_api_key() == "NEW API KEY" + def test_env_vars(self, monkeypatch): + hdx_url = "https://testurl" + hdx_key = "TEST_HDX_KEY" + monkeypatch.setenv("HDX_URL", hdx_url) + monkeypatch.setenv("HDX_KEY", hdx_key) + Configuration._create( + user_agent="test", + hdx_base_config_dict={}, + ) + configuration = Configuration.read() + assert configuration.get_hdx_site_url() == hdx_url + assert configuration.get_api_key() == hdx_key + monkeypatch.delenv("HDX_URL") + monkeypatch.delenv("HDX_KEY") + + hdx_key = "TEST_HDX_KEY_STAGE" + monkeypatch.setenv("HDX_KEY_STAGE", hdx_key) + Configuration._create( + user_agent="test", + hdx_site="stage", + hdx_base_config_dict={}, + ) + configuration = Configuration.read() + assert configuration.get_api_key() == hdx_key + def test_create_set_configuration(self, project_config_yaml): Configuration._create( user_agent="test", diff --git a/tests/hdx/api/test_locations.py b/tests/hdx/api/test_locations.py index 3f9998b..5b3e8ce 100755 --- a/tests/hdx/api/test_locations.py +++ b/tests/hdx/api/test_locations.py @@ -1,4 +1,5 @@ """HDX Location Tests""" + from hdx.api.configuration import Configuration from hdx.api.locations import Locations from hdx.location.country import Country diff --git a/tests/hdx/conftest.py b/tests/hdx/conftest.py index a4a6ce5..23a9f99 100755 --- a/tests/hdx/conftest.py +++ b/tests/hdx/conftest.py @@ -1,4 +1,5 @@ """Global fixtures""" + import re import smtplib from os.path import join diff --git a/tests/hdx/data/test_dataset_core.py b/tests/hdx/data/test_dataset_core.py index a365997..dfe06c8 100755 --- a/tests/hdx/data/test_dataset_core.py +++ b/tests/hdx/data/test_dataset_core.py @@ -1,4 +1,5 @@ """Dataset Tests (core methods)""" + import copy import json import re diff --git a/tests/hdx/data/test_dataset_noncore.py b/tests/hdx/data/test_dataset_noncore.py index e8c06fa..5adaf6a 100755 --- a/tests/hdx/data/test_dataset_noncore.py +++ b/tests/hdx/data/test_dataset_noncore.py @@ -1,4 +1,5 @@ """Dataset Tests (noncore methods)""" + import copy import json from datetime import datetime, timezone diff --git a/tests/hdx/data/test_dataset_title_helper.py b/tests/hdx/data/test_dataset_title_helper.py index eb9d2c3..8f396e3 100755 --- a/tests/hdx/data/test_dataset_title_helper.py +++ b/tests/hdx/data/test_dataset_title_helper.py @@ -1,4 +1,5 @@ """Dataset Title Helper Tests""" + from datetime import datetime, timezone import pytest diff --git a/tests/hdx/data/test_organization.py b/tests/hdx/data/test_organization.py index 1cff035..221c6d3 100755 --- a/tests/hdx/data/test_organization.py +++ b/tests/hdx/data/test_organization.py @@ -1,4 +1,5 @@ """Organization Tests""" + import copy import json from os.path import join diff --git a/tests/hdx/data/test_resource.py b/tests/hdx/data/test_resource.py index 44e0a6a..a68d09f 100755 --- a/tests/hdx/data/test_resource.py +++ b/tests/hdx/data/test_resource.py @@ -1,4 +1,5 @@ """Resource Tests""" + import copy import json import os @@ -393,9 +394,9 @@ def post(url, data, headers, files, allow_redirects, auth=None): resultdictcopy["url_type"] = "upload" resultdictcopy["resource_type"] = "file.upload" filename = os.path.basename(files["upload"].name) - resultdictcopy[ - "url" - ] = f"http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/{filename}" + resultdictcopy["url"] = ( + f"http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/{filename}" + ) resultdictcopy["state"] = datadict["state"] result = json.dumps(resultdictcopy) @@ -456,9 +457,9 @@ def post(url, data, headers, files, allow_redirects, auth=None): resultdictcopy["url_type"] = "upload" resultdictcopy["resource_type"] = "file.upload" filename = os.path.basename(files["upload"].name) - resultdictcopy[ - "url" - ] = f"http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/{filename}" + resultdictcopy["url"] = ( + f"http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/{filename}" + ) result = json.dumps(resultdictcopy) return MockResponse( 200, diff --git a/tests/hdx/data/test_resource_view.py b/tests/hdx/data/test_resource_view.py index c28f283..a6653ad 100755 --- a/tests/hdx/data/test_resource_view.py +++ b/tests/hdx/data/test_resource_view.py @@ -1,4 +1,5 @@ """Resource view Tests""" + import copy import json from os.path import join diff --git a/tests/hdx/data/test_showcase.py b/tests/hdx/data/test_showcase.py index ada8c12..d1197e1 100755 --- a/tests/hdx/data/test_showcase.py +++ b/tests/hdx/data/test_showcase.py @@ -1,4 +1,5 @@ """Showcase Tests""" + import copy import json from os.path import join diff --git a/tests/hdx/data/test_user.py b/tests/hdx/data/test_user.py index 2c3e8de..0b1b2fb 100755 --- a/tests/hdx/data/test_user.py +++ b/tests/hdx/data/test_user.py @@ -1,4 +1,5 @@ """User Tests""" + import copy import json from os.path import join diff --git a/tests/hdx/data/test_vocabulary.py b/tests/hdx/data/test_vocabulary.py index b8778fa..097da3f 100755 --- a/tests/hdx/data/test_vocabulary.py +++ b/tests/hdx/data/test_vocabulary.py @@ -1,4 +1,5 @@ """Vocabulary Tests""" + import copy import json from os.path import join diff --git a/tests/hdx/facades/test_infer_arguments.py b/tests/hdx/facades/test_infer_arguments.py index 768187d..b93fd37 100755 --- a/tests/hdx/facades/test_infer_arguments.py +++ b/tests/hdx/facades/test_infer_arguments.py @@ -1,4 +1,5 @@ """Simple Facade Tests""" + import sys import pytest diff --git a/tests/hdx/facades/test_keyword_arguments.py b/tests/hdx/facades/test_keyword_arguments.py index 3d6b671..b09afd4 100755 --- a/tests/hdx/facades/test_keyword_arguments.py +++ b/tests/hdx/facades/test_keyword_arguments.py @@ -1,4 +1,5 @@ """Simple Facade Tests""" + import pytest from . import my_testfnkw, testresult diff --git a/tests/hdx/facades/test_simple.py b/tests/hdx/facades/test_simple.py index 8ebf04a..37c0e4c 100755 --- a/tests/hdx/facades/test_simple.py +++ b/tests/hdx/facades/test_simple.py @@ -1,4 +1,5 @@ """Simple Facade Tests""" + import pytest from . import my_excfn, my_testfn, my_testkeyfn, my_testuafn, testresult diff --git a/workingexample/my_code.py b/workingexample/my_code.py index 3df5651..716f1a9 100755 --- a/workingexample/my_code.py +++ b/workingexample/my_code.py @@ -2,6 +2,7 @@ Generate a dataset """ + import logging logger = logging.getLogger(__name__) diff --git a/workingexample/run.py b/workingexample/run.py index 9452563..9fc6d6d 100755 --- a/workingexample/run.py +++ b/workingexample/run.py @@ -2,6 +2,7 @@ Calls a function that generates a dataset and creates it in HDX. """ + import logging from .my_code import generate_dataset