-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'develop' into d/DX-469/spark-s3-deployment
* develop: [MAINTENANCE] Add check to `CloudDataContext` to ensure using latest PyPI version (#7753) [RELEASE] 0.16.10 (#7774)
- Loading branch information
Showing
8 changed files
with
181 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| export default { | ||
| release_version: 'great_expectations, version 0.16.9', | ||
| release_version: 'great_expectations, version 0.16.10', | ||
| min_python: '3.7', | ||
| max_python: '3.10' | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import json | ||
| import logging | ||
| import sys | ||
|
|
||
| import requests | ||
| from packaging import version | ||
| from typing_extensions import TypedDict | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class _PyPIPackageInfo(TypedDict): | ||
| version: str | ||
|
|
||
|
|
||
| class _PyPIPackageData(TypedDict): | ||
| info: _PyPIPackageInfo | ||
|
|
||
|
|
||
| # Should only run in prod and in specific tests | ||
| # This flag let's us conditionally turn on the feature | ||
| _ENABLE_VERSION_CHECK_IN_TESTS = False | ||
|
|
||
|
|
||
| class _VersionChecker: | ||
|
|
||
| _BASE_PYPI_URL = "https://pypi.org/pypi" | ||
| _PYPI_GX_ENDPOINT = f"{_BASE_PYPI_URL}/great_expectations/json" | ||
|
|
||
| def __init__(self, user_version: str) -> None: | ||
| self._user_version = version.Version(user_version) | ||
|
|
||
| def check_if_using_latest_gx(self) -> bool: | ||
| if self._running_non_version_check_tests(): | ||
| return True | ||
|
|
||
| pypi_version = self._get_latest_version_from_pypi() | ||
| if not pypi_version: | ||
| logger.debug("Could not compare with latest PyPI version; skipping check.") | ||
| return True | ||
|
|
||
| if self._is_using_outdated_release(pypi_version): | ||
| self._warn_user(pypi_version) | ||
| return False | ||
| return True | ||
|
|
||
| def _running_non_version_check_tests(self) -> bool: | ||
| return "pytest" in sys.modules and _ENABLE_VERSION_CHECK_IN_TESTS | ||
|
|
||
| def _get_latest_version_from_pypi(self) -> version.Version | None: | ||
| response_json: _PyPIPackageData | None = None | ||
| try: | ||
| response = requests.get(self._PYPI_GX_ENDPOINT) | ||
| response.raise_for_status() | ||
| response_json = response.json() | ||
| except json.JSONDecodeError as jsonError: | ||
| logger.debug(f"Failed to parse PyPI API response into JSON: {jsonError}") | ||
| except requests.HTTPError as http_err: | ||
| logger.debug( | ||
| f"An HTTP error occurred when trying to hit PyPI API: {http_err}" | ||
| ) | ||
| except requests.Timeout as timeout_exc: | ||
| logger.debug( | ||
| f"Failed to hit the PyPI API due a timeout error: {timeout_exc}" | ||
| ) | ||
|
|
||
| if not response_json: | ||
| return None | ||
|
|
||
| # Structure should be guaranteed but let's be defensive in case PyPI changes. | ||
| info = response_json.get("info", {}) | ||
| pkg_version = info.get("version") | ||
| if not pkg_version: | ||
| logger.debug( | ||
| "Successfully hit PyPI API but payload structure is not as expected." | ||
| ) | ||
| return None | ||
|
|
||
| return version.Version(pkg_version) | ||
|
|
||
| def _is_using_outdated_release(self, pypi_version: version.Version) -> bool: | ||
| return pypi_version > self._user_version | ||
|
|
||
| def _warn_user(self, pypi_version: version.Version) -> None: | ||
| logger.warning( | ||
| f"You are using great_expectations version {self._user_version}; " | ||
| f"however, version {pypi_version} is available.\nYou should consider " | ||
| "upgrading via `pip install great_expectations --upgrade`\n." | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| 0.16.9 | ||
| 0.16.10 |
52 changes: 52 additions & 0 deletions
52
tests/data_context/cloud_data_context/test_version_checker.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| import pytest | ||
| import responses | ||
|
|
||
| import great_expectations.data_context._version_checker as vc | ||
| from great_expectations.data_context._version_checker import _VersionChecker | ||
|
|
||
| # Set to some arbitrary value so tests will continue to work regardless of GX's actual version | ||
| _MOCK_PYPI_VERSION = "0.16.8" | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def enable_pypi_version_check(): | ||
| vc._ENABLE_VERSION_CHECK_IN_TESTS = False | ||
| yield | ||
| vc._ENABLE_VERSION_CHECK_IN_TESTS = True | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "version,expected,status", | ||
| [ | ||
| pytest.param("0.15.0", False, 200, id="outdated"), | ||
| pytest.param(_MOCK_PYPI_VERSION, True, 200, id="up-to-date"), | ||
| # packaging.version should take care of dirty hashes but worth checking against | ||
| pytest.param( | ||
| f"{_MOCK_PYPI_VERSION}+59.g1ff4de04d.dirty", | ||
| True, | ||
| 200, | ||
| id="up-to-date local dev", | ||
| ), | ||
| # If we error, we shouldn't raise a warning to the user | ||
| pytest.param(_MOCK_PYPI_VERSION, True, 400, id="bad request"), | ||
| ], | ||
| ) | ||
| @pytest.mark.unit | ||
| @responses.activate | ||
| def test_check_if_using_latest_gx( | ||
| enable_pypi_version_check, version: str, expected: bool, status: int, caplog | ||
| ): | ||
| pypi_payload = {"info": {"version": _MOCK_PYPI_VERSION}} | ||
| responses.add( | ||
| responses.GET, | ||
| _VersionChecker._PYPI_GX_ENDPOINT, | ||
| json=pypi_payload, | ||
| status=status, | ||
| ) | ||
|
|
||
| checker = _VersionChecker(version) | ||
| actual = checker.check_if_using_latest_gx() | ||
|
|
||
| assert actual is expected | ||
| if not actual: | ||
| assert "upgrade" in caplog.text |