From 73ffaf933c3e65b239b0baa97ddf6719d1b90d1f Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Wed, 3 Nov 2021 12:20:24 -0400 Subject: [PATCH 1/7] [WIP] version-api-pypi integration --- src/sparsezoo/requests/base.py | 1 + src/sparsezoo/version.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/sparsezoo/requests/base.py b/src/sparsezoo/requests/base.py index 67f075ba..ef46ae05 100644 --- a/src/sparsezoo/requests/base.py +++ b/src/sparsezoo/requests/base.py @@ -43,6 +43,7 @@ ) MODELS_API_URL = f"{BASE_API_URL}/models" RECIPES_API_URL = f"{BASE_API_URL}/recipes" +LATEST_PACKAGE_VERSION_URL = f"{BASE_API_URL}/packages/check-latest" # optional prefix for stubs ZOO_STUB_PREFIX = "zoo:" diff --git a/src/sparsezoo/version.py b/src/sparsezoo/version.py index d24bd879..5ebc0a5b 100644 --- a/src/sparsezoo/version.py +++ b/src/sparsezoo/version.py @@ -16,8 +16,16 @@ Functionality for storing and setting the version info for SparseZoo """ + +import logging from datetime import date +import requests + +from sparsezoo.requests.base import LATEST_PACKAGE_VERSION_URL + + +_LOGGER = logging.getLogger(__name__) version_base = "0.9.0" is_release = False # change to True to set the generated version as a release version @@ -31,6 +39,25 @@ def _generate_version(): ) +def _version_check(version, package_name, package_integration): + + url = f"{LATEST_PACKAGE_VERSION_URL}/\ + packages={package_name}\ + &versions={version}\ + &integrations={package_integration}" + response = requests.get(url) # no token-headers required + response.raise_for_status() + + response_json = response.json() + for checkedPackage in response_json["checkedPackages"]: + if not checkedPackage["isLatest"]: + _LOGGER.warning( + f"Latest version {checkedPackage.isLatestVersion} \ + available for {checkedPackage.packageName}. \ + Current version {checkedPackage.userVersion}" + ) + + __all__ = [ "__version__", "version_base", @@ -48,3 +75,6 @@ def _generate_version(): [None] if len(version.split(".")) < 4 else [] ) # handle conditional for version being 3 parts or 4 (4 containing build date) version_major_minor = f"{version_major}.{version_minor}" + + +# thread running in the background to run [_version_check] here From bc943ab701913e30b25c942946599741f6fdd681 Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Wed, 3 Nov 2021 16:26:12 -0400 Subject: [PATCH 2/7] added thread to run version check in the background --- src/sparsezoo/version.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sparsezoo/version.py b/src/sparsezoo/version.py index 5ebc0a5b..0a809031 100644 --- a/src/sparsezoo/version.py +++ b/src/sparsezoo/version.py @@ -18,6 +18,7 @@ import logging +import threading from datetime import date import requests @@ -39,11 +40,11 @@ def _generate_version(): ) -def _version_check(version, package_name, package_integration): +def _version_check(package_name, package_version, package_integration): url = f"{LATEST_PACKAGE_VERSION_URL}/\ packages={package_name}\ - &versions={version}\ + &versions={package_version}\ &integrations={package_integration}" response = requests.get(url) # no token-headers required response.raise_for_status() @@ -77,4 +78,13 @@ def _version_check(version, package_name, package_integration): version_major_minor = f"{version_major}.{version_minor}" -# thread running in the background to run [_version_check] here +package_integration = "" # temp holder + +threading.Thread( + target=_version_check, + kwargs={ + "package_name": __name__, + "package_version": version, + "package_integration": package_integration, + }, +).start() From 04912929c9e9254147b8c5ddf055f2e43b2957cf Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Sat, 6 Nov 2021 11:22:38 -0400 Subject: [PATCH 3/7] successful api call and added env var --- setup.py | 3 +- src/sparsezoo/__init__.py | 59 +++++++++++++++++++++++++++++++++- src/sparsezoo/requests/base.py | 1 + src/sparsezoo/version.py | 41 +---------------------- 4 files changed, 62 insertions(+), 42 deletions(-) diff --git a/setup.py b/setup.py index 8c480d3e..eaae81fe 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,8 @@ """ import os -from typing import Tuple, List, Dict +from typing import Dict, List, Tuple + from setuptools import find_packages, setup # default variables to be overwritten by the version.py file diff --git a/src/sparsezoo/__init__.py b/src/sparsezoo/__init__.py index acdfe7a6..0e611fce 100644 --- a/src/sparsezoo/__init__.py +++ b/src/sparsezoo/__init__.py @@ -15,7 +15,8 @@ # limitations under the License. """ -Functionality for accessing models, recipes, and supporting files in the SparseZoo +- Functionality for accessing models, recipes, and supporting files in the SparseZoo +- Notify the user the last pypi package version """ # flake8: noqa @@ -25,3 +26,59 @@ from .main import * from .models.zoo import * from .objects import * + +from .requests.base import LATEST_PACKAGE_VERSION_URL + +import logging +import requests +import threading +import os + +_LOGGER = logging.getLogger(__name__) + +def _version_check(package_name, package_version, package_integration): + url = f"{LATEST_PACKAGE_VERSION_URL}?"\ + f"packages={package_name}"\ + f"&versions={package_version}"\ + f"&integrations={package_integration}" + try: + response = requests.get(url) # no token-headers required + response.raise_for_status() + + response_json = response.json() + for checkedPackage in response_json["checkedPackages"]: + if not checkedPackage["isLatest"]: + _LOGGER.warning( + "WARNING: "\ + f"You are using {checkedPackage['packageName']} "\ + f"version {checkedPackage['userVersion']} "\ + f"however version {checkedPackage['latestVersion']} "\ + "is available.\n"\ + "You should consider upgrading via executing the "\ + f"'pip install {checkedPackage['packageName']}'"\ + "command" + ) + _LOGGER.warning( + "WARNING: "\ + "To turn off version check, set an environmental variable " \ + "NM_VERSION_CHECK=false"\ + ) + except: + pass + +package_integration = "HuggingFace" # temp holder + +# Run thread in the background +if os.getenv("NM_VERSION_CHECK") == None or os.getenv("NM_VERSION_CHECK").lower().strip() != 'false': + threading.Thread( + target=_version_check, + kwargs={ + "package_name": __name__, + "package_version": version, + "package_integration": package_integration, + }, + ).start() +else: + print('skipping version check') + + diff --git a/src/sparsezoo/requests/base.py b/src/sparsezoo/requests/base.py index ef46ae05..b59c5174 100644 --- a/src/sparsezoo/requests/base.py +++ b/src/sparsezoo/requests/base.py @@ -31,6 +31,7 @@ "RecipeArgs", "RECIPES_API_URL", "parse_zoo_stub", + "LATEST_PACKAGE_VERSION_URL", ] diff --git a/src/sparsezoo/version.py b/src/sparsezoo/version.py index 0a809031..6c23b5af 100644 --- a/src/sparsezoo/version.py +++ b/src/sparsezoo/version.py @@ -17,16 +17,8 @@ """ -import logging -import threading -from datetime import date - -import requests - -from sparsezoo.requests.base import LATEST_PACKAGE_VERSION_URL - -_LOGGER = logging.getLogger(__name__) +from datetime import date version_base = "0.9.0" is_release = False # change to True to set the generated version as a release version @@ -40,25 +32,6 @@ def _generate_version(): ) -def _version_check(package_name, package_version, package_integration): - - url = f"{LATEST_PACKAGE_VERSION_URL}/\ - packages={package_name}\ - &versions={package_version}\ - &integrations={package_integration}" - response = requests.get(url) # no token-headers required - response.raise_for_status() - - response_json = response.json() - for checkedPackage in response_json["checkedPackages"]: - if not checkedPackage["isLatest"]: - _LOGGER.warning( - f"Latest version {checkedPackage.isLatestVersion} \ - available for {checkedPackage.packageName}. \ - Current version {checkedPackage.userVersion}" - ) - - __all__ = [ "__version__", "version_base", @@ -76,15 +49,3 @@ def _version_check(package_name, package_version, package_integration): [None] if len(version.split(".")) < 4 else [] ) # handle conditional for version being 3 parts or 4 (4 containing build date) version_major_minor = f"{version_major}.{version_minor}" - - -package_integration = "" # temp holder - -threading.Thread( - target=_version_check, - kwargs={ - "package_name": __name__, - "package_version": version, - "package_integration": package_integration, - }, -).start() From c473230dcb180365de5e4814c63fbab6d2e89331 Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Fri, 12 Nov 2021 16:59:04 -0500 Subject: [PATCH 4/7] code run in background thread, imported as a private function --- src/sparsezoo/__init__.py | 58 ++------------------ src/sparsezoo/package.py | 112 ++++++++++++++++++++++++++++++++++++++ src/sparsezoo/version.py | 2 +- 3 files changed, 118 insertions(+), 54 deletions(-) create mode 100644 src/sparsezoo/package.py diff --git a/src/sparsezoo/__init__.py b/src/sparsezoo/__init__.py index 0e611fce..88683435 100644 --- a/src/sparsezoo/__init__.py +++ b/src/sparsezoo/__init__.py @@ -26,59 +26,11 @@ from .main import * from .models.zoo import * from .objects import * +from .package import * -from .requests.base import LATEST_PACKAGE_VERSION_URL - -import logging -import requests -import threading -import os - -_LOGGER = logging.getLogger(__name__) - -def _version_check(package_name, package_version, package_integration): - url = f"{LATEST_PACKAGE_VERSION_URL}?"\ - f"packages={package_name}"\ - f"&versions={package_version}"\ - f"&integrations={package_integration}" - try: - response = requests.get(url) # no token-headers required - response.raise_for_status() - - response_json = response.json() - for checkedPackage in response_json["checkedPackages"]: - if not checkedPackage["isLatest"]: - _LOGGER.warning( - "WARNING: "\ - f"You are using {checkedPackage['packageName']} "\ - f"version {checkedPackage['userVersion']} "\ - f"however version {checkedPackage['latestVersion']} "\ - "is available.\n"\ - "You should consider upgrading via executing the "\ - f"'pip install {checkedPackage['packageName']}'"\ - "command" - ) - _LOGGER.warning( - "WARNING: "\ - "To turn off version check, set an environmental variable " \ - "NM_VERSION_CHECK=false"\ - ) - except: - pass - -package_integration = "HuggingFace" # temp holder - -# Run thread in the background -if os.getenv("NM_VERSION_CHECK") == None or os.getenv("NM_VERSION_CHECK").lower().strip() != 'false': - threading.Thread( - target=_version_check, - kwargs={ - "package_name": __name__, - "package_version": version, - "package_integration": package_integration, - }, - ).start() -else: - print('skipping version check') +from sparsezoo.package import check_package_version as _check_package_version +_check_package_version( + package_name=__name__, package_integration="Null", package_version=version_base +) diff --git a/src/sparsezoo/package.py b/src/sparsezoo/package.py new file mode 100644 index 00000000..b8a8bb0c --- /dev/null +++ b/src/sparsezoo/package.py @@ -0,0 +1,112 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import threading + +import requests + +from sparsezoo.requests import LATEST_PACKAGE_VERSION_URL + + +LOGGER = logging.getLogger(__name__) + + +def package_version_check_request( + package_name: str, package_integration: str, package_version: str +): + """ + Make an api call to api-neuralmagic.com, retrieve payload and check if the + user is on the latest package version. Lambda: nm-get-latest-version + + :param package_name: package name of the client + :param package_version: package version of the client + :param package_integration: package integration of the client + """ + url = ( + f"{LATEST_PACKAGE_VERSION_URL}?" + f"packages={package_name}" + f"&integrations={package_integration}" + f"&versions={package_version}" + ) + try: + response = requests.get(url) # no token-headers required + response.raise_for_status() + response_json = response.json() + + for checked_package in response_json["checked_packages"]: + if not checked_package["is_latest"]: + LOGGER.warning( + "WARNING: " + f"You are using {checked_package['package_name']} " + f"version {checked_package['user_package_version']} " + f"however version {checked_package['latest_package_version']} " + "is available.\n" + "Consider upgrading via executing the " + f"'pip install {checked_package['package_name']}' " + "command.\n" + "To turn off set an environmental variable " + "NM_VERSION_CHECK=false" + ) + except Exception as err: + raise RuntimeError( + f"Execption occured in the Neural Magic's internal version-api check\n{err}" + ) + + +def version_check_execution_condition( + package_name: str, package_integration: str, package_version: str +): + """ + Check if conditions are met to run the version-check api + + :param package_name: package name of the client + :param package_version: package version of the client + :param package_integration: package integration of the client + """ + if ( + os.getenv("NM_VERSION_CHECK") is None + or os.getenv("NM_VERSION_CHECK").lower().strip() != "false" + ): + package_version_check_request( + package_name=package_name, + package_integration=package_integration, + package_version=package_version, + ) + else: + LOGGER.info( + "Skipping Neural Magic's internal code exection: " + "latest package version check" + ) + + +def check_package_version( + package_name: str, package_version: str, package_integration: str +): + """ + Run a background thread to run version-check api + + :param package_name: package name of the client + :param package_version: package version of the client + :param package_integration: package integration of the client + """ + threading.Thread( + target=version_check_execution_condition, + kwargs={ + "package_name": package_name, + "package_integration": package_integration, + "package_version": package_version, + }, + ).start() diff --git a/src/sparsezoo/version.py b/src/sparsezoo/version.py index 6c23b5af..0d8d8c23 100644 --- a/src/sparsezoo/version.py +++ b/src/sparsezoo/version.py @@ -17,9 +17,9 @@ """ - from datetime import date + version_base = "0.9.0" is_release = False # change to True to set the generated version as a release version From 1cd190f2f35c0d9a3dfbef50cea5b05f8cfeaf49 Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Fri, 12 Nov 2021 17:36:42 -0500 Subject: [PATCH 5/7] changed pip command in logger --- src/sparsezoo/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sparsezoo/package.py b/src/sparsezoo/package.py index b8a8bb0c..7008b576 100644 --- a/src/sparsezoo/package.py +++ b/src/sparsezoo/package.py @@ -55,8 +55,7 @@ def package_version_check_request( f"however version {checked_package['latest_package_version']} " "is available.\n" "Consider upgrading via executing the " - f"'pip install {checked_package['package_name']}' " - "command.\n" + f"'pip install --upgrade' command.\n" "To turn off set an environmental variable " "NM_VERSION_CHECK=false" ) From 5a040e1b7ea1e9655e99a14bd2cfb61376cac46b Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Mon, 15 Nov 2021 14:20:30 -0500 Subject: [PATCH 6/7] fix issues with nightly not being pushed to the package version api --- src/sparsezoo/__init__.py | 3 ++- src/sparsezoo/package.py | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/sparsezoo/__init__.py b/src/sparsezoo/__init__.py index 88683435..bb28e52b 100644 --- a/src/sparsezoo/__init__.py +++ b/src/sparsezoo/__init__.py @@ -32,5 +32,6 @@ from sparsezoo.package import check_package_version as _check_package_version _check_package_version( - package_name=__name__, package_integration="Null", package_version=version_base + package_name=__name__ if is_release else f"{__name__}-nightly", + package_version=version_base, ) diff --git a/src/sparsezoo/package.py b/src/sparsezoo/package.py index 7008b576..8c8d049d 100644 --- a/src/sparsezoo/package.py +++ b/src/sparsezoo/package.py @@ -15,6 +15,7 @@ import logging import os import threading +from typing import Optional import requests @@ -25,7 +26,7 @@ def package_version_check_request( - package_name: str, package_integration: str, package_version: str + package_name: str, package_version: str, package_integration: Optional[str] ): """ Make an api call to api-neuralmagic.com, retrieve payload and check if the @@ -66,7 +67,7 @@ def package_version_check_request( def version_check_execution_condition( - package_name: str, package_integration: str, package_version: str + package_name: str, package_version: str, package_integration: Optional[str] ): """ Check if conditions are met to run the version-check api @@ -92,7 +93,7 @@ def version_check_execution_condition( def check_package_version( - package_name: str, package_version: str, package_integration: str + package_name: str, package_version: str, package_integration: Optional[str] = None ): """ Run a background thread to run version-check api @@ -105,7 +106,7 @@ def check_package_version( target=version_check_execution_condition, kwargs={ "package_name": package_name, - "package_integration": package_integration, "package_version": package_version, + "package_integration": package_integration, }, ).start() From f7d0fd61eba6f03020882996183dfc476c558127 Mon Sep 17 00:00:00 2001 From: Mark Kurtz Date: Mon, 15 Nov 2021 14:38:32 -0500 Subject: [PATCH 7/7] change base_version to version --- src/sparsezoo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sparsezoo/__init__.py b/src/sparsezoo/__init__.py index bb28e52b..78d04453 100644 --- a/src/sparsezoo/__init__.py +++ b/src/sparsezoo/__init__.py @@ -33,5 +33,5 @@ _check_package_version( package_name=__name__ if is_release else f"{__name__}-nightly", - package_version=version_base, + package_version=version, )