From 73a7cb468755bf874f027d9053ca1c8d5b84ede5 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 15 May 2024 10:52:22 -0400 Subject: [PATCH] CHIA-420 remove ignore for `pkg_resources` deprecation warning (#17933) * remove ignore for `pkg_resources` deprecation warning * replace pkg_resources usages * fixup * fixup * fixup * a couple * imports * cleanup * maybe --- chia/_tests/clvm/test_condition_codes.py | 10 ++--- chia/_tests/cmds/wallet/test_wallet.py | 15 ++++--- chia/_tests/core/daemon/test_daemon.py | 4 +- chia/_tests/core/util/test_keychain.py | 12 +++--- chia/_tests/simulation/test_simulation.py | 4 +- chia/_tests/util/benchmarks.py | 7 ++-- chia/plotters/chiapos.py | 2 +- chia/ssl/create_ssl.py | 7 ++-- chia/timelord/timelord_launcher.py | 14 ++++--- chia/util/config.py | 6 ++- chia/util/keychain.py | 6 ++- chia/wallet/puzzles/load_clvm.py | 50 +++++++++++------------ install-timelord.sh | 2 +- installhelper.py | 18 ++++---- pytest.ini | 1 - setup.py | 1 + 16 files changed, 82 insertions(+), 77 deletions(-) diff --git a/chia/_tests/clvm/test_condition_codes.py b/chia/_tests/clvm/test_condition_codes.py index ac64edd8e0e6..8d20e4d387c4 100644 --- a/chia/_tests/clvm/test_condition_codes.py +++ b/chia/_tests/clvm/test_condition_codes.py @@ -1,13 +1,13 @@ from __future__ import annotations -import pkg_resources +import importlib_resources from clvm.casts import int_from_bytes from chia.types.condition_opcodes import ConditionOpcode def test_condition_codes_is_complete() -> None: - with open(pkg_resources.resource_filename("chia.wallet.puzzles", "condition_codes.clib")) as f: - contents: str = f.read() - for name, value in ConditionOpcode.__members__.items(): - assert f"(defconstant {name} {int_from_bytes(value)})" in contents + condition_codes_path = importlib_resources.files("chia.wallet.puzzles").joinpath("condition_codes.clib") + contents = condition_codes_path.read_text(encoding="utf-8") + for name, value in ConditionOpcode.__members__.items(): + assert f"(defconstant {name} {int_from_bytes(value)})" in contents diff --git a/chia/_tests/cmds/wallet/test_wallet.py b/chia/_tests/cmds/wallet/test_wallet.py index 043a32f65d12..9411b08c141d 100644 --- a/chia/_tests/cmds/wallet/test_wallet.py +++ b/chia/_tests/cmds/wallet/test_wallet.py @@ -1,9 +1,10 @@ from __future__ import annotations +import os from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union, cast -import pkg_resources +import importlib_resources import pytest from chia_rs import Coin, G2Element @@ -37,9 +38,8 @@ from chia.wallet.util.wallet_types import WalletType from chia.wallet.wallet_coin_store import GetCoinRecords -test_offer_file_path: Path = Path(pkg_resources.resource_filename(__name__, "test_offer.toffer")) -test_offer_file_name: str = str(test_offer_file_path) -test_offer_file_bech32: str = open(test_offer_file_name).read() +test_offer_file_path = importlib_resources.files(__name__).joinpath("test_offer.toffer") +test_offer_file_bech32 = test_offer_file_path.read_text(encoding="utf-8") test_offer_id: str = "0xdfb7e8643376820ec995b0bcdb3fc1f764c16b814df5e074631263fcf1e00839" test_offer_id_bytes: bytes32 = bytes32.from_hexstr(test_offer_id) @@ -921,7 +921,6 @@ async def take_offer( inst_rpc_client = TakeOfferRpcClient() # pylint: disable=no-value-for-parameter test_rpc_clients.wallet_rpc_client = inst_rpc_client - command_args = ["wallet", "take_offer", test_offer_file_name, FINGERPRINT_ARG, "-m1", "--reuse"] # these are various things that should be in the output cat1 = bytes32.from_hexstr("fd6a341ed39c05c31157d5bfea395a0e142398ced24deea1e82f836d7ec2909c") cat2 = bytes32.from_hexstr("dc59bcd60ce5fc9c93a5d3b11875486b03efb53a53da61e453f5cf61a7746860") @@ -934,7 +933,11 @@ async def take_offer( " - accce8e1c71b56624f2ecaeff5af57eac41365080449904d0717bd333c04806d: 0.001 (1 mojo)", "Accepted offer with ID dfb7e8643376820ec995b0bcdb3fc1f764c16b814df5e074631263fcf1e00839", ] - run_cli_command_and_assert(capsys, root_dir, command_args, assert_list) + + with importlib_resources.as_file(test_offer_file_path) as test_offer_file_name: + command_args = ["wallet", "take_offer", os.fspath(test_offer_file_name), FINGERPRINT_ARG, "-m1", "--reuse"] + run_cli_command_and_assert(capsys, root_dir, command_args, assert_list) + expected_calls: logType = { "cat_asset_id_to_name": [ (cat1,), diff --git a/chia/_tests/core/daemon/test_daemon.py b/chia/_tests/core/daemon/test_daemon.py index 2c8f8154fb01..71f20e8a3b4c 100644 --- a/chia/_tests/core/daemon/test_daemon.py +++ b/chia/_tests/core/daemon/test_daemon.py @@ -1,12 +1,12 @@ from __future__ import annotations +import importlib.metadata import json from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast import aiohttp -import pkg_resources import pytest from aiohttp import WSMessage from aiohttp.web_ws import WebSocketResponse @@ -37,7 +37,7 @@ from chia.util.ws_message import create_payload, create_payload_dict from chia.wallet.derive_keys import master_sk_to_farmer_sk, master_sk_to_pool_sk -chiapos_version = pkg_resources.get_distribution("chiapos").version +chiapos_version = importlib.metadata.version("chiapos") @dataclass diff --git a/chia/_tests/core/util/test_keychain.py b/chia/_tests/core/util/test_keychain.py index cea338da1d11..934901221003 100644 --- a/chia/_tests/core/util/test_keychain.py +++ b/chia/_tests/core/util/test_keychain.py @@ -5,7 +5,7 @@ from dataclasses import replace from typing import Callable, List, Optional, Tuple -import pkg_resources +import importlib_resources import pytest from chia_rs import AugSchemeMPL, G1Element, PrivateKey @@ -156,9 +156,8 @@ def test_bip39_eip2333_test_vector(self, empty_temp_file_keyring: TempKeyring): assert child_sk == PrivateKey.from_bytes(tv_child_int.to_bytes(32, "big")) def test_bip39_test_vectors(self): - test_vectors_path = pkg_resources.resource_filename(chia._tests.util.__name__, "bip39_test_vectors.json") - with open(test_vectors_path) as f: - all_vectors = json.loads(f.read()) + test_vectors_path = importlib_resources.files(chia._tests.util.__name__).joinpath("bip39_test_vectors.json") + all_vectors = json.loads(test_vectors_path.read_text(encoding="utf-8")) for vector_list in all_vectors["english"]: entropy_bytes = bytes.fromhex(vector_list[0]) @@ -173,9 +172,8 @@ def test_bip39_test_vectors_short(self): """ Tests that the first 4 letters of each mnemonic phrase matches as if it were the full phrase """ - test_vectors_path = pkg_resources.resource_filename(chia._tests.util.__name__, "bip39_test_vectors.json") - with open(test_vectors_path) as f: - all_vectors = json.load(f) + test_vectors_path = importlib_resources.files(chia._tests.util.__name__).joinpath("bip39_test_vectors.json") + all_vectors = json.loads(test_vectors_path.read_text(encoding="utf-8")) for idx, [entropy_hex, full_mnemonic, seed, short_mnemonic] in enumerate(all_vectors["english"]): entropy_bytes = bytes.fromhex(entropy_hex) diff --git a/chia/_tests/simulation/test_simulation.py b/chia/_tests/simulation/test_simulation.py index f97c72b560a5..f9e097d67971 100644 --- a/chia/_tests/simulation/test_simulation.py +++ b/chia/_tests/simulation/test_simulation.py @@ -1,10 +1,10 @@ from __future__ import annotations +import importlib.metadata import json from typing import AsyncIterator, List, Tuple import aiohttp -import pkg_resources import pytest from chia._tests.core.node_height import node_height_at_least @@ -30,7 +30,7 @@ from chia.wallet.util.tx_config import DEFAULT_TX_CONFIG from chia.wallet.wallet_node import WalletNode -chiapos_version = pkg_resources.get_distribution("chiapos").version +chiapos_version = importlib.metadata.version("chiapos") test_constants_modified = test_constants.replace( DIFFICULTY_STARTING=uint64(2**8), diff --git a/chia/_tests/util/benchmarks.py b/chia/_tests/util/benchmarks.py index 12d8e1f384d4..54d24e055836 100644 --- a/chia/_tests/util/benchmarks.py +++ b/chia/_tests/util/benchmarks.py @@ -3,7 +3,7 @@ import random from typing import Tuple -import pkg_resources +import importlib_resources from chia_rs import AugSchemeMPL, ClassgroupElement, Coin, G1Element, G2Element, VDFInfo, VDFProof from chia.consensus.coinbase import create_farmer_coin, create_pool_coin @@ -20,9 +20,8 @@ # farmer puzzle hash ph = bytes32(b"a" * 32) -clvm_generator_bin_path = pkg_resources.resource_filename(__name__, "clvm_generator.bin") -with open(clvm_generator_bin_path, "rb") as f: - clvm_generator = f.read() +clvm_generator_bin_path = importlib_resources.files(__name__).joinpath("clvm_generator.bin") +clvm_generator = clvm_generator_bin_path.read_bytes() def rewards(height: uint32) -> Tuple[Coin, Coin]: diff --git a/chia/plotters/chiapos.py b/chia/plotters/chiapos.py index a390984d82db..bed2d23643f9 100644 --- a/chia/plotters/chiapos.py +++ b/chia/plotters/chiapos.py @@ -19,7 +19,7 @@ def get_chiapos_install_info() -> Optional[Dict[str, Any]]: - chiapos_version: str = importlib.metadata.version("chiapos") + chiapos_version = importlib.metadata.version("chiapos") return {"display_name": "Chia Proof of Space", "version": chiapos_version, "installed": True} diff --git a/chia/ssl/create_ssl.py b/chia/ssl/create_ssl.py index 33c28dd494bd..13c01448fd52 100644 --- a/chia/ssl/create_ssl.py +++ b/chia/ssl/create_ssl.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -import pkg_resources +import importlib_resources from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization @@ -29,8 +29,9 @@ def get_chia_ca_crt_key() -> Tuple[Any, Any]: - crt = pkg_resources.resource_string(__name__, "chia_ca.crt") - key = pkg_resources.resource_string(__name__, "chia_ca.key") + here = importlib_resources.files(__name__) + crt = here.joinpath("chia_ca.crt").read_bytes() + key = here.joinpath("chia_ca.key").read_bytes() return crt, key diff --git a/chia/timelord/timelord_launcher.py b/chia/timelord/timelord_launcher.py index 69c7754a3941..d476a3a50fb4 100644 --- a/chia/timelord/timelord_launcher.py +++ b/chia/timelord/timelord_launcher.py @@ -12,8 +12,6 @@ from types import FrameType from typing import Any, AsyncIterator, Dict, List, Optional -import pkg_resources - from chia.util.chia_logging import initialize_logging from chia.util.config import load_config from chia.util.default_root import DEFAULT_ROOT_PATH @@ -66,11 +64,17 @@ async def manage_proc(self, proc: asyncio.subprocess.Process) -> AsyncIterator[N def find_vdf_client() -> pathlib.Path: - location = pkg_resources.get_distribution("chiavdf").location - if location is None: + try: + import chiavdf + except ImportError: + raise Exception("Cannot import chiavdf package") + + file_string = getattr(chiavdf, "__file__", None) + if file_string is None: raise Exception("Cannot find chiavdf package location") - p = pathlib.Path(location, "vdf_client") + location = pathlib.Path(file_string).parent + p = location.joinpath("vdf_client") if p.is_file(): return p raise FileNotFoundError("Cannot find vdf_client binary. Is Timelord installed? See install-timelord.sh") diff --git a/chia/util/config.py b/chia/util/config.py index 380c9d287721..c6484c803565 100644 --- a/chia/util/config.py +++ b/chia/util/config.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Union, cast -import pkg_resources +import importlib_resources import yaml from typing_extensions import Literal @@ -25,7 +25,9 @@ def initial_config_file(filename: Union[str, Path]) -> str: - return pkg_resources.resource_string(__name__, f"initial-{filename}").decode() + initial_config_path = importlib_resources.files(__name__).joinpath(f"initial-{filename}") + contents: str = initial_config_path.read_text(encoding="utf-8") + return contents def create_default_chia_config(root_path: Path, filenames: List[str] = ["config.yaml"]) -> None: diff --git a/chia/util/keychain.py b/chia/util/keychain.py index c3d1f36d67c6..472c230c62e6 100644 --- a/chia/util/keychain.py +++ b/chia/util/keychain.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Tuple, Union, overload -import pkg_resources +import importlib_resources from bitstring import BitArray # pyright: reportMissingImports=false from chia_rs import AugSchemeMPL, G1Element, PrivateKey # pyright: reportMissingImports=false from typing_extensions import final @@ -54,7 +54,9 @@ def set_keys_root_path(keys_root_path: Path) -> None: def bip39_word_list() -> str: - return pkg_resources.resource_string(__name__, "english.txt").decode() + word_list_path = importlib_resources.files(__name__).joinpath("english.txt") + contents: str = word_list_path.read_text(encoding="utf-8") + return contents def generate_mnemonic() -> str: diff --git a/chia/wallet/puzzles/load_clvm.py b/chia/wallet/puzzles/load_clvm.py index 1c99042d75a2..89fc4396fbef 100644 --- a/chia/wallet/puzzles/load_clvm.py +++ b/chia/wallet/puzzles/load_clvm.py @@ -8,7 +8,7 @@ import tempfile from typing import List -import pkg_resources +import importlib_resources from clvm_tools_rs import compile_clvm as compile_clvm_rust from chia.types.blockchain_format.program import Program @@ -21,6 +21,8 @@ (os.environ.get("CHIA_DEV_COMPILE_CLVM_ON_IMPORT", "") != "") or ("pytest" in sys.modules) ) and os.environ.get("CHIA_DEV_COMPILE_CLVM_DISABLED", None) is None +here_name = __name__.rpartition(".")[0] + def translate_path(p_): p = str(p_) @@ -78,7 +80,7 @@ def compile_clvm(full_path: pathlib.Path, output: pathlib.Path, search_paths: Li def load_serialized_clvm( - clvm_filename, package_or_requirement=__name__, include_standard_libraries: bool = True, recompile: bool = True + clvm_filename, package_or_requirement=here_name, include_standard_libraries: bool = True, recompile: bool = True ) -> SerializedProgram: """ This function takes a .clsp file in the given package and compiles it to a @@ -92,28 +94,22 @@ def load_serialized_clvm( # Set the CHIA_DEV_COMPILE_CLVM_ON_IMPORT environment variable to anything except # "" or "0" to trigger automatic recompilation of the Chialisp on load. - if recompile: - try: - if pkg_resources.resource_exists(package_or_requirement, clvm_filename): - # Establish whether the size is zero on entry - full_path = pathlib.Path(pkg_resources.resource_filename(package_or_requirement, clvm_filename)) - output = full_path.parent / hex_filename - if not output.exists() or os.stat(full_path).st_mtime > os.stat(output).st_mtime: - search_paths = [full_path.parent] - if include_standard_libraries: - # we can't get the dir, but we can get a file then get its parent. - chia_puzzles_path = pathlib.Path( - pkg_resources.resource_filename(__name__, "__init__.py") - ).parent - search_paths.append(chia_puzzles_path) - compile_clvm(full_path, output, search_paths=search_paths) - - except NotImplementedError: - # pyinstaller doesn't support `pkg_resources.resource_exists` - # so we just fall through to loading the hex clvm - pass - - clvm_hex = pkg_resources.resource_string(package_or_requirement, hex_filename).decode("utf8") + resources = importlib_resources.files(package_or_requirement) + if recompile and not getattr(sys, "frozen", False): + full_path = resources.joinpath(clvm_filename) + if full_path.exists(): + # Establish whether the size is zero on entry + output = full_path.parent / hex_filename + if not output.exists() or os.stat(full_path).st_mtime > os.stat(output).st_mtime: + search_paths = [full_path.parent] + if include_standard_libraries: + # we can't get the dir, but we can get a file then get its parent. + chia_puzzles_path = pathlib.Path(__file__).parent + search_paths.append(chia_puzzles_path) + compile_clvm(full_path, output, search_paths=search_paths) + + clvm_path = resources.joinpath(hex_filename) + clvm_hex = clvm_path.read_text(encoding="utf-8") assert len(clvm_hex.strip()) != 0 clvm_blob = bytes.fromhex(clvm_hex) return SerializedProgram.from_bytes(clvm_blob) @@ -121,7 +117,7 @@ def load_serialized_clvm( def load_clvm( clvm_filename, - package_or_requirement=__name__, + package_or_requirement=here_name, include_standard_libraries: bool = True, recompile: bool = True, ) -> Program: @@ -139,7 +135,7 @@ def load_clvm( def load_clvm_maybe_recompile( clvm_filename, - package_or_requirement=__name__, + package_or_requirement=here_name, include_standard_libraries: bool = True, recompile: bool = recompile_requested, ) -> Program: @@ -153,7 +149,7 @@ def load_clvm_maybe_recompile( def load_serialized_clvm_maybe_recompile( clvm_filename, - package_or_requirement=__name__, + package_or_requirement=here_name, include_standard_libraries: bool = True, recompile: bool = recompile_requested, ) -> SerializedProgram: diff --git a/install-timelord.sh b/install-timelord.sh index a9654ee7baca..a6364a35a2fc 100755 --- a/install-timelord.sh +++ b/install-timelord.sh @@ -49,7 +49,7 @@ else fi export BUILD_VDF_BENCH=Y # Installs the useful vdf_bench test of CPU squaring speed -THE_PATH=$(python -c 'import pkg_resources; print( pkg_resources.get_distribution("chiavdf").location)' 2>/dev/null)/vdf_client +THE_PATH=$(python -c 'import pathlib, chiavdf, importlib_resources; print(pathlib.Path(chiavdf.__file__).parent)')/vdf_client CHIAVDF_VERSION=$(python -c 'import os; os.environ["CHIA_SKIP_SETUP"] = "1"; from setup import dependencies; t = [_ for _ in dependencies if _.startswith("chiavdf")][0]; print(t)') ubuntu_cmake_install() { diff --git a/installhelper.py b/installhelper.py index 552f2a844c63..1a09f31402ca 100644 --- a/installhelper.py +++ b/installhelper.py @@ -11,7 +11,7 @@ import subprocess from os.path import exists -from pkg_resources import parse_version +from packaging.version import Version # @@ -20,22 +20,22 @@ # Copyright (C) 2015-2018 CERN. # def make_semver(version_str: str) -> str: - v = parse_version(version_str) - major = v._version.release[0] # type: ignore[attr-defined] + v = Version(version_str) + major = v.release[0] try: - minor = v._version.release[1] # type: ignore[attr-defined] + minor = v.release[1] except IndexError: minor = 0 try: - patch = v._version.release[2] # type: ignore[attr-defined] + patch = v.release[2] except IndexError: patch = 0 prerelease = [] - if v._version.pre: # type: ignore[attr-defined] - prerelease.append("".join(str(x) for x in v._version.pre)) # type: ignore[attr-defined] - if v._version.dev: # type: ignore[attr-defined] - prerelease.append("".join(str(x) for x in v._version.dev)) # type: ignore[attr-defined] + if v.pre: + prerelease.append("".join(str(x) for x in v.pre)) + if v.dev is not None: + prerelease.append(f"dev{v.dev}") local = v.local diff --git a/pytest.ini b/pytest.ini index 89070e0e1bbf..bdb4d9fd6b1c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -29,5 +29,4 @@ filterwarnings = ignore:BackendFinder.find_spec\(\) not found; falling back to find_module\(\):ImportWarning ignore:BackendLoader.exec_module\(\) not found; falling back to load_module\(\):ImportWarning ignore:The --rsyncdir command line argument and rsyncdirs config variable are deprecated.:DeprecationWarning - ignore:pkg_resources is deprecated as an API:DeprecationWarning ignore:record_property is incompatible with junit_family:pytest.PytestWarning diff --git a/setup.py b/setup.py index a315ef43890f..abf27d1a1bfc 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ "concurrent-log-handler==0.9.25", # Concurrently log and rotate logs "cryptography==42.0.5", # Python cryptography library for TLS - keyring conflict "filelock==3.14.0", # For reading and writing config multiprocess and multithread safely (non-reentrant locks) + "importlib-resources==6.4.0", "keyring==25.1.0", # Store keys in MacOS Keychain, Windows Credential Locker "PyYAML==6.0.1", # Used for config file format "setproctitle==1.3.3", # Gives the chia processes readable names