From 3b2e5f6882e174fc2581969cb071daa42b4d1bdf Mon Sep 17 00:00:00 2001 From: Florin Chirica Date: Tue, 15 Aug 2023 19:29:13 +0300 Subject: [PATCH 1/4] Timeout for retrieving a decompressor. --- chia/harvester/harvester.py | 3 +++ chia/harvester/harvester_api.py | 24 ++++++++++++++++++++++++ chia/plotting/check_plots.py | 2 ++ chia/plotting/manager.py | 2 ++ chia/plotting/util.py | 1 + chia/util/initial-config.yaml | 2 ++ 6 files changed, 34 insertions(+) diff --git a/chia/harvester/harvester.py b/chia/harvester/harvester.py index 259726794c63..c93d939d3ee6 100644 --- a/chia/harvester/harvester.py +++ b/chia/harvester/harvester.py @@ -16,6 +16,7 @@ from chia.plotting.manager import PlotManager from chia.plotting.util import ( DEFAULT_DECOMPRESSOR_THREAD_COUNT, + DEFAULT_DECOMPRESSOR_TIMEOUT, DEFAULT_DISABLE_CPU_AFFINITY, DEFAULT_ENFORCE_GPU_INDEX, DEFAULT_GPU_INDEX, @@ -103,6 +104,7 @@ def __init__(self, root_path: Path, config: Dict[str, Any], constants: Consensus use_gpu_harvesting = config.get("use_gpu_harvesting", DEFAULT_USE_GPU_HARVESTING) gpu_index = config.get("gpu_index", DEFAULT_GPU_INDEX) enforce_gpu_index = config.get("enforce_gpu_index", DEFAULT_ENFORCE_GPU_INDEX) + decompressor_timeout = config.get("decompressor_timeout", DEFAULT_DECOMPRESSOR_TIMEOUT) try: self._mode = self.plot_manager.configure_decompressor( @@ -113,6 +115,7 @@ def __init__(self, root_path: Path, config: Dict[str, Any], constants: Consensus use_gpu_harvesting, gpu_index, enforce_gpu_index, + decompressor_timeout, ) except Exception as e: self.log.error(f"{type(e)} {e} while configuring decompressor.") diff --git a/chia/harvester/harvester_api.py b/chia/harvester/harvester_api.py index 49b5240ad1aa..fabaeeda4d92 100644 --- a/chia/harvester/harvester_api.py +++ b/chia/harvester/harvester_api.py @@ -99,6 +99,22 @@ def blocking_lookup(filename: Path, plot_info: PlotInfo) -> List[Tuple[bytes32, ) try: quality_strings = plot_info.prover.get_qualities_for_challenge(sp_challenge_hash) + except RuntimeError as e: + if str(e) == "Timeout waiting for context queue.": + self.harvester.log.warning( + f"No decompressor available. Cancelling qualities retrieving for {filename}" + ) + self.harvester.log.warning( + f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, " + f"plot_info: {plot_info}" + ) + else: + self.harvester.log.error(f"Exception fetching qualities for {filename}. {e}") + self.harvester.log.error( + f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, " + f"plot_info: {plot_info}" + ) + return [] except Exception as e: self.harvester.log.error(f"Error using prover object {e}") self.harvester.log.error( @@ -146,6 +162,14 @@ def blocking_lookup(filename: Path, plot_info: PlotInfo) -> List[Tuple[bytes32, f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, " f"plot_info: {plot_info}" ) + elif str(e) == "Timeout waiting for context queue.": + self.harvester.log.warning( + f"No decompressor available. Cancelling full proof retrieving for {filename}" + ) + self.harvester.log.warning( + f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, " + f"plot_info: {plot_info}" + ) else: self.harvester.log.error(f"Exception fetching full proof for {filename}. {e}") self.harvester.log.error( diff --git a/chia/plotting/check_plots.py b/chia/plotting/check_plots.py index 34caf7e92bae..50763a7ab1dc 100644 --- a/chia/plotting/check_plots.py +++ b/chia/plotting/check_plots.py @@ -64,6 +64,7 @@ def check_plots( use_gpu_harvesting = config["harvester"].get("use_gpu_harvesting", False) gpu_index = config["harvester"].get("gpu_index", 0) enforce_gpu_index = config["harvester"].get("enforce_gpu_index", False) + decompressor_timeout = config["harvester"].get("decompressor_timeout", 20) plot_manager.configure_decompressor( context_count, @@ -73,6 +74,7 @@ def check_plots( use_gpu_harvesting, gpu_index, enforce_gpu_index, + decompressor_timeout, ) if num is not None: diff --git a/chia/plotting/manager.py b/chia/plotting/manager.py index 3acd314bf5d9..cb6ebd68e16f 100644 --- a/chia/plotting/manager.py +++ b/chia/plotting/manager.py @@ -97,6 +97,7 @@ def configure_decompressor( use_gpu_harvesting: bool, gpu_index: int, enforce_gpu_index: bool, + decompressor_timeout: int, ) -> HarvestingMode: if max_compression_level_allowed > 7: log.error( @@ -113,6 +114,7 @@ def configure_decompressor( use_gpu_harvesting, gpu_index, enforce_gpu_index, + decompressor_timeout, ) if not is_using_gpu and use_gpu_harvesting: log.error( diff --git a/chia/plotting/util.py b/chia/plotting/util.py index dbcdc597d66c..b84d8634d921 100644 --- a/chia/plotting/util.py +++ b/chia/plotting/util.py @@ -19,6 +19,7 @@ DEFAULT_PARALLEL_DECOMPRESSOR_COUNT = 0 DEFAULT_DECOMPRESSOR_THREAD_COUNT = 0 +DEFAULT_DECOMPRESSOR_TIMEOUT = 20 DEFAULT_DISABLE_CPU_AFFINITY = False DEFAULT_MAX_COMPRESSION_LEVEL_ALLOWED = 7 DEFAULT_USE_GPU_HARVESTING = False diff --git a/chia/util/initial-config.yaml b/chia/util/initial-config.yaml index c1a5e362f91b..13c826172616 100644 --- a/chia/util/initial-config.yaml +++ b/chia/util/initial-config.yaml @@ -222,6 +222,8 @@ harvester: use_gpu_harvesting: False gpu_index: 0 enforce_gpu_index: False + # If no decompressor is available after `decompressor_timeout` seconds, abort the current operation. + decompressor_timeout: 20 pool: # Replace this with a real receive address From 9d3d19685a2f93d762f452b69475577ec7471b9a Mon Sep 17 00:00:00 2001 From: wallentx Date: Tue, 15 Aug 2023 14:17:36 -0500 Subject: [PATCH 2/4] Pre-commit fix --- chia/util/initial-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/util/initial-config.yaml b/chia/util/initial-config.yaml index 13c826172616..8e7c6707730f 100644 --- a/chia/util/initial-config.yaml +++ b/chia/util/initial-config.yaml @@ -222,7 +222,7 @@ harvester: use_gpu_harvesting: False gpu_index: 0 enforce_gpu_index: False - # If no decompressor is available after `decompressor_timeout` seconds, abort the current operation. + # If no decompressor is available after `decompressor_timeout` seconds, abort the current operation. decompressor_timeout: 20 pool: From 93bc09ee89ba96f54dc10e14d46331310aba8ff4 Mon Sep 17 00:00:00 2001 From: Florin Chirica Date: Tue, 15 Aug 2023 22:49:43 +0300 Subject: [PATCH 3/4] Use new chiapos --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ca6ef19ae199..5cb7992f02a5 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ "boto3==1.26.161", # AWS S3 for DL s3 plugin "chiavdf==1.0.10", # timelord and vdf verification "chiabip158==1.2", # bip158-style wallet filters - "chiapos==2.0.0", # proof of space + "chiapos==2.0.1b1", # proof of space "clvm==0.9.7", "clvm_tools==0.4.6", # Currying, Program.to, other conveniences "chia_rs==0.2.10", From e0c0adf1d07544831611ff732eaa4df4e7d7ca55 Mon Sep 17 00:00:00 2001 From: wallentx Date: Wed, 16 Aug 2023 15:56:43 -0500 Subject: [PATCH 4/4] Use chiapos 2.0.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5cb7992f02a5..d1b0cfac1b3d 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ "boto3==1.26.161", # AWS S3 for DL s3 plugin "chiavdf==1.0.10", # timelord and vdf verification "chiabip158==1.2", # bip158-style wallet filters - "chiapos==2.0.1b1", # proof of space + "chiapos==2.0.1", # proof of space "clvm==0.9.7", "clvm_tools==0.4.6", # Currying, Program.to, other conveniences "chia_rs==0.2.10",