diff --git a/chia/harvester/harvester.py b/chia/harvester/harvester.py index 259726794c63..c93d939d3ee6 100644 --- a/chia/harvester/harvester.py +++ b/chia/harvester/harvester.py @@ -16,6 +16,7 @@ from chia.plotting.manager import PlotManager from chia.plotting.util import ( DEFAULT_DECOMPRESSOR_THREAD_COUNT, + DEFAULT_DECOMPRESSOR_TIMEOUT, DEFAULT_DISABLE_CPU_AFFINITY, DEFAULT_ENFORCE_GPU_INDEX, DEFAULT_GPU_INDEX, @@ -103,6 +104,7 @@ def __init__(self, root_path: Path, config: Dict[str, Any], constants: Consensus use_gpu_harvesting = config.get("use_gpu_harvesting", DEFAULT_USE_GPU_HARVESTING) gpu_index = config.get("gpu_index", DEFAULT_GPU_INDEX) enforce_gpu_index = config.get("enforce_gpu_index", DEFAULT_ENFORCE_GPU_INDEX) + decompressor_timeout = config.get("decompressor_timeout", DEFAULT_DECOMPRESSOR_TIMEOUT) try: self._mode = self.plot_manager.configure_decompressor( @@ -113,6 +115,7 @@ def __init__(self, root_path: Path, config: Dict[str, Any], constants: Consensus use_gpu_harvesting, gpu_index, enforce_gpu_index, + decompressor_timeout, ) except Exception as e: self.log.error(f"{type(e)} {e} while configuring decompressor.") diff --git a/chia/harvester/harvester_api.py b/chia/harvester/harvester_api.py index 49b5240ad1aa..fabaeeda4d92 100644 --- a/chia/harvester/harvester_api.py +++ b/chia/harvester/harvester_api.py @@ -99,6 +99,22 @@ def blocking_lookup(filename: Path, plot_info: PlotInfo) -> List[Tuple[bytes32, ) try: quality_strings = plot_info.prover.get_qualities_for_challenge(sp_challenge_hash) + except RuntimeError as e: + if str(e) == "Timeout waiting for context queue.": + self.harvester.log.warning( + f"No decompressor available. Cancelling qualities retrieving for {filename}" + ) + self.harvester.log.warning( + f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, " + f"plot_info: {plot_info}" + ) + else: + self.harvester.log.error(f"Exception fetching qualities for {filename}. {e}") + self.harvester.log.error( + f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, " + f"plot_info: {plot_info}" + ) + return [] except Exception as e: self.harvester.log.error(f"Error using prover object {e}") self.harvester.log.error( @@ -146,6 +162,14 @@ def blocking_lookup(filename: Path, plot_info: PlotInfo) -> List[Tuple[bytes32, f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, " f"plot_info: {plot_info}" ) + elif str(e) == "Timeout waiting for context queue.": + self.harvester.log.warning( + f"No decompressor available. Cancelling full proof retrieving for {filename}" + ) + self.harvester.log.warning( + f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, " + f"plot_info: {plot_info}" + ) else: self.harvester.log.error(f"Exception fetching full proof for {filename}. {e}") self.harvester.log.error( diff --git a/chia/plotting/check_plots.py b/chia/plotting/check_plots.py index 34caf7e92bae..50763a7ab1dc 100644 --- a/chia/plotting/check_plots.py +++ b/chia/plotting/check_plots.py @@ -64,6 +64,7 @@ def check_plots( use_gpu_harvesting = config["harvester"].get("use_gpu_harvesting", False) gpu_index = config["harvester"].get("gpu_index", 0) enforce_gpu_index = config["harvester"].get("enforce_gpu_index", False) + decompressor_timeout = config["harvester"].get("decompressor_timeout", 20) plot_manager.configure_decompressor( context_count, @@ -73,6 +74,7 @@ def check_plots( use_gpu_harvesting, gpu_index, enforce_gpu_index, + decompressor_timeout, ) if num is not None: diff --git a/chia/plotting/manager.py b/chia/plotting/manager.py index 3acd314bf5d9..cb6ebd68e16f 100644 --- a/chia/plotting/manager.py +++ b/chia/plotting/manager.py @@ -97,6 +97,7 @@ def configure_decompressor( use_gpu_harvesting: bool, gpu_index: int, enforce_gpu_index: bool, + decompressor_timeout: int, ) -> HarvestingMode: if max_compression_level_allowed > 7: log.error( @@ -113,6 +114,7 @@ def configure_decompressor( use_gpu_harvesting, gpu_index, enforce_gpu_index, + decompressor_timeout, ) if not is_using_gpu and use_gpu_harvesting: log.error( diff --git a/chia/plotting/util.py b/chia/plotting/util.py index dbcdc597d66c..b84d8634d921 100644 --- a/chia/plotting/util.py +++ b/chia/plotting/util.py @@ -19,6 +19,7 @@ DEFAULT_PARALLEL_DECOMPRESSOR_COUNT = 0 DEFAULT_DECOMPRESSOR_THREAD_COUNT = 0 +DEFAULT_DECOMPRESSOR_TIMEOUT = 20 DEFAULT_DISABLE_CPU_AFFINITY = False DEFAULT_MAX_COMPRESSION_LEVEL_ALLOWED = 7 DEFAULT_USE_GPU_HARVESTING = False diff --git a/chia/util/initial-config.yaml b/chia/util/initial-config.yaml index c1a5e362f91b..8e7c6707730f 100644 --- a/chia/util/initial-config.yaml +++ b/chia/util/initial-config.yaml @@ -222,6 +222,8 @@ harvester: use_gpu_harvesting: False gpu_index: 0 enforce_gpu_index: False + # If no decompressor is available after `decompressor_timeout` seconds, abort the current operation. + decompressor_timeout: 20 pool: # Replace this with a real receive address diff --git a/setup.py b/setup.py index ca6ef19ae199..d1b0cfac1b3d 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ "boto3==1.26.161", # AWS S3 for DL s3 plugin "chiavdf==1.0.10", # timelord and vdf verification "chiabip158==1.2", # bip158-style wallet filters - "chiapos==2.0.0", # proof of space + "chiapos==2.0.1", # proof of space "clvm==0.9.7", "clvm_tools==0.4.6", # Currying, Program.to, other conveniences "chia_rs==0.2.10",