Timeout for retrieving a decompressor. (#16047)

* Timeout for retrieving a decompressor.

* Pre-commit fix

* Use new chiapos

* Use chiapos 2.0.1

---------

Co-authored-by: wallentx <william.allentx@gmail.com>

chia/harvester/harvester.py

@@ -16,6 +16,7 @@
 from chia.plotting.manager import PlotManager
 from chia.plotting.util import (
     DEFAULT_DECOMPRESSOR_THREAD_COUNT,
+    DEFAULT_DECOMPRESSOR_TIMEOUT,
     DEFAULT_DISABLE_CPU_AFFINITY,
     DEFAULT_ENFORCE_GPU_INDEX,
     DEFAULT_GPU_INDEX,
@@ -103,6 +104,7 @@ def __init__(self, root_path: Path, config: Dict[str, Any], constants: Consensus
         use_gpu_harvesting = config.get("use_gpu_harvesting", DEFAULT_USE_GPU_HARVESTING)
         gpu_index = config.get("gpu_index", DEFAULT_GPU_INDEX)
         enforce_gpu_index = config.get("enforce_gpu_index", DEFAULT_ENFORCE_GPU_INDEX)
+        decompressor_timeout = config.get("decompressor_timeout", DEFAULT_DECOMPRESSOR_TIMEOUT)
 
         try:
             self._mode = self.plot_manager.configure_decompressor(
@@ -113,6 +115,7 @@ def __init__(self, root_path: Path, config: Dict[str, Any], constants: Consensus
                 use_gpu_harvesting,
                 gpu_index,
                 enforce_gpu_index,
+                decompressor_timeout,
             )
         except Exception as e:
             self.log.error(f"{type(e)} {e} while configuring decompressor.")

chia/harvester/harvester_api.py

@@ -99,6 +99,22 @@ def blocking_lookup(filename: Path, plot_info: PlotInfo) -> List[Tuple[bytes32,
                 )
                 try:
                     quality_strings = plot_info.prover.get_qualities_for_challenge(sp_challenge_hash)
+                except RuntimeError as e:
+                    if str(e) == "Timeout waiting for context queue.":
+                        self.harvester.log.warning(
+                            f"No decompressor available. Cancelling qualities retrieving for {filename}"
+                        )
+                        self.harvester.log.warning(
+                            f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, "
+                            f"plot_info: {plot_info}"
+                        )
+                    else:
+                        self.harvester.log.error(f"Exception fetching qualities for {filename}. {e}")
+                        self.harvester.log.error(
+                            f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, "
+                            f"plot_info: {plot_info}"
+                        )
+                    return []
                 except Exception as e:
                     self.harvester.log.error(f"Error using prover object {e}")
                     self.harvester.log.error(
@@ -146,6 +162,14 @@ def blocking_lookup(filename: Path, plot_info: PlotInfo) -> List[Tuple[bytes32,
                                         f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, "
                                         f"plot_info: {plot_info}"
                                     )
+                                elif str(e) == "Timeout waiting for context queue.":
+                                    self.harvester.log.warning(
+                                        f"No decompressor available. Cancelling full proof retrieving for {filename}"
+                                    )
+                                    self.harvester.log.warning(
+                                        f"File: {filename} Plot ID: {plot_id.hex()}, challenge: {sp_challenge_hash}, "
+                                        f"plot_info: {plot_info}"
+                                    )
                                 else:
                                     self.harvester.log.error(f"Exception fetching full proof for {filename}. {e}")
                                     self.harvester.log.error(

chia/plotting/check_plots.py

@@ -64,6 +64,7 @@ def check_plots(
     use_gpu_harvesting = config["harvester"].get("use_gpu_harvesting", False)
     gpu_index = config["harvester"].get("gpu_index", 0)
     enforce_gpu_index = config["harvester"].get("enforce_gpu_index", False)
+    decompressor_timeout = config["harvester"].get("decompressor_timeout", 20)
 
     plot_manager.configure_decompressor(
         context_count,
@@ -73,6 +74,7 @@ def check_plots(
         use_gpu_harvesting,
         gpu_index,
         enforce_gpu_index,
+        decompressor_timeout,
     )
 
     if num is not None:

chia/plotting/manager.py

@@ -97,6 +97,7 @@ def configure_decompressor(
         use_gpu_harvesting: bool,
         gpu_index: int,
         enforce_gpu_index: bool,
+        decompressor_timeout: int,
     ) -> HarvestingMode:
         if max_compression_level_allowed > 7:
             log.error(
@@ -113,6 +114,7 @@ def configure_decompressor(
             use_gpu_harvesting,
             gpu_index,
             enforce_gpu_index,
+            decompressor_timeout,
         )
         if not is_using_gpu and use_gpu_harvesting:
             log.error(

chia/plotting/util.py

@@ -19,6 +19,7 @@
 
 DEFAULT_PARALLEL_DECOMPRESSOR_COUNT = 0
 DEFAULT_DECOMPRESSOR_THREAD_COUNT = 0
+DEFAULT_DECOMPRESSOR_TIMEOUT = 20
 DEFAULT_DISABLE_CPU_AFFINITY = False
 DEFAULT_MAX_COMPRESSION_LEVEL_ALLOWED = 7
 DEFAULT_USE_GPU_HARVESTING = False

chia/util/initial-config.yaml

@@ -222,6 +222,8 @@ harvester:
   use_gpu_harvesting: False
   gpu_index: 0
   enforce_gpu_index: False
+  # If no decompressor is available after `decompressor_timeout` seconds, abort the current operation.
+  decompressor_timeout: 20
 
 pool:
   # Replace this with a real receive address

setup.py

@@ -12,7 +12,7 @@
     "boto3==1.26.161",  # AWS S3 for DL s3 plugin
     "chiavdf==1.0.10",  # timelord and vdf verification
     "chiabip158==1.2",  # bip158-style wallet filters
-    "chiapos==2.0.0",  # proof of space
+    "chiapos==2.0.1",  # proof of space
     "clvm==0.9.7",
     "clvm_tools==0.4.6",  # Currying, Program.to, other conveniences
     "chia_rs==0.2.10",