Config: Add option to change recursion limit in daemon workers (#6044)

Add the `daemon.recursion_limit` config option which is used by a daemon
worker to set the recursion limit using `sys.setrecursionlimit`. This is
to help with the current problem where heavy workloads with many process
functions can create deep stacks which would result in `RecursionError`
exceptions being thrown. The default is set to 3000, increased from the
built-in default of 1000. This value has precedent as it is also used by
`jedi` which is used for `ipython`.

aiida/engine/daemon/worker.py

@@ -11,11 +11,12 @@
 import asyncio
 import logging
 import signal
+import sys
 
 from aiida.common.log import configure_logging
 from aiida.engine.daemon.client import get_daemon_client
 from aiida.engine.runners import Runner
-from aiida.manage import get_manager
+from aiida.manage import get_config_option, get_manager
 
 LOGGER = logging.getLogger(__name__)
 
@@ -49,6 +50,10 @@ def start_daemon_worker() -> None:
         LOGGER.exception('daemon worker failed to start')
         raise
 
+    if isinstance(rlimit := get_config_option('daemon.recursion_limit'), int):
+        LOGGER.info('Setting maximum recursion limit of daemon worker to %s', rlimit)
+        sys.setrecursionlimit(rlimit)
+
     signals = (signal.SIGTERM, signal.SIGINT)
     for s in signals:  # pylint: disable=invalid-name
         runner.loop.add_signal_handler(s, lambda s=s: asyncio.create_task(shutdown_worker(runner)))

aiida/manage/configuration/schema/config-v9.schema.json

@@ -30,6 +30,13 @@
           "minimum": 1,
           "description": "Maximum number of concurrent process tasks that each daemon worker can handle"
         },
+        "daemon.recursion_limit": {
+          "type": "integer",
+          "default": 3000,
+          "maximum": 100000,
+          "minimum": 1,
+          "description": "Maximum recursion depth for the daemon workers"
+        },
         "db.batch_size": {
           "type": "integer",
           "default": 100000,