From 94bd76316dcc240523236f11b8665152053d7e0e Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Fri, 17 Apr 2026 15:28:13 +0530 Subject: [PATCH 1/3] [FIX] Use importlib.util.find_spec for pluggable worker discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _verify_pluggable_worker_exists() previously checked for the literal file `pluggable_worker//worker.py` on disk, which breaks when the plugin has been compiled to a .so (Nuitka, Cython, or any C extension) — the module is perfectly importable but the pre-check rejects it because only the .py extension is considered. Replace the filesystem check with importlib.util.find_spec(), which is Python's standard way to ask "is this module resolvable by the import system?". It honors every registered finder — source .py, compiled .so, bytecode .pyc, namespace packages, zipimports — so the function now matches what its docstring claims: verifying the module can be loaded, not that a specific file extension is present. Behavior is preserved for existing deployments: - Images with no `pluggable_worker//` subpackage → find_spec raises ModuleNotFoundError (ImportError subclass) → returns False. - Images with source .py → find_spec resolves the .py → returns True. - Images with compiled .so → find_spec resolves the .so → returns True. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../shared/infrastructure/config/builder.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/workers/shared/infrastructure/config/builder.py b/workers/shared/infrastructure/config/builder.py index 2d9491177f..a4be324ad5 100644 --- a/workers/shared/infrastructure/config/builder.py +++ b/workers/shared/infrastructure/config/builder.py @@ -323,24 +323,21 @@ def _verify_pluggable_worker_exists(worker_type: WorkerType) -> bool: try: import importlib - from pathlib import Path - - # Check if the worker.py file exists - # Path resolution: builder.py is at /app/workers/shared/infrastructure/config/ - # We need to get to /app/workers/, so go up 3 levels - pluggable_worker_path = ( - Path(__file__).resolve().parents[3] - / "pluggable_worker" - / worker_type.value - / "worker.py" - ) + import importlib.util + + # Ask Python's import system whether the module is resolvable. + # find_spec() consults all registered finders and handles every + # module representation (source .py, Nuitka/Cython .so, .pyc, + # namespace packages, zipimports) — unlike a filesystem check + # for a specific file extension, which breaks for compiled plugins. + module_path = f"pluggable_worker.{worker_type.value}.worker" - if not pluggable_worker_path.exists(): - logger.error(f"Pluggable worker file not found: {pluggable_worker_path}") + if importlib.util.find_spec(module_path) is None: + logger.error(f"Pluggable worker module not importable: {module_path}") return False - # Try to import the module to verify it's valid - module_path = f"pluggable_worker.{worker_type.value}.worker" + # Load it to catch findable-but-broken modules (e.g. import errors + # inside worker.py that find_spec wouldn't surface). importlib.import_module(module_path) except ImportError: From 52e342dbad2dcf4ea2175f7ab5290e3101a51389 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Fri, 17 Apr 2026 18:41:14 +0530 Subject: [PATCH 2/3] [FIX] Handle ValueError from find_spec in pluggable worker verification Greptile-flagged edge case: importlib.util.find_spec() can raise ValueError (not just ImportError) when sys.modules has a partially initialised module entry with __spec__ = None from a prior failed import. Broaden the except to catch both. Co-Authored-By: Claude Opus 4.7 (1M context) --- workers/shared/infrastructure/config/builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workers/shared/infrastructure/config/builder.py b/workers/shared/infrastructure/config/builder.py index a4be324ad5..859d823b98 100644 --- a/workers/shared/infrastructure/config/builder.py +++ b/workers/shared/infrastructure/config/builder.py @@ -340,7 +340,9 @@ def _verify_pluggable_worker_exists(worker_type: WorkerType) -> bool: # inside worker.py that find_spec wouldn't surface). importlib.import_module(module_path) - except ImportError: + except (ImportError, ValueError): + # ValueError: find_spec raises this when sys.modules[module_path] is + # populated but has __spec__ = None (from a prior failed import). logger.exception(f"Failed to import pluggable worker {worker_type.value}") return False except (OSError, AttributeError): From 3179fb6701a20d4f6653dd8de4b28d852ef560e6 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Sat, 18 Apr 2026 11:10:53 +0530 Subject: [PATCH 3/3] [FIX] Resolve api-deployment worker directory from enum import path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit worker.py:452 did worker_type.value.replace("-", "_") to derive the on-disk dir name. All WorkerType enum values already use underscores, so the replace was a no-op; for API_DEPLOYMENT whose dir is "api-deployment" (hyphen), it resolved to "api_deployment" and the os.path.exists() check failed. Boot then logged a spurious "❌ Worker directory not found: /app/api_deployment" at ERROR level. The task registration path (builder + celery autodiscover via to_import_path) is unaffected, so this was purely log noise — but noise at ERROR level that masks real failures in log scans. Fix: derive the directory from the authoritative to_import_path() which already handles the hyphen case (api_deployment -> api-deployment). Co-Authored-By: Claude Opus 4.7 (1M context) --- workers/worker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/workers/worker.py b/workers/worker.py index ed2a605848..3822e81a16 100755 --- a/workers/worker.py +++ b/workers/worker.py @@ -448,8 +448,10 @@ def on_task_postrun(sender=None, task_id=None, **kwargs): worker_directory = os.path.join("pluggable_worker", worker_type.value) worker_path = os.path.join(base_dir, worker_directory) else: - # Core workers use their value directly (with hyphens converted to underscores where needed) - worker_directory = worker_type.value.replace("-", "_") + # Enum values use underscores (Python module names); a few on-disk dirs + # still use hyphens (e.g. api-deployment). Derive the directory from the + # authoritative import-path map on WorkerType instead of a blind replace. + worker_directory = worker_type.to_import_path().rsplit(".", 1)[0] worker_path = os.path.join(base_dir, worker_directory) # Add worker directory to path for task imports