feat: plugin bundling, catalog, installation and versioning

.env.example

@@ -6,9 +6,25 @@
 #   `allow`, `deny`
 # PLUGINS_DEFAULT_HOOK_POLICY=allow
 
+# Path to plugins folder
+# PLUGINS_FOLDER=plugins
 # Path to main plugins configuration file
 # PLUGINS_CONFIG_FILE=plugins/config.yaml
 
+### Plugin installation
+# Comma Separated Values used by install with --type monorepo
+# PLUGINS_REPO_URLS="https://github.com/ibm/cpex-plugins"
+
+# registry path
+# PLUGIN_REGISTRY_FOLDER=data
+
+# Github API
+# PLUGINS_GITHUB_API=api.github.com
+
+# PLUGINS_GITHUB_TOKEN=<github token>
+### end Plugin installation
+
+
 # Logging level for plugin framework components
 # PLUGINS_LOG_LEVEL=INFO
 
@@ -148,3 +164,15 @@
 # PLUGINS_GRPC_SERVER_SSL_ENABLED=
 
 
+
+### Package Integrity Verification
+# Enable SHA256 hash verification for PyPI packages (default: True)
+# When enabled, downloaded packages are verified against hashes from PyPI's JSON API
+# Recommended: Keep enabled for security
+# PLUGINS_VERIFY_PACKAGE_INTEGRITY=True
+
+# Strict integrity mode (default: False)
+# When True: Fail installation if package hashes are unavailable
+# When False: Warn but continue if hashes are unavailable
+# Recommended: False for development, True for production
+# PLUGINS_STRICT_INTEGRITY_MODE=False

.gitignore

@@ -250,4 +250,4 @@ db_path/
 tmp/
 
 .continue
-
+plugin-catalog

cpex/framework/isolated/client.py

@@ -27,6 +27,7 @@
 from cpex.framework.hooks.registry import get_hook_registry
 from cpex.framework.isolated.venv_comm import VenvProcessCommunicator
 from cpex.framework.models import PluginConfig, PluginContext, PluginErrorModel, PluginPayload, PluginResult
+from cpex.framework.utils import find_package_path
 
 logger = logging.getLogger(__name__)
 
@@ -43,10 +44,10 @@ def __init__(self, config: PluginConfig, plugin_dirs) -> None:
         # use the first plugin dir specified in the plugin configuration file.
         path = Path(self.plugin_dirs[0]).resolve()
         class_root = self.config.config.get("class_name").split(".")[0]
-        cache_root = path / class_root
-        self.plugin_path = cache_root
+        cache_root: Path = path / class_root
+        self.plugin_path: Path = cache_root
         if not cache_root.exists():
-            raise RuntimeError(f"plugin path does not exist: {str(cache_root)}")
+            cache_root.mkdir(parents=True, exist_ok=True)
         self.cache_dir: Path = cache_root / ".cpex" / "venv_cache"
         self.cache_dir.mkdir(parents=True, exist_ok=True)
 
@@ -217,21 +218,17 @@ async def initialize(self) -> None:
         else:
             requirements_file = Path(requirements_file_input)
 
-        # If it's a relative path, resolve it relative to plugin_path
-        if not requirements_file.is_absolute():
-            requirements_file = (self.plugin_path / requirements_file).resolve()
-        else:
-            # If absolute, resolve it to normalize
-            requirements_file = requirements_file.resolve()
-
-        # Validate that the resolved path is within plugin_path (security check)
+        # Try to find the package location where plugin-manifest.yaml resides
+        # Fall back to self.plugin_path if package is not installed (e.g., in tests)
         try:
-            requirements_file.relative_to(self.plugin_path.resolve())
-        except ValueError as ve:
-            raise RuntimeError(
-                f"Invalid requirements_file path: {requirements_file_input}. "
-                f"Path must be within plugin directory: {self.plugin_path}"
-            ) from ve
+            package_path = find_package_path(self.config.name)
+            logger.debug("Found installed package %s at %s", self.config.name, package_path)
+        except RuntimeError:
+            # Package not installed (e.g., in test environment), use plugin_path
+            package_path = self.plugin_path
+            logger.debug("Package %s not installed, using plugin_path: %s", self.config.name, package_path)
+
+        requirements_file = package_path / requirements_file_input
 
         # Create venv with caching support
         new_venv = await self.create_venv(venv_path=venv_path, requirements_file=requirements_file, use_cache=True)
@@ -339,3 +336,10 @@ async def invoke_hook(self, hook_type: str, payload: PluginPayload, context: Plu
         except Exception as e:
             logger.exception("Unexpected error invoking hook '%s' for plugin '%s'", hook_type, self.name)
             raise PluginError(error=convert_exception_to_error(e, plugin_name=self.name)) from e
+
+    def remove_venv(self):
+        """
+        Remove the virtual environment associated with the plugin.
+        """
+        shutil.rmtree(self.plugin_path.joinpath(".cpex"))
+        shutil.rmtree(self.plugin_path.joinpath(".venv"))

cpex/framework/isolated/venv_comm.py

@@ -53,6 +53,13 @@ def _get_python_executable(self):
 
         return str(python_exe)
 
+    def upgrade_pip(self) -> None:
+        """Upgrade pip in the target venv."""
+        try:
+            subprocess.check_call([self.python_executable, "-m", "pip", "install", "--upgrade", "pip"])
+        except Exception as e:
+            raise RuntimeError("Failed to upgrade pip") from e
+
     def install_requirements(self, requirements_file: str) -> None:
         """
         Install Python requirements from a file in the target venv.
@@ -62,6 +69,7 @@ def install_requirements(self, requirements_file: str) -> None:
         requirements_path = Path(requirements_file)
         if requirements_path.exists():
             try:
+                self.upgrade_pip()
                 subprocess.check_call([self.python_executable, "-m", "pip", "install", "-r", requirements_file])
             except Exception as e:
                 raise RuntimeError(f"Failed to install requirements from {requirements_file}") from e

cpex/framework/isolated/worker.py

@@ -24,7 +24,7 @@
 from cpex.framework.loader.plugin import ALLOWED_PLUGIN_DIRS
 from cpex.framework.manager import PluginExecutor
 from cpex.framework.models import PluginConfig, PluginContext
-from cpex.framework.utils import parse_class_name
+from cpex.framework.utils import import_module, parse_class_name
 
 logger = logging.getLogger(__name__)
 
@@ -115,7 +115,7 @@ async def process_task(task_data, tp: TaskProcessor):
             hook_type = task_data.get(HOOK_TYPE)
             cls_name: str = task_data.get("class_name")
             mod_name, n_cls_name = parse_class_name(cls_name)
-            module: ModuleType = importlib.import_module(mod_name)
+            module: ModuleType = import_module(mod_name)
             # cool, we found the module, and verified it implemented the hook type.
             class_ = getattr(module, n_cls_name)
             plugin_type = cast(Type[Plugin], class_)
@@ -162,7 +162,7 @@ async def main():
         while True:
             try:
                 # Read one line at a time
-                if tp.plugin_config:
+                if tp.plugin_config and "max_content_size" in tp.plugin_config:
                     line = sys.stdin.readline(limit=int(tp.plugin_config.max_content_size))
                 else:
                     # on the first read, the plugin_config has not yet been initialized so just read.
@@ -198,14 +198,17 @@ async def main():
                 serialized_response = json.dumps(serializable_response)
                 # Send response back to parent (one line per response)
                 if tp.plugin_config:
-                    if len(serialized_response) > tp.plugin_config.max_content_size:
-                        logger.error("Serialized response exceeds max content size")
-                        error_response = {
-                            "status": "error",
-                            "message": "Serialized response exceeds max content size",
-                            "request_id": request_id,
-                        }
-                        serialized_response = json.dumps(error_response)
+                    # workaround until cpex is updated beyond dev11
+                    # cpex is a dependency of the plugin and as such it's PluginConfig does not contain the max_content_size yet.
+                    if "max_content_size" in tp.plugin_config:
+                        if len(serialized_response) > tp.plugin_config.max_content_size:
+                            logger.error("Serialized response exceeds max content size")
+                            error_response = {
+                                "status": "error",
+                                "message": "Serialized response exceeds max content size",
+                                "request_id": request_id,
+                            }
+                            serialized_response = json.dumps(error_response)
                 print(serialized_response, flush=True)
 
             except json.JSONDecodeError as e:

cpex/framework/loader/config.py

@@ -79,3 +79,22 @@ def load_config(config: str, use_jinja: bool = True) -> Config:
         except FileNotFoundError:
             # Graceful fallback for tests and minimal environments without plugin config
             return Config(plugins=[], plugin_dirs=[])
+
+
+class ConfigSaver:
+    """
+    A configuration saver
+    """
+
+    @staticmethod
+    def save_config(config: Config, config_path: str) -> None:
+        """
+        Save the supplied configuration data to the filesystem
+        """
+        try:
+            updated_content = yaml.safe_dump(config.model_dump(mode="json"), default_flow_style=False)
+            with open(os.path.normpath(config_path), "w", encoding="utf-8") as file:
+                file.write(updated_content)
+                file.flush()
+        except OSError as ose:
+            raise RuntimeError(f"Error saving PluginConfig to {config_path}") from ose