Changed: Enhance DataLayer Plugin Registration System for Improved Third-Party Integration

chia/_tests/core/data_layer/test_plugin.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import pytest
+
+from chia.data_layer.data_layer_util import PluginRemote
+from chia.data_layer.util.plugin import load_plugin_configurations
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.anyio
+async def test_load_plugin_configurations(tmp_path: Path) -> None:
+    # Setup test environment
+    plugin_type = "downloaders"
+    root_path = tmp_path / "plugins_root"
+    config_path = root_path / "plugins" / plugin_type
+    config_path.mkdir(parents=True)
+
+    # Create valid and invalid config files
+    valid_config = [
+        {"url": "https://example.com/plugin1"},
+        {"url": "https://example.com/plugin2", "headers": {"Authorization": "Bearer token"}},
+    ]
+    invalid_config = {"config": "invalid"}
+    with open(config_path / "valid.conf", "w") as file:
+        json.dump(valid_config, file)
+    with open(config_path / "invalid.conf", "w") as file:
+        json.dump(invalid_config, file)
+
+    # Test loading configurations
+    loaded_configs = await load_plugin_configurations(root_path, plugin_type, log)
+
+    expected_configs = [
+        PluginRemote.unmarshal(marshalled=config) if isinstance(config, dict) else None for config in valid_config
+    ]
+    # Filter out None values that may have been added due to invalid config structures
+    expected_configs = list(filter(None, expected_configs))
+    assert set(loaded_configs) == set(expected_configs), "Should only load valid configurations"
+
+
+@pytest.mark.anyio
+async def test_load_plugin_configurations_no_configs(tmp_path: Path) -> None:
+    # Setup test environment with no config files
+    plugin_type = "uploaders"
+    root_path = tmp_path / "plugins_root"
+
+    # Test loading configurations with no config files
+    loaded_configs = await load_plugin_configurations(root_path, plugin_type, log)
+
+    assert loaded_configs == [], "Should return an empty list when no configurations are present"
+
+
+@pytest.mark.anyio
+async def test_load_plugin_configurations_unreadable_file(tmp_path: Path) -> None:
+    # Setup test environment
+    plugin_type = "downloaders"
+    root_path = tmp_path / "plugins_root"
+    config_path = root_path / "plugins" / plugin_type
+    config_path.mkdir(parents=True)
+
+    # Create an unreadable config file
+    unreadable_config_file = config_path / "unreadable.conf"
+    unreadable_config_file.touch()
+    unreadable_config_file.chmod(0)  # Make the file unreadable
+
+    # Test loading configurations
+    loaded_configs = await load_plugin_configurations(root_path, plugin_type, log)
+
+    assert loaded_configs == [], "Should gracefully handle unreadable files"
+
+
+@pytest.mark.anyio
+async def test_load_plugin_configurations_improper_json(tmp_path: Path) -> None:
+    # Setup test environment
+    plugin_type = "downloaders"
+    root_path = tmp_path / "plugins_root"
+    config_path = root_path / "plugins" / plugin_type
+    config_path.mkdir(parents=True)
+
+    # Create a config file with improper JSON
+    with open(config_path / "improper_json.conf", "w") as file:
+        file.write("{not: 'a valid json'}")
+
+    # Test loading configurations
+    loaded_configs = await load_plugin_configurations(root_path, plugin_type, log)
+
+    assert loaded_configs == [], "Should gracefully handle files with improper JSON"

chia/data_layer/data_layer_util.py

@@ -741,7 +741,7 @@ class PluginRemote:
     def unmarshal(cls, marshalled: Dict[str, Any]) -> PluginRemote:
         return cls(
             url=marshalled["url"],
-            headers=marshalled["headers"],
+            headers=marshalled.get("headers", {}),
         )
 
 

chia/data_layer/util/plugin.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import List
+
+import yaml
+
+from chia.data_layer.data_layer_util import PluginRemote
+from chia.util.log_exceptions import log_exceptions
+
+
+async def load_plugin_configurations(root_path: Path, config_type: str, log: logging.Logger) -> List[PluginRemote]:
+    """
+    Loads plugin configurations from the specified directory and validates that the contents
+    are in the expected JSON format (an array of PluginRemote objects). It gracefully handles errors
+    and ensures that the necessary directories exist, creating them if they do not.
+
+    Args:
+        root_path (Path): The root path where the plugins directory is located.
+        config_type (str): The type of plugins to load ('downloaders' or 'uploaders').
+
+    Returns:
+        List[PluginRemote]: A list of valid PluginRemote instances for the specified plugin type.
+    """
+    config_path = root_path / "plugins" / config_type
+    config_path.mkdir(parents=True, exist_ok=True)  # Ensure the config directory exists
+
+    valid_configs = []
+    for conf_file in config_path.glob("*.conf"):
+        with log_exceptions(
+            log=log,
+            consume=True,
+            message=f"Skipping config file due to failure loading or parsing: {conf_file}",
+        ):
+            with open(conf_file) as file:
+                data = yaml.safe_load(file)
+
+            valid_configs.extend([PluginRemote.unmarshal(marshalled=item) for item in data])
+            log.info(f"loaded plugin configuration: {conf_file}")
+    return valid_configs

chia/server/start_data_layer.py

@@ -8,6 +8,7 @@
 from chia.data_layer.data_layer import DataLayer
 from chia.data_layer.data_layer_api import DataLayerAPI
 from chia.data_layer.data_layer_util import PluginRemote
+from chia.data_layer.util.plugin import load_plugin_configurations
 from chia.rpc.data_layer_rpc_api import DataLayerRpcApi
 from chia.rpc.wallet_rpc_client import WalletRpcClient
 from chia.server.outbound_message import NodeType
@@ -100,19 +101,24 @@ async def async_main() -> int:
     )
 
     plugins_config = config["data_layer"].get("plugins", {})
+    service_dir = DEFAULT_ROOT_PATH / SERVICE_NAME
 
     old_uploaders = config["data_layer"].get("uploaders", [])
     new_uploaders = plugins_config.get("uploaders", [])
+    conf_file_uploaders = await load_plugin_configurations(service_dir, "uploaders", log)
     uploaders: List[PluginRemote] = [
         *(PluginRemote(url=url) for url in old_uploaders),
         *(PluginRemote.unmarshal(marshalled=marshalled) for marshalled in new_uploaders),
+        *conf_file_uploaders,
     ]
 
     old_downloaders = config["data_layer"].get("downloaders", [])
     new_downloaders = plugins_config.get("downloaders", [])
+    conf_file_uploaders = await load_plugin_configurations(service_dir, "downloaders", log)
     downloaders: List[PluginRemote] = [
         *(PluginRemote(url=url) for url in old_downloaders),
         *(PluginRemote.unmarshal(marshalled=marshalled) for marshalled in new_downloaders),
+        *conf_file_uploaders,
     ]
 
     service = create_data_layer_service(DEFAULT_ROOT_PATH, config, downloaders, uploaders)