chore: minor improvements to working on manifest only connectors locally

bin/generate-component-manifest-dagger.sh

@@ -7,5 +7,4 @@
 
 set -e
 
-pip install dagger-io==0.13.3
 python bin/generate_component_manifest_files.py

debug_manifest/README.md

@@ -22,11 +22,19 @@ To configure the debugger in VSCode to run the `debug_manifest`, follow these st
       "request": "launch",
       "console": "integratedTerminal",
       "cwd": "${workspaceFolder}/debug_manifest",
-      "python": "<PATH_TO_CDK_ENV>/bin/python",
+      "python": "<PATH_TO_CDK_ENV>/bin/python", // REPLACE ME
       "module": "debug_manifest",
       "args": [
         // SPECIFY THE COMMAND: [spec, check, discover, read]
         "read",
+        // SPECIFY THE MANIFEST FILE
+        "--manifest-path",
+        // PATH TO THE MANIFEST FILE
+        "resources/manifest.yaml",
+        // SPECIFY A COMPONENTS.PY FILE (OPTIONAL)
+        "--components-path",
+        // PATH TO THE COMPONENTS FILE
+        "resources/components.py",
         // SPECIFY THE CONFIG
         "--config",
         // PATH TO THE CONFIG FILE

debug_manifest/debug_manifest.py

@@ -3,17 +3,12 @@
 #
 
 import sys
-from typing import Any, Mapping
 
 from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
 from airbyte_cdk.sources.declarative.yaml_declarative_source import (
     YamlDeclarativeSource,
 )
 
-configuration: Mapping[str, Any] = {
-    "path_to_yaml": "resources/manifest.yaml",
-}
-
 
 def debug_manifest(source: YamlDeclarativeSource, args: list[str]) -> None:
     """
@@ -22,15 +17,56 @@ def debug_manifest(source: YamlDeclarativeSource, args: list[str]) -> None:
     launch(source, args)
 
 
+def _register_components_from_file(filepath: str) -> None:
+    """
+    Dynamically load a Python file containing custom component definitions and register it
+    under specific module names in sys.modules to ensure that these classes can be properly
+    resolved during hydration of the manifest yaml file.
+
+    This is a somewhat hacky replacement for the file structure manipulation we do when building
+    connector images to ensure the custom components can be imported.
+    """
+    import importlib.util
+    import sys
+    from pathlib import Path
+
+    components_path = Path(filepath)
+    if not components_path.exists():
+        raise FileNotFoundError(f"Components file not found: {components_path}")
+
+    module_name = "components"
+    sdm_module_name = "source_declarative_manifest.components"
+
+    spec = importlib.util.spec_from_file_location(module_name, components_path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Could not load module from {components_path}")
+
+    # Create module and execute code
+    module = importlib.util.module_from_spec(spec)
+
+    # Register then execute the module
+    # we dual-register the module to mirror what is done elsewhere in the CDK
+    sys.modules[module_name] = module
+    sys.modules[sdm_module_name] = module
+
+    spec.loader.exec_module(module)
+
+
 if __name__ == "__main__":
     args = sys.argv[1:]
+    parsed_args = AirbyteEntrypoint.parse_args(args)
+
+    manifest_path = getattr(parsed_args, "manifest_path", None) or "resources/manifest.yaml"
+    components_path = getattr(parsed_args, "components_path", None)
+    if components_path:
+        _register_components_from_file(components_path)
     catalog_path = AirbyteEntrypoint.extract_catalog(args)
     config_path = AirbyteEntrypoint.extract_config(args)
     state_path = AirbyteEntrypoint.extract_state(args)
 
     debug_manifest(
         YamlDeclarativeSource(
-            path_to_yaml="resources/manifest.yaml",
+            path_to_yaml=manifest_path,
             catalog=YamlDeclarativeSource.read_catalog(catalog_path) if catalog_path else None,
             config=YamlDeclarativeSource.read_config(config_path) if config_path else None,
             state=YamlDeclarativeSource.read_state(state_path) if state_path else None,