airbyte.sources

Sources connectors module for PyAirbyte.

 1"""Sources connectors module for PyAirbyte."""
 2from __future__ import annotations
 3
 4from airbyte.sources import base, util
 5from airbyte.sources.registry import (
 6    ConnectorMetadata,
 7    get_available_connectors,
 8    get_connector_metadata,
 9)
10from airbyte.sources.util import get_source
11
12
13__all__ = [
14    # Submodules
15    "base",
16    "util",
17    # Factories
18    "get_source",
19    # Helper Functions
20    "get_available_connectors",
21    "get_connector_metadata",
22    # Classes
23    "Source",
24    "ConnectorMetadata",
25]
def get_source( name: str, config: dict[str, typing.Any] | None = None, *, streams: str | list[str] | None = None, version: str | None = None, pip_url: str | None = None, local_executable: pathlib.Path | str | None = None, install_if_missing: bool = True) -> airbyte.sources.base.Source:
 46def get_source(
 47    name: str,
 48    config: dict[str, Any] | None = None,
 49    *,
 50    streams: str | list[str] | None = None,
 51    version: str | None = None,
 52    pip_url: str | None = None,
 53    local_executable: Path | str | None = None,
 54    install_if_missing: bool = True,
 55) -> Source:
 56    """Get a connector by name and version.
 57
 58    Args:
 59        name: connector name
 60        config: connector config - if not provided, you need to set it later via the set_config
 61            method.
 62        streams: list of stream names to select for reading. If set to "*", all streams will be
 63            selected. If not provided, you can set it later via the `select_streams()` or
 64            `select_all_streams()` method.
 65        version: connector version - if not provided, the currently installed version will be used.
 66            If no version is installed, the latest available version will be used. The version can
 67            also be set to "latest" to force the use of the latest available version.
 68        pip_url: connector pip URL - if not provided, the pip url will be inferred from the
 69            connector name.
 70        local_executable: If set, the connector will be assumed to already be installed and will be
 71            executed using this path or executable name. Otherwise, the connector will be installed
 72            automatically in a virtual environment.
 73        install_if_missing: Whether to install the connector if it is not available locally. This
 74            parameter is ignored when local_executable is set.
 75    """
 76    if local_executable:
 77        if pip_url:
 78            raise exc.PyAirbyteInputError(
 79                message="Param 'pip_url' is not supported when 'local_executable' is set."
 80            )
 81        if version:
 82            raise exc.PyAirbyteInputError(
 83                message="Param 'version' is not supported when 'local_executable' is set."
 84            )
 85
 86        if isinstance(local_executable, str):
 87            if "/" in local_executable or "\\" in local_executable:
 88                # Assume this is a path
 89                local_executable = Path(local_executable).absolute()
 90            else:
 91                which_executable: str | None = None
 92                which_executable = shutil.which(local_executable)
 93                if not which_executable and sys.platform == "win32":
 94                    # Try with the .exe extension
 95                    local_executable = f"{local_executable}.exe"
 96                    which_executable = shutil.which(local_executable)
 97
 98                if which_executable is None:
 99                    raise exc.AirbyteConnectorExecutableNotFoundError(
100                        connector_name=name,
101                        context={
102                            "executable": local_executable,
103                            "working_directory": Path.cwd().absolute(),
104                        },
105                    ) from FileNotFoundError(local_executable)
106                local_executable = Path(which_executable).absolute()
107
108        print(f"Using local `{name}` executable: {local_executable!s}")
109        return Source(
110            name=name,
111            config=config,
112            streams=streams,
113            executor=PathExecutor(
114                name=name,
115                path=local_executable,
116            ),
117        )
118
119    # else: we are installing a connector in a virtual environment:
120
121    metadata: ConnectorMetadata | None = None
122    try:
123        metadata = get_connector_metadata(name)
124    except exc.AirbyteConnectorNotRegisteredError as ex:
125        if not pip_url:
126            log_install_state(name, state=EventState.FAILED, exception=ex)
127            # We don't have a pip url or registry entry, so we can't install the connector
128            raise
129
130    try:
131        executor = VenvExecutor(
132            name=name,
133            metadata=metadata,
134            target_version=version,
135            pip_url=pip_url,
136        )
137        if install_if_missing:
138            executor.ensure_installation()
139
140        return Source(
141            name=name,
142            config=config,
143            streams=streams,
144            executor=executor,
145        )
146    except Exception as e:
147        log_install_state(name, state=EventState.FAILED, exception=e)
148        raise

Get a connector by name and version.

Arguments:
  • name: connector name
  • config: connector config - if not provided, you need to set it later via the set_config method.
  • streams: list of stream names to select for reading. If set to "*", all streams will be selected. If not provided, you can set it later via the select_streams() or select_all_streams() method.
  • version: connector version - if not provided, the currently installed version will be used. If no version is installed, the latest available version will be used. The version can also be set to "latest" to force the use of the latest available version.
  • pip_url: connector pip URL - if not provided, the pip url will be inferred from the connector name.
  • local_executable: If set, the connector will be assumed to already be installed and will be executed using this path or executable name. Otherwise, the connector will be installed automatically in a virtual environment.
  • install_if_missing: Whether to install the connector if it is not available locally. This parameter is ignored when local_executable is set.
def get_available_connectors() -> list[str]:
118def get_available_connectors() -> list[str]:
119    """Return a list of all available connectors.
120
121    Connectors will be returned in alphabetical order, with the standard prefix "source-".
122    """
123    return sorted(
124        conn.name for conn in _get_registry_cache().values() if conn.pypi_package_name is not None
125    )

Return a list of all available connectors.

Connectors will be returned in alphabetical order, with the standard prefix "source-".

def get_connector_metadata(name: str) -> ConnectorMetadata:
 94def get_connector_metadata(name: str) -> ConnectorMetadata:
 95    """Check the cache for the connector.
 96
 97    If the cache is empty, populate by calling update_cache.
 98    """
 99    cache = copy(_get_registry_cache())
100    if not cache:
101        raise exc.PyAirbyteInternalError(
102            message="Connector registry could not be loaded.",
103            context={
104                "registry_url": _get_registry_url(),
105            },
106        )
107    if name not in cache:
108        raise exc.AirbyteConnectorNotRegisteredError(
109            connector_name=name,
110            context={
111                "registry_url": _get_registry_url(),
112                "available_connectors": get_available_connectors(),
113            },
114        )
115    return cache[name]

Check the cache for the connector.

If the cache is empty, populate by calling update_cache.

Source
@dataclass
class ConnectorMetadata:
24@dataclass
25class ConnectorMetadata:
26    """Metadata for a connector."""
27
28    name: str
29    """Connector name. For example, "source-google-sheets"."""
30
31    latest_available_version: str
32    """The latest available version of the connector."""
33
34    pypi_package_name: str | None
35    """The name of the PyPI package for the connector, if it exists."""

Metadata for a connector.

ConnectorMetadata( name: str, latest_available_version: str, pypi_package_name: str | None)
name: str

Connector name. For example, "source-google-sheets".

latest_available_version: str

The latest available version of the connector.

pypi_package_name: str | None

The name of the PyPI package for the connector, if it exists.