airbyte.sources.util

Utility functions for working with sources.

  1# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
  2"""Utility functions for working with sources."""
  3
  4from __future__ import annotations
  5
  6import shutil
  7import sys
  8import warnings
  9from pathlib import Path
 10from typing import Any
 11
 12from airbyte import exceptions as exc
 13from airbyte._executor import PathExecutor, VenvExecutor
 14from airbyte._util.telemetry import EventState, log_install_state
 15from airbyte.sources.base import Source
 16from airbyte.sources.registry import ConnectorMetadata, get_connector_metadata
 17
 18
 19def get_connector(
 20    name: str,
 21    config: dict[str, Any] | None = None,
 22    *,
 23    version: str | None = None,
 24    pip_url: str | None = None,
 25    local_executable: Path | str | None = None,
 26    install_if_missing: bool = True,
 27) -> Source:
 28    """Deprecated. Use get_source instead."""
 29    warnings.warn(
 30        "The `get_connector()` function is deprecated and will be removed in a future version."
 31        "Please use `get_source()` instead.",
 32        DeprecationWarning,
 33        stacklevel=2,
 34    )
 35    return get_source(
 36        name=name,
 37        config=config,
 38        version=version,
 39        pip_url=pip_url,
 40        local_executable=local_executable,
 41        install_if_missing=install_if_missing,
 42    )
 43
 44
 45def get_source(
 46    name: str,
 47    config: dict[str, Any] | None = None,
 48    *,
 49    streams: str | list[str] | None = None,
 50    version: str | None = None,
 51    pip_url: str | None = None,
 52    local_executable: Path | str | None = None,
 53    install_if_missing: bool = True,
 54) -> Source:
 55    """Get a connector by name and version.
 56
 57    Args:
 58        name: connector name
 59        config: connector config - if not provided, you need to set it later via the set_config
 60            method.
 61        streams: list of stream names to select for reading. If set to "*", all streams will be
 62            selected. If not provided, you can set it later via the `select_streams()` or
 63            `select_all_streams()` method.
 64        version: connector version - if not provided, the currently installed version will be used.
 65            If no version is installed, the latest available version will be used. The version can
 66            also be set to "latest" to force the use of the latest available version.
 67        pip_url: connector pip URL - if not provided, the pip url will be inferred from the
 68            connector name.
 69        local_executable: If set, the connector will be assumed to already be installed and will be
 70            executed using this path or executable name. Otherwise, the connector will be installed
 71            automatically in a virtual environment.
 72        install_if_missing: Whether to install the connector if it is not available locally. This
 73            parameter is ignored when local_executable is set.
 74    """
 75    if local_executable:
 76        if pip_url:
 77            raise exc.PyAirbyteInputError(
 78                message="Param 'pip_url' is not supported when 'local_executable' is set."
 79            )
 80        if version:
 81            raise exc.PyAirbyteInputError(
 82                message="Param 'version' is not supported when 'local_executable' is set."
 83            )
 84
 85        if isinstance(local_executable, str):
 86            if "/" in local_executable or "\\" in local_executable:
 87                # Assume this is a path
 88                local_executable = Path(local_executable).absolute()
 89            else:
 90                which_executable: str | None = None
 91                which_executable = shutil.which(local_executable)
 92                if not which_executable and sys.platform == "win32":
 93                    # Try with the .exe extension
 94                    local_executable = f"{local_executable}.exe"
 95                    which_executable = shutil.which(local_executable)
 96
 97                if which_executable is None:
 98                    raise exc.AirbyteConnectorExecutableNotFoundError(
 99                        connector_name=name,
100                        context={
101                            "executable": local_executable,
102                            "working_directory": Path.cwd().absolute(),
103                        },
104                    ) from FileNotFoundError(local_executable)
105                local_executable = Path(which_executable).absolute()
106
107        print(f"Using local `{name}` executable: {local_executable!s}")
108        return Source(
109            name=name,
110            config=config,
111            streams=streams,
112            executor=PathExecutor(
113                name=name,
114                path=local_executable,
115            ),
116        )
117
118    # else: we are installing a connector in a virtual environment:
119
120    metadata: ConnectorMetadata | None = None
121    try:
122        metadata = get_connector_metadata(name)
123    except exc.AirbyteConnectorNotRegisteredError as ex:
124        if not pip_url:
125            log_install_state(name, state=EventState.FAILED, exception=ex)
126            # We don't have a pip url or registry entry, so we can't install the connector
127            raise
128
129    try:
130        executor = VenvExecutor(
131            name=name,
132            metadata=metadata,
133            target_version=version,
134            pip_url=pip_url,
135        )
136        if install_if_missing:
137            executor.ensure_installation()
138
139        return Source(
140            name=name,
141            config=config,
142            streams=streams,
143            executor=executor,
144        )
145    except Exception as e:
146        log_install_state(name, state=EventState.FAILED, exception=e)
147        raise
148
149
150__all__ = [
151    "get_source",
152]
def get_source( name: str, config: dict[str, typing.Any] | None = None, *, streams: str | list[str] | None = None, version: str | None = None, pip_url: str | None = None, local_executable: pathlib.Path | str | None = None, install_if_missing: bool = True) -> airbyte.sources.base.Source:
 46def get_source(
 47    name: str,
 48    config: dict[str, Any] | None = None,
 49    *,
 50    streams: str | list[str] | None = None,
 51    version: str | None = None,
 52    pip_url: str | None = None,
 53    local_executable: Path | str | None = None,
 54    install_if_missing: bool = True,
 55) -> Source:
 56    """Get a connector by name and version.
 57
 58    Args:
 59        name: connector name
 60        config: connector config - if not provided, you need to set it later via the set_config
 61            method.
 62        streams: list of stream names to select for reading. If set to "*", all streams will be
 63            selected. If not provided, you can set it later via the `select_streams()` or
 64            `select_all_streams()` method.
 65        version: connector version - if not provided, the currently installed version will be used.
 66            If no version is installed, the latest available version will be used. The version can
 67            also be set to "latest" to force the use of the latest available version.
 68        pip_url: connector pip URL - if not provided, the pip url will be inferred from the
 69            connector name.
 70        local_executable: If set, the connector will be assumed to already be installed and will be
 71            executed using this path or executable name. Otherwise, the connector will be installed
 72            automatically in a virtual environment.
 73        install_if_missing: Whether to install the connector if it is not available locally. This
 74            parameter is ignored when local_executable is set.
 75    """
 76    if local_executable:
 77        if pip_url:
 78            raise exc.PyAirbyteInputError(
 79                message="Param 'pip_url' is not supported when 'local_executable' is set."
 80            )
 81        if version:
 82            raise exc.PyAirbyteInputError(
 83                message="Param 'version' is not supported when 'local_executable' is set."
 84            )
 85
 86        if isinstance(local_executable, str):
 87            if "/" in local_executable or "\\" in local_executable:
 88                # Assume this is a path
 89                local_executable = Path(local_executable).absolute()
 90            else:
 91                which_executable: str | None = None
 92                which_executable = shutil.which(local_executable)
 93                if not which_executable and sys.platform == "win32":
 94                    # Try with the .exe extension
 95                    local_executable = f"{local_executable}.exe"
 96                    which_executable = shutil.which(local_executable)
 97
 98                if which_executable is None:
 99                    raise exc.AirbyteConnectorExecutableNotFoundError(
100                        connector_name=name,
101                        context={
102                            "executable": local_executable,
103                            "working_directory": Path.cwd().absolute(),
104                        },
105                    ) from FileNotFoundError(local_executable)
106                local_executable = Path(which_executable).absolute()
107
108        print(f"Using local `{name}` executable: {local_executable!s}")
109        return Source(
110            name=name,
111            config=config,
112            streams=streams,
113            executor=PathExecutor(
114                name=name,
115                path=local_executable,
116            ),
117        )
118
119    # else: we are installing a connector in a virtual environment:
120
121    metadata: ConnectorMetadata | None = None
122    try:
123        metadata = get_connector_metadata(name)
124    except exc.AirbyteConnectorNotRegisteredError as ex:
125        if not pip_url:
126            log_install_state(name, state=EventState.FAILED, exception=ex)
127            # We don't have a pip url or registry entry, so we can't install the connector
128            raise
129
130    try:
131        executor = VenvExecutor(
132            name=name,
133            metadata=metadata,
134            target_version=version,
135            pip_url=pip_url,
136        )
137        if install_if_missing:
138            executor.ensure_installation()
139
140        return Source(
141            name=name,
142            config=config,
143            streams=streams,
144            executor=executor,
145        )
146    except Exception as e:
147        log_install_state(name, state=EventState.FAILED, exception=e)
148        raise

Get a connector by name and version.

Arguments:
  • name: connector name
  • config: connector config - if not provided, you need to set it later via the set_config method.
  • streams: list of stream names to select for reading. If set to "*", all streams will be selected. If not provided, you can set it later via the select_streams() or select_all_streams() method.
  • version: connector version - if not provided, the currently installed version will be used. If no version is installed, the latest available version will be used. The version can also be set to "latest" to force the use of the latest available version.
  • pip_url: connector pip URL - if not provided, the pip url will be inferred from the connector name.
  • local_executable: If set, the connector will be assumed to already be installed and will be executed using this path or executable name. Otherwise, the connector will be installed automatically in a virtual environment.
  • install_if_missing: Whether to install the connector if it is not available locally. This parameter is ignored when local_executable is set.