airbyte.sources.util
Utility functions for working with sources.
1# Copyright (c) 2023 Airbyte, Inc., all rights reserved. 2"""Utility functions for working with sources.""" 3 4from __future__ import annotations 5 6import shutil 7import sys 8import warnings 9from pathlib import Path 10from typing import Any 11 12from airbyte import exceptions as exc 13from airbyte._executor import PathExecutor, VenvExecutor 14from airbyte._util.telemetry import EventState, log_install_state 15from airbyte.sources.base import Source 16from airbyte.sources.registry import ConnectorMetadata, get_connector_metadata 17 18 19def get_connector( 20 name: str, 21 config: dict[str, Any] | None = None, 22 *, 23 version: str | None = None, 24 pip_url: str | None = None, 25 local_executable: Path | str | None = None, 26 install_if_missing: bool = True, 27) -> Source: 28 """Deprecated. Use get_source instead.""" 29 warnings.warn( 30 "The `get_connector()` function is deprecated and will be removed in a future version." 31 "Please use `get_source()` instead.", 32 DeprecationWarning, 33 stacklevel=2, 34 ) 35 return get_source( 36 name=name, 37 config=config, 38 version=version, 39 pip_url=pip_url, 40 local_executable=local_executable, 41 install_if_missing=install_if_missing, 42 ) 43 44 45def get_source( 46 name: str, 47 config: dict[str, Any] | None = None, 48 *, 49 streams: str | list[str] | None = None, 50 version: str | None = None, 51 pip_url: str | None = None, 52 local_executable: Path | str | None = None, 53 install_if_missing: bool = True, 54) -> Source: 55 """Get a connector by name and version. 56 57 Args: 58 name: connector name 59 config: connector config - if not provided, you need to set it later via the set_config 60 method. 61 streams: list of stream names to select for reading. If set to "*", all streams will be 62 selected. If not provided, you can set it later via the `select_streams()` or 63 `select_all_streams()` method. 64 version: connector version - if not provided, the currently installed version will be used. 65 If no version is installed, the latest available version will be used. The version can 66 also be set to "latest" to force the use of the latest available version. 67 pip_url: connector pip URL - if not provided, the pip url will be inferred from the 68 connector name. 69 local_executable: If set, the connector will be assumed to already be installed and will be 70 executed using this path or executable name. Otherwise, the connector will be installed 71 automatically in a virtual environment. 72 install_if_missing: Whether to install the connector if it is not available locally. This 73 parameter is ignored when local_executable is set. 74 """ 75 if local_executable: 76 if pip_url: 77 raise exc.PyAirbyteInputError( 78 message="Param 'pip_url' is not supported when 'local_executable' is set." 79 ) 80 if version: 81 raise exc.PyAirbyteInputError( 82 message="Param 'version' is not supported when 'local_executable' is set." 83 ) 84 85 if isinstance(local_executable, str): 86 if "/" in local_executable or "\\" in local_executable: 87 # Assume this is a path 88 local_executable = Path(local_executable).absolute() 89 else: 90 which_executable: str | None = None 91 which_executable = shutil.which(local_executable) 92 if not which_executable and sys.platform == "win32": 93 # Try with the .exe extension 94 local_executable = f"{local_executable}.exe" 95 which_executable = shutil.which(local_executable) 96 97 if which_executable is None: 98 raise exc.AirbyteConnectorExecutableNotFoundError( 99 connector_name=name, 100 context={ 101 "executable": local_executable, 102 "working_directory": Path.cwd().absolute(), 103 }, 104 ) from FileNotFoundError(local_executable) 105 local_executable = Path(which_executable).absolute() 106 107 print(f"Using local `{name}` executable: {local_executable!s}") 108 return Source( 109 name=name, 110 config=config, 111 streams=streams, 112 executor=PathExecutor( 113 name=name, 114 path=local_executable, 115 ), 116 ) 117 118 # else: we are installing a connector in a virtual environment: 119 120 metadata: ConnectorMetadata | None = None 121 try: 122 metadata = get_connector_metadata(name) 123 except exc.AirbyteConnectorNotRegisteredError as ex: 124 if not pip_url: 125 log_install_state(name, state=EventState.FAILED, exception=ex) 126 # We don't have a pip url or registry entry, so we can't install the connector 127 raise 128 129 try: 130 executor = VenvExecutor( 131 name=name, 132 metadata=metadata, 133 target_version=version, 134 pip_url=pip_url, 135 ) 136 if install_if_missing: 137 executor.ensure_installation() 138 139 return Source( 140 name=name, 141 config=config, 142 streams=streams, 143 executor=executor, 144 ) 145 except Exception as e: 146 log_install_state(name, state=EventState.FAILED, exception=e) 147 raise 148 149 150__all__ = [ 151 "get_source", 152]
def
get_source( name: str, config: dict[str, typing.Any] | None = None, *, streams: str | list[str] | None = None, version: str | None = None, pip_url: str | None = None, local_executable: pathlib.Path | str | None = None, install_if_missing: bool = True) -> airbyte.sources.base.Source:
46def get_source( 47 name: str, 48 config: dict[str, Any] | None = None, 49 *, 50 streams: str | list[str] | None = None, 51 version: str | None = None, 52 pip_url: str | None = None, 53 local_executable: Path | str | None = None, 54 install_if_missing: bool = True, 55) -> Source: 56 """Get a connector by name and version. 57 58 Args: 59 name: connector name 60 config: connector config - if not provided, you need to set it later via the set_config 61 method. 62 streams: list of stream names to select for reading. If set to "*", all streams will be 63 selected. If not provided, you can set it later via the `select_streams()` or 64 `select_all_streams()` method. 65 version: connector version - if not provided, the currently installed version will be used. 66 If no version is installed, the latest available version will be used. The version can 67 also be set to "latest" to force the use of the latest available version. 68 pip_url: connector pip URL - if not provided, the pip url will be inferred from the 69 connector name. 70 local_executable: If set, the connector will be assumed to already be installed and will be 71 executed using this path or executable name. Otherwise, the connector will be installed 72 automatically in a virtual environment. 73 install_if_missing: Whether to install the connector if it is not available locally. This 74 parameter is ignored when local_executable is set. 75 """ 76 if local_executable: 77 if pip_url: 78 raise exc.PyAirbyteInputError( 79 message="Param 'pip_url' is not supported when 'local_executable' is set." 80 ) 81 if version: 82 raise exc.PyAirbyteInputError( 83 message="Param 'version' is not supported when 'local_executable' is set." 84 ) 85 86 if isinstance(local_executable, str): 87 if "/" in local_executable or "\\" in local_executable: 88 # Assume this is a path 89 local_executable = Path(local_executable).absolute() 90 else: 91 which_executable: str | None = None 92 which_executable = shutil.which(local_executable) 93 if not which_executable and sys.platform == "win32": 94 # Try with the .exe extension 95 local_executable = f"{local_executable}.exe" 96 which_executable = shutil.which(local_executable) 97 98 if which_executable is None: 99 raise exc.AirbyteConnectorExecutableNotFoundError( 100 connector_name=name, 101 context={ 102 "executable": local_executable, 103 "working_directory": Path.cwd().absolute(), 104 }, 105 ) from FileNotFoundError(local_executable) 106 local_executable = Path(which_executable).absolute() 107 108 print(f"Using local `{name}` executable: {local_executable!s}") 109 return Source( 110 name=name, 111 config=config, 112 streams=streams, 113 executor=PathExecutor( 114 name=name, 115 path=local_executable, 116 ), 117 ) 118 119 # else: we are installing a connector in a virtual environment: 120 121 metadata: ConnectorMetadata | None = None 122 try: 123 metadata = get_connector_metadata(name) 124 except exc.AirbyteConnectorNotRegisteredError as ex: 125 if not pip_url: 126 log_install_state(name, state=EventState.FAILED, exception=ex) 127 # We don't have a pip url or registry entry, so we can't install the connector 128 raise 129 130 try: 131 executor = VenvExecutor( 132 name=name, 133 metadata=metadata, 134 target_version=version, 135 pip_url=pip_url, 136 ) 137 if install_if_missing: 138 executor.ensure_installation() 139 140 return Source( 141 name=name, 142 config=config, 143 streams=streams, 144 executor=executor, 145 ) 146 except Exception as e: 147 log_install_state(name, state=EventState.FAILED, exception=e) 148 raise
Get a connector by name and version.
Arguments:
- name: connector name
- config: connector config - if not provided, you need to set it later via the set_config method.
- streams: list of stream names to select for reading. If set to "*", all streams will be
selected. If not provided, you can set it later via the
select_streams()
orselect_all_streams()
method. - version: connector version - if not provided, the currently installed version will be used. If no version is installed, the latest available version will be used. The version can also be set to "latest" to force the use of the latest available version.
- pip_url: connector pip URL - if not provided, the pip url will be inferred from the connector name.
- local_executable: If set, the connector will be assumed to already be installed and will be executed using this path or executable name. Otherwise, the connector will be installed automatically in a virtual environment.
- install_if_missing: Whether to install the connector if it is not available locally. This parameter is ignored when local_executable is set.