airbyte.sources
Sources connectors module for PyAirbyte.
1"""Sources connectors module for PyAirbyte.""" 2from __future__ import annotations 3 4from airbyte.sources import base, util 5from airbyte.sources.registry import ( 6 ConnectorMetadata, 7 get_available_connectors, 8 get_connector_metadata, 9) 10from airbyte.sources.util import get_source 11 12 13__all__ = [ 14 # Submodules 15 "base", 16 "util", 17 # Factories 18 "get_source", 19 # Helper Functions 20 "get_available_connectors", 21 "get_connector_metadata", 22 # Classes 23 "Source", 24 "ConnectorMetadata", 25]
def
get_source( name: str, config: dict[str, typing.Any] | None = None, *, streams: str | list[str] | None = None, version: str | None = None, pip_url: str | None = None, local_executable: pathlib.Path | str | None = None, install_if_missing: bool = True) -> airbyte.sources.base.Source:
46def get_source( 47 name: str, 48 config: dict[str, Any] | None = None, 49 *, 50 streams: str | list[str] | None = None, 51 version: str | None = None, 52 pip_url: str | None = None, 53 local_executable: Path | str | None = None, 54 install_if_missing: bool = True, 55) -> Source: 56 """Get a connector by name and version. 57 58 Args: 59 name: connector name 60 config: connector config - if not provided, you need to set it later via the set_config 61 method. 62 streams: list of stream names to select for reading. If set to "*", all streams will be 63 selected. If not provided, you can set it later via the `select_streams()` or 64 `select_all_streams()` method. 65 version: connector version - if not provided, the currently installed version will be used. 66 If no version is installed, the latest available version will be used. The version can 67 also be set to "latest" to force the use of the latest available version. 68 pip_url: connector pip URL - if not provided, the pip url will be inferred from the 69 connector name. 70 local_executable: If set, the connector will be assumed to already be installed and will be 71 executed using this path or executable name. Otherwise, the connector will be installed 72 automatically in a virtual environment. 73 install_if_missing: Whether to install the connector if it is not available locally. This 74 parameter is ignored when local_executable is set. 75 """ 76 if local_executable: 77 if pip_url: 78 raise exc.PyAirbyteInputError( 79 message="Param 'pip_url' is not supported when 'local_executable' is set." 80 ) 81 if version: 82 raise exc.PyAirbyteInputError( 83 message="Param 'version' is not supported when 'local_executable' is set." 84 ) 85 86 if isinstance(local_executable, str): 87 if "/" in local_executable or "\\" in local_executable: 88 # Assume this is a path 89 local_executable = Path(local_executable).absolute() 90 else: 91 which_executable: str | None = None 92 which_executable = shutil.which(local_executable) 93 if not which_executable and sys.platform == "win32": 94 # Try with the .exe extension 95 local_executable = f"{local_executable}.exe" 96 which_executable = shutil.which(local_executable) 97 98 if which_executable is None: 99 raise exc.AirbyteConnectorExecutableNotFoundError( 100 connector_name=name, 101 context={ 102 "executable": local_executable, 103 "working_directory": Path.cwd().absolute(), 104 }, 105 ) from FileNotFoundError(local_executable) 106 local_executable = Path(which_executable).absolute() 107 108 print(f"Using local `{name}` executable: {local_executable!s}") 109 return Source( 110 name=name, 111 config=config, 112 streams=streams, 113 executor=PathExecutor( 114 name=name, 115 path=local_executable, 116 ), 117 ) 118 119 # else: we are installing a connector in a virtual environment: 120 121 metadata: ConnectorMetadata | None = None 122 try: 123 metadata = get_connector_metadata(name) 124 except exc.AirbyteConnectorNotRegisteredError as ex: 125 if not pip_url: 126 log_install_state(name, state=EventState.FAILED, exception=ex) 127 # We don't have a pip url or registry entry, so we can't install the connector 128 raise 129 130 try: 131 executor = VenvExecutor( 132 name=name, 133 metadata=metadata, 134 target_version=version, 135 pip_url=pip_url, 136 ) 137 if install_if_missing: 138 executor.ensure_installation() 139 140 return Source( 141 name=name, 142 config=config, 143 streams=streams, 144 executor=executor, 145 ) 146 except Exception as e: 147 log_install_state(name, state=EventState.FAILED, exception=e) 148 raise
Get a connector by name and version.
Arguments:
- name: connector name
- config: connector config - if not provided, you need to set it later via the set_config method.
- streams: list of stream names to select for reading. If set to "*", all streams will be
selected. If not provided, you can set it later via the
select_streams()
orselect_all_streams()
method. - version: connector version - if not provided, the currently installed version will be used. If no version is installed, the latest available version will be used. The version can also be set to "latest" to force the use of the latest available version.
- pip_url: connector pip URL - if not provided, the pip url will be inferred from the connector name.
- local_executable: If set, the connector will be assumed to already be installed and will be executed using this path or executable name. Otherwise, the connector will be installed automatically in a virtual environment.
- install_if_missing: Whether to install the connector if it is not available locally. This parameter is ignored when local_executable is set.
def
get_available_connectors() -> list[str]:
118def get_available_connectors() -> list[str]: 119 """Return a list of all available connectors. 120 121 Connectors will be returned in alphabetical order, with the standard prefix "source-". 122 """ 123 return sorted( 124 conn.name for conn in _get_registry_cache().values() if conn.pypi_package_name is not None 125 )
Return a list of all available connectors.
Connectors will be returned in alphabetical order, with the standard prefix "source-".
94def get_connector_metadata(name: str) -> ConnectorMetadata: 95 """Check the cache for the connector. 96 97 If the cache is empty, populate by calling update_cache. 98 """ 99 cache = copy(_get_registry_cache()) 100 if not cache: 101 raise exc.PyAirbyteInternalError( 102 message="Connector registry could not be loaded.", 103 context={ 104 "registry_url": _get_registry_url(), 105 }, 106 ) 107 if name not in cache: 108 raise exc.AirbyteConnectorNotRegisteredError( 109 connector_name=name, 110 context={ 111 "registry_url": _get_registry_url(), 112 "available_connectors": get_available_connectors(), 113 }, 114 ) 115 return cache[name]
Check the cache for the connector.
If the cache is empty, populate by calling update_cache.
Source
@dataclass
class
ConnectorMetadata:
24@dataclass 25class ConnectorMetadata: 26 """Metadata for a connector.""" 27 28 name: str 29 """Connector name. For example, "source-google-sheets".""" 30 31 latest_available_version: str 32 """The latest available version of the connector.""" 33 34 pypi_package_name: str | None 35 """The name of the PyPI package for the connector, if it exists."""
Metadata for a connector.