airbyte.cloud.connections
Cloud Connections.
1# Copyright (c) 2024 Airbyte, Inc., all rights reserved. 2"""Cloud Connections.""" 3 4from __future__ import annotations 5 6from typing import TYPE_CHECKING, cast 7 8from airbyte._util import api_util 9from airbyte.cloud.sync_results import SyncResult 10 11 12if TYPE_CHECKING: 13 from airbyte_api.models.shared.connectionresponse import ConnectionResponse 14 from airbyte_api.models.shared.jobresponse import JobResponse 15 16 from airbyte.cloud.workspaces import CloudWorkspace 17 18 19class CloudConnection: 20 """A connection is a link between a source and a destination. 21 22 Do not instantiate this class directly. Instead, use 23 `CloudWorkspace.deploy_connection` 24 or `.CloudWorkspace.get_connection` methods. 25 """ 26 27 def __init__( 28 self, 29 workspace: CloudWorkspace, 30 connection_id: str, 31 source: str | None = None, 32 destination: str | None = None, 33 ) -> None: 34 self.connection_id = connection_id 35 """The ID of the connection.""" 36 37 self.workspace = workspace 38 """The workspace that the connection belongs to.""" 39 40 self._source_id = source 41 """The ID of the source.""" 42 43 self._destination_id = destination 44 """The ID of the destination.""" 45 46 self._connection_info: ConnectionResponse | None = None 47 48 def _fetch_connection_info(self) -> ConnectionResponse: 49 """Populate the connection with data from the API.""" 50 return api_util.get_connection( 51 workspace_id=self.workspace.workspace_id, 52 connection_id=self.connection_id, 53 api_root=self.workspace.api_root, 54 api_key=self.workspace.api_key, 55 ) 56 57 # Properties 58 59 @property 60 def source_id(self) -> str: 61 """The ID of the source.""" 62 if not self._source_id: 63 if not self._connection_info: 64 self._connection_info = self._fetch_connection_info() 65 66 self._source_id = self._connection_info.source_id 67 68 return cast(str, self._source_id) 69 70 @property 71 def destination_id(self) -> str: 72 """The ID of the destination.""" 73 if not self._destination_id: 74 if not self._connection_info: 75 self._connection_info = self._fetch_connection_info() 76 77 self._destination_id = self._connection_info.source_id 78 79 return cast(str, self._destination_id) 80 81 @property 82 def stream_names(self) -> list[str]: 83 """The stream names.""" 84 if not self._connection_info: 85 self._connection_info = self._fetch_connection_info() 86 87 return [stream.name for stream in self._connection_info.configurations.streams] 88 89 @property 90 def table_prefix(self) -> str: 91 """The table prefix.""" 92 if not self._connection_info: 93 self._connection_info = self._fetch_connection_info() 94 95 return self._connection_info.configurations.prefix 96 97 @property 98 def connection_url(self) -> str | None: 99 return f"{self.workspace.workspace_url}/connections/{self.connection_id}" 100 101 @property 102 def job_history_url(self) -> str | None: 103 return f"{self.connection_url}/job-history" 104 105 # Run Sync 106 107 def run_sync( 108 self, 109 *, 110 wait: bool = True, 111 wait_timeout: int = 300, 112 ) -> SyncResult: 113 """Run a sync.""" 114 connection_response = api_util.run_connection( 115 connection_id=self.connection_id, 116 api_root=self.workspace.api_root, 117 api_key=self.workspace.api_key, 118 workspace_id=self.workspace.workspace_id, 119 ) 120 sync_result = SyncResult( 121 workspace=self.workspace, 122 connection=self, 123 job_id=connection_response.job_id, 124 ) 125 126 if wait: 127 sync_result.wait_for_completion( 128 wait_timeout=wait_timeout, 129 raise_failure=True, 130 raise_timeout=True, 131 ) 132 133 return sync_result 134 135 # Logs 136 137 def get_previous_sync_logs( 138 self, 139 *, 140 limit: int = 10, 141 ) -> list[SyncResult]: 142 """Get the previous sync logs for a connection.""" 143 sync_logs: list[JobResponse] = api_util.get_job_logs( 144 connection_id=self.connection_id, 145 api_root=self.workspace.api_root, 146 api_key=self.workspace.api_key, 147 workspace_id=self.workspace.workspace_id, 148 limit=limit, 149 ) 150 return [ 151 SyncResult( 152 workspace=self.workspace, 153 connection=self, 154 job_id=sync_log.job_id, 155 _latest_status=sync_log.status, 156 ) 157 for sync_log in sync_logs 158 ] 159 160 def get_sync_result( 161 self, 162 job_id: str | None = None, 163 ) -> SyncResult | None: 164 """Get the sync result for the connection. 165 166 If `job_id` is not provided, the most recent sync job will be used. 167 168 Returns `None` if job_id is omitted and no previous jobs are found. 169 """ 170 if job_id is None: 171 # Get the most recent sync job 172 results = self.get_previous_sync_logs( 173 limit=1, 174 ) 175 if results: 176 return results[0] 177 178 return None 179 180 # Get the sync job by ID (lazy loaded) 181 return SyncResult( 182 workspace=self.workspace, 183 connection=self, 184 job_id=job_id, 185 ) 186 187 # Deletions 188 189 def delete( 190 self, 191 *, 192 delete_source: bool = False, 193 delete_destination: bool = False, 194 ) -> None: 195 """Delete the connection. 196 197 Args: 198 delete_source: Whether to also delete the source. 199 delete_destination: Whether to also delete the destination. 200 """ 201 self.workspace.delete_connection(connection_id=self.connection_id) 202 203 if delete_source: 204 self.workspace.delete_source(source=self.source_id) 205 206 if delete_destination: 207 self.workspace.delete_destination(destination=self.destination_id)
class
CloudConnection:
20class CloudConnection: 21 """A connection is a link between a source and a destination. 22 23 Do not instantiate this class directly. Instead, use 24 `CloudWorkspace.deploy_connection` 25 or `.CloudWorkspace.get_connection` methods. 26 """ 27 28 def __init__( 29 self, 30 workspace: CloudWorkspace, 31 connection_id: str, 32 source: str | None = None, 33 destination: str | None = None, 34 ) -> None: 35 self.connection_id = connection_id 36 """The ID of the connection.""" 37 38 self.workspace = workspace 39 """The workspace that the connection belongs to.""" 40 41 self._source_id = source 42 """The ID of the source.""" 43 44 self._destination_id = destination 45 """The ID of the destination.""" 46 47 self._connection_info: ConnectionResponse | None = None 48 49 def _fetch_connection_info(self) -> ConnectionResponse: 50 """Populate the connection with data from the API.""" 51 return api_util.get_connection( 52 workspace_id=self.workspace.workspace_id, 53 connection_id=self.connection_id, 54 api_root=self.workspace.api_root, 55 api_key=self.workspace.api_key, 56 ) 57 58 # Properties 59 60 @property 61 def source_id(self) -> str: 62 """The ID of the source.""" 63 if not self._source_id: 64 if not self._connection_info: 65 self._connection_info = self._fetch_connection_info() 66 67 self._source_id = self._connection_info.source_id 68 69 return cast(str, self._source_id) 70 71 @property 72 def destination_id(self) -> str: 73 """The ID of the destination.""" 74 if not self._destination_id: 75 if not self._connection_info: 76 self._connection_info = self._fetch_connection_info() 77 78 self._destination_id = self._connection_info.source_id 79 80 return cast(str, self._destination_id) 81 82 @property 83 def stream_names(self) -> list[str]: 84 """The stream names.""" 85 if not self._connection_info: 86 self._connection_info = self._fetch_connection_info() 87 88 return [stream.name for stream in self._connection_info.configurations.streams] 89 90 @property 91 def table_prefix(self) -> str: 92 """The table prefix.""" 93 if not self._connection_info: 94 self._connection_info = self._fetch_connection_info() 95 96 return self._connection_info.configurations.prefix 97 98 @property 99 def connection_url(self) -> str | None: 100 return f"{self.workspace.workspace_url}/connections/{self.connection_id}" 101 102 @property 103 def job_history_url(self) -> str | None: 104 return f"{self.connection_url}/job-history" 105 106 # Run Sync 107 108 def run_sync( 109 self, 110 *, 111 wait: bool = True, 112 wait_timeout: int = 300, 113 ) -> SyncResult: 114 """Run a sync.""" 115 connection_response = api_util.run_connection( 116 connection_id=self.connection_id, 117 api_root=self.workspace.api_root, 118 api_key=self.workspace.api_key, 119 workspace_id=self.workspace.workspace_id, 120 ) 121 sync_result = SyncResult( 122 workspace=self.workspace, 123 connection=self, 124 job_id=connection_response.job_id, 125 ) 126 127 if wait: 128 sync_result.wait_for_completion( 129 wait_timeout=wait_timeout, 130 raise_failure=True, 131 raise_timeout=True, 132 ) 133 134 return sync_result 135 136 # Logs 137 138 def get_previous_sync_logs( 139 self, 140 *, 141 limit: int = 10, 142 ) -> list[SyncResult]: 143 """Get the previous sync logs for a connection.""" 144 sync_logs: list[JobResponse] = api_util.get_job_logs( 145 connection_id=self.connection_id, 146 api_root=self.workspace.api_root, 147 api_key=self.workspace.api_key, 148 workspace_id=self.workspace.workspace_id, 149 limit=limit, 150 ) 151 return [ 152 SyncResult( 153 workspace=self.workspace, 154 connection=self, 155 job_id=sync_log.job_id, 156 _latest_status=sync_log.status, 157 ) 158 for sync_log in sync_logs 159 ] 160 161 def get_sync_result( 162 self, 163 job_id: str | None = None, 164 ) -> SyncResult | None: 165 """Get the sync result for the connection. 166 167 If `job_id` is not provided, the most recent sync job will be used. 168 169 Returns `None` if job_id is omitted and no previous jobs are found. 170 """ 171 if job_id is None: 172 # Get the most recent sync job 173 results = self.get_previous_sync_logs( 174 limit=1, 175 ) 176 if results: 177 return results[0] 178 179 return None 180 181 # Get the sync job by ID (lazy loaded) 182 return SyncResult( 183 workspace=self.workspace, 184 connection=self, 185 job_id=job_id, 186 ) 187 188 # Deletions 189 190 def delete( 191 self, 192 *, 193 delete_source: bool = False, 194 delete_destination: bool = False, 195 ) -> None: 196 """Delete the connection. 197 198 Args: 199 delete_source: Whether to also delete the source. 200 delete_destination: Whether to also delete the destination. 201 """ 202 self.workspace.delete_connection(connection_id=self.connection_id) 203 204 if delete_source: 205 self.workspace.delete_source(source=self.source_id) 206 207 if delete_destination: 208 self.workspace.delete_destination(destination=self.destination_id)
A connection is a link between a source and a destination.
Do not instantiate this class directly. Instead, use
CloudWorkspace.deploy_connection
or airbyte.cloud.CloudWorkspace.get_connection
methods.
CloudConnection( workspace: airbyte.cloud.workspaces.CloudWorkspace, connection_id: str, source: str | None = None, destination: str | None = None)
28 def __init__( 29 self, 30 workspace: CloudWorkspace, 31 connection_id: str, 32 source: str | None = None, 33 destination: str | None = None, 34 ) -> None: 35 self.connection_id = connection_id 36 """The ID of the connection.""" 37 38 self.workspace = workspace 39 """The workspace that the connection belongs to.""" 40 41 self._source_id = source 42 """The ID of the source.""" 43 44 self._destination_id = destination 45 """The ID of the destination.""" 46 47 self._connection_info: ConnectionResponse | None = None
source_id: str
60 @property 61 def source_id(self) -> str: 62 """The ID of the source.""" 63 if not self._source_id: 64 if not self._connection_info: 65 self._connection_info = self._fetch_connection_info() 66 67 self._source_id = self._connection_info.source_id 68 69 return cast(str, self._source_id)
The ID of the source.
destination_id: str
71 @property 72 def destination_id(self) -> str: 73 """The ID of the destination.""" 74 if not self._destination_id: 75 if not self._connection_info: 76 self._connection_info = self._fetch_connection_info() 77 78 self._destination_id = self._connection_info.source_id 79 80 return cast(str, self._destination_id)
The ID of the destination.
stream_names: list[str]
82 @property 83 def stream_names(self) -> list[str]: 84 """The stream names.""" 85 if not self._connection_info: 86 self._connection_info = self._fetch_connection_info() 87 88 return [stream.name for stream in self._connection_info.configurations.streams]
The stream names.
table_prefix: str
90 @property 91 def table_prefix(self) -> str: 92 """The table prefix.""" 93 if not self._connection_info: 94 self._connection_info = self._fetch_connection_info() 95 96 return self._connection_info.configurations.prefix
The table prefix.
def
run_sync( self, *, wait: bool = True, wait_timeout: int = 300) -> airbyte.cloud.sync_results.SyncResult:
108 def run_sync( 109 self, 110 *, 111 wait: bool = True, 112 wait_timeout: int = 300, 113 ) -> SyncResult: 114 """Run a sync.""" 115 connection_response = api_util.run_connection( 116 connection_id=self.connection_id, 117 api_root=self.workspace.api_root, 118 api_key=self.workspace.api_key, 119 workspace_id=self.workspace.workspace_id, 120 ) 121 sync_result = SyncResult( 122 workspace=self.workspace, 123 connection=self, 124 job_id=connection_response.job_id, 125 ) 126 127 if wait: 128 sync_result.wait_for_completion( 129 wait_timeout=wait_timeout, 130 raise_failure=True, 131 raise_timeout=True, 132 ) 133 134 return sync_result
Run a sync.
def
get_previous_sync_logs(self, *, limit: int = 10) -> list[airbyte.cloud.sync_results.SyncResult]:
138 def get_previous_sync_logs( 139 self, 140 *, 141 limit: int = 10, 142 ) -> list[SyncResult]: 143 """Get the previous sync logs for a connection.""" 144 sync_logs: list[JobResponse] = api_util.get_job_logs( 145 connection_id=self.connection_id, 146 api_root=self.workspace.api_root, 147 api_key=self.workspace.api_key, 148 workspace_id=self.workspace.workspace_id, 149 limit=limit, 150 ) 151 return [ 152 SyncResult( 153 workspace=self.workspace, 154 connection=self, 155 job_id=sync_log.job_id, 156 _latest_status=sync_log.status, 157 ) 158 for sync_log in sync_logs 159 ]
Get the previous sync logs for a connection.
def
get_sync_result( self, job_id: str | None = None) -> airbyte.cloud.sync_results.SyncResult | None:
161 def get_sync_result( 162 self, 163 job_id: str | None = None, 164 ) -> SyncResult | None: 165 """Get the sync result for the connection. 166 167 If `job_id` is not provided, the most recent sync job will be used. 168 169 Returns `None` if job_id is omitted and no previous jobs are found. 170 """ 171 if job_id is None: 172 # Get the most recent sync job 173 results = self.get_previous_sync_logs( 174 limit=1, 175 ) 176 if results: 177 return results[0] 178 179 return None 180 181 # Get the sync job by ID (lazy loaded) 182 return SyncResult( 183 workspace=self.workspace, 184 connection=self, 185 job_id=job_id, 186 )
Get the sync result for the connection.
If job_id
is not provided, the most recent sync job will be used.
Returns None
if job_id is omitted and no previous jobs are found.
def
delete( self, *, delete_source: bool = False, delete_destination: bool = False) -> None:
190 def delete( 191 self, 192 *, 193 delete_source: bool = False, 194 delete_destination: bool = False, 195 ) -> None: 196 """Delete the connection. 197 198 Args: 199 delete_source: Whether to also delete the source. 200 delete_destination: Whether to also delete the destination. 201 """ 202 self.workspace.delete_connection(connection_id=self.connection_id) 203 204 if delete_source: 205 self.workspace.delete_source(source=self.source_id) 206 207 if delete_destination: 208 self.workspace.delete_destination(destination=self.destination_id)
Delete the connection.
Arguments:
- delete_source: Whether to also delete the source.
- delete_destination: Whether to also delete the destination.