airbyte.cloud.connections

Cloud Connections.

  1# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
  2"""Cloud Connections."""
  3
  4from __future__ import annotations
  5
  6from typing import TYPE_CHECKING, cast
  7
  8from airbyte._util import api_util
  9from airbyte.cloud.sync_results import SyncResult
 10
 11
 12if TYPE_CHECKING:
 13    from airbyte_api.models.shared.connectionresponse import ConnectionResponse
 14    from airbyte_api.models.shared.jobresponse import JobResponse
 15
 16    from airbyte.cloud.workspaces import CloudWorkspace
 17
 18
 19class CloudConnection:
 20    """A connection is a link between a source and a destination.
 21
 22    Do not instantiate this class directly. Instead, use
 23    `CloudWorkspace.deploy_connection`
 24    or `.CloudWorkspace.get_connection` methods.
 25    """
 26
 27    def __init__(
 28        self,
 29        workspace: CloudWorkspace,
 30        connection_id: str,
 31        source: str | None = None,
 32        destination: str | None = None,
 33    ) -> None:
 34        self.connection_id = connection_id
 35        """The ID of the connection."""
 36
 37        self.workspace = workspace
 38        """The workspace that the connection belongs to."""
 39
 40        self._source_id = source
 41        """The ID of the source."""
 42
 43        self._destination_id = destination
 44        """The ID of the destination."""
 45
 46        self._connection_info: ConnectionResponse | None = None
 47
 48    def _fetch_connection_info(self) -> ConnectionResponse:
 49        """Populate the connection with data from the API."""
 50        return api_util.get_connection(
 51            workspace_id=self.workspace.workspace_id,
 52            connection_id=self.connection_id,
 53            api_root=self.workspace.api_root,
 54            api_key=self.workspace.api_key,
 55        )
 56
 57    # Properties
 58
 59    @property
 60    def source_id(self) -> str:
 61        """The ID of the source."""
 62        if not self._source_id:
 63            if not self._connection_info:
 64                self._connection_info = self._fetch_connection_info()
 65
 66            self._source_id = self._connection_info.source_id
 67
 68        return cast(str, self._source_id)
 69
 70    @property
 71    def destination_id(self) -> str:
 72        """The ID of the destination."""
 73        if not self._destination_id:
 74            if not self._connection_info:
 75                self._connection_info = self._fetch_connection_info()
 76
 77            self._destination_id = self._connection_info.source_id
 78
 79        return cast(str, self._destination_id)
 80
 81    @property
 82    def stream_names(self) -> list[str]:
 83        """The stream names."""
 84        if not self._connection_info:
 85            self._connection_info = self._fetch_connection_info()
 86
 87        return [stream.name for stream in self._connection_info.configurations.streams]
 88
 89    @property
 90    def table_prefix(self) -> str:
 91        """The table prefix."""
 92        if not self._connection_info:
 93            self._connection_info = self._fetch_connection_info()
 94
 95        return self._connection_info.configurations.prefix
 96
 97    @property
 98    def connection_url(self) -> str | None:
 99        return f"{self.workspace.workspace_url}/connections/{self.connection_id}"
100
101    @property
102    def job_history_url(self) -> str | None:
103        return f"{self.connection_url}/job-history"
104
105    # Run Sync
106
107    def run_sync(
108        self,
109        *,
110        wait: bool = True,
111        wait_timeout: int = 300,
112    ) -> SyncResult:
113        """Run a sync."""
114        connection_response = api_util.run_connection(
115            connection_id=self.connection_id,
116            api_root=self.workspace.api_root,
117            api_key=self.workspace.api_key,
118            workspace_id=self.workspace.workspace_id,
119        )
120        sync_result = SyncResult(
121            workspace=self.workspace,
122            connection=self,
123            job_id=connection_response.job_id,
124        )
125
126        if wait:
127            sync_result.wait_for_completion(
128                wait_timeout=wait_timeout,
129                raise_failure=True,
130                raise_timeout=True,
131            )
132
133        return sync_result
134
135    # Logs
136
137    def get_previous_sync_logs(
138        self,
139        *,
140        limit: int = 10,
141    ) -> list[SyncResult]:
142        """Get the previous sync logs for a connection."""
143        sync_logs: list[JobResponse] = api_util.get_job_logs(
144            connection_id=self.connection_id,
145            api_root=self.workspace.api_root,
146            api_key=self.workspace.api_key,
147            workspace_id=self.workspace.workspace_id,
148            limit=limit,
149        )
150        return [
151            SyncResult(
152                workspace=self.workspace,
153                connection=self,
154                job_id=sync_log.job_id,
155                _latest_status=sync_log.status,
156            )
157            for sync_log in sync_logs
158        ]
159
160    def get_sync_result(
161        self,
162        job_id: str | None = None,
163    ) -> SyncResult | None:
164        """Get the sync result for the connection.
165
166        If `job_id` is not provided, the most recent sync job will be used.
167
168        Returns `None` if job_id is omitted and no previous jobs are found.
169        """
170        if job_id is None:
171            # Get the most recent sync job
172            results = self.get_previous_sync_logs(
173                limit=1,
174            )
175            if results:
176                return results[0]
177
178            return None
179
180        # Get the sync job by ID (lazy loaded)
181        return SyncResult(
182            workspace=self.workspace,
183            connection=self,
184            job_id=job_id,
185        )
186
187    # Deletions
188
189    def delete(
190        self,
191        *,
192        delete_source: bool = False,
193        delete_destination: bool = False,
194    ) -> None:
195        """Delete the connection.
196
197        Args:
198            delete_source: Whether to also delete the source.
199            delete_destination: Whether to also delete the destination.
200        """
201        self.workspace.delete_connection(connection_id=self.connection_id)
202
203        if delete_source:
204            self.workspace.delete_source(source=self.source_id)
205
206        if delete_destination:
207            self.workspace.delete_destination(destination=self.destination_id)
class CloudConnection:
 20class CloudConnection:
 21    """A connection is a link between a source and a destination.
 22
 23    Do not instantiate this class directly. Instead, use
 24    `CloudWorkspace.deploy_connection`
 25    or `.CloudWorkspace.get_connection` methods.
 26    """
 27
 28    def __init__(
 29        self,
 30        workspace: CloudWorkspace,
 31        connection_id: str,
 32        source: str | None = None,
 33        destination: str | None = None,
 34    ) -> None:
 35        self.connection_id = connection_id
 36        """The ID of the connection."""
 37
 38        self.workspace = workspace
 39        """The workspace that the connection belongs to."""
 40
 41        self._source_id = source
 42        """The ID of the source."""
 43
 44        self._destination_id = destination
 45        """The ID of the destination."""
 46
 47        self._connection_info: ConnectionResponse | None = None
 48
 49    def _fetch_connection_info(self) -> ConnectionResponse:
 50        """Populate the connection with data from the API."""
 51        return api_util.get_connection(
 52            workspace_id=self.workspace.workspace_id,
 53            connection_id=self.connection_id,
 54            api_root=self.workspace.api_root,
 55            api_key=self.workspace.api_key,
 56        )
 57
 58    # Properties
 59
 60    @property
 61    def source_id(self) -> str:
 62        """The ID of the source."""
 63        if not self._source_id:
 64            if not self._connection_info:
 65                self._connection_info = self._fetch_connection_info()
 66
 67            self._source_id = self._connection_info.source_id
 68
 69        return cast(str, self._source_id)
 70
 71    @property
 72    def destination_id(self) -> str:
 73        """The ID of the destination."""
 74        if not self._destination_id:
 75            if not self._connection_info:
 76                self._connection_info = self._fetch_connection_info()
 77
 78            self._destination_id = self._connection_info.source_id
 79
 80        return cast(str, self._destination_id)
 81
 82    @property
 83    def stream_names(self) -> list[str]:
 84        """The stream names."""
 85        if not self._connection_info:
 86            self._connection_info = self._fetch_connection_info()
 87
 88        return [stream.name for stream in self._connection_info.configurations.streams]
 89
 90    @property
 91    def table_prefix(self) -> str:
 92        """The table prefix."""
 93        if not self._connection_info:
 94            self._connection_info = self._fetch_connection_info()
 95
 96        return self._connection_info.configurations.prefix
 97
 98    @property
 99    def connection_url(self) -> str | None:
100        return f"{self.workspace.workspace_url}/connections/{self.connection_id}"
101
102    @property
103    def job_history_url(self) -> str | None:
104        return f"{self.connection_url}/job-history"
105
106    # Run Sync
107
108    def run_sync(
109        self,
110        *,
111        wait: bool = True,
112        wait_timeout: int = 300,
113    ) -> SyncResult:
114        """Run a sync."""
115        connection_response = api_util.run_connection(
116            connection_id=self.connection_id,
117            api_root=self.workspace.api_root,
118            api_key=self.workspace.api_key,
119            workspace_id=self.workspace.workspace_id,
120        )
121        sync_result = SyncResult(
122            workspace=self.workspace,
123            connection=self,
124            job_id=connection_response.job_id,
125        )
126
127        if wait:
128            sync_result.wait_for_completion(
129                wait_timeout=wait_timeout,
130                raise_failure=True,
131                raise_timeout=True,
132            )
133
134        return sync_result
135
136    # Logs
137
138    def get_previous_sync_logs(
139        self,
140        *,
141        limit: int = 10,
142    ) -> list[SyncResult]:
143        """Get the previous sync logs for a connection."""
144        sync_logs: list[JobResponse] = api_util.get_job_logs(
145            connection_id=self.connection_id,
146            api_root=self.workspace.api_root,
147            api_key=self.workspace.api_key,
148            workspace_id=self.workspace.workspace_id,
149            limit=limit,
150        )
151        return [
152            SyncResult(
153                workspace=self.workspace,
154                connection=self,
155                job_id=sync_log.job_id,
156                _latest_status=sync_log.status,
157            )
158            for sync_log in sync_logs
159        ]
160
161    def get_sync_result(
162        self,
163        job_id: str | None = None,
164    ) -> SyncResult | None:
165        """Get the sync result for the connection.
166
167        If `job_id` is not provided, the most recent sync job will be used.
168
169        Returns `None` if job_id is omitted and no previous jobs are found.
170        """
171        if job_id is None:
172            # Get the most recent sync job
173            results = self.get_previous_sync_logs(
174                limit=1,
175            )
176            if results:
177                return results[0]
178
179            return None
180
181        # Get the sync job by ID (lazy loaded)
182        return SyncResult(
183            workspace=self.workspace,
184            connection=self,
185            job_id=job_id,
186        )
187
188    # Deletions
189
190    def delete(
191        self,
192        *,
193        delete_source: bool = False,
194        delete_destination: bool = False,
195    ) -> None:
196        """Delete the connection.
197
198        Args:
199            delete_source: Whether to also delete the source.
200            delete_destination: Whether to also delete the destination.
201        """
202        self.workspace.delete_connection(connection_id=self.connection_id)
203
204        if delete_source:
205            self.workspace.delete_source(source=self.source_id)
206
207        if delete_destination:
208            self.workspace.delete_destination(destination=self.destination_id)

A connection is a link between a source and a destination.

Do not instantiate this class directly. Instead, use CloudWorkspace.deploy_connection or airbyte.cloud.CloudWorkspace.get_connection methods.

CloudConnection( workspace: airbyte.cloud.workspaces.CloudWorkspace, connection_id: str, source: str | None = None, destination: str | None = None)
28    def __init__(
29        self,
30        workspace: CloudWorkspace,
31        connection_id: str,
32        source: str | None = None,
33        destination: str | None = None,
34    ) -> None:
35        self.connection_id = connection_id
36        """The ID of the connection."""
37
38        self.workspace = workspace
39        """The workspace that the connection belongs to."""
40
41        self._source_id = source
42        """The ID of the source."""
43
44        self._destination_id = destination
45        """The ID of the destination."""
46
47        self._connection_info: ConnectionResponse | None = None
connection_id

The ID of the connection.

workspace

The workspace that the connection belongs to.

source_id: str
60    @property
61    def source_id(self) -> str:
62        """The ID of the source."""
63        if not self._source_id:
64            if not self._connection_info:
65                self._connection_info = self._fetch_connection_info()
66
67            self._source_id = self._connection_info.source_id
68
69        return cast(str, self._source_id)

The ID of the source.

destination_id: str
71    @property
72    def destination_id(self) -> str:
73        """The ID of the destination."""
74        if not self._destination_id:
75            if not self._connection_info:
76                self._connection_info = self._fetch_connection_info()
77
78            self._destination_id = self._connection_info.source_id
79
80        return cast(str, self._destination_id)

The ID of the destination.

stream_names: list[str]
82    @property
83    def stream_names(self) -> list[str]:
84        """The stream names."""
85        if not self._connection_info:
86            self._connection_info = self._fetch_connection_info()
87
88        return [stream.name for stream in self._connection_info.configurations.streams]

The stream names.

table_prefix: str
90    @property
91    def table_prefix(self) -> str:
92        """The table prefix."""
93        if not self._connection_info:
94            self._connection_info = self._fetch_connection_info()
95
96        return self._connection_info.configurations.prefix

The table prefix.

connection_url: str | None
 98    @property
 99    def connection_url(self) -> str | None:
100        return f"{self.workspace.workspace_url}/connections/{self.connection_id}"
job_history_url: str | None
102    @property
103    def job_history_url(self) -> str | None:
104        return f"{self.connection_url}/job-history"
def run_sync( self, *, wait: bool = True, wait_timeout: int = 300) -> airbyte.cloud.sync_results.SyncResult:
108    def run_sync(
109        self,
110        *,
111        wait: bool = True,
112        wait_timeout: int = 300,
113    ) -> SyncResult:
114        """Run a sync."""
115        connection_response = api_util.run_connection(
116            connection_id=self.connection_id,
117            api_root=self.workspace.api_root,
118            api_key=self.workspace.api_key,
119            workspace_id=self.workspace.workspace_id,
120        )
121        sync_result = SyncResult(
122            workspace=self.workspace,
123            connection=self,
124            job_id=connection_response.job_id,
125        )
126
127        if wait:
128            sync_result.wait_for_completion(
129                wait_timeout=wait_timeout,
130                raise_failure=True,
131                raise_timeout=True,
132            )
133
134        return sync_result

Run a sync.

def get_previous_sync_logs(self, *, limit: int = 10) -> list[airbyte.cloud.sync_results.SyncResult]:
138    def get_previous_sync_logs(
139        self,
140        *,
141        limit: int = 10,
142    ) -> list[SyncResult]:
143        """Get the previous sync logs for a connection."""
144        sync_logs: list[JobResponse] = api_util.get_job_logs(
145            connection_id=self.connection_id,
146            api_root=self.workspace.api_root,
147            api_key=self.workspace.api_key,
148            workspace_id=self.workspace.workspace_id,
149            limit=limit,
150        )
151        return [
152            SyncResult(
153                workspace=self.workspace,
154                connection=self,
155                job_id=sync_log.job_id,
156                _latest_status=sync_log.status,
157            )
158            for sync_log in sync_logs
159        ]

Get the previous sync logs for a connection.

def get_sync_result( self, job_id: str | None = None) -> airbyte.cloud.sync_results.SyncResult | None:
161    def get_sync_result(
162        self,
163        job_id: str | None = None,
164    ) -> SyncResult | None:
165        """Get the sync result for the connection.
166
167        If `job_id` is not provided, the most recent sync job will be used.
168
169        Returns `None` if job_id is omitted and no previous jobs are found.
170        """
171        if job_id is None:
172            # Get the most recent sync job
173            results = self.get_previous_sync_logs(
174                limit=1,
175            )
176            if results:
177                return results[0]
178
179            return None
180
181        # Get the sync job by ID (lazy loaded)
182        return SyncResult(
183            workspace=self.workspace,
184            connection=self,
185            job_id=job_id,
186        )

Get the sync result for the connection.

If job_id is not provided, the most recent sync job will be used.

Returns None if job_id is omitted and no previous jobs are found.

def delete( self, *, delete_source: bool = False, delete_destination: bool = False) -> None:
190    def delete(
191        self,
192        *,
193        delete_source: bool = False,
194        delete_destination: bool = False,
195    ) -> None:
196        """Delete the connection.
197
198        Args:
199            delete_source: Whether to also delete the source.
200            delete_destination: Whether to also delete the destination.
201        """
202        self.workspace.delete_connection(connection_id=self.connection_id)
203
204        if delete_source:
205            self.workspace.delete_source(source=self.source_id)
206
207        if delete_destination:
208            self.workspace.delete_destination(destination=self.destination_id)

Delete the connection.

Arguments:
  • delete_source: Whether to also delete the source.
  • delete_destination: Whether to also delete the destination.