airbyte.cloud.workspaces
PyAirbyte classes and methods for interacting with the Airbyte Cloud API.
By overriding api_root
, you can use this module to interact with self-managed Airbyte instances,
both OSS and Enterprise.
1# Copyright (c) 2024 Airbyte, Inc., all rights reserved. 2"""PyAirbyte classes and methods for interacting with the Airbyte Cloud API. 3 4By overriding `api_root`, you can use this module to interact with self-managed Airbyte instances, 5both OSS and Enterprise. 6""" 7 8from __future__ import annotations 9 10from dataclasses import dataclass 11from typing import TYPE_CHECKING 12 13from airbyte import exceptions as exc 14from airbyte._util.api_util import ( 15 CLOUD_API_ROOT, 16 create_connection, 17 create_destination, 18 create_source, 19 delete_connection, 20 delete_destination, 21 delete_source, 22 get_connection, 23 get_workspace, 24) 25from airbyte.cloud._destination_util import get_destination_config_from_cache 26from airbyte.cloud.connections import CloudConnection 27from airbyte.cloud.sync_results import SyncResult 28from airbyte.sources.base import Source 29 30 31if TYPE_CHECKING: 32 from airbyte_api.models.shared.connectionresponse import ConnectionResponse 33 from airbyte_api.models.shared.destinationresponse import DestinationResponse 34 35 from airbyte.caches.base import CacheBase 36 37 38@dataclass 39class CloudWorkspace: 40 """A remote workspace on the Airbyte Cloud. 41 42 By overriding `api_root`, you can use this class to interact with self-managed Airbyte 43 instances, both OSS and Enterprise. 44 """ 45 46 workspace_id: str 47 api_key: str 48 api_root: str = CLOUD_API_ROOT 49 50 @property 51 def workspace_url(self) -> str | None: 52 return f"{self.api_root}/workspaces/{self.workspace_id}" 53 54 # Test connection and creds 55 56 def connect(self) -> None: 57 """Check that the workspace is reachable and raise an exception otherwise. 58 59 Note: It is not necessary to call this method before calling other operations. It 60 serves primarily as a simple check to ensure that the workspace is reachable 61 and credentials are correct. 62 """ 63 _ = get_workspace( 64 api_root=self.api_root, 65 api_key=self.api_key, 66 workspace_id=self.workspace_id, 67 ) 68 print(f"Successfully connected to workspace: {self.workspace_url}") 69 70 # Deploy and delete sources 71 72 def deploy_source( 73 self, 74 source: Source, 75 ) -> str: 76 """Deploy a source to the workspace. 77 78 Returns the newly deployed source ID. 79 """ 80 source_configuration = source.get_config().copy() 81 source_configuration["sourceType"] = source.name.replace("source-", "") 82 83 deployed_source = create_source( 84 name=f"{source.name.replace('-', ' ').title()} (Deployed by PyAirbyte)", 85 api_root=self.api_root, 86 api_key=self.api_key, 87 workspace_id=self.workspace_id, 88 config=source_configuration, 89 ) 90 91 # Set the deployment Ids on the source object 92 source._deployed_api_root = self.api_root # noqa: SLF001 # Accessing nn-public API 93 source._deployed_workspace_id = self.workspace_id # noqa: SLF001 # Accessing nn-public API 94 source._deployed_source_id = deployed_source.source_id # noqa: SLF001 # Accessing nn-public API 95 96 return deployed_source.source_id 97 98 def delete_source( 99 self, 100 source: str | Source, 101 ) -> None: 102 """Delete a source from the workspace. 103 104 You can pass either the source ID `str` or a deployed `Source` object. 105 """ 106 if not isinstance(source, (str, Source)): 107 raise ValueError(f"Invalid source type: {type(source)}") # noqa: TRY004, TRY003 108 109 if isinstance(source, Source): 110 if not source._deployed_source_id: # noqa: SLF001 111 raise ValueError("Source has not been deployed.") # noqa: TRY003 112 113 source_id = source._deployed_source_id # noqa: SLF001 114 115 elif isinstance(source, str): 116 source_id = source 117 118 delete_source( 119 source_id=source_id, 120 api_root=self.api_root, 121 api_key=self.api_key, 122 ) 123 124 # Deploy and delete destinations 125 126 def deploy_cache_as_destination( 127 self, 128 cache: CacheBase, 129 ) -> str: 130 """Deploy a cache to the workspace as a new destination. 131 132 Returns the newly deployed destination ID. 133 """ 134 cache_type_name = cache.__class__.__name__.replace("Cache", "") 135 136 deployed_destination: DestinationResponse = create_destination( 137 name=f"Destination {cache_type_name} (Deployed by PyAirbyte)", 138 api_root=self.api_root, 139 api_key=self.api_key, 140 workspace_id=self.workspace_id, 141 config=get_destination_config_from_cache(cache), 142 ) 143 144 # Set the deployment Ids on the source object 145 cache._deployed_api_root = self.api_root # noqa: SLF001 # Accessing nn-public API 146 cache._deployed_workspace_id = self.workspace_id # noqa: SLF001 # Accessing nn-public API 147 cache._deployed_destination_id = deployed_destination.destination_id # noqa: SLF001 # Accessing nn-public API 148 149 return deployed_destination.destination_id 150 151 def delete_destination( 152 self, 153 *, 154 destination: str | None = None, 155 cache: CacheBase | None = None, 156 ) -> None: 157 """Delete a deployed destination from the workspace. 158 159 You can pass either the `Cache` class or the deployed destination ID as a `str`. 160 """ 161 if destination is None and cache is None: 162 raise ValueError("You must provide either a destination ID or a cache object.") # noqa: TRY003 163 if destination is not None and cache is not None: 164 raise ValueError( # noqa: TRY003 165 "You must provide either a destination ID or a cache object, not both." 166 ) 167 168 if cache: 169 if not cache._deployed_destination_id: # noqa: SLF001 170 raise ValueError("Cache has not been deployed.") # noqa: TRY003 171 172 destination = cache._deployed_destination_id # noqa: SLF001 173 174 if destination is None: 175 raise ValueError("No destination ID provided.") # noqa: TRY003 176 177 delete_destination( 178 destination_id=destination, 179 api_root=self.api_root, 180 api_key=self.api_key, 181 ) 182 183 # Deploy and delete connections 184 185 def deploy_connection( 186 self, 187 source: Source | str, 188 cache: CacheBase | None = None, 189 destination: str | None = None, 190 table_prefix: str | None = None, 191 selected_streams: list[str] | None = None, 192 ) -> str: 193 """Deploy a source and cache to the workspace as a new connection. 194 195 Returns the newly deployed connection ID as a `str`. 196 197 Args: 198 source (Source | str): The source to deploy. You can pass either an already deployed 199 source ID `str` or a PyAirbyte `Source` object. If you pass a `Source` object, 200 it will be deployed automatically. 201 cache (CacheBase, optional): The cache to deploy as a new destination. You can provide 202 `cache` or `destination`, but not both. 203 destination (str, optional): The destination ID to use. You can provide 204 `cache` or `destination`, but not both. 205 """ 206 # Resolve source ID 207 source_id: str 208 if isinstance(source, Source): 209 selected_streams = selected_streams or source.get_selected_streams() 210 if source._deployed_source_id: # noqa: SLF001 211 source_id = source._deployed_source_id # noqa: SLF001 212 else: 213 source_id = self.deploy_source(source) 214 else: 215 source_id = source 216 if not selected_streams: 217 raise exc.PyAirbyteInputError( 218 guidance="You must provide `selected_streams` when deploying a source ID." 219 ) 220 221 # Resolve destination ID 222 destination_id: str 223 if destination: 224 destination_id = destination 225 elif cache: 226 table_prefix = table_prefix if table_prefix is not None else (cache.table_prefix or "") 227 if not cache._deployed_destination_id: # noqa: SLF001 228 destination_id = self.deploy_cache_as_destination(cache) 229 else: 230 destination_id = cache._deployed_destination_id # noqa: SLF001 231 else: 232 raise exc.PyAirbyteInputError( 233 guidance="You must provide either a destination ID or a cache object." 234 ) 235 236 assert source_id is not None 237 assert destination_id is not None 238 239 deployed_connection = create_connection( 240 name="Connection (Deployed by PyAirbyte)", 241 source_id=source_id, 242 destination_id=destination_id, 243 api_root=self.api_root, 244 api_key=self.api_key, 245 workspace_id=self.workspace_id, 246 selected_stream_names=selected_streams, 247 prefix=table_prefix or "", 248 ) 249 250 if isinstance(source, Source): 251 source._deployed_connection_id = deployed_connection.connection_id # noqa: SLF001 252 if cache: 253 cache._deployed_connection_id = deployed_connection.connection_id # noqa: SLF001 254 255 return deployed_connection.connection_id 256 257 def get_connection( 258 self, 259 connection_id: str, 260 ) -> CloudConnection: 261 """Get a connection by ID. 262 263 This method does not fetch data from the API. It returns a `CloudConnection` object, 264 which will be loaded lazily as needed. 265 """ 266 return CloudConnection( 267 workspace=self, 268 connection_id=connection_id, 269 ) 270 271 def delete_connection( 272 self, 273 connection_id: str | None, 274 *, 275 delete_source: bool = False, 276 delete_destination: bool = False, 277 ) -> None: 278 """Delete a deployed connection from the workspace.""" 279 if connection_id is None: 280 raise ValueError("No connection ID provided.") # noqa: TRY003 281 282 connection: ConnectionResponse = get_connection( 283 connection_id=connection_id, 284 api_root=self.api_root, 285 api_key=self.api_key, 286 workspace_id=self.workspace_id, 287 ) 288 delete_connection( 289 connection_id=connection_id, 290 api_root=self.api_root, 291 api_key=self.api_key, 292 workspace_id=self.workspace_id, 293 ) 294 if delete_source: 295 self.delete_source(source=connection.source_id) 296 297 if delete_destination: 298 self.delete_destination(destination=connection.destination_id) 299 300 # Run syncs 301 302 def run_sync( 303 self, 304 connection_id: str, 305 *, 306 wait: bool = True, 307 wait_timeout: int = 300, 308 ) -> SyncResult: 309 """Run a sync on a deployed connection.""" 310 connection = CloudConnection( 311 workspace=self, 312 connection_id=connection_id, 313 ) 314 return connection.run_sync(wait=wait, wait_timeout=wait_timeout) 315 316 # Get sync results and previous sync logs 317 318 def get_sync_result( 319 self, 320 connection_id: str, 321 job_id: str | None = None, 322 ) -> SyncResult | None: 323 """Get the sync result for a connection job. 324 325 If `job_id` is not provided, the most recent sync job will be used. 326 327 Returns `None` if job_id is omitted and no previous jobs are found. 328 """ 329 connection = CloudConnection( 330 workspace=self, 331 connection_id=connection_id, 332 ) 333 if job_id is None: 334 results = self.get_previous_sync_logs( 335 connection_id=connection_id, 336 limit=1, 337 ) 338 if results: 339 return results[0] 340 341 return None 342 connection = CloudConnection( 343 workspace=self, 344 connection_id=connection_id, 345 ) 346 return SyncResult( 347 workspace=self, 348 connection=connection, 349 job_id=job_id, 350 ) 351 352 def get_previous_sync_logs( 353 self, 354 connection_id: str, 355 *, 356 limit: int = 10, 357 ) -> list[SyncResult]: 358 """Get the previous sync logs for a connection.""" 359 connection = CloudConnection( 360 workspace=self, 361 connection_id=connection_id, 362 ) 363 return connection.get_previous_sync_logs( 364 limit=limit, 365 )
39@dataclass 40class CloudWorkspace: 41 """A remote workspace on the Airbyte Cloud. 42 43 By overriding `api_root`, you can use this class to interact with self-managed Airbyte 44 instances, both OSS and Enterprise. 45 """ 46 47 workspace_id: str 48 api_key: str 49 api_root: str = CLOUD_API_ROOT 50 51 @property 52 def workspace_url(self) -> str | None: 53 return f"{self.api_root}/workspaces/{self.workspace_id}" 54 55 # Test connection and creds 56 57 def connect(self) -> None: 58 """Check that the workspace is reachable and raise an exception otherwise. 59 60 Note: It is not necessary to call this method before calling other operations. It 61 serves primarily as a simple check to ensure that the workspace is reachable 62 and credentials are correct. 63 """ 64 _ = get_workspace( 65 api_root=self.api_root, 66 api_key=self.api_key, 67 workspace_id=self.workspace_id, 68 ) 69 print(f"Successfully connected to workspace: {self.workspace_url}") 70 71 # Deploy and delete sources 72 73 def deploy_source( 74 self, 75 source: Source, 76 ) -> str: 77 """Deploy a source to the workspace. 78 79 Returns the newly deployed source ID. 80 """ 81 source_configuration = source.get_config().copy() 82 source_configuration["sourceType"] = source.name.replace("source-", "") 83 84 deployed_source = create_source( 85 name=f"{source.name.replace('-', ' ').title()} (Deployed by PyAirbyte)", 86 api_root=self.api_root, 87 api_key=self.api_key, 88 workspace_id=self.workspace_id, 89 config=source_configuration, 90 ) 91 92 # Set the deployment Ids on the source object 93 source._deployed_api_root = self.api_root # noqa: SLF001 # Accessing nn-public API 94 source._deployed_workspace_id = self.workspace_id # noqa: SLF001 # Accessing nn-public API 95 source._deployed_source_id = deployed_source.source_id # noqa: SLF001 # Accessing nn-public API 96 97 return deployed_source.source_id 98 99 def delete_source( 100 self, 101 source: str | Source, 102 ) -> None: 103 """Delete a source from the workspace. 104 105 You can pass either the source ID `str` or a deployed `Source` object. 106 """ 107 if not isinstance(source, (str, Source)): 108 raise ValueError(f"Invalid source type: {type(source)}") # noqa: TRY004, TRY003 109 110 if isinstance(source, Source): 111 if not source._deployed_source_id: # noqa: SLF001 112 raise ValueError("Source has not been deployed.") # noqa: TRY003 113 114 source_id = source._deployed_source_id # noqa: SLF001 115 116 elif isinstance(source, str): 117 source_id = source 118 119 delete_source( 120 source_id=source_id, 121 api_root=self.api_root, 122 api_key=self.api_key, 123 ) 124 125 # Deploy and delete destinations 126 127 def deploy_cache_as_destination( 128 self, 129 cache: CacheBase, 130 ) -> str: 131 """Deploy a cache to the workspace as a new destination. 132 133 Returns the newly deployed destination ID. 134 """ 135 cache_type_name = cache.__class__.__name__.replace("Cache", "") 136 137 deployed_destination: DestinationResponse = create_destination( 138 name=f"Destination {cache_type_name} (Deployed by PyAirbyte)", 139 api_root=self.api_root, 140 api_key=self.api_key, 141 workspace_id=self.workspace_id, 142 config=get_destination_config_from_cache(cache), 143 ) 144 145 # Set the deployment Ids on the source object 146 cache._deployed_api_root = self.api_root # noqa: SLF001 # Accessing nn-public API 147 cache._deployed_workspace_id = self.workspace_id # noqa: SLF001 # Accessing nn-public API 148 cache._deployed_destination_id = deployed_destination.destination_id # noqa: SLF001 # Accessing nn-public API 149 150 return deployed_destination.destination_id 151 152 def delete_destination( 153 self, 154 *, 155 destination: str | None = None, 156 cache: CacheBase | None = None, 157 ) -> None: 158 """Delete a deployed destination from the workspace. 159 160 You can pass either the `Cache` class or the deployed destination ID as a `str`. 161 """ 162 if destination is None and cache is None: 163 raise ValueError("You must provide either a destination ID or a cache object.") # noqa: TRY003 164 if destination is not None and cache is not None: 165 raise ValueError( # noqa: TRY003 166 "You must provide either a destination ID or a cache object, not both." 167 ) 168 169 if cache: 170 if not cache._deployed_destination_id: # noqa: SLF001 171 raise ValueError("Cache has not been deployed.") # noqa: TRY003 172 173 destination = cache._deployed_destination_id # noqa: SLF001 174 175 if destination is None: 176 raise ValueError("No destination ID provided.") # noqa: TRY003 177 178 delete_destination( 179 destination_id=destination, 180 api_root=self.api_root, 181 api_key=self.api_key, 182 ) 183 184 # Deploy and delete connections 185 186 def deploy_connection( 187 self, 188 source: Source | str, 189 cache: CacheBase | None = None, 190 destination: str | None = None, 191 table_prefix: str | None = None, 192 selected_streams: list[str] | None = None, 193 ) -> str: 194 """Deploy a source and cache to the workspace as a new connection. 195 196 Returns the newly deployed connection ID as a `str`. 197 198 Args: 199 source (Source | str): The source to deploy. You can pass either an already deployed 200 source ID `str` or a PyAirbyte `Source` object. If you pass a `Source` object, 201 it will be deployed automatically. 202 cache (CacheBase, optional): The cache to deploy as a new destination. You can provide 203 `cache` or `destination`, but not both. 204 destination (str, optional): The destination ID to use. You can provide 205 `cache` or `destination`, but not both. 206 """ 207 # Resolve source ID 208 source_id: str 209 if isinstance(source, Source): 210 selected_streams = selected_streams or source.get_selected_streams() 211 if source._deployed_source_id: # noqa: SLF001 212 source_id = source._deployed_source_id # noqa: SLF001 213 else: 214 source_id = self.deploy_source(source) 215 else: 216 source_id = source 217 if not selected_streams: 218 raise exc.PyAirbyteInputError( 219 guidance="You must provide `selected_streams` when deploying a source ID." 220 ) 221 222 # Resolve destination ID 223 destination_id: str 224 if destination: 225 destination_id = destination 226 elif cache: 227 table_prefix = table_prefix if table_prefix is not None else (cache.table_prefix or "") 228 if not cache._deployed_destination_id: # noqa: SLF001 229 destination_id = self.deploy_cache_as_destination(cache) 230 else: 231 destination_id = cache._deployed_destination_id # noqa: SLF001 232 else: 233 raise exc.PyAirbyteInputError( 234 guidance="You must provide either a destination ID or a cache object." 235 ) 236 237 assert source_id is not None 238 assert destination_id is not None 239 240 deployed_connection = create_connection( 241 name="Connection (Deployed by PyAirbyte)", 242 source_id=source_id, 243 destination_id=destination_id, 244 api_root=self.api_root, 245 api_key=self.api_key, 246 workspace_id=self.workspace_id, 247 selected_stream_names=selected_streams, 248 prefix=table_prefix or "", 249 ) 250 251 if isinstance(source, Source): 252 source._deployed_connection_id = deployed_connection.connection_id # noqa: SLF001 253 if cache: 254 cache._deployed_connection_id = deployed_connection.connection_id # noqa: SLF001 255 256 return deployed_connection.connection_id 257 258 def get_connection( 259 self, 260 connection_id: str, 261 ) -> CloudConnection: 262 """Get a connection by ID. 263 264 This method does not fetch data from the API. It returns a `CloudConnection` object, 265 which will be loaded lazily as needed. 266 """ 267 return CloudConnection( 268 workspace=self, 269 connection_id=connection_id, 270 ) 271 272 def delete_connection( 273 self, 274 connection_id: str | None, 275 *, 276 delete_source: bool = False, 277 delete_destination: bool = False, 278 ) -> None: 279 """Delete a deployed connection from the workspace.""" 280 if connection_id is None: 281 raise ValueError("No connection ID provided.") # noqa: TRY003 282 283 connection: ConnectionResponse = get_connection( 284 connection_id=connection_id, 285 api_root=self.api_root, 286 api_key=self.api_key, 287 workspace_id=self.workspace_id, 288 ) 289 delete_connection( 290 connection_id=connection_id, 291 api_root=self.api_root, 292 api_key=self.api_key, 293 workspace_id=self.workspace_id, 294 ) 295 if delete_source: 296 self.delete_source(source=connection.source_id) 297 298 if delete_destination: 299 self.delete_destination(destination=connection.destination_id) 300 301 # Run syncs 302 303 def run_sync( 304 self, 305 connection_id: str, 306 *, 307 wait: bool = True, 308 wait_timeout: int = 300, 309 ) -> SyncResult: 310 """Run a sync on a deployed connection.""" 311 connection = CloudConnection( 312 workspace=self, 313 connection_id=connection_id, 314 ) 315 return connection.run_sync(wait=wait, wait_timeout=wait_timeout) 316 317 # Get sync results and previous sync logs 318 319 def get_sync_result( 320 self, 321 connection_id: str, 322 job_id: str | None = None, 323 ) -> SyncResult | None: 324 """Get the sync result for a connection job. 325 326 If `job_id` is not provided, the most recent sync job will be used. 327 328 Returns `None` if job_id is omitted and no previous jobs are found. 329 """ 330 connection = CloudConnection( 331 workspace=self, 332 connection_id=connection_id, 333 ) 334 if job_id is None: 335 results = self.get_previous_sync_logs( 336 connection_id=connection_id, 337 limit=1, 338 ) 339 if results: 340 return results[0] 341 342 return None 343 connection = CloudConnection( 344 workspace=self, 345 connection_id=connection_id, 346 ) 347 return SyncResult( 348 workspace=self, 349 connection=connection, 350 job_id=job_id, 351 ) 352 353 def get_previous_sync_logs( 354 self, 355 connection_id: str, 356 *, 357 limit: int = 10, 358 ) -> list[SyncResult]: 359 """Get the previous sync logs for a connection.""" 360 connection = CloudConnection( 361 workspace=self, 362 connection_id=connection_id, 363 ) 364 return connection.get_previous_sync_logs( 365 limit=limit, 366 )
A remote workspace on the Airbyte Cloud.
By overriding api_root
, you can use this class to interact with self-managed Airbyte
instances, both OSS and Enterprise.
57 def connect(self) -> None: 58 """Check that the workspace is reachable and raise an exception otherwise. 59 60 Note: It is not necessary to call this method before calling other operations. It 61 serves primarily as a simple check to ensure that the workspace is reachable 62 and credentials are correct. 63 """ 64 _ = get_workspace( 65 api_root=self.api_root, 66 api_key=self.api_key, 67 workspace_id=self.workspace_id, 68 ) 69 print(f"Successfully connected to workspace: {self.workspace_url}")
Check that the workspace is reachable and raise an exception otherwise.
Note: It is not necessary to call this method before calling other operations. It serves primarily as a simple check to ensure that the workspace is reachable and credentials are correct.
73 def deploy_source( 74 self, 75 source: Source, 76 ) -> str: 77 """Deploy a source to the workspace. 78 79 Returns the newly deployed source ID. 80 """ 81 source_configuration = source.get_config().copy() 82 source_configuration["sourceType"] = source.name.replace("source-", "") 83 84 deployed_source = create_source( 85 name=f"{source.name.replace('-', ' ').title()} (Deployed by PyAirbyte)", 86 api_root=self.api_root, 87 api_key=self.api_key, 88 workspace_id=self.workspace_id, 89 config=source_configuration, 90 ) 91 92 # Set the deployment Ids on the source object 93 source._deployed_api_root = self.api_root # noqa: SLF001 # Accessing nn-public API 94 source._deployed_workspace_id = self.workspace_id # noqa: SLF001 # Accessing nn-public API 95 source._deployed_source_id = deployed_source.source_id # noqa: SLF001 # Accessing nn-public API 96 97 return deployed_source.source_id
Deploy a source to the workspace.
Returns the newly deployed source ID.
99 def delete_source( 100 self, 101 source: str | Source, 102 ) -> None: 103 """Delete a source from the workspace. 104 105 You can pass either the source ID `str` or a deployed `Source` object. 106 """ 107 if not isinstance(source, (str, Source)): 108 raise ValueError(f"Invalid source type: {type(source)}") # noqa: TRY004, TRY003 109 110 if isinstance(source, Source): 111 if not source._deployed_source_id: # noqa: SLF001 112 raise ValueError("Source has not been deployed.") # noqa: TRY003 113 114 source_id = source._deployed_source_id # noqa: SLF001 115 116 elif isinstance(source, str): 117 source_id = source 118 119 delete_source( 120 source_id=source_id, 121 api_root=self.api_root, 122 api_key=self.api_key, 123 )
Delete a source from the workspace.
You can pass either the source ID str
or a deployed Source
object.
127 def deploy_cache_as_destination( 128 self, 129 cache: CacheBase, 130 ) -> str: 131 """Deploy a cache to the workspace as a new destination. 132 133 Returns the newly deployed destination ID. 134 """ 135 cache_type_name = cache.__class__.__name__.replace("Cache", "") 136 137 deployed_destination: DestinationResponse = create_destination( 138 name=f"Destination {cache_type_name} (Deployed by PyAirbyte)", 139 api_root=self.api_root, 140 api_key=self.api_key, 141 workspace_id=self.workspace_id, 142 config=get_destination_config_from_cache(cache), 143 ) 144 145 # Set the deployment Ids on the source object 146 cache._deployed_api_root = self.api_root # noqa: SLF001 # Accessing nn-public API 147 cache._deployed_workspace_id = self.workspace_id # noqa: SLF001 # Accessing nn-public API 148 cache._deployed_destination_id = deployed_destination.destination_id # noqa: SLF001 # Accessing nn-public API 149 150 return deployed_destination.destination_id
Deploy a cache to the workspace as a new destination.
Returns the newly deployed destination ID.
152 def delete_destination( 153 self, 154 *, 155 destination: str | None = None, 156 cache: CacheBase | None = None, 157 ) -> None: 158 """Delete a deployed destination from the workspace. 159 160 You can pass either the `Cache` class or the deployed destination ID as a `str`. 161 """ 162 if destination is None and cache is None: 163 raise ValueError("You must provide either a destination ID or a cache object.") # noqa: TRY003 164 if destination is not None and cache is not None: 165 raise ValueError( # noqa: TRY003 166 "You must provide either a destination ID or a cache object, not both." 167 ) 168 169 if cache: 170 if not cache._deployed_destination_id: # noqa: SLF001 171 raise ValueError("Cache has not been deployed.") # noqa: TRY003 172 173 destination = cache._deployed_destination_id # noqa: SLF001 174 175 if destination is None: 176 raise ValueError("No destination ID provided.") # noqa: TRY003 177 178 delete_destination( 179 destination_id=destination, 180 api_root=self.api_root, 181 api_key=self.api_key, 182 )
Delete a deployed destination from the workspace.
You can pass either the Cache
class or the deployed destination ID as a str
.
186 def deploy_connection( 187 self, 188 source: Source | str, 189 cache: CacheBase | None = None, 190 destination: str | None = None, 191 table_prefix: str | None = None, 192 selected_streams: list[str] | None = None, 193 ) -> str: 194 """Deploy a source and cache to the workspace as a new connection. 195 196 Returns the newly deployed connection ID as a `str`. 197 198 Args: 199 source (Source | str): The source to deploy. You can pass either an already deployed 200 source ID `str` or a PyAirbyte `Source` object. If you pass a `Source` object, 201 it will be deployed automatically. 202 cache (CacheBase, optional): The cache to deploy as a new destination. You can provide 203 `cache` or `destination`, but not both. 204 destination (str, optional): The destination ID to use. You can provide 205 `cache` or `destination`, but not both. 206 """ 207 # Resolve source ID 208 source_id: str 209 if isinstance(source, Source): 210 selected_streams = selected_streams or source.get_selected_streams() 211 if source._deployed_source_id: # noqa: SLF001 212 source_id = source._deployed_source_id # noqa: SLF001 213 else: 214 source_id = self.deploy_source(source) 215 else: 216 source_id = source 217 if not selected_streams: 218 raise exc.PyAirbyteInputError( 219 guidance="You must provide `selected_streams` when deploying a source ID." 220 ) 221 222 # Resolve destination ID 223 destination_id: str 224 if destination: 225 destination_id = destination 226 elif cache: 227 table_prefix = table_prefix if table_prefix is not None else (cache.table_prefix or "") 228 if not cache._deployed_destination_id: # noqa: SLF001 229 destination_id = self.deploy_cache_as_destination(cache) 230 else: 231 destination_id = cache._deployed_destination_id # noqa: SLF001 232 else: 233 raise exc.PyAirbyteInputError( 234 guidance="You must provide either a destination ID or a cache object." 235 ) 236 237 assert source_id is not None 238 assert destination_id is not None 239 240 deployed_connection = create_connection( 241 name="Connection (Deployed by PyAirbyte)", 242 source_id=source_id, 243 destination_id=destination_id, 244 api_root=self.api_root, 245 api_key=self.api_key, 246 workspace_id=self.workspace_id, 247 selected_stream_names=selected_streams, 248 prefix=table_prefix or "", 249 ) 250 251 if isinstance(source, Source): 252 source._deployed_connection_id = deployed_connection.connection_id # noqa: SLF001 253 if cache: 254 cache._deployed_connection_id = deployed_connection.connection_id # noqa: SLF001 255 256 return deployed_connection.connection_id
Deploy a source and cache to the workspace as a new connection.
Returns the newly deployed connection ID as a str
.
Arguments:
- source (Source | str): The source to deploy. You can pass either an already deployed
source ID
str
or a PyAirbyteSource
object. If you pass aSource
object, it will be deployed automatically. - cache (CacheBase, optional): The cache to deploy as a new destination. You can provide
cache
ordestination
, but not both. - destination (str, optional): The destination ID to use. You can provide
cache
ordestination
, but not both.
258 def get_connection( 259 self, 260 connection_id: str, 261 ) -> CloudConnection: 262 """Get a connection by ID. 263 264 This method does not fetch data from the API. It returns a `CloudConnection` object, 265 which will be loaded lazily as needed. 266 """ 267 return CloudConnection( 268 workspace=self, 269 connection_id=connection_id, 270 )
Get a connection by ID.
This method does not fetch data from the API. It returns a CloudConnection
object,
which will be loaded lazily as needed.
272 def delete_connection( 273 self, 274 connection_id: str | None, 275 *, 276 delete_source: bool = False, 277 delete_destination: bool = False, 278 ) -> None: 279 """Delete a deployed connection from the workspace.""" 280 if connection_id is None: 281 raise ValueError("No connection ID provided.") # noqa: TRY003 282 283 connection: ConnectionResponse = get_connection( 284 connection_id=connection_id, 285 api_root=self.api_root, 286 api_key=self.api_key, 287 workspace_id=self.workspace_id, 288 ) 289 delete_connection( 290 connection_id=connection_id, 291 api_root=self.api_root, 292 api_key=self.api_key, 293 workspace_id=self.workspace_id, 294 ) 295 if delete_source: 296 self.delete_source(source=connection.source_id) 297 298 if delete_destination: 299 self.delete_destination(destination=connection.destination_id)
Delete a deployed connection from the workspace.
303 def run_sync( 304 self, 305 connection_id: str, 306 *, 307 wait: bool = True, 308 wait_timeout: int = 300, 309 ) -> SyncResult: 310 """Run a sync on a deployed connection.""" 311 connection = CloudConnection( 312 workspace=self, 313 connection_id=connection_id, 314 ) 315 return connection.run_sync(wait=wait, wait_timeout=wait_timeout)
Run a sync on a deployed connection.
319 def get_sync_result( 320 self, 321 connection_id: str, 322 job_id: str | None = None, 323 ) -> SyncResult | None: 324 """Get the sync result for a connection job. 325 326 If `job_id` is not provided, the most recent sync job will be used. 327 328 Returns `None` if job_id is omitted and no previous jobs are found. 329 """ 330 connection = CloudConnection( 331 workspace=self, 332 connection_id=connection_id, 333 ) 334 if job_id is None: 335 results = self.get_previous_sync_logs( 336 connection_id=connection_id, 337 limit=1, 338 ) 339 if results: 340 return results[0] 341 342 return None 343 connection = CloudConnection( 344 workspace=self, 345 connection_id=connection_id, 346 ) 347 return SyncResult( 348 workspace=self, 349 connection=connection, 350 job_id=job_id, 351 )
Get the sync result for a connection job.
If job_id
is not provided, the most recent sync job will be used.
Returns None
if job_id is omitted and no previous jobs are found.
353 def get_previous_sync_logs( 354 self, 355 connection_id: str, 356 *, 357 limit: int = 10, 358 ) -> list[SyncResult]: 359 """Get the previous sync logs for a connection.""" 360 connection = CloudConnection( 361 workspace=self, 362 connection_id=connection_id, 363 ) 364 return connection.get_previous_sync_logs( 365 limit=limit, 366 )
Get the previous sync logs for a connection.