airbyte.secrets.google_gsm
Secret manager that retrieves secrets from Google Secrets Manager (GSM).
Usage Example:
gsm_secrets_manager = GoogleGSMSecretManager(
project=AIRBYTE_INTERNAL_GCP_PROJECT,
credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"),
)
first_secret: SecretHandle = next(
gsm_secrets_manager.fetch_connector_secrets(
connector_name=connector_name,
),
None,
)
print(f"Found '{connector_name}' credential secret '${first_secret.secret_name}'.")
return first_secret.get_value().parse_json()
More compact example:
gsm_secrets_manager = GoogleGSMSecretManager(
project=AIRBYTE_INTERNAL_GCP_PROJECT,
credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"),
)
connector_config: dict = (
next(
gsm_secrets_manager.fetch_connector_secrets(
connector_name=connector_name,
),
None,
)
.get_value()
.parse_json()
)
1# Copyright (c) 2024 Airbyte, Inc., all rights reserved. 2"""Secret manager that retrieves secrets from Google Secrets Manager (GSM). 3 4Usage Example: 5 6```python 7gsm_secrets_manager = GoogleGSMSecretManager( 8 project=AIRBYTE_INTERNAL_GCP_PROJECT, 9 credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"), 10) 11first_secret: SecretHandle = next( 12 gsm_secrets_manager.fetch_connector_secrets( 13 connector_name=connector_name, 14 ), 15 None, 16) 17 18print(f"Found '{connector_name}' credential secret '${first_secret.secret_name}'.") 19return first_secret.get_value().parse_json() 20``` 21 22More compact example: 23 24```python 25gsm_secrets_manager = GoogleGSMSecretManager( 26 project=AIRBYTE_INTERNAL_GCP_PROJECT, 27 credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"), 28) 29connector_config: dict = ( 30 next( 31 gsm_secrets_manager.fetch_connector_secrets( 32 connector_name=connector_name, 33 ), 34 None, 35 ) 36 .get_value() 37 .parse_json() 38) 39``` 40""" 41 42from __future__ import annotations 43 44import json 45import os 46from pathlib import Path 47from typing import TYPE_CHECKING 48 49from google.cloud import secretmanager_v1 as secretmanager 50 51from airbyte import exceptions as exc 52from airbyte.secrets.base import SecretHandle, SecretSourceEnum, SecretString 53from airbyte.secrets.custom import CustomSecretManager 54 55 56if TYPE_CHECKING: 57 from collections.abc import Iterable 58 59 from google.cloud.secretmanager_v1.services.secret_manager_service.pagers import ( 60 ListSecretsPager, 61 ) 62 63 64class GoogleGSMSecretManager(CustomSecretManager): 65 """Secret manager that retrieves secrets from Google Secrets Manager (GSM). 66 67 This class inherits from `CustomSecretManager` and also adds methods 68 that are specific to this implementation: `fetch_secrets()`, 69 `fetch_secrets_by_label()` and `fetch_connector_secrets()`. 70 71 This secret manager is not enabled by default. To use it, you must provide the project ID and 72 the credentials for a service account with the necessary permissions to access the secrets. 73 74 The `fetch_connector_secret()` method assumes a label name of `connector` 75 matches the name of the connector (`source-github`, `destination-snowflake`, etc.) 76 """ 77 78 name = SecretSourceEnum.GOOGLE_GSM.value 79 auto_register = False 80 as_backup = False 81 replace_existing = False 82 83 CONNECTOR_LABEL = "connector" 84 """The label key used to filter secrets by connector name.""" 85 86 def __init__( 87 self, 88 project: str, 89 *, 90 credentials_path: str | None = None, 91 credentials_json: str | SecretString | None = None, 92 auto_register: bool = False, 93 as_backup: bool = False, 94 ) -> None: 95 """Instantiate a new Google GSM secret manager instance. 96 97 You can provide either the path to the credentials file or the JSON contents of the 98 credentials file. If both are provided, a `PyAirbyteInputError` will be raised. 99 """ 100 if credentials_path and credentials_json: 101 raise exc.PyAirbyteInputError( 102 guidance=("You can provide `credentials_path` or `credentials_json` but not both."), 103 ) 104 105 self.project = project 106 107 if credentials_json is not None and not isinstance(credentials_json, SecretString): 108 credentials_json = SecretString(credentials_json) 109 110 if not credentials_json and not credentials_path: 111 if "GOOGLE_APPLICATION_CREDENTIALS" in os.environ: 112 credentials_path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] 113 114 elif "GCP_GSM_CREDENTIALS" in os.environ: 115 credentials_json = SecretString(os.environ["GCP_GSM_CREDENTIALS"]) 116 117 if credentials_path: 118 credentials_json = SecretString(Path(credentials_path).read_text()) 119 120 if not credentials_json: 121 raise exc.PyAirbyteInputError( 122 guidance=( 123 "No Google Cloud credentials found. You can provide the path to the " 124 "credentials file using the `credentials_path` argument, or provide the JSON " 125 "contents of the credentials file using the `credentials_json` argument." 126 ), 127 ) 128 129 self.secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info( 130 json.loads(credentials_json) 131 ) 132 133 if auto_register: 134 self.auto_register = auto_register 135 136 if as_backup: 137 self.as_backup = as_backup 138 139 super().__init__() # Handles the registration if needed 140 141 def get_secret(self, secret_name: str) -> SecretString | None: 142 """Get a named secret from Google Colab user secrets.""" 143 return SecretString( 144 self.secret_client.access_secret_version( 145 name=f"projects/{self.project}/secrets/{secret_name}/versions/latest" 146 ).payload.data.decode("UTF-8") 147 ) 148 149 def fetch_secrets( 150 self, 151 *, 152 filter_string: str, 153 ) -> Iterable[SecretHandle]: 154 """List all available secrets in the secret manager. 155 156 Example filter strings: 157 - `labels.connector=source-bigquery`: Filter for secrets with the labe 'source-bigquery'. 158 159 Args: 160 filter_string (str): A filter string to apply to the list of secrets, following the 161 format described in the Google Secret Manager documentation: 162 https://cloud.google.com/secret-manager/docs/filtering 163 164 Returns: 165 Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets. 166 """ 167 gsm_secrets: ListSecretsPager = self.secret_client.list_secrets( 168 secretmanager.ListSecretsRequest( 169 request={ 170 "filter": filter_string, 171 } 172 ) 173 ) 174 175 return [ 176 SecretHandle( 177 parent=self, 178 secret_name=secret.name, 179 ) 180 for secret in gsm_secrets 181 ] 182 183 def fetch_secrets_by_label( 184 self, 185 label_key: str, 186 label_value: str, 187 ) -> Iterable[SecretHandle]: 188 """List all available secrets in the secret manager. 189 190 Args: 191 label_key (str): The key of the label to filter by. 192 label_value (str): The value of the label to filter by. 193 194 Returns: 195 Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets. 196 """ 197 return self.fetch_secrets(filter_string=f"labels.{label_key}={label_value}") 198 199 def fetch_connector_secrets( 200 self, 201 connector_name: str, 202 ) -> Iterable[SecretHandle]: 203 """Fetch secrets in the secret manager, using the connector name as a filter for the label. 204 205 The label key used to filter the secrets is defined by the `CONNECTOR_LABEL` attribute, 206 which defaults to 'connector'. 207 208 Args: 209 connector_name (str): The name of the connector to filter by. 210 211 Returns: 212 Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets. 213 """ 214 return self.fetch_secrets_by_label( 215 label_key=self.CONNECTOR_LABEL, 216 label_value=connector_name, 217 )
65class GoogleGSMSecretManager(CustomSecretManager): 66 """Secret manager that retrieves secrets from Google Secrets Manager (GSM). 67 68 This class inherits from `CustomSecretManager` and also adds methods 69 that are specific to this implementation: `fetch_secrets()`, 70 `fetch_secrets_by_label()` and `fetch_connector_secrets()`. 71 72 This secret manager is not enabled by default. To use it, you must provide the project ID and 73 the credentials for a service account with the necessary permissions to access the secrets. 74 75 The `fetch_connector_secret()` method assumes a label name of `connector` 76 matches the name of the connector (`source-github`, `destination-snowflake`, etc.) 77 """ 78 79 name = SecretSourceEnum.GOOGLE_GSM.value 80 auto_register = False 81 as_backup = False 82 replace_existing = False 83 84 CONNECTOR_LABEL = "connector" 85 """The label key used to filter secrets by connector name.""" 86 87 def __init__( 88 self, 89 project: str, 90 *, 91 credentials_path: str | None = None, 92 credentials_json: str | SecretString | None = None, 93 auto_register: bool = False, 94 as_backup: bool = False, 95 ) -> None: 96 """Instantiate a new Google GSM secret manager instance. 97 98 You can provide either the path to the credentials file or the JSON contents of the 99 credentials file. If both are provided, a `PyAirbyteInputError` will be raised. 100 """ 101 if credentials_path and credentials_json: 102 raise exc.PyAirbyteInputError( 103 guidance=("You can provide `credentials_path` or `credentials_json` but not both."), 104 ) 105 106 self.project = project 107 108 if credentials_json is not None and not isinstance(credentials_json, SecretString): 109 credentials_json = SecretString(credentials_json) 110 111 if not credentials_json and not credentials_path: 112 if "GOOGLE_APPLICATION_CREDENTIALS" in os.environ: 113 credentials_path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] 114 115 elif "GCP_GSM_CREDENTIALS" in os.environ: 116 credentials_json = SecretString(os.environ["GCP_GSM_CREDENTIALS"]) 117 118 if credentials_path: 119 credentials_json = SecretString(Path(credentials_path).read_text()) 120 121 if not credentials_json: 122 raise exc.PyAirbyteInputError( 123 guidance=( 124 "No Google Cloud credentials found. You can provide the path to the " 125 "credentials file using the `credentials_path` argument, or provide the JSON " 126 "contents of the credentials file using the `credentials_json` argument." 127 ), 128 ) 129 130 self.secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info( 131 json.loads(credentials_json) 132 ) 133 134 if auto_register: 135 self.auto_register = auto_register 136 137 if as_backup: 138 self.as_backup = as_backup 139 140 super().__init__() # Handles the registration if needed 141 142 def get_secret(self, secret_name: str) -> SecretString | None: 143 """Get a named secret from Google Colab user secrets.""" 144 return SecretString( 145 self.secret_client.access_secret_version( 146 name=f"projects/{self.project}/secrets/{secret_name}/versions/latest" 147 ).payload.data.decode("UTF-8") 148 ) 149 150 def fetch_secrets( 151 self, 152 *, 153 filter_string: str, 154 ) -> Iterable[SecretHandle]: 155 """List all available secrets in the secret manager. 156 157 Example filter strings: 158 - `labels.connector=source-bigquery`: Filter for secrets with the labe 'source-bigquery'. 159 160 Args: 161 filter_string (str): A filter string to apply to the list of secrets, following the 162 format described in the Google Secret Manager documentation: 163 https://cloud.google.com/secret-manager/docs/filtering 164 165 Returns: 166 Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets. 167 """ 168 gsm_secrets: ListSecretsPager = self.secret_client.list_secrets( 169 secretmanager.ListSecretsRequest( 170 request={ 171 "filter": filter_string, 172 } 173 ) 174 ) 175 176 return [ 177 SecretHandle( 178 parent=self, 179 secret_name=secret.name, 180 ) 181 for secret in gsm_secrets 182 ] 183 184 def fetch_secrets_by_label( 185 self, 186 label_key: str, 187 label_value: str, 188 ) -> Iterable[SecretHandle]: 189 """List all available secrets in the secret manager. 190 191 Args: 192 label_key (str): The key of the label to filter by. 193 label_value (str): The value of the label to filter by. 194 195 Returns: 196 Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets. 197 """ 198 return self.fetch_secrets(filter_string=f"labels.{label_key}={label_value}") 199 200 def fetch_connector_secrets( 201 self, 202 connector_name: str, 203 ) -> Iterable[SecretHandle]: 204 """Fetch secrets in the secret manager, using the connector name as a filter for the label. 205 206 The label key used to filter the secrets is defined by the `CONNECTOR_LABEL` attribute, 207 which defaults to 'connector'. 208 209 Args: 210 connector_name (str): The name of the connector to filter by. 211 212 Returns: 213 Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets. 214 """ 215 return self.fetch_secrets_by_label( 216 label_key=self.CONNECTOR_LABEL, 217 label_value=connector_name, 218 )
Secret manager that retrieves secrets from Google Secrets Manager (GSM).
This class inherits from CustomSecretManager
and also adds methods
that are specific to this implementation: fetch_secrets()
,
fetch_secrets_by_label()
and fetch_connector_secrets()
.
This secret manager is not enabled by default. To use it, you must provide the project ID and the credentials for a service account with the necessary permissions to access the secrets.
The fetch_connector_secret()
method assumes a label name of connector
matches the name of the connector (source-github
, destination-snowflake
, etc.)
87 def __init__( 88 self, 89 project: str, 90 *, 91 credentials_path: str | None = None, 92 credentials_json: str | SecretString | None = None, 93 auto_register: bool = False, 94 as_backup: bool = False, 95 ) -> None: 96 """Instantiate a new Google GSM secret manager instance. 97 98 You can provide either the path to the credentials file or the JSON contents of the 99 credentials file. If both are provided, a `PyAirbyteInputError` will be raised. 100 """ 101 if credentials_path and credentials_json: 102 raise exc.PyAirbyteInputError( 103 guidance=("You can provide `credentials_path` or `credentials_json` but not both."), 104 ) 105 106 self.project = project 107 108 if credentials_json is not None and not isinstance(credentials_json, SecretString): 109 credentials_json = SecretString(credentials_json) 110 111 if not credentials_json and not credentials_path: 112 if "GOOGLE_APPLICATION_CREDENTIALS" in os.environ: 113 credentials_path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] 114 115 elif "GCP_GSM_CREDENTIALS" in os.environ: 116 credentials_json = SecretString(os.environ["GCP_GSM_CREDENTIALS"]) 117 118 if credentials_path: 119 credentials_json = SecretString(Path(credentials_path).read_text()) 120 121 if not credentials_json: 122 raise exc.PyAirbyteInputError( 123 guidance=( 124 "No Google Cloud credentials found. You can provide the path to the " 125 "credentials file using the `credentials_path` argument, or provide the JSON " 126 "contents of the credentials file using the `credentials_json` argument." 127 ), 128 ) 129 130 self.secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info( 131 json.loads(credentials_json) 132 ) 133 134 if auto_register: 135 self.auto_register = auto_register 136 137 if as_backup: 138 self.as_backup = as_backup 139 140 super().__init__() # Handles the registration if needed
Instantiate a new Google GSM secret manager instance.
You can provide either the path to the credentials file or the JSON contents of the
credentials file. If both are provided, a PyAirbyteInputError
will be raised.
142 def get_secret(self, secret_name: str) -> SecretString | None: 143 """Get a named secret from Google Colab user secrets.""" 144 return SecretString( 145 self.secret_client.access_secret_version( 146 name=f"projects/{self.project}/secrets/{secret_name}/versions/latest" 147 ).payload.data.decode("UTF-8") 148 )
Get a named secret from Google Colab user secrets.
150 def fetch_secrets( 151 self, 152 *, 153 filter_string: str, 154 ) -> Iterable[SecretHandle]: 155 """List all available secrets in the secret manager. 156 157 Example filter strings: 158 - `labels.connector=source-bigquery`: Filter for secrets with the labe 'source-bigquery'. 159 160 Args: 161 filter_string (str): A filter string to apply to the list of secrets, following the 162 format described in the Google Secret Manager documentation: 163 https://cloud.google.com/secret-manager/docs/filtering 164 165 Returns: 166 Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets. 167 """ 168 gsm_secrets: ListSecretsPager = self.secret_client.list_secrets( 169 secretmanager.ListSecretsRequest( 170 request={ 171 "filter": filter_string, 172 } 173 ) 174 ) 175 176 return [ 177 SecretHandle( 178 parent=self, 179 secret_name=secret.name, 180 ) 181 for secret in gsm_secrets 182 ]
List all available secrets in the secret manager.
Example filter strings:
labels.connector=source-bigquery
: Filter for secrets with the labe 'source-bigquery'.
Arguments:
- filter_string (str): A filter string to apply to the list of secrets, following the format described in the Google Secret Manager documentation: https://cloud.google.com/secret-manager/docs/filtering
Returns:
Iterable[SecretHandle]: An iterable of
SecretHandle
objects for the matching secrets.
184 def fetch_secrets_by_label( 185 self, 186 label_key: str, 187 label_value: str, 188 ) -> Iterable[SecretHandle]: 189 """List all available secrets in the secret manager. 190 191 Args: 192 label_key (str): The key of the label to filter by. 193 label_value (str): The value of the label to filter by. 194 195 Returns: 196 Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets. 197 """ 198 return self.fetch_secrets(filter_string=f"labels.{label_key}={label_value}")
List all available secrets in the secret manager.
Arguments:
- label_key (str): The key of the label to filter by.
- label_value (str): The value of the label to filter by.
Returns:
Iterable[SecretHandle]: An iterable of
SecretHandle
objects for the matching secrets.
200 def fetch_connector_secrets( 201 self, 202 connector_name: str, 203 ) -> Iterable[SecretHandle]: 204 """Fetch secrets in the secret manager, using the connector name as a filter for the label. 205 206 The label key used to filter the secrets is defined by the `CONNECTOR_LABEL` attribute, 207 which defaults to 'connector'. 208 209 Args: 210 connector_name (str): The name of the connector to filter by. 211 212 Returns: 213 Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets. 214 """ 215 return self.fetch_secrets_by_label( 216 label_key=self.CONNECTOR_LABEL, 217 label_value=connector_name, 218 )
Fetch secrets in the secret manager, using the connector name as a filter for the label.
The label key used to filter the secrets is defined by the CONNECTOR_LABEL
attribute,
which defaults to 'connector'.
Arguments:
- connector_name (str): The name of the connector to filter by.
Returns:
Iterable[SecretHandle]: An iterable of
SecretHandle
objects for the matching secrets.