airbyte.secrets.google_gsm

Secret manager that retrieves secrets from Google Secrets Manager (GSM).

Usage Example:

gsm_secrets_manager = GoogleGSMSecretManager(
    project=AIRBYTE_INTERNAL_GCP_PROJECT,
    credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"),
)
first_secret: SecretHandle = next(
    gsm_secrets_manager.fetch_connector_secrets(
        connector_name=connector_name,
    ),
    None,
)

print(f"Found '{connector_name}' credential secret '${first_secret.secret_name}'.")
return first_secret.get_value().parse_json()

More compact example:

gsm_secrets_manager = GoogleGSMSecretManager(
    project=AIRBYTE_INTERNAL_GCP_PROJECT,
    credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"),
)
connector_config: dict = (
    next(
        gsm_secrets_manager.fetch_connector_secrets(
            connector_name=connector_name,
        ),
        None,
    )
    .get_value()
    .parse_json()
)
  1# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
  2"""Secret manager that retrieves secrets from Google Secrets Manager (GSM).
  3
  4Usage Example:
  5
  6```python
  7gsm_secrets_manager = GoogleGSMSecretManager(
  8    project=AIRBYTE_INTERNAL_GCP_PROJECT,
  9    credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"),
 10)
 11first_secret: SecretHandle = next(
 12    gsm_secrets_manager.fetch_connector_secrets(
 13        connector_name=connector_name,
 14    ),
 15    None,
 16)
 17
 18print(f"Found '{connector_name}' credential secret '${first_secret.secret_name}'.")
 19return first_secret.get_value().parse_json()
 20```
 21
 22More compact example:
 23
 24```python
 25gsm_secrets_manager = GoogleGSMSecretManager(
 26    project=AIRBYTE_INTERNAL_GCP_PROJECT,
 27    credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"),
 28)
 29connector_config: dict = (
 30    next(
 31        gsm_secrets_manager.fetch_connector_secrets(
 32            connector_name=connector_name,
 33        ),
 34        None,
 35    )
 36    .get_value()
 37    .parse_json()
 38)
 39```
 40"""
 41
 42from __future__ import annotations
 43
 44import json
 45import os
 46from pathlib import Path
 47from typing import TYPE_CHECKING
 48
 49from google.cloud import secretmanager_v1 as secretmanager
 50
 51from airbyte import exceptions as exc
 52from airbyte.secrets.base import SecretHandle, SecretSourceEnum, SecretString
 53from airbyte.secrets.custom import CustomSecretManager
 54
 55
 56if TYPE_CHECKING:
 57    from collections.abc import Iterable
 58
 59    from google.cloud.secretmanager_v1.services.secret_manager_service.pagers import (
 60        ListSecretsPager,
 61    )
 62
 63
 64class GoogleGSMSecretManager(CustomSecretManager):
 65    """Secret manager that retrieves secrets from Google Secrets Manager (GSM).
 66
 67    This class inherits from `CustomSecretManager` and also adds methods
 68    that are specific to this implementation: `fetch_secrets()`,
 69    `fetch_secrets_by_label()` and `fetch_connector_secrets()`.
 70
 71    This secret manager is not enabled by default. To use it, you must provide the project ID and
 72    the credentials for a service account with the necessary permissions to access the secrets.
 73
 74    The `fetch_connector_secret()` method assumes a label name of `connector`
 75    matches the name of the connector (`source-github`, `destination-snowflake`, etc.)
 76    """
 77
 78    name = SecretSourceEnum.GOOGLE_GSM.value
 79    auto_register = False
 80    as_backup = False
 81    replace_existing = False
 82
 83    CONNECTOR_LABEL = "connector"
 84    """The label key used to filter secrets by connector name."""
 85
 86    def __init__(
 87        self,
 88        project: str,
 89        *,
 90        credentials_path: str | None = None,
 91        credentials_json: str | SecretString | None = None,
 92        auto_register: bool = False,
 93        as_backup: bool = False,
 94    ) -> None:
 95        """Instantiate a new Google GSM secret manager instance.
 96
 97        You can provide either the path to the credentials file or the JSON contents of the
 98        credentials file. If both are provided, a `PyAirbyteInputError` will be raised.
 99        """
100        if credentials_path and credentials_json:
101            raise exc.PyAirbyteInputError(
102                guidance=("You can provide `credentials_path` or `credentials_json` but not both."),
103            )
104
105        self.project = project
106
107        if credentials_json is not None and not isinstance(credentials_json, SecretString):
108            credentials_json = SecretString(credentials_json)
109
110        if not credentials_json and not credentials_path:
111            if "GOOGLE_APPLICATION_CREDENTIALS" in os.environ:
112                credentials_path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
113
114            elif "GCP_GSM_CREDENTIALS" in os.environ:
115                credentials_json = SecretString(os.environ["GCP_GSM_CREDENTIALS"])
116
117        if credentials_path:
118            credentials_json = SecretString(Path(credentials_path).read_text())
119
120        if not credentials_json:
121            raise exc.PyAirbyteInputError(
122                guidance=(
123                    "No Google Cloud credentials found. You can provide the path to the "
124                    "credentials file using the `credentials_path` argument, or provide the JSON "
125                    "contents of the credentials file using the `credentials_json` argument."
126                ),
127            )
128
129        self.secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info(
130            json.loads(credentials_json)
131        )
132
133        if auto_register:
134            self.auto_register = auto_register
135
136        if as_backup:
137            self.as_backup = as_backup
138
139        super().__init__()  # Handles the registration if needed
140
141    def get_secret(self, secret_name: str) -> SecretString | None:
142        """Get a named secret from Google Colab user secrets."""
143        return SecretString(
144            self.secret_client.access_secret_version(
145                name=f"projects/{self.project}/secrets/{secret_name}/versions/latest"
146            ).payload.data.decode("UTF-8")
147        )
148
149    def fetch_secrets(
150        self,
151        *,
152        filter_string: str,
153    ) -> Iterable[SecretHandle]:
154        """List all available secrets in the secret manager.
155
156        Example filter strings:
157        - `labels.connector=source-bigquery`: Filter for secrets with the labe 'source-bigquery'.
158
159        Args:
160            filter_string (str): A filter string to apply to the list of secrets, following the
161                format described in the Google Secret Manager documentation:
162                https://cloud.google.com/secret-manager/docs/filtering
163
164        Returns:
165            Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets.
166        """
167        gsm_secrets: ListSecretsPager = self.secret_client.list_secrets(
168            secretmanager.ListSecretsRequest(
169                request={
170                    "filter": filter_string,
171                }
172            )
173        )
174
175        return [
176            SecretHandle(
177                parent=self,
178                secret_name=secret.name,
179            )
180            for secret in gsm_secrets
181        ]
182
183    def fetch_secrets_by_label(
184        self,
185        label_key: str,
186        label_value: str,
187    ) -> Iterable[SecretHandle]:
188        """List all available secrets in the secret manager.
189
190        Args:
191            label_key (str): The key of the label to filter by.
192            label_value (str): The value of the label to filter by.
193
194        Returns:
195            Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets.
196        """
197        return self.fetch_secrets(filter_string=f"labels.{label_key}={label_value}")
198
199    def fetch_connector_secrets(
200        self,
201        connector_name: str,
202    ) -> Iterable[SecretHandle]:
203        """Fetch secrets in the secret manager, using the connector name as a filter for the label.
204
205        The label key used to filter the secrets is defined by the `CONNECTOR_LABEL` attribute,
206        which defaults to 'connector'.
207
208        Args:
209            connector_name (str): The name of the connector to filter by.
210
211        Returns:
212            Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets.
213        """
214        return self.fetch_secrets_by_label(
215            label_key=self.CONNECTOR_LABEL,
216            label_value=connector_name,
217        )
class GoogleGSMSecretManager(airbyte.secrets.custom.CustomSecretManager):
 65class GoogleGSMSecretManager(CustomSecretManager):
 66    """Secret manager that retrieves secrets from Google Secrets Manager (GSM).
 67
 68    This class inherits from `CustomSecretManager` and also adds methods
 69    that are specific to this implementation: `fetch_secrets()`,
 70    `fetch_secrets_by_label()` and `fetch_connector_secrets()`.
 71
 72    This secret manager is not enabled by default. To use it, you must provide the project ID and
 73    the credentials for a service account with the necessary permissions to access the secrets.
 74
 75    The `fetch_connector_secret()` method assumes a label name of `connector`
 76    matches the name of the connector (`source-github`, `destination-snowflake`, etc.)
 77    """
 78
 79    name = SecretSourceEnum.GOOGLE_GSM.value
 80    auto_register = False
 81    as_backup = False
 82    replace_existing = False
 83
 84    CONNECTOR_LABEL = "connector"
 85    """The label key used to filter secrets by connector name."""
 86
 87    def __init__(
 88        self,
 89        project: str,
 90        *,
 91        credentials_path: str | None = None,
 92        credentials_json: str | SecretString | None = None,
 93        auto_register: bool = False,
 94        as_backup: bool = False,
 95    ) -> None:
 96        """Instantiate a new Google GSM secret manager instance.
 97
 98        You can provide either the path to the credentials file or the JSON contents of the
 99        credentials file. If both are provided, a `PyAirbyteInputError` will be raised.
100        """
101        if credentials_path and credentials_json:
102            raise exc.PyAirbyteInputError(
103                guidance=("You can provide `credentials_path` or `credentials_json` but not both."),
104            )
105
106        self.project = project
107
108        if credentials_json is not None and not isinstance(credentials_json, SecretString):
109            credentials_json = SecretString(credentials_json)
110
111        if not credentials_json and not credentials_path:
112            if "GOOGLE_APPLICATION_CREDENTIALS" in os.environ:
113                credentials_path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
114
115            elif "GCP_GSM_CREDENTIALS" in os.environ:
116                credentials_json = SecretString(os.environ["GCP_GSM_CREDENTIALS"])
117
118        if credentials_path:
119            credentials_json = SecretString(Path(credentials_path).read_text())
120
121        if not credentials_json:
122            raise exc.PyAirbyteInputError(
123                guidance=(
124                    "No Google Cloud credentials found. You can provide the path to the "
125                    "credentials file using the `credentials_path` argument, or provide the JSON "
126                    "contents of the credentials file using the `credentials_json` argument."
127                ),
128            )
129
130        self.secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info(
131            json.loads(credentials_json)
132        )
133
134        if auto_register:
135            self.auto_register = auto_register
136
137        if as_backup:
138            self.as_backup = as_backup
139
140        super().__init__()  # Handles the registration if needed
141
142    def get_secret(self, secret_name: str) -> SecretString | None:
143        """Get a named secret from Google Colab user secrets."""
144        return SecretString(
145            self.secret_client.access_secret_version(
146                name=f"projects/{self.project}/secrets/{secret_name}/versions/latest"
147            ).payload.data.decode("UTF-8")
148        )
149
150    def fetch_secrets(
151        self,
152        *,
153        filter_string: str,
154    ) -> Iterable[SecretHandle]:
155        """List all available secrets in the secret manager.
156
157        Example filter strings:
158        - `labels.connector=source-bigquery`: Filter for secrets with the labe 'source-bigquery'.
159
160        Args:
161            filter_string (str): A filter string to apply to the list of secrets, following the
162                format described in the Google Secret Manager documentation:
163                https://cloud.google.com/secret-manager/docs/filtering
164
165        Returns:
166            Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets.
167        """
168        gsm_secrets: ListSecretsPager = self.secret_client.list_secrets(
169            secretmanager.ListSecretsRequest(
170                request={
171                    "filter": filter_string,
172                }
173            )
174        )
175
176        return [
177            SecretHandle(
178                parent=self,
179                secret_name=secret.name,
180            )
181            for secret in gsm_secrets
182        ]
183
184    def fetch_secrets_by_label(
185        self,
186        label_key: str,
187        label_value: str,
188    ) -> Iterable[SecretHandle]:
189        """List all available secrets in the secret manager.
190
191        Args:
192            label_key (str): The key of the label to filter by.
193            label_value (str): The value of the label to filter by.
194
195        Returns:
196            Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets.
197        """
198        return self.fetch_secrets(filter_string=f"labels.{label_key}={label_value}")
199
200    def fetch_connector_secrets(
201        self,
202        connector_name: str,
203    ) -> Iterable[SecretHandle]:
204        """Fetch secrets in the secret manager, using the connector name as a filter for the label.
205
206        The label key used to filter the secrets is defined by the `CONNECTOR_LABEL` attribute,
207        which defaults to 'connector'.
208
209        Args:
210            connector_name (str): The name of the connector to filter by.
211
212        Returns:
213            Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets.
214        """
215        return self.fetch_secrets_by_label(
216            label_key=self.CONNECTOR_LABEL,
217            label_value=connector_name,
218        )

Secret manager that retrieves secrets from Google Secrets Manager (GSM).

This class inherits from CustomSecretManager and also adds methods that are specific to this implementation: fetch_secrets(), fetch_secrets_by_label() and fetch_connector_secrets().

This secret manager is not enabled by default. To use it, you must provide the project ID and the credentials for a service account with the necessary permissions to access the secrets.

The fetch_connector_secret() method assumes a label name of connector matches the name of the connector (source-github, destination-snowflake, etc.)

GoogleGSMSecretManager( project: str, *, credentials_path: str | None = None, credentials_json: str | airbyte.secrets.base.SecretString | None = None, auto_register: bool = False, as_backup: bool = False)
 87    def __init__(
 88        self,
 89        project: str,
 90        *,
 91        credentials_path: str | None = None,
 92        credentials_json: str | SecretString | None = None,
 93        auto_register: bool = False,
 94        as_backup: bool = False,
 95    ) -> None:
 96        """Instantiate a new Google GSM secret manager instance.
 97
 98        You can provide either the path to the credentials file or the JSON contents of the
 99        credentials file. If both are provided, a `PyAirbyteInputError` will be raised.
100        """
101        if credentials_path and credentials_json:
102            raise exc.PyAirbyteInputError(
103                guidance=("You can provide `credentials_path` or `credentials_json` but not both."),
104            )
105
106        self.project = project
107
108        if credentials_json is not None and not isinstance(credentials_json, SecretString):
109            credentials_json = SecretString(credentials_json)
110
111        if not credentials_json and not credentials_path:
112            if "GOOGLE_APPLICATION_CREDENTIALS" in os.environ:
113                credentials_path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
114
115            elif "GCP_GSM_CREDENTIALS" in os.environ:
116                credentials_json = SecretString(os.environ["GCP_GSM_CREDENTIALS"])
117
118        if credentials_path:
119            credentials_json = SecretString(Path(credentials_path).read_text())
120
121        if not credentials_json:
122            raise exc.PyAirbyteInputError(
123                guidance=(
124                    "No Google Cloud credentials found. You can provide the path to the "
125                    "credentials file using the `credentials_path` argument, or provide the JSON "
126                    "contents of the credentials file using the `credentials_json` argument."
127                ),
128            )
129
130        self.secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info(
131            json.loads(credentials_json)
132        )
133
134        if auto_register:
135            self.auto_register = auto_register
136
137        if as_backup:
138            self.as_backup = as_backup
139
140        super().__init__()  # Handles the registration if needed

Instantiate a new Google GSM secret manager instance.

You can provide either the path to the credentials file or the JSON contents of the credentials file. If both are provided, a PyAirbyteInputError will be raised.

name = 'google_gsm'
auto_register = False
as_backup = False
replace_existing = False
CONNECTOR_LABEL = 'connector'

The label key used to filter secrets by connector name.

project
secret_client
def get_secret(self, secret_name: str) -> airbyte.secrets.base.SecretString | None:
142    def get_secret(self, secret_name: str) -> SecretString | None:
143        """Get a named secret from Google Colab user secrets."""
144        return SecretString(
145            self.secret_client.access_secret_version(
146                name=f"projects/{self.project}/secrets/{secret_name}/versions/latest"
147            ).payload.data.decode("UTF-8")
148        )

Get a named secret from Google Colab user secrets.

def fetch_secrets( self, *, filter_string: str) -> collections.abc.Iterable[airbyte.secrets.base.SecretHandle]:
150    def fetch_secrets(
151        self,
152        *,
153        filter_string: str,
154    ) -> Iterable[SecretHandle]:
155        """List all available secrets in the secret manager.
156
157        Example filter strings:
158        - `labels.connector=source-bigquery`: Filter for secrets with the labe 'source-bigquery'.
159
160        Args:
161            filter_string (str): A filter string to apply to the list of secrets, following the
162                format described in the Google Secret Manager documentation:
163                https://cloud.google.com/secret-manager/docs/filtering
164
165        Returns:
166            Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets.
167        """
168        gsm_secrets: ListSecretsPager = self.secret_client.list_secrets(
169            secretmanager.ListSecretsRequest(
170                request={
171                    "filter": filter_string,
172                }
173            )
174        )
175
176        return [
177            SecretHandle(
178                parent=self,
179                secret_name=secret.name,
180            )
181            for secret in gsm_secrets
182        ]

List all available secrets in the secret manager.

Example filter strings:

  • labels.connector=source-bigquery: Filter for secrets with the labe 'source-bigquery'.
Arguments:
Returns:

Iterable[SecretHandle]: An iterable of SecretHandle objects for the matching secrets.

def fetch_secrets_by_label( self, label_key: str, label_value: str) -> collections.abc.Iterable[airbyte.secrets.base.SecretHandle]:
184    def fetch_secrets_by_label(
185        self,
186        label_key: str,
187        label_value: str,
188    ) -> Iterable[SecretHandle]:
189        """List all available secrets in the secret manager.
190
191        Args:
192            label_key (str): The key of the label to filter by.
193            label_value (str): The value of the label to filter by.
194
195        Returns:
196            Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets.
197        """
198        return self.fetch_secrets(filter_string=f"labels.{label_key}={label_value}")

List all available secrets in the secret manager.

Arguments:
  • label_key (str): The key of the label to filter by.
  • label_value (str): The value of the label to filter by.
Returns:

Iterable[SecretHandle]: An iterable of SecretHandle objects for the matching secrets.

def fetch_connector_secrets( self, connector_name: str) -> collections.abc.Iterable[airbyte.secrets.base.SecretHandle]:
200    def fetch_connector_secrets(
201        self,
202        connector_name: str,
203    ) -> Iterable[SecretHandle]:
204        """Fetch secrets in the secret manager, using the connector name as a filter for the label.
205
206        The label key used to filter the secrets is defined by the `CONNECTOR_LABEL` attribute,
207        which defaults to 'connector'.
208
209        Args:
210            connector_name (str): The name of the connector to filter by.
211
212        Returns:
213            Iterable[SecretHandle]: An iterable of `SecretHandle` objects for the matching secrets.
214        """
215        return self.fetch_secrets_by_label(
216            label_key=self.CONNECTOR_LABEL,
217            label_value=connector_name,
218        )

Fetch secrets in the secret manager, using the connector name as a filter for the label.

The label key used to filter the secrets is defined by the CONNECTOR_LABEL attribute, which defaults to 'connector'.

Arguments:
  • connector_name (str): The name of the connector to filter by.
Returns:

Iterable[SecretHandle]: An iterable of SecretHandle objects for the matching secrets.