diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 9d45d2dc2c2e0..b7b08bdf4d9d9 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -107,6 +107,7 @@ jobs: GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }} GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }} + GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }} GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index cf685295ee0af..1490808c33fe7 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -102,6 +102,7 @@ jobs: GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }} GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }} + GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }} GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json index c15dd270d14b0..5abea4626c530 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "d19ae824-e289-4b14-995a-0632eb46d246", "name": "Google Directory", "dockerRepository": "airbyte/source-google-directory", - "dockerImageTag": "0.1.6", + "dockerImageTag": "0.1.8", "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-directory" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d4f0bfecf1e5b..e3ab0f83124c1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -198,7 +198,7 @@ - name: Google Directory sourceDefinitionId: d19ae824-e289-4b14-995a-0632eb46d246 dockerRepository: airbyte/source-google-directory - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/sources/google-directory sourceType: api - name: Google Search Console diff --git a/airbyte-integrations/connectors/source-google-directory/Dockerfile b/airbyte-integrations/connectors/source-google-directory/Dockerfile index 995b550e28eba..cc6e9f2f1a690 100644 --- a/airbyte-integrations/connectors/source-google-directory/Dockerfile +++ b/airbyte-integrations/connectors/source-google-directory/Dockerfile @@ -34,5 +34,5 @@ COPY source_google_directory ./source_google_directory ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-google-directory diff --git a/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml index fb8a23bcf7aeb..8d3a0596d25d1 100644 --- a/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml @@ -7,16 +7,22 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" + - config_path: "integration_tests/invalid_config_oauth.json" + status: "failed" discovery: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - # API returns different lastLoginTime for some users, eteg is generated on all data and also same time different + # API returns different lastLoginTime for some users, eteg is generated based on all data, so also sometime are different ignored_fields: "users": ["etag", "lastLoginTime"] diff --git a/airbyte-integrations/connectors/source-google-directory/integration_tests/invalid_config_oauth.json b/airbyte-integrations/connectors/source-google-directory/integration_tests/invalid_config_oauth.json new file mode 100644 index 0000000000000..ade2aa38d0116 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-directory/integration_tests/invalid_config_oauth.json @@ -0,0 +1,7 @@ +{ + "credentials": { + "client_id": "", + "client_secret": "", + "refresh_token": "" + } +} diff --git a/airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config.json index 361b1de29e923..229ddbd5e6043 100644 --- a/airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config.json @@ -1,4 +1,6 @@ { - "credentials_json": "", - "email": "test@test.test" + "credentials": { + "credentials_json": "", + "email": "test@test.test" + } } diff --git a/airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config_oauth.json b/airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config_oauth.json new file mode 100644 index 0000000000000..baca9caa3c4c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config_oauth.json @@ -0,0 +1,7 @@ +{ + "credentials": { + "client_id": "", + "client_secret": "", + "refresh_token": "" + } +} diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py index 8083eb3080445..f3a5ba0ab4154 100644 --- a/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py @@ -6,10 +6,12 @@ import json from abc import ABC, abstractmethod from functools import partial -from typing import Callable, Dict, Iterator, Sequence +from typing import Any, Callable, Dict, Iterator, Mapping, Sequence import backoff +from google.auth.transport.requests import Request from google.oauth2 import service_account +from google.oauth2.credentials import Credentials from googleapiclient.discovery import build from googleapiclient.errors import HttpError as GoogleApiHttpError @@ -19,20 +21,41 @@ class API: - def __init__(self, credentials_json: str, email: str): + def __init__(self, credentials: Mapping[str, Any]): self._creds = None - self._credentials_json = credentials_json - self._admin_email = email + self._raw_credentials = credentials self._service = None - def _load_account_info(self) -> Dict: - account_info = json.loads(self._credentials_json) + @staticmethod + def _load_account_info(credentials_json: str) -> Dict: + account_info = json.loads(credentials_json) return account_info - def _obtain_creds(self): - account_info = self._load_account_info() + def _obtain_service_account_creds(self) -> service_account.Credentials: + """Obtaining creds based on Service account scenario""" + credentials_json = self._raw_credentials.get("credentials_json") + admin_email = self._raw_credentials.get("email") + account_info = self._load_account_info(credentials_json) creds = service_account.Credentials.from_service_account_info(account_info, scopes=SCOPES) - self._creds = creds.with_subject(self._admin_email) + self._creds = creds.with_subject(admin_email) + + def _obtain_web_app_creds(self) -> Credentials: + """Obtaining creds based on Web server application scenario""" + info = { + "client_id": self._raw_credentials.get("client_id"), + "client_secret": self._raw_credentials.get("client_secret"), + "refresh_token": self._raw_credentials.get("refresh_token"), + } + creds = Credentials.from_authorized_user_info(info) + if creds.expired: + creds.refresh(Request()) + self._creds = creds + + def _obtain_creds(self): + if "credentials_json" in self._raw_credentials: + self._obtain_service_account_creds() + elif "client_id" and "client_secret" in self._raw_credentials: + self._obtain_web_app_creds() def _construct_resource(self): if not self._creds: diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/client.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/client.py index 32b48f2007053..086e51c1077bb 100644 --- a/airbyte-integrations/connectors/source-google-directory/source_google_directory/client.py +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/client.py @@ -11,8 +11,11 @@ class Client(BaseClient): - def __init__(self, credentials_json: str, email: str): - self._api = API(credentials_json, email) + def __init__(self, credentials: Mapping[str, Any] = None, credentials_json: str = None, email: str = None): + # supporting old config format + if not credentials: + credentials = {"credentials_json": credentials_json, "email": email} + self._api = API(credentials) self._apis = {"users": UsersAPI(self._api), "groups": GroupsAPI(self._api), "group_members": GroupMembersAPI(self._api)} super().__init__() diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json b/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json index c848cdde6dae0..5b59a13256374 100644 --- a/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json @@ -4,18 +4,87 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Google Directory Spec", "type": "object", - "required": ["credentials_json", "email"], - "additionalProperties": false, + "required": [], + "additionalProperties": true, "properties": { - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. See the docs for more information on how to generate this key.", - "airbyte_secret": true - }, - "email": { - "type": "string", - "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs." + "credentials": { + "title": "Google Credentials", + "description": "Google APIs use the OAuth 2.0 protocol for authentication and authorization. The Source supports Web server application and Service accounts scenarios", + "type": "object", + "oneOf": [ + { + "title": "Sign in via Google (Oauth)", + "description": "For these scenario user only needs to give permission to read Google Directory data", + "type": "object", + "required": ["client_id", "client_secret", "refresh_token"], + "properties": { + "credentials_title": { + "type": "string", + "title": "Credentials title", + "description": "Authentication scenario", + "const": "Web server app", + "enum": ["Web server app"], + "default": "Web server app", + "order": 0 + }, + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The client ID of developer application", + "airbyte_secret": true + }, + "client_secret": { + "title": "Client secret", + "type": "string", + "description": "The client secret of developer application", + "airbyte_secret": true + }, + "refresh_token": { + "title": "Refresh Token", + "type": "string", + "description": "The token for obtaining new access token", + "airbyte_secret": true + } + } + }, + { + "title": "Service account Key", + "description": "For these scenario user should obtain service account's credentials from the Google API Console and provide delegated email", + "type": "object", + "required": ["credentials_json", "email"], + "properties": { + "credentials_title": { + "type": "string", + "title": "Credentials title", + "description": "Authentication scenario", + "const": "Service accounts", + "enum": ["Service accounts"], + "default": "Service accounts", + "order": 0 + }, + "credentials_json": { + "type": "string", + "title": "Credentials JSON", + "description": "The contents of the JSON service account key. See the docs for more information on how to generate this key.", + "airbyte_secret": true + }, + "email": { + "type": "string", + "title": "Email", + "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs." + } + } + } + ] } } + }, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", 0], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["refresh_token"]] + } } } diff --git a/docs/integrations/sources/google-directory.md b/docs/integrations/sources/google-directory.md index bb52676d054df..e80608000e364 100644 --- a/docs/integrations/sources/google-directory.md +++ b/docs/integrations/sources/google-directory.md @@ -35,9 +35,19 @@ This Source is capable of syncing the following Streams: This connector attempts to back off gracefully when it hits Directory API's rate limits. To find more information about limits, see [Google Directory's Limits and Quotas](https://developers.google.com/admin-sdk/directory/v1/limits) documentation. -## Getting started +## Getting Started \(Airbyte Cloud\) -### Requirements +1. Click `OAuth2.0 authorization` then `Authenticate your Google Directory account`. +2. You're done. + +## Getting Started \(Airbyte Open-Source\) + +Google APIs use the OAuth 2.0 protocol for authentication and authorization. This connector supports [Web server application](https://developers.google.com/identity/protocols/oauth2#webserver) and [Service accounts](https://developers.google.com/identity/protocols/oauth2#serviceaccount) scenarios. Therefore, there are 2 options of setting up authorization for this source: + +* Use your Google account and authorize over Google's OAuth on connection setup. Select "Default OAuth2.0 authorization" from dropdown list. +* Create service account specifically for Airbyte. + +### Service account requirements * Credentials to a Google Service Account with delegated Domain Wide Authority * Email address of the workspace admin which created the Service Account @@ -58,6 +68,8 @@ You should now be ready to use the Google Directory connector in Airbyte. | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.8 | 2021-11-02 | [7409](https://github.com/airbytehq/airbyte/pull/7409) | Support oauth (update publish) | +| 0.1.7 | 2021-11-02 | [7409](https://github.com/airbytehq/airbyte/pull/7409) | Support oauth | | 0.1.6 | 2021-11-02 | [7464](https://github.com/airbytehq/airbyte/pull/7464) | Migrate to the CDK | | 0.1.5 | 2021-10-20 | [6930](https://github.com/airbytehq/airbyte/pull/6930) | Fix crash when a group don't have members | | 0.1.4 | 2021-10-19 | [7167](https://github.com/airbytehq/airbyte/pull/7167) | Add organizations and phones to `users` schema | diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index bb5e4da501503..6936d2e3a54a3 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -86,6 +86,7 @@ write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json" write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json" write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS" +write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS_OAUTH" "config_oauth.json" write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS" write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json" write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS"