From a1f128543bf64c248e8de30959a88dac5762b8e6 Mon Sep 17 00:00:00 2001 From: alvova Date: Mon, 11 Jul 2022 18:16:13 +0300 Subject: [PATCH 01/12] add oauth --- .../source-okta/source_okta/source.py | 16 ++- .../source-okta/source_okta/spec.json | 126 +++++++++++++++-- .../oauth/OAuthImplementationFactory.java | 1 + .../io/airbyte/oauth/flows/OktaOAuthFlow.java | 132 ++++++++++++++++++ 4 files changed, 260 insertions(+), 15 deletions(-) create mode 100644 airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java diff --git a/airbyte-integrations/connectors/source-okta/source_okta/source.py b/airbyte-integrations/connectors/source-okta/source_okta/source.py index 97c97bec379c..c17b67895ace 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/source.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/source.py @@ -148,11 +148,17 @@ def path(self, **kwargs) -> str: class SourceOkta(AbstractSource): - def initialize_authenticator(self, config: Mapping[str, Any]) -> TokenAuthenticator: - return TokenAuthenticator(config["token"], auth_method="SSWS") - - def get_url_base(self, config: Mapping[str, Any]) -> str: - return parse.urljoin(config["base_url"], "/api/v1/") + @staticmethod + def initialize_authenticator(config: Mapping[str, Any]) -> TokenAuthenticator: + if "token" in config or config["credentials"]["auth_type"] == "api_token": + token = config.get("token") or config["credentials"]["token"] + return TokenAuthenticator(token, auth_method="SSWS") + return TokenAuthenticator(config["credentials"]["access_token"]) + + @staticmethod + def get_url_base(config: Mapping[str, Any]) -> str: + base_url = config.get("base_url") or f"https://{config['domain']}.okta.com" + return parse.urljoin(base_url, "/api/v1/") def check_connection(self, logger, config) -> Tuple[bool, any]: try: diff --git a/airbyte-integrations/connectors/source-okta/source_okta/spec.json b/airbyte-integrations/connectors/source-okta/source_okta/spec.json index 8d83004b1958..f0cfb033bf20 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/spec.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/spec.json @@ -4,20 +4,126 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Okta Spec", "type": "object", - "required": ["token", "base_url"], + "required": [], "additionalProperties": false, "properties": { - "token": { + "domain": { "type": "string", - "title": "API Token", - "description": "A Okta token. See the docs for instructions on how to generate it.", - "airbyte_secret": true + "title": "Okta domain", + "description": "The Okta domain. See the docs for instructions on how to find it.", + "airbyte_secret": false }, - "base_url": { - "type": "string", - "title": "Base URL", - "description": "The Okta base URL.", - "airbyte_secret": true + "credentials": { + "title": "Authorization Method", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": [ + "auth_type", + "client_id", + "client_secret", + "access_token" + ], + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0", + "order": 0 + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The Client ID of your OAuth application.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "The Client Secret of your OAuth application.", + "airbyte_secret": true + }, + "access_token": { + "type": "string", + "title": "Access Token", + "description": "Access Token for making authenticated requests.", + "airbyte_secret": true + } + } + }, + { + "type": "object", + "title": "API Token", + "required": ["auth_type", "api_token"], + "properties": { + "auth_type": { + "type": "string", + "const": "api_token", + "order": 0 + }, + "api_token": { + "type": "string", + "title": "Personal API Token", + "description": "An Okta token. See the docs for instructions on how to generate it.", + "airbyte_secret": true + } + } + } + ] + } + } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": true, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + }, + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "domain": { + "type": "string", + "path_in_connector_config": ["domain"] + } + } } } } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index ed95f433a0bd..b6b7f090ea42 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -61,6 +61,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/destination-snowflake", new DestinationSnowflakeOAuthFlow(configRepository, httpClient)) .put("airbyte/destination-google-sheets", new DestinationGoogleSheetsOAuthFlow(configRepository, httpClient)) .put("airbyte/source-snowflake", new SourceSnowflakeOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-okta", new OktaOAuthFlow(configRepository, httpClient)) .build(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java new file mode 100644 index 000000000000..50f75709c51f --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +/** + * Following docs from + * https://developer.okta.com/docs/guides/implement-grant-type/authcode/main/#request-an-authorization-code + */ +public class OktaOAuthFlow extends BaseOAuth2Flow { + + public OktaOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { + super(configRepository, httpClient); + } + + @VisibleForTesting + public OktaOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(final UUID definitionId, + final String clientId, + final String redirectUrl, + final JsonNode inputOAuthConfiguration) + throws IOException { + + // getting domain value from user's config + final String domain = getConfigValueUnsafe(inputOAuthConfiguration, "domain"); + + final URIBuilder builder = new URIBuilder() + .setScheme("https") + .setHost(domain + ".okta.com") + .setPath("oauth2/default/v1/authorize") + // required + .addParameter("client_id", clientId) + .addParameter("redirect_uri", redirectUrl) + .addParameter("scope", "openid") + .addParameter("response_type", "code") + .addParameter("state", getState()); + + try { + return builder.build().toString(); + } catch (final URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected Map getAccessTokenQueryParameters(String clientId, + String clientSecret, + String authCode, + String redirectUrl) { + return ImmutableMap.builder() + // required + .put("code", authCode) + .put("redirect_uri", redirectUrl) + .put("grant_type", "authorization_code") + .build(); + } + + @Override + protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { + // getting domain value from user's config + final String domain = getConfigValueUnsafe(inputOAuthConfiguration, "domain"); + return "https://" + domain + ".okta.com/oauth2/default/v1/token"; + } + + @Override + protected Map completeOAuthFlow(final String clientId, + final String clientSecret, + final String authCode, + final String redirectUrl, + final JsonNode inputOAuthConfiguration, + final JsonNode oAuthParamConfig) + throws IOException { + final var accessTokenUrl = getAccessTokenUrl(inputOAuthConfiguration); + final byte[] authorization = Base64.getEncoder() + .encode((clientId + ":" + clientSecret).getBytes(StandardCharsets.UTF_8)); + final HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers + .ofString(tokenReqContentType.getConverter().apply( + getAccessTokenQueryParameters(clientId, clientSecret, authCode, redirectUrl)))) + .uri(URI.create(accessTokenUrl)) + .header("Content-Type", tokenReqContentType.getContentType()) + .header("Accept", "application/json") + .header("Authorization", "Basic " + new String(authorization, StandardCharsets.UTF_8)) + .build(); + try { + final HttpResponse response = httpClient.send(request, + HttpResponse.BodyHandlers.ofString()); + + return extractOAuthOutput(Jsons.deserialize(response.body()), accessTokenUrl); + } catch (final InterruptedException e) { + throw new IOException("Failed to complete OAuth flow", e); + } + } + + @Override + protected Map extractOAuthOutput(final JsonNode data, final String accessTokenUrl) throws IOException { + final Map result = new HashMap<>(); + // getting out access_token + if (data.has("access_token")) { + result.put("access_token", data.get("access_token").asText()); + } else { + throw new IOException(String.format("Missing 'access_token' in query params from %s", accessTokenUrl)); + } + return result; + } + +} From a49fcbef8f94328f2adf3bfb1a545376a24e45dc Mon Sep 17 00:00:00 2001 From: alvova Date: Thu, 14 Jul 2022 15:34:43 +0300 Subject: [PATCH 02/12] upd --- .../connectors/source-okta/source_okta/spec.json | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-okta/source_okta/spec.json b/airbyte-integrations/connectors/source-okta/source_okta/spec.json index f0cfb033bf20..be730899634a 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/spec.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/spec.json @@ -5,7 +5,7 @@ "title": "Okta Spec", "type": "object", "required": [], - "additionalProperties": false, + "additionalProperties": true, "properties": { "domain": { "type": "string", @@ -14,18 +14,13 @@ "airbyte_secret": false }, "credentials": { - "title": "Authorization Method", + "title": "Authorization Method *", "type": "object", "oneOf": [ { "type": "object", "title": "OAuth2.0", - "required": [ - "auth_type", - "client_id", - "client_secret", - "access_token" - ], + "required": ["auth_type", "access_token"], "properties": { "auth_type": { "type": "string", @@ -91,7 +86,7 @@ }, "complete_oauth_server_input_specification": { "type": "object", - "additionalProperties": true, + "additionalProperties": false, "properties": { "client_id": { "type": "string" From aa956d3483b4c18adc34ccd77b04176986f1c9c5 Mon Sep 17 00:00:00 2001 From: alvova Date: Thu, 14 Jul 2022 17:00:45 +0300 Subject: [PATCH 03/12] upd --- .../src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java index 50f75709c51f..52574dd34b2b 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java @@ -52,11 +52,11 @@ protected String formatConsentUrl(final UUID definitionId, final URIBuilder builder = new URIBuilder() .setScheme("https") .setHost(domain + ".okta.com") - .setPath("oauth2/default/v1/authorize") + .setPath("oauth2/v1/authorize") // required .addParameter("client_id", clientId) .addParameter("redirect_uri", redirectUrl) - .addParameter("scope", "openid") + .addParameter("scope", "okta.users.read okta.logs.read okta.groups.read") .addParameter("response_type", "code") .addParameter("state", getState()); @@ -84,7 +84,7 @@ protected Map getAccessTokenQueryParameters(String clientId, protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { // getting domain value from user's config final String domain = getConfigValueUnsafe(inputOAuthConfiguration, "domain"); - return "https://" + domain + ".okta.com/oauth2/default/v1/token"; + return "https://" + domain + ".okta.com/oauth2/v1/token"; } @Override From 2bdb0a4a0f3c198ba26f3ae01bd76fe2324134cd Mon Sep 17 00:00:00 2001 From: alvova Date: Thu, 14 Jul 2022 17:01:29 +0300 Subject: [PATCH 04/12] upd --- .../src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java index 52574dd34b2b..2895938f1f64 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java @@ -26,7 +26,7 @@ /** * Following docs from - * https://developer.okta.com/docs/guides/implement-grant-type/authcode/main/#request-an-authorization-code + * https://developer.okta.com/docs/guides/implement-oauth-for-okta/main/ */ public class OktaOAuthFlow extends BaseOAuth2Flow { From 4b8cad91a85b16a80e792b191a0ba73f06644513 Mon Sep 17 00:00:00 2001 From: alvova Date: Thu, 14 Jul 2022 17:25:26 +0300 Subject: [PATCH 05/12] upd doc --- docs/02-integrations/01-sources/okta.md | 39 ++++++++++++------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/docs/02-integrations/01-sources/okta.md b/docs/02-integrations/01-sources/okta.md index 1f3dbc64be9b..1518608591ae 100644 --- a/docs/02-integrations/01-sources/okta.md +++ b/docs/02-integrations/01-sources/okta.md @@ -37,33 +37,30 @@ The connector is restricted by normal Okta [requests limitation](https://develop ### Requirements -* Okta API Token +You can use [OAuth2.0](https://developer.okta.com/docs/guides/implement-grant-type/authcodepkce/main/) +or an [API token](https://developer.okta.com/docs/guides/create-an-api-token/overview/) to authenticate your Okta account. +If you choose to authenticate with OAuth2.0, [register](https://dev-01177082-admin.okta.com/admin/apps/active) your Okta application. ### Setup guide -In order to pull data out of your Okta instance, you need to create an [API Token](https://developer.okta.com/docs/guides/create-an-api-token/overview/). - -:::info +1. Use API token from requirements and Okta [domain](https://developer.okta.com/docs/guides/find-your-domain/-/main/). +2. Go to local Airbyte page. +3. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +4. On the Set up the source page select **Okta** from the Source type dropdown. +5. Paste all data to required fields. +6. Click `Set up source`. +**Note:** Different Okta APIs require different admin privilege levels. API tokens inherit the privilege level of the admin account used to create them -::: - -1. Sign in to your Okta organization as a user with [administrator privileges](https://help.okta.com/en/prod/okta_help_CSH.htm#ext_Security_Administrators) -2. Access the API page: In the Admin Console, select API from the Security menu and then select the Tokens tab. -3. Click Create Token. -4. Name your token and click Create Token. -5. Record the token value. This is the only opportunity to see it and record it. -6. In Airbyte, create a Okta source. -7. You can now pull data from your Okta instance! - ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.4 | 2021-11-02 | [7584](https://github.com/airbytehq/airbyte/pull/7584) | Fix incremental params for log stream | -| 0.1.3 | 2021-09-08 | [5905](https://github.com/airbytehq/airbyte/pull/5905) | Fix incremental stream defect | -| 0.1.2 | 2021-07-01 | [4456](https://github.com/airbytehq/airbyte/pull/4456) | Bugfix infinite pagination in logs stream | -| 0.1.1 | 2021-06-09 | [3937](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` env variable for kubernetes support | -| 0.1.0 | 2021-05-30 | [3563](https://github.com/airbytehq/airbyte/pull/3563) | Initial Release | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:------------------------------------------------------------|:-------------------------------------------------------------| +| 0.1.5 | 2022-07-14 | [14710](https://github.com/airbytehq/airbyte/pull/14710) | Implement OAuth2.0 authorization method | +| 0.1.4 | 2021-11-02 | [7584](https://github.com/airbytehq/airbyte/pull/7584) | Fix incremental params for log stream | +| 0.1.3 | 2021-09-08 | [5905](https://github.com/airbytehq/airbyte/pull/5905) | Fix incremental stream defect | +| 0.1.2 | 2021-07-01 | [4456](https://github.com/airbytehq/airbyte/pull/4456) | Bugfix infinite pagination in logs stream | +| 0.1.1 | 2021-06-09 | [3937](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` env variable for kubernetes support | +| 0.1.0 | 2021-05-30 | [3563](https://github.com/airbytehq/airbyte/pull/3563) | Initial Release | From 00222887557cf2284471faf1c147b86b119a522e Mon Sep 17 00:00:00 2001 From: alvova Date: Thu, 14 Jul 2022 17:25:34 +0300 Subject: [PATCH 06/12] upd version --- airbyte-integrations/connectors/source-okta/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-okta/Dockerfile b/airbyte-integrations/connectors/source-okta/Dockerfile index 50d5d9cb04a5..a53374fe8c0e 100644 --- a/airbyte-integrations/connectors/source-okta/Dockerfile +++ b/airbyte-integrations/connectors/source-okta/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-okta From 1821363fde465b89b09c764ec3fc80aa3c0d2344 Mon Sep 17 00:00:00 2001 From: alvova Date: Fri, 15 Jul 2022 01:35:29 +0300 Subject: [PATCH 07/12] upd --- .../source-okta/acceptance-test-config.yml | 12 +++- .../source-okta/source_okta/schemas/logs.json | 60 ++++------------ .../source-okta/source_okta/source.py | 68 +++++++++++++++---- .../source-okta/source_okta/spec.json | 12 ++-- .../io/airbyte/oauth/flows/OktaOAuthFlow.java | 18 +---- 5 files changed, 87 insertions(+), 83 deletions(-) diff --git a/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml b/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml index 1e60fe03ef3a..fd51a2d2f2d1 100644 --- a/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml @@ -5,6 +5,10 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" + - config_path: "secrets/config_api_token.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: @@ -12,13 +16,17 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["logs"] + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_api_token.json" + configured_catalog_path: "integration_tests/configured_catalog.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + # Test is skipped because requests for Logs stream fails when published is in the future + # future_state_path: "integration_tests/abnormal_state.json" cursor_paths: users: ["lastUpdated"] diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json index 5872b10c06eb..020ade16e657 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json @@ -5,10 +5,7 @@ "alternateId": { "type": ["string", "null"] }, - "detail": { - "additionalProperties": { - "type": ["object", "null"] - }, + "detailEntry": { "type": ["object", "null"] }, "displayName": { @@ -26,42 +23,15 @@ "authenticationContext": { "properties": { "authenticationProvider": { - "enum": [ - "OKTA_AUTHENTICATION_PROVIDER", - "ACTIVE_DIRECTORY", - "LDAP", - "FEDERATION", - "SOCIAL", - "FACTOR_PROVIDER" - ], "type": ["string", "null"] }, "authenticationStep": { - "type": ["integer", "null"] + "type": ["integer"] }, "credentialProvider": { - "enum": [ - "OKTA_AUTHENTICATION_PROVIDER", - "OKTA_CREDENTIAL_PROVIDER", - "RSA", - "SYMANTEC", - "GOOGLE", - "DUO", - "YUBIKEY" - ], "type": ["string", "null"] }, "credentialType": { - "enum": [ - "OTP", - "SMS", - "PASSWORD", - "ASSERTION", - "IWA", - "EMAIL", - "OAUTH2", - "JWT" - ], "type": ["string", "null"] }, "externalSessionId": { @@ -148,9 +118,6 @@ "debugContext": { "properties": { "debugData": { - "additionalProperties": { - "type": ["object", "null"] - }, "type": ["object", "null"] } }, @@ -171,7 +138,16 @@ "type": ["string", "null"] }, "result": { - "type": ["string", "null"] + "enum": [ + "SUCCESS", + "FAILURE", + "SKIPPED", + "ALLOW", + "DENY", + "CHALLENGE", + "UNKNOWN" + ], + "type": "string" } }, "type": ["object", "null"] @@ -222,7 +198,7 @@ "type": ["string", "null"] }, "version": { - "type": "string" + "type": ["string", "null"] } }, "type": ["object", "null"] @@ -263,19 +239,16 @@ "type": ["string", "null"] }, "detailEntry": { - "additionalProperties": { - "type": ["object", "null"] - }, "type": ["object", "null"] }, "displayName": { "type": ["string", "null"] }, "id": { - "type": ["string", "null"] + "type": "string" }, "type": { - "type": ["string", "null"] + "type": "string" } }, "type": ["object", "null"] @@ -285,9 +258,6 @@ "transaction": { "properties": { "detail": { - "additionalProperties": { - "type": ["object", "null"] - }, "type": ["object", "null"] }, "id": { diff --git a/airbyte-integrations/connectors/source-okta/source_okta/source.py b/airbyte-integrations/connectors/source-okta/source_okta/source.py index c17b67895ace..0c9590b3f0c7 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/source.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/source.py @@ -12,7 +12,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator class OktaStream(HttpStream, ABC): @@ -147,24 +147,63 @@ def path(self, **kwargs) -> str: return "users" +class OktaOauth2Authenticator(Oauth2Authenticator): + def get_refresh_request_body(self) -> Mapping[str, Any]: + return { + "grant_type": "refresh_token", + "refresh_token": self.refresh_token, + } + + def refresh_access_token(self) -> Tuple[str, int]: + try: + response = requests.request(method="POST", url=self.token_refresh_endpoint, data=self.get_refresh_request_body(), + auth=(self.client_id, self.client_secret)) + response.raise_for_status() + response_json = response.json() + return response_json["access_token"], response_json["expires_in"] + except Exception as e: + raise Exception(f"Error while refreshing access token: {e}") from e + + class SourceOkta(AbstractSource): - @staticmethod - def initialize_authenticator(config: Mapping[str, Any]) -> TokenAuthenticator: - if "token" in config or config["credentials"]["auth_type"] == "api_token": - token = config.get("token") or config["credentials"]["token"] - return TokenAuthenticator(token, auth_method="SSWS") - return TokenAuthenticator(config["credentials"]["access_token"]) + def initialize_authenticator(self, config: Mapping[str, Any]): + if "token" in config: + return TokenAuthenticator(config["token"], auth_method="SSWS") + + creds = config.get("credentials") + if not creds: + raise "Config validation error. `credentials` not specified." + + auth_type = creds.get("auth_type") + if not auth_type: + raise "Config validation error. `auth_type` not specified." + + if auth_type == "api_token": + return TokenAuthenticator(creds["api_token"], auth_method="SSWS") + + if auth_type == "oauth2.0": + return OktaOauth2Authenticator( + token_refresh_endpoint=self.get_token_refresh_endpoint(config), + client_secret=creds["client_secret"], + client_id=creds["client_id"], + refresh_token=creds["refresh_token"], + ) @staticmethod def get_url_base(config: Mapping[str, Any]) -> str: - base_url = config.get("base_url") or f"https://{config['domain']}.okta.com" - return parse.urljoin(base_url, "/api/v1/") + return config.get("base_url") or f"https://{config['domain']}.okta.com" + + def get_api_endpoint(self, config: Mapping[str, Any]) -> str: + return parse.urljoin(self.get_url_base(config), "/api/v1/") + + def get_token_refresh_endpoint(self, config: Mapping[str, Any]) -> str: + return parse.urljoin(self.get_url_base(config), "/oauth2/v1/token") def check_connection(self, logger, config) -> Tuple[bool, any]: try: auth = self.initialize_authenticator(config) - base_url = self.get_url_base(config) - url = parse.urljoin(base_url, "users") + api_endpoint = self.get_api_endpoint(config) + url = parse.urljoin(api_endpoint, "users") response = requests.get( url, @@ -177,15 +216,16 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return False, response.json() except Exception: - return False, "Failed to authenticate with the provided credentials" + import traceback + return False, traceback.format_exc() def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = self.initialize_authenticator(config) - url_base = self.get_url_base(config) + api_endpoint = self.get_api_endpoint(config) initialization_params = { "authenticator": auth, - "url_base": url_base, + "url_base": api_endpoint, } return [ diff --git a/airbyte-integrations/connectors/source-okta/source_okta/spec.json b/airbyte-integrations/connectors/source-okta/source_okta/spec.json index be730899634a..8253185230bc 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/spec.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/spec.json @@ -20,7 +20,7 @@ { "type": "object", "title": "OAuth2.0", - "required": ["auth_type", "access_token"], + "required": ["auth_type", "client_id", "client_secret", "refresh_token"], "properties": { "auth_type": { "type": "string", @@ -39,10 +39,10 @@ "description": "The Client Secret of your OAuth application.", "airbyte_secret": true }, - "access_token": { + "refresh_token": { "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", + "title": "Refresh Token", + "description": "Refresh Token to obtain new Access Token, when it's expired.", "airbyte_secret": true } } @@ -78,9 +78,9 @@ "type": "object", "additionalProperties": false, "properties": { - "access_token": { + "refresh_token": { "type": "string", - "path_in_connector_config": ["credentials", "access_token"] + "path_in_connector_config": ["credentials", "refresh_token"] } } }, diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java index 2895938f1f64..5b9641d3c9b0 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java @@ -18,15 +18,13 @@ import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.util.Base64; -import java.util.HashMap; import java.util.Map; import java.util.UUID; import java.util.function.Supplier; import org.apache.http.client.utils.URIBuilder; /** - * Following docs from - * https://developer.okta.com/docs/guides/implement-oauth-for-okta/main/ + * Following docs from https://developer.okta.com/docs/guides/implement-oauth-for-okta/main/ */ public class OktaOAuthFlow extends BaseOAuth2Flow { @@ -56,7 +54,7 @@ protected String formatConsentUrl(final UUID definitionId, // required .addParameter("client_id", clientId) .addParameter("redirect_uri", redirectUrl) - .addParameter("scope", "okta.users.read okta.logs.read okta.groups.read") + .addParameter("scope", "okta.users.read okta.logs.read okta.groups.read offline_access") .addParameter("response_type", "code") .addParameter("state", getState()); @@ -117,16 +115,4 @@ protected Map completeOAuthFlow(final String clientId, } } - @Override - protected Map extractOAuthOutput(final JsonNode data, final String accessTokenUrl) throws IOException { - final Map result = new HashMap<>(); - // getting out access_token - if (data.has("access_token")) { - result.put("access_token", data.get("access_token").asText()); - } else { - throw new IOException(String.format("Missing 'access_token' in query params from %s", accessTokenUrl)); - } - return result; - } - } From bd4c77d308ecca52d56d0c285abf3b2abadd2a60 Mon Sep 17 00:00:00 2001 From: alvova Date: Fri, 15 Jul 2022 02:06:25 +0300 Subject: [PATCH 08/12] upd --- .../connectors/source-okta/acceptance-test-config.yml | 5 +---- .../src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml b/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml index fd51a2d2f2d1..9ac6d86acad7 100644 --- a/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml @@ -26,7 +26,4 @@ tests: incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - # Test is skipped because requests for Logs stream fails when published is in the future - # future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - users: ["lastUpdated"] + future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java index 5b9641d3c9b0..a877c682e76a 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/OktaOAuthFlow.java @@ -54,7 +54,7 @@ protected String formatConsentUrl(final UUID definitionId, // required .addParameter("client_id", clientId) .addParameter("redirect_uri", redirectUrl) - .addParameter("scope", "okta.users.read okta.logs.read okta.groups.read offline_access") + .addParameter("scope", "okta.users.read okta.logs.read okta.groups.read okta.roles.read offline_access") .addParameter("response_type", "code") .addParameter("state", getState()); From 4906efe323d2a76a84199a2aab22d3b6afa53a4f Mon Sep 17 00:00:00 2001 From: alvova Date: Fri, 15 Jul 2022 02:12:56 +0300 Subject: [PATCH 09/12] upd logs schema --- .../connectors/source-okta/source_okta/schemas/logs.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json index 24d708fe1220..069d60a6a765 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json @@ -39,7 +39,6 @@ }, "credentialProvider": { "enum": [ - "OKTA_AUTHENTICATION_PROVIDER", "OKTA_CREDENTIAL_PROVIDER", "RSA", "SYMANTEC", @@ -60,6 +59,10 @@ "EMAIL", "OAUTH2", "JWT", + "CERTIFICATE", + "PRE_SHARED_SYMMETRIC_KEY", + "OKTA_CLIENT_SESSION", + "DEVICE_UDID", null ], "type": ["string", "null"] From a6375434f3d1a2617fe65f9929e9e8412de0f36b Mon Sep 17 00:00:00 2001 From: alvova Date: Tue, 19 Jul 2022 19:21:03 +0300 Subject: [PATCH 10/12] remove oauth --- airbyte-integrations/connectors/source-okta/Dockerfile | 2 +- .../connectors/source-okta/acceptance-test-config.yml | 2 -- .../connectors/source-okta/source_okta/source.py | 10 +++++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-okta/Dockerfile b/airbyte-integrations/connectors/source-okta/Dockerfile index ed4dd1de885d..a853344ee684 100644 --- a/airbyte-integrations/connectors/source-okta/Dockerfile +++ b/airbyte-integrations/connectors/source-okta/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-okta diff --git a/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml b/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml index 9ac6d86acad7..8e4c9a87ab84 100644 --- a/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml @@ -16,8 +16,6 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - config_path: "secrets/config_api_token.json" configured_catalog_path: "integration_tests/configured_catalog.json" full_refresh: diff --git a/airbyte-integrations/connectors/source-okta/source_okta/source.py b/airbyte-integrations/connectors/source-okta/source_okta/source.py index eaeb0b687146..bf3602dad259 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/source.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/source.py @@ -13,7 +13,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator class OktaStream(HttpStream, ABC): @@ -241,8 +241,12 @@ def get_refresh_request_body(self) -> Mapping[str, Any]: def refresh_access_token(self) -> Tuple[str, int]: try: - response = requests.request(method="POST", url=self.token_refresh_endpoint, data=self.get_refresh_request_body(), - auth=(self.client_id, self.client_secret)) + response = requests.request( + method="POST", + url=self.token_refresh_endpoint, + data=self.get_refresh_request_body(), + auth=(self.client_id, self.client_secret), + ) response.raise_for_status() response_json = response.json() return response_json["access_token"], response_json["expires_in"] From 88acf1f8039cf1945385cab83ac50aef0614c631 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 19 Jul 2022 16:46:27 +0000 Subject: [PATCH 11/12] auto-bump connector version [ci skip] --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 117 +++++++++++++++--- 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f13da31cbdac..35dd94329d31 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -620,7 +620,7 @@ - name: Okta sourceDefinitionId: 1d4fdb25-64fc-4569-92da-fcdca79a8372 dockerRepository: airbyte/source-okta - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/sources/okta icon: okta.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index d9b9d43b4393..b4db3d4ec3c2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6030,32 +6030,119 @@ - - "client_secret" oauthFlowOutputParameters: - - "access_token" -- dockerImage: "airbyte/source-okta:0.1.7" +- dockerImage: "airbyte/source-okta:0.1.8" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/okta" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" title: "Okta Spec" type: "object" - required: - - "token" - - "base_url" - additionalProperties: false + required: [] + additionalProperties: true properties: - token: - type: "string" - title: "API Token" - description: "A Okta token. See the docs for instructions on how to generate it." - airbyte_secret: true - base_url: + domain: type: "string" - title: "Base URL" - description: "The Okta base URL." - airbyte_secret: true + title: "Okta domain" + description: "The Okta domain. See the docs for instructions on how to find it." + airbyte_secret: false + credentials: + title: "Authorization Method *" + type: "object" + oneOf: + - type: "object" + title: "OAuth2.0" + required: + - "auth_type" + - "client_id" + - "client_secret" + - "refresh_token" + properties: + auth_type: + type: "string" + const: "oauth2.0" + order: 0 + client_id: + type: "string" + title: "Client ID" + description: "The Client ID of your OAuth application." + airbyte_secret: true + client_secret: + type: "string" + title: "Client Secret" + description: "The Client Secret of your OAuth application." + airbyte_secret: true + refresh_token: + type: "string" + title: "Refresh Token" + description: "Refresh Token to obtain new Access Token, when it's\ + \ expired." + airbyte_secret: true + - type: "object" + title: "API Token" + required: + - "auth_type" + - "api_token" + properties: + auth_type: + type: "string" + const: "api_token" + order: 0 + api_token: + type: "string" + title: "Personal API Token" + description: "An Okta token. See the docs for instructions on how to generate it." + airbyte_secret: true supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] + advanced_auth: + auth_flow_type: "oauth2.0" + predicate_key: + - "credentials" + - "auth_type" + predicate_value: "oauth2.0" + oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: "object" + additionalProperties: false + properties: + domain: + type: "string" + path_in_connector_config: + - "domain" + complete_oauth_output_specification: + type: "object" + additionalProperties: false + properties: + refresh_token: + type: "string" + path_in_connector_config: + - "credentials" + - "refresh_token" + complete_oauth_server_input_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + client_secret: + type: "string" + complete_oauth_server_output_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + path_in_connector_config: + - "credentials" + - "client_id" + client_secret: + type: "string" + path_in_connector_config: + - "credentials" + - "client_secret" - dockerImage: "airbyte/source-onesignal:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/onesignal" From fe1ef29041aa21c4f87c8f1889bd8890b576dba4 Mon Sep 17 00:00:00 2001 From: alvova Date: Tue, 19 Jul 2022 20:01:06 +0300 Subject: [PATCH 12/12] format --- .../connectors/source-okta/source_okta/spec.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-okta/source_okta/spec.json b/airbyte-integrations/connectors/source-okta/source_okta/spec.json index 8253185230bc..b258cbdc21af 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/spec.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/spec.json @@ -20,7 +20,12 @@ { "type": "object", "title": "OAuth2.0", - "required": ["auth_type", "client_id", "client_secret", "refresh_token"], + "required": [ + "auth_type", + "client_id", + "client_secret", + "refresh_token" + ], "properties": { "auth_type": { "type": "string",