diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index e62268e8f26f2e..35b495d33b47fb 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -121,6 +121,7 @@ jobs: HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH }} INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }} INTERCOM_INTEGRATION_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_TEST_CREDS }} + INTERCOM_INTEGRATION_OAUTH_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_OAUTH_TEST_CREDS }} ITERABLE_INTEGRATION_TEST_CREDS: ${{ secrets.ITERABLE_INTEGRATION_TEST_CREDS }} JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }} KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 339f72c80b1c59..976b2fdfc9e75d 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -116,6 +116,7 @@ jobs: HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH }} INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }} INTERCOM_INTEGRATION_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_TEST_CREDS }} + INTERCOM_INTEGRATION_OAUTH_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_OAUTH_TEST_CREDS }} ITERABLE_INTEGRATION_TEST_CREDS: ${{ secrets.ITERABLE_INTEGRATION_TEST_CREDS }} JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }} KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json index aee79b998133c3..ba703f0fd4aede 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a", "name": "Intercom", "dockerRepository": "airbyte/source-intercom", - "dockerImageTag": "0.1.6", + "dockerImageTag": "0.1.7", "documentationUrl": "https://docs.airbyte.io/integrations/sources/intercom", "icon": "intercom.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d9e2e6636e465f..a29552821529d6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -261,7 +261,7 @@ - name: Intercom sourceDefinitionId: d8313939-3782-41b0-be29-b3ca20d8dd3a dockerRepository: airbyte/source-intercom - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/sources/intercom icon: intercom.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 17e979bdd903ec..17bdf9d69e87f9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1132,7 +1132,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.22" +- dockerImage: "airbyte/source-facebook-marketing:0.2.24" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" @@ -2535,7 +2535,7 @@ oauthFlowInitParameters: [] oauthFlowOutputParameters: - - "access_token" -- dockerImage: "airbyte/source-intercom:0.1.6" +- dockerImage: "airbyte/source-intercom:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/intercom" connectionSpecification: @@ -4968,7 +4968,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-salesforce:0.1.3" +- dockerImage: "airbyte/source-salesforce:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/salesforce" connectionSpecification: @@ -5000,9 +5000,9 @@ airbyte_secret: true start_date: description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated. Priority for filtering\ - \ by `updated` fields, and only then by `created` fields if they are available\ - \ for stream." + \ data before this date will not be replicated. This field uses the \"\ + updated\" field if available, otherwise the \"created\" fields if they\ + \ are available for a stream." type: "string" pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" examples: diff --git a/airbyte-integrations/connectors/source-intercom/Dockerfile b/airbyte-integrations/connectors/source-intercom/Dockerfile index 84043639949d87..5a7317b571aec7 100644 --- a/airbyte-integrations/connectors/source-intercom/Dockerfile +++ b/airbyte-integrations/connectors/source-intercom/Dockerfile @@ -35,5 +35,5 @@ COPY source_intercom ./source_intercom ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-intercom diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 966dd38fb1a423..c8f0769e64b713 100755 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -309,9 +309,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: - AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") - config["start_date"] = datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%SZ").timestamp() + AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") auth = TokenAuthenticator(token=config["access_token"]) return [ diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json b/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json index a233d03a9084a0..ad04c0891be4dc 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json @@ -4,20 +4,29 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Source Intercom Spec", "type": "object", - "required": ["access_token", "start_date"], - "additionalProperties": false, + "required": ["start_date", "access_token"], + "additionalProperties": true, "properties": { - "access_token": { - "type": "string", - "description": "Intercom Access Token. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - }, "start_date": { "type": "string", "description": "The date from which you'd like to replicate data for Intercom API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", "examples": ["2020-11-16T00:00:00Z"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "access_token": { + "title": "Access Token", + "type": "string", + "description": "Access token generated either from an oauth flow or from the Intercom Developer dashboard. See the docs for more information on how to obtain this key manually.", + "airbyte_secret": true } } + }, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": [], + "oauthFlowInitParameters": [], + "oauthFlowOutputParameters": [["access_token"]] + } } } diff --git a/airbyte-oauth/build.gradle b/airbyte-oauth/build.gradle index a5b74ae07a9533..f24a6af55042f3 100644 --- a/airbyte-oauth/build.gradle +++ b/airbyte-oauth/build.gradle @@ -8,4 +8,6 @@ dependencies { implementation project(':airbyte-config:persistence') implementation project(':airbyte-json-validation') testImplementation project(':airbyte-oauth') + + implementation group: 'org.seleniumhq.selenium', name: 'selenium-java', version: '3.141.59' } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 3db545c1685dcb..7c613dfcd646ae 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -11,6 +11,7 @@ import io.airbyte.oauth.flows.AsanaOAuthFlow; import io.airbyte.oauth.flows.GithubOAuthFlow; import io.airbyte.oauth.flows.HubspotOAuthFlow; +import io.airbyte.oauth.flows.IntercomOAuthFlow; import io.airbyte.oauth.flows.SalesforceOAuthFlow; import io.airbyte.oauth.flows.SlackOAuthFlow; import io.airbyte.oauth.flows.SurveymonkeyOAuthFlow; @@ -41,6 +42,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/source-google-search-console", new GoogleSearchConsoleOAuthFlow(configRepository, httpClient)) .put("airbyte/source-google-sheets", new GoogleSheetsOAuthFlow(configRepository, httpClient)) .put("airbyte/source-hubspot", new HubspotOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-intercom", new IntercomOAuthFlow(configRepository, httpClient)) .put("airbyte/source-instagram", new InstagramOAuthFlow(configRepository, httpClient)) .put("airbyte/source-salesforce", new SalesforceOAuthFlow(configRepository, httpClient)) .put("airbyte/source-slack", new SlackOAuthFlow(configRepository, httpClient)) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/IntercomOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/IntercomOAuthFlow.java new file mode 100644 index 00000000000000..25d0daa66316be --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/IntercomOAuthFlow.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +public class IntercomOAuthFlow extends BaseOAuth2Flow { + + private static final String AUTHORIZE_URL = "https://app.intercom.com/a/oauth/connect"; + private static final String ACCESS_TOKEN_URL = "https://api.intercom.io/auth/eagle/token"; + + public IntercomOAuthFlow(ConfigRepository configRepository, HttpClient httpClient) { + super(configRepository, httpClient); + } + + @VisibleForTesting + public IntercomOAuthFlow(ConfigRepository configRepository, final HttpClient httpClient, Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(UUID definitionId, String clientId, String redirectUrl) throws IOException { + try { + return new URIBuilder(AUTHORIZE_URL) + .addParameter("client_id", clientId) + .addParameter("redirect_uri", redirectUrl) + .addParameter("response_type", "code") + .addParameter("state", getState()) + .build().toString(); + } catch (URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected String getAccessTokenUrl() { + return ACCESS_TOKEN_URL; + } + + @Override + protected Map extractOAuthOutput(final JsonNode data, final String accessTokenUrl) { + // Intercom does not have refresh token but calls it "long lived access token" instead: + // see https://developers.intercom.com/building-apps/docs/setting-up-oauth + Preconditions.checkArgument(data.has("access_token"), "Missing 'access_token' in query params from %s", ACCESS_TOKEN_URL); + return Map.of("access_token", data.get("access_token").asText()); + } + + @Override + protected List getDefaultOAuthOutputPath() { + return List.of(); + } + +} diff --git a/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/IntercomOAuthFlowIntegrationTest.java b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/IntercomOAuthFlowIntegrationTest.java new file mode 100644 index 00000000000000..25fa792047e44a --- /dev/null +++ b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/IntercomOAuthFlowIntegrationTest.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.OAuthFlowImplementation; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.net.http.HttpClient; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class IntercomOAuthFlowIntegrationTest extends OAuthFlowIntegrationTest { + + protected static final Path CREDENTIALS_PATH = Path.of("secrets/intercom.json"); + protected static final String REDIRECT_URL = "http://localhost:8000/code"; + protected static final int SERVER_LISTENING_PORT = 8000; + + @Override + protected Path getCredentialsPath() { + return CREDENTIALS_PATH; + } + + @Override + protected OAuthFlowImplementation getFlowImplementation(ConfigRepository configRepository, HttpClient httpClient) { + return new IntercomOAuthFlow(configRepository, httpClient); + } + + @Override + protected int getServerListeningPort() { + return SERVER_LISTENING_PORT; + } + + @BeforeEach + public void setup() throws IOException { + super.setup(); + } + + @Test + public void testFullIntercomOAuthFlow() throws InterruptedException, ConfigNotFoundException, IOException, JsonValidationException { + int limit = 20; + final UUID workspaceId = UUID.randomUUID(); + final UUID definitionId = UUID.randomUUID(); + final String fullConfigAsString = new String(Files.readAllBytes(CREDENTIALS_PATH)); + final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode( + Map.of("authorization", + ImmutableMap.builder() + .put("client_id", credentialsJson.get("client_id").asText()) + .put("client_secret", credentialsJson.get("client_secret").asText()) + .build()))))); + + final String url = flow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + LOGGER.info("Waiting for user consent at: {}", url); + + // TODO: To automate, start a selenium job to navigate to the Consent URL and click on allowing + // access... + while (!serverHandler.isSucceeded() && limit > 0) { + Thread.sleep(1000); + limit -= 1; + } + assertTrue(serverHandler.isSucceeded(), "Failed to get User consent on time"); + final Map params = flow.completeSourceOAuth(workspaceId, definitionId, + Map.of("code", serverHandler.getParamValue()), REDIRECT_URL); + LOGGER.info("Response from completing OAuth Flow is: {}", params.toString()); + assertTrue(params.containsKey("access_token")); + assertTrue(params.get("access_token").toString().length() > 0); + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/IntercomOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/IntercomOAuthFlowTest.java new file mode 100644 index 00000000000000..8a109cd21f74d5 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/IntercomOAuthFlowTest.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class IntercomOAuthFlowTest { + + private UUID workspaceId; + private UUID definitionId; + private IntercomOAuthFlow intercomoAuthFlow; + private HttpClient httpClient; + + private static final String REDIRECT_URL = "https://airbyte.io"; + + private static String getConstantState() { + return "state"; + } + + @BeforeEach + public void setup() throws IOException, JsonValidationException { + workspaceId = UUID.randomUUID(); + definitionId = UUID.randomUUID(); + ConfigRepository configRepository = mock(ConfigRepository.class); + httpClient = mock(HttpClient.class); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode( + ImmutableMap.builder() + .put("client_id", "test_client_id") + .put("client_secret", "test_client_secret") + .build())))); + intercomoAuthFlow = new IntercomOAuthFlow(configRepository, httpClient, IntercomOAuthFlowTest::getConstantState); + + } + + @Test + public void testGetSourceConcentUrl() throws IOException, ConfigNotFoundException { + final String concentUrl = + intercomoAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + assertEquals(concentUrl, + "https://app.intercom.com/a/oauth/connect?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state"); + } + + @Test + public void testCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + + Map returnedCredentials = Map.of("access_token", "refresh_token_response"); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = + intercomoAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); + } + +} diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md index 6d1b556aa389b1..1eb1554ae570db 100644 --- a/docs/integrations/sources/intercom.md +++ b/docs/integrations/sources/intercom.md @@ -55,6 +55,7 @@ Please read [How to get your Access Token](https://developers.intercom.com/build | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.8 | 2021-09-28 | [7060](https://github.com/airbytehq/airbyte/pull/7060) | Added oauth support | | 0.1.6 | 2021-10-07 | [6879](https://github.com/airbytehq/airbyte/pull/6879) | Corrected pagination for contacts | | 0.1.5 | 2021-09-28 | [6082](https://github.com/airbytehq/airbyte/pull/6082) | Corrected android\_last\_seen\_at field data type in schemas | | 0.1.4 | 2021-09-20 | [6087](https://github.com/airbytehq/airbyte/pull/6087) | Corrected updated\_at field data type in schemas | diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 9ef3a287208ab2..95741a3bf8fe96 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -101,6 +101,7 @@ write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" write_standard_creds source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" write_standard_creds source-intercom "$INTERCOM_INTEGRATION_TEST_CREDS" +write_standard_creds source-intercom "$INTERCOM_INTEGRATION_OAUTH_TEST_CREDS" "config_apikey.json" write_standard_creds source-iterable "$ITERABLE_INTEGRATION_TEST_CREDS" write_standard_creds source-jira "$JIRA_INTEGRATION_TEST_CREDS" write_standard_creds source-klaviyo "$KLAVIYO_TEST_CREDS"